def update(self, att, val):
"""
Update a specifed column value for this Scene object
with a new value
:param att: column to update
:param val: new value
:return: updated value from self
"""
sql = 'update ordering_scene set %s = %s where id = %s'
log_sql = ''
try:
with db_instance() as db:
log_sql = db.cursor.mogrify(sql,
(db_extns.AsIs(att), val, self.id))
logger.info("\n*** Updating scene: \n" + log_sql + '\n***\n"')
db.execute(sql, (db_extns.AsIs(att), val, self.id))
db.commit()
except DBConnectException as e:
raise SceneException('Error updating scene: {}\nSQL: {}'.format(
e.message, log_sql))
self.__setattr__(att, val)
return self.__getattribute__(att)
def delete(self, orderid, filename=None):
"""
Removes an order from physical online cache disk
:param filename: file to delete inside of an order
:param orderid: associated order to delete
"""
if not self.exists(orderid, filename):
msg = 'Invalid orderid {} or filename {}'.format(orderid, filename)
logger.critical(msg)
return False
if filename:
path = os.path.join(self.orderpath, orderid, filename)
else:
path = os.path.join(self.orderpath, orderid)
# this should be the dir where the order is held
logger.info('Deleting {} from online cache'.format(path))
# TODO: if storage system supports immutable options
# >>> sudo chattr -fR -i {0};rm -rf {0}
# However, nfs does not support this extended attributes
try:
cmd = 'chmod -R 744 {0};rm -rf {0}'.format(path)
self.execute_command(cmd)
except OnlineCacheException as exc:
logger.critical('Failed to remove files from output cache. '
'Command: {} Error: {}'.format(cmd, exc))
return False
return True
def post(version):
try:
user = flask.g.user
order = request.get_json(force=True)
if not order:
message = {"status": 400, "msg": "Unable to parse json data. Please ensure your order follows json "
"conventions and your http call is correct. If you believe this "
"message is in error please email customer service"}
else:
try:
order = lowercase_all(order)
orderid = espa.place_order(order, user)
if isinstance(orderid, str) and "@" in orderid:
# if order submission was successful, orderid is returned as a string
# which includes the submitters email address
message = {"status": 200, "orderid": orderid}
else:
# there was a problem, and orderid is a dict with the problem details
logger.info("problem with user submitted order. user: {0}\n details: {1}".format(user.username, orderid))
message = {"status": 400, "message": orderid}
except Exception as e:
logger.debug("exception posting order: {0}\nuser: {1}\n msg: {2}".format(order, user.username, e.message))
message = {"status": 500, "msg": "the system experienced an exception. the admins have been notified"}
except BadRequest as e:
# request.get_json throws a BadRequest
logger.debug("BadRequest, could not parse request into json {}\nuser: {}\nform data {}\n".format(e.description, user.username, request.form))
message = {"status": 400, "msg": "Could not parse the request into json"}
except Exception as e:
logger.debug("ERR posting order. user: {0}\n error: {1}".format(user.username, e))
message = {"status": 500, "msg": "the system has experienced an exception. the admins have been notified."}
response = jsonify(message)
response.status_code = message['status']
return response
def update(self, att, val):
"""
Update a specified column value for this Order object
:param att: column to update
:param val: new value
:return: updated value from self
"""
sql = 'update ordering_order set %s = %s where id = %s'
log_sql = ''
try:
with db_instance() as db:
log_sql = db.cursor.mogrify(sql, (db_extns.AsIs(att),
val, self.id))
logger.info(log_sql)
db.execute(sql, (db_extns.AsIs(att), val, self.id))
db.commit()
except DBConnectException as e:
logger.debug('Error updating order: {}\nSQL: {}'
.format(e.message, log_sql))
self.__setattr__(att, val)
return self.__getattribute__(att)
def __init__(self, *args, **kwargs):
super(LTASoapService, self).__init__(*args, **kwargs)
if config.mode in ('dev', 'tst'):
logger.info('Building SoapClient for:{0}'.format(self.url))
self.client = SoapClient(self.url,
location=self.location,
cache=self.build_object_cache())
def delete(self, orderid, filename=None):
"""
Removes an order from physical online cache disk
:param filename: file to delete inside of an order
:param orderid: associated order to delete
"""
if not self.exists(orderid, filename):
msg = 'Invalid orderid {} or filename {}'.format(orderid, filename)
logger.critical(msg)
return False
if filename:
path = os.path.join(self.orderpath, orderid, filename)
else:
path = os.path.join(self.orderpath, orderid)
# this should be the dir where the order is held
logger.info('Deleting {} from online cache'.format(path))
try:
self.execute_command(
'sudo chattr -fR -i {0};rm -rf {0}'.format(path))
except OnlineCacheException:
# in the event /lustre is mounted to an NFS system
logger.info(
"onlinecache delete, chattr error, attempting chmod instead..."
)
self.execute_command('chmod -R 644 {0};rm -rf {0}'.format(path))
return True
def update(self, att, val):
"""
Update a specified column value for this Order object
:param att: column to update
:param val: new value
:return: updated value from self
"""
sql = 'update ordering_order set %s = %s where id = %s'
log_sql = ''
try:
with db_instance() as db:
log_sql = db.cursor.mogrify(sql,
(db_extns.AsIs(att), val, self.id))
logger.info(log_sql)
db.execute(sql, (db_extns.AsIs(att), val, self.id))
db.commit()
except DBConnectException as e:
logger.debug('Error updating order: {}\nSQL: {}'.format(
e.message, log_sql))
self.__setattr__(att, val)
return self.__getattribute__(att)
def where(cls, params):
"""
Query for particular users
:param params: dictionary of column: value parameters
:return: list of matching User objects
"""
if not isinstance(params, dict):
raise UserException(
'Where arguments must be passed as a dictionary')
sql, values = format_sql_params(cls.base_sql, params)
ret = []
log_sql = ''
try:
with db_instance() as db:
log_sql = db.cursor.mogrify(sql, values)
logger.info('user.py where sql: {}'.format(log_sql))
db.select(sql, values)
for i in db:
obj = User(i["username"], i["email"], i["first_name"],
i["last_name"], i["contactid"])
ret.append(obj)
except DBConnectException as e:
logger.debug('Error querying for users: {}\n'
'sql: {}'.format(e.message, log_sql))
raise UserException(e)
return ret
def verify_scenes(self, product_ids, dataset):
"""
Check if supplied IDs successfully map to M2M entity IDs
:param product_ids: Landsat Collection IDs ['LC08_..', ...]
:type product_ids: list
:return: dict
"""
entity_ids = self.id_lookup(product_ids,
dataset) # gather M2M entity ids
# M2M entity ids do not correlate to product availability though. And product download url
# requests will Fail a request if any of the products requested are not actually
# available (as of M2M api version 1.4.1). So we need to make a downloadOptions request
# to the M2M Api to ensure said product is actually available
availability = self.download_available(
entity_ids, dataset) # dict of {entityid:availability, ...}
response = dict()
for k, v in entity_ids.items():
if v:
response[k] = availability[v]
else:
logger.info("id: {} is not available".format(k))
response[k] = False
return response
def update(self, att, val):
"""
Update a specifed column value for this Scene object
with a new value
:param att: column to update
:param val: new value
:return: updated value from self
"""
sql = 'update ordering_scene set %s = %s where id = %s'
log_sql = ''
try:
with db_instance() as db:
log_sql = db.cursor.mogrify(sql, (db_extns.AsIs(att),
val, self.id))
logger.info("\n*** Updating scene: \n" + log_sql + '\n***\n"')
db.execute(sql, (db_extns.AsIs(att), val, self.id))
db.commit()
except DBConnectException as e:
raise SceneException('Error updating scene: {}\nSQL: {}'
.format(e.message, log_sql))
self.__setattr__(att, val)
return self.__getattribute__(att)
def where(cls, params):
"""
Query for a particular row in the ordering_scene table
:param params: dictionary of column: value parameter to select on
:return: list of matching Scene objects
"""
if not isinstance(params, dict):
raise SceneException('Where arguments must be '
'passed as a dictionary')
sql, values = format_sql_params(cls.base_sql, params)
ret = []
log_sql = ''
try:
with db_instance() as db:
log_sql = db.cursor.mogrify(sql, values)
logger.info('scene.py where sql: {}'.format(log_sql))
db.select(sql, values)
for i in db:
sd = dict(i)
obj = Scene(**sd)
ret.append(obj)
except DBConnectException as e:
logger.critical('Error retrieving scenes: {}\n'
'sql: {}'.format(e.message, log_sql))
raise SceneException(e)
return ret
def verify_user(username, password):
try:
# usernames with spaces are valid in EE, though they can't be used for cache keys
cache_key = '{}-credentials'.format(username.replace(' ', '_espa_cred_insert_'))
cache_entry = cache.get(cache_key)
if cache_entry:
# Need to be encrypted?
if cache_entry['password'] == password:
user_entry = cache_entry['user_entry']
# User may have changed their password while it was still cached
else:
user_entry = User.get(username, password)
else:
user_entry = User.get(username, password)
cache_entry = {'password': password,
'user_entry': user_entry}
cache.set(cache_key, cache_entry, 7200)
user = User(*user_entry)
flask.g.user = user # Replace usage with cached version
except Exception:
logger.info('Invalid login attempt, username: {}'.format(username))
return False
return True
def where(cls, params):
"""
Query for a particular row in the ordering_oder table
:param params: dictionary of column: value parameter to select on
:return: list of matching Order objects
"""
if not isinstance(params, dict):
raise OrderException('Where arguments must be '
'passed as a dictionary')
sql, values = format_sql_params(cls.base_sql, params)
ret = []
log_sql = ''
try:
with db_instance() as db:
log_sql = db.cursor.mogrify(sql, values)
logger.info('order.py where sql: {}'.format(log_sql))
db.select(sql, values)
for i in db:
od = dict(i)
obj = Order(**od)
ret.append(obj)
except DBConnectException as e:
logger.debug('Error order where: {}\n'
'sql: {}'.format(e.message, log_sql))
raise OrderException(e)
return ret
def check_LTA(prod_ls):
not_avail = nlaps.products_are_nlaps(prod_ls)
if not_avail:
raise InventoryException(not_avail)
logger.info('@@ VERIFY: FETCH via OrderWrapperService')
results = lta.verify_scenes(prod_ls)
logger.info('@@ VERIFY: COMPLETE via OrderWrapperService')
return results
def check_dmid(prod_ls, contactid=None):
# find all the submitted products that are nlaps and reject them
not_avail = nlaps.products_are_nlaps(prod_ls)
if not_avail:
raise InventoryException(not_avail)
logger.info('@@ VERIFY: FETCH via MACHINE-TO-MACHINE')
token = inventory.get_cached_session()
results = inventory.get_cached_verify_scenes(token, prod_ls)
logger.info('@@ VERIFY: COMPLETE via MACHINE-TO-MACHINE')
return results
def verify_user(username, password):
if (username is None) or (not (str(username).strip())):
logger.warning('Invalid username supplied: %s', username)
flask.g.error_reason = 'auth'
return False
try:
# usernames with spaces are valid in EE, though they can't be used for cache keys
cache_key = '{}-credentials'.format(
username.replace(' ', '_espa_cred_insert_'))
cache_entry = cache.get(cache_key)
if cache_entry:
# Need to be encrypted?
if cache_entry['password'] == password:
user_entry = cache_entry['user_entry']
# User may have changed their password while it was still cached
else:
user_entry = User.get(username, password)
else:
user_entry = User.get(username, password)
cache_entry = {'password': password, 'user_entry': user_entry}
cache.set(cache_key, cache_entry, 7200)
user = User(*user_entry)
flask.g.user = user # Replace usage with cached version
except UserException as e:
logger.info('Invalid login attempt, username: {}, {}'.format(
username, e))
flask.g.error_reason = 'unknown'
return False
except ERSApiAuthFailedException as e:
logger.info('Invalid login attempt, username: {}, {}'.format(
username, e))
flask.g.error_reason = 'auth'
return False
except ERSApiErrorException as e:
logger.info('ERS lookup failed, username: {}, {}'.format(username, e))
flask.g.error_reason = 'unknown'
return False
except ERSApiConnectionException as e:
logger.info('ERS is down {}'.format(e))
flask.g.error_reason = 'conn'
return False
except DBConnectException as e:
logger.critical('! Database reported a problem: {}'.format(e))
flask.g.error_reasons = 'db'
return False
except Exception:
logger.info('Invalid login attempt, username: {}'.format(username))
flask.g.error_reason = 'unknown'
return False
return True
def get(cls, col_name, scene_name, orderid):
"""
Retrieve a value for a particular column based on
the long name of the order
:param col_name: column value to retrieve
:param scene_name: scene/collection id
:param orderid: long name for the related order,
[email protected]
:return: column value
"""
sql = ('select %s '
'from ordering_scene '
'join ordering_order '
'on ordering_order.id = ordering_scene.order_id '
'where ordering_scene.name = %s '
'and ordering_order.orderid = %s')
if '.' in col_name:
_, col = col_name.split('.')
else:
col = col_name
log_sql = ''
try:
with db_instance() as db:
log_sql = (db.cursor.
mogrify(sql, (db_extns.AsIs(col_name),
scene_name, orderid)))
logger.info(log_sql)
db.select(sql, (db_extns.AsIs(col_name),
scene_name, orderid))
ret = db[0][col]
except DBConnectException as e:
logger.debug('Error scene get\n'
'msg: {0}\n'
'sql: {1}'.format(e.message, log_sql))
raise SceneException(e.message)
except KeyError as e:
logger.debug('Scene.get returned no results\n'
'sql: {}'.format(log_sql))
raise SceneException('Key Error: {}'
.format(e.message))
return ret
def get(cls, col_name, scene_name, orderid):
"""
Retrieve a value for a particular column based on
the long name of the order
:param col_name: column value to retrieve
:param scene_name: scene/collection id
:param orderid: long name for the related order,
[email protected]
:return: column value
"""
sql = ('select %s '
'from ordering_scene '
'join ordering_order '
'on ordering_order.id = ordering_scene.order_id '
'where ordering_scene.name = %s '
'and ordering_order.orderid = %s')
if '.' in col_name:
_, col = col_name.split('.')
else:
col = col_name
log_sql = ''
try:
with db_instance() as db:
log_sql = (db.cursor.
mogrify(sql, (db_extns.AsIs(col_name),
scene_name, orderid)))
logger.info(log_sql)
db.select(sql, (db_extns.AsIs(col_name),
scene_name, orderid))
ret = db[0][col]
except DBConnectException as e:
logger.debug('Error scene get\n'
'msg: {0}\n'
'sql: {1}'.format(e.message, log_sql))
raise SceneException(e.message)
except KeyError as e:
logger.debug('Scene.get returned no results\n'
'sql: {}'.format(log_sql))
raise SceneException('Key Error: {}'
.format(e.message))
return ret
def save(self):
"""
Upsert self to the database
"""
sql = ('INSERT INTO ordering_order %s VALUES %s '
'ON CONFLICT (orderid) '
'DO UPDATE '
'SET %s = %s')
attr_tup = ('orderid', 'status', 'order_source',
'product_options', 'product_opts', 'order_type',
'initial_email_sent', 'completion_email_sent',
'note', 'completion_date', 'order_date', 'user_id',
'ee_order_id', 'email', 'priority')
vals = tuple(self.__getattribute__(v)
if v != 'product_opts'
else json.dumps(self.__getattribute__(v))
for v in attr_tup)
cols = '({})'.format(','.join(attr_tup))
log_sql = ''
try:
with db_instance() as db:
log_sql = db.cursor.mogrify(sql, (db_extns.AsIs(cols),
vals,
db_extns.AsIs(cols),
vals))
db.execute(sql, (db_extns.AsIs(cols), vals,
db_extns.AsIs(cols), vals))
db.commit()
logger.info('Saved updates to order id: {}\n'
'order.id: {}\nsql: {}\nargs: {}'
.format(self.orderid, self.id, log_sql,
zip(attr_tup, vals)))
except DBConnectException as e:
logger.debug('Error saving order: {}\nsql: {}'
.format(e.message, log_sql))
raise OrderException(e)
new = Order.find(self.id)
for att in attr_tup:
self.__setattr__(att, new.__getattribute__(att))
def create(cls, params):
"""
Create a new scene entry in the ordering_scene table
Also supports a bulk insert for large sets of scenes to insert
dict{'name': ,
'order_id': ,
'status': ,
'sensor_type': ,
'ee_unit_id': }
:param params: dictionary representation of a scene to insert
into the system or a list of dictionary objects
"""
if isinstance(params, (list, tuple)):
template = ','.join(['%s'] * len(params))
args = [(s['name'], s['order_id'],
s['status'], s['sensor_type'],
s['ee_unit_id'], '', '', '', '', '')
for s in params]
else:
template = '%s'
args = [(params['name'], params['order_id'],
params['status'], params['sensor_type'],
params['ee_unit_id'], '', '', '', '', '')]
sql = ('INSERT INTO ordering_scene '
'(name, order_id, status, sensor_type, ee_unit_id, '
'product_distro_location, product_dload_url, '
'cksum_distro_location, cksum_download_url, '
'processing_location) VALUES {}'.format(template))
log_sql = ''
try:
with db_instance() as db:
log_sql = db.cursor.mogrify(sql, args)
logger.info('scene creation sql: {}'
.format(log_sql))
db.execute(sql, args)
db.commit()
except DBConnectException as e:
logger.debug('error creating new scene(s): {}\n'
'sql: {}\n'
.format(e.message, log_sql))
raise SceneException(e.message)
def save(self):
"""
Upsert self to the database
"""
sql = ('INSERT INTO ordering_order %s VALUES %s '
'ON CONFLICT (orderid) '
'DO UPDATE '
'SET %s = %s')
attr_tup = ('orderid', 'status', 'order_source', 'product_options',
'product_opts', 'order_type', 'initial_email_sent',
'completion_email_sent', 'note', 'completion_date',
'order_date', 'user_id', 'ee_order_id', 'email',
'priority')
vals = tuple(
self.__getattribute__(v) if v != 'product_opts' else json.
dumps(self.__getattribute__(v)) for v in attr_tup)
cols = '({})'.format(','.join(attr_tup))
log_sql = ''
try:
with db_instance() as db:
log_sql = db.cursor.mogrify(
sql,
(db_extns.AsIs(cols), vals, db_extns.AsIs(cols), vals))
db.execute(
sql,
(db_extns.AsIs(cols), vals, db_extns.AsIs(cols), vals))
db.commit()
logger.info('Saved updates to order id: {}\n'
'order.id: {}\nsql: {}\nargs: {}'.format(
self.orderid, self.id, log_sql,
zip(attr_tup, vals)))
except DBConnectException as e:
logger.debug('Error saving order: {}\nsql: {}'.format(
e.message, log_sql))
raise OrderException(e)
new = Order.find(self.id)
for att in attr_tup:
self.__setattr__(att, new.__getattribute__(att))
def create(cls, params):
"""
Create a new scene entry in the ordering_scene table
Also supports a bulk insert for large sets of scenes to insert
dict{'name': ,
'order_id': ,
'status': ,
'sensor_type': ,
'ee_unit_id': }
:param params: dictionary representation of a scene to insert
into the system or a list of dictionary objects
"""
if isinstance(params, (list, tuple)):
template = ','.join(['%s'] * len(params))
args = [(s['name'], s['order_id'],
s['status'], s['sensor_type'],
s['ee_unit_id'], '', '', '', '', '')
for s in params]
else:
template = '%s'
args = [(params['name'], params['order_id'],
params['status'], params['sensor_type'],
params['ee_unit_id'], '', '', '', '', '')]
sql = ('INSERT INTO ordering_scene '
'(name, order_id, status, sensor_type, ee_unit_id, '
'product_distro_location, product_dload_url, '
'cksum_distro_location, cksum_download_url, '
'processing_location) VALUES {}'.format(template))
log_sql = ''
try:
with db_instance() as db:
log_sql = db.cursor.mogrify(sql, args)
logger.info('scene creation sql: {}'
.format(log_sql))
db.execute(sql, args)
db.commit()
except DBConnectException as e:
logger.debug('error creating new scene(s): {}\n'
'sql: {}\n'
.format(e.message, log_sql))
raise SceneException(e.message)
def place_order(self, order, user):
"""Enters a new order into the system.
Args:
:keyword order (api.domain.order.Order): The order to be entered into the system
Returns:
Order: The generated order
Raises:
api.api_exceptions.ValidationException: Error occurred validating params
api.api_exceptions.InventoryException: Items were not found/unavailable
"""
try:
# perform validation, raises ValidationException
order = self.validation.validate(order, user.username)
# perform check on open scene limit
self.ordering.check_open_scenes(order=order,
user_id=user.id,
filters={
'status':
('submitted', 'oncache',
'onorder', 'queued',
'processing')
})
# performs inventory check, raises InventoryException
self.inventory.check(order, user.contactid)
# track metrics
self.metrics.collect(order)
# capture the order
response = self.ordering.place_order(order, user)
except (InventoryException, ValidationException,
InventoryConnectionException, OpenSceneLimitException) as e:
logger.info('Bad order submission: User %s Order %s\nexception %s',
user.username, order, traceback.format_exc())
raise
except:
logger.critical(
"ERR version1 place_order arg: %s: %s\n"
"exception %s", user.username, order, traceback.format_exc())
response = default_error_message
return response
def cancel_order(self, orderid, request_ip_address):
"""
Cancels an order, and all scenes contained within it
:return:
"""
order = Order.where({'id': orderid})
if len(order) != 1:
raise OrderingProviderException('Order not found')
else:
order = order.pop()
logger.info('Received request to cancel {} from {}'.format(
orderid, request_ip_address))
killable_scene_states = ('submitted', 'oncache', 'onorder', 'queued',
'retry', 'error', 'unavailable', 'complete')
scenes = order.scenes(sql_dict={'status': killable_scene_states})
if len(scenes) > 0:
Scene.bulk_update([s.id for s in scenes], Scene.cancel_opts())
else:
logger.info('No scenes to cancel for order {}'.format(
orderid, request_ip_address))
order.status = 'cancelled'
order.save()
logger.info('Request to cancel {} from {} successful.'.format(
orderid, request_ip_address))
return order
def save(self):
"""
Save the current configuration of the scene object to the DB
"""
sql = 'UPDATE ordering_scene SET %s = %s WHERE id = %s'
attr_tup = ('status', 'cksum_download_url', 'log_file_contents',
'processing_location', 'retry_after', 'job_name',
'note', 'retry_count', 'sensor_type',
'product_dload_url', 'tram_order_id',
'completion_date', 'ee_unit_id', 'retry_limit',
'cksum_distro_location', 'product_distro_location',
'reported_orphan', 'orphaned', 'failed_lta_status_update',
'download_size')
vals = tuple(self.__getattribute__(v) for v in attr_tup)
cols = '({})'.format(','.join(attr_tup))
log_sql = ''
try:
with db_instance() as db:
log_sql = db.cursor.mogrify(sql, (db_extns.AsIs(cols),
vals, self.id))
db.execute(sql, (db_extns.AsIs(cols), vals, self.id))
db.commit()
logger.info('\n*** Saved updates to scene id: {}, name:{}\n'
'sql: {}\n args: {}\n***'
.format(self.id, self.name,
log_sql, zip(attr_tup, vals)))
except DBConnectException as e:
logger.debug("Error saving scene: {}\n"
"sql: {}".format(e.message, log_sql))
raise SceneException(e)
new = Scene.where({'id': self.id})[0]
for att in attr_tup:
self.__setattr__(att, new.__getattribute__(att))
def delete(self, orderid, filename=None):
"""
Removes an order from physical online cache disk
:param filename: file to delete inside of an order
:param orderid: associated order to delete
"""
if filename:
path = os.path.join(self.orderpath, orderid, filename)
else:
path = os.path.join(self.orderpath, orderid)
# this should be the dir where the order is held
logger.info('Deleting {} from online cache'.format(path))
try:
self.execute_command('sudo chattr -fR -i {0};rm -rf {0}'.format(path))
except OnlineCacheException:
# in the event /lustre is mounted to an NFS system
logger.info("onlinecache delete, chattr error, attempting chmod instead...")
self.execute_command('chmod -R 644 {0};rm -rf {0}'.format(path))
return True
def place_order(self, order, user):
"""Enters a new order into the system.
Args:
:keyword order (api.domain.order.Order): The order to be entered into the system
Returns:
str: The generated order id
Raises:
api.api_exceptions.ValidationException: Error occurred validating params
api.api_exceptions.InventoryException: Items were not found/unavailable
"""
try:
# perform validation, raises ValidationException
order = self.validation.validate(order, user.username)
# performs inventory check, raises InventoryException
self.inventory.check(order)
# track metrics
self.metrics.collect(order)
# capture the order
response = self.ordering.place_order(order, user)
except ValidationException as e:
logger.info('Invalid order received: {0}\nresponse {1}'.format(
order, e.response))
# Need to format the string repr of the exception for end user consumption
response = e.response
except InventoryException as e:
logger.info(
'Requested inputs not available: {0}\nresponse {1}'.format(
order, e.response))
response = e.response
except:
logger.debug(
"ERR version1 place_order arg: {0}\nexception {1}".format(
order, traceback.format_exc()))
response = default_error_message
return response
def save(self):
"""
Save the current configuration of the scene object to the DB
"""
sql = 'UPDATE ordering_scene SET %s = %s WHERE id = %s'
attr_tup = ('status', 'cksum_download_url', 'log_file_contents',
'processing_location', 'retry_after', 'job_name', 'note',
'retry_count', 'sensor_type', 'product_dload_url',
'tram_order_id', 'completion_date', 'ee_unit_id',
'retry_limit', 'cksum_distro_location',
'product_distro_location', 'reported_orphan', 'orphaned',
'failed_lta_status_update', 'download_size',
'status_modified')
vals = tuple(self.__getattribute__(v) for v in attr_tup)
cols = '({})'.format(','.join(attr_tup))
log_sql = ''
try:
with db_instance() as db:
log_sql = db.cursor.mogrify(
sql, (db_extns.AsIs(cols), vals, self.id))
db.execute(sql, (db_extns.AsIs(cols), vals, self.id))
db.commit()
logger.info('\n*** Saved updates to scene id: {}, name:{}\n'
'sql: {}\n args: {}\n***'.format(
self.id, self.name, log_sql,
zip(attr_tup, vals)))
except DBConnectException as e:
logger.critical("Error saving scene: {}\n"
"sql: {}".format(e.message, log_sql))
raise SceneException(e)
new = Scene.where({'id': self.id})[0]
for att in attr_tup:
self.__setattr__(att, new.__getattribute__(att))
def user_email(self):
"""
Retrieve the email address associated with this order
:return: email address
"""
sql = 'select email from auth_user where id = %s'
ret = None
log_sql = ''
try:
with db_instance() as db:
log_sql = db.cursor.mogrify(sql, (self.user_id, ))
logger.info('order.py user_email: {}'.format(log_sql))
db.select(sql, self.user_id)
ret = db[0]['email']
except DBConnectException as e:
logger.debug('Error retrieving user_email: {}'.format(log_sql))
raise OrderException(e)
return ret
def bulk_update(cls, ids=None, updates=None):
"""
Update a list of scenes with
:param ids: ids of scenes to update
:param updates: attributes to update
:return: True
"""
if not isinstance(ids, (list, tuple)):
raise TypeError('Scene.bulk_update ids should be a list')
if not isinstance(updates, dict):
raise TypeError('Scene.bulk_update updates should be a dict')
sql = 'UPDATE ordering_scene SET %s = %s WHERE id in %s'
fields = '({})'.format(','.join(updates.keys()))
vals = tuple(updates.values())
ids = tuple(ids)
if ",)" in sql:
sql = sql.replace(",)", ")")
log_sql = ''
try:
with db_instance() as db:
log_sql = db.cursor.mogrify(sql,
(db_extns.AsIs(fields), vals, ids))
logger.info('\n*** Bulk Updating scenes: \n' + log_sql +
"\n\***\n")
db.execute(sql, (db_extns.AsIs(fields), vals, ids))
db.commit()
except DBConnectException as e:
logger.critical('Error scene bulk_update: {}\nSQL: {}'.format(
e.message, log_sql))
raise SceneException(e)
return True
def bulk_update(cls, ids=None, updates=None):
"""
Update a list of scenes with
:param ids: ids of scenes to update
:param updates: attributes to update
:return: True
"""
if not isinstance(ids, (list, tuple)):
raise TypeError('Scene.bulk_update ids should be a list')
if not isinstance(updates, dict):
raise TypeError('Scene.bulk_update updates should be a dict')
sql = 'UPDATE ordering_scene SET %s = %s WHERE id in %s'
fields = '({})'.format(','.join(updates.keys()))
vals = tuple(updates.values())
ids = tuple(ids)
if ",)" in sql:
sql = sql.replace(",)", ")")
log_sql = ''
try:
with db_instance() as db:
log_sql = db.cursor.mogrify(sql, (db_extns.AsIs(fields),
vals, ids))
logger.info('\n*** Bulk Updating scenes: \n' + log_sql + "\n\***\n")
db.execute(sql, (db_extns.AsIs(fields), vals, ids))
db.commit()
except DBConnectException as e:
logger.debug('Error scene bulk_update: {}\nSQL: {}'
.format(e.message, log_sql))
raise SceneException(e)
return True
def user_email(self):
"""
Retrieve the email address associated with this order
:return: email address
"""
sql = 'select email from auth_user where id = %s'
ret = None
log_sql = ''
try:
with db_instance() as db:
log_sql = db.cursor.mogrify(sql, (self.user_id,))
logger.info('order.py user_email: {}'.format(log_sql))
db.select(sql, self.user_id)
ret = db[0]['email']
except DBConnectException as e:
logger.debug('Error retrieving user_email: {}'
.format(log_sql))
raise OrderException(e)
return ret
def create(cls, params):
"""
Place a new order into the system
:param params: dict of required parameters to be used
params = {'product_opts': {dictionary object of the order received}
'orderid': id generated from generate_order_id
'user_id': EE user id
'order_type': typically 'level2_ondemand'
'status': 'ordered'
'note': user notes
'ee_order_id': earth explorer order id, or ''
'order_source': 'espa' or 'ee'
'order_date': date time string
'priority': legacy item, should be 'normal'
'email': user's contact email
'product_options': legacy column}
:return: order object
"""
opts = params['product_opts']
params['product_opts'] = json.dumps(params['product_opts'])
sql = ('INSERT INTO ordering_order '
'(orderid, user_id, order_type, status, note, '
'product_opts, ee_order_id, order_source, order_date, '
'priority, email, product_options) '
'VALUES (%(orderid)s, %(user_id)s, %(order_type)s, '
'%(status)s, %(note)s, %(product_opts)s, '
'%(ee_order_id)s, %(order_source)s, %(order_date)s, '
'%(priority)s, %(email)s, %(product_options)s)')
logger.info('Order creation parameters: {}'.format(params))
log_sql = ''
try:
with db_instance() as db:
log_sql = db.cursor.mogrify(sql, params)
logger.info('New order complete SQL: {}'
.format(log_sql))
db.execute(sql, params)
db.commit()
except DBConnectException as e:
logger.debug('Error creating new order: {}\n'
'sql: {}'.format(e.message, log_sql))
raise OrderException(e)
order = Order.find(params['orderid'])
# Let the load_ee_order method handle the scene injection
# as there is special logic for interacting with LTA
if params['ee_order_id']:
return order
sensor_keys = sensor.SensorCONST.instances.keys()
bulk_ls = []
for key in opts:
if key in sensor_keys:
sensor_type = ''
item1 = opts[key]['inputs'][0]
if isinstance(sensor.instance(item1), sensor.Landsat):
sensor_type = 'landsat'
elif isinstance(sensor.instance(item1), sensor.Modis):
sensor_type = 'modis'
for s in opts[key]['inputs']:
scene_dict = {'name': s,
'sensor_type': sensor_type,
'order_id': order.id,
'status': 'submitted',
'ee_unit_id': None}
bulk_ls.append(scene_dict)
if 'plot_statistics' in opts and opts['plot_statistics']:
scene_dict = {'name': 'plot',
'sensor_type': 'plot',
'order_id': order.id,
'status': 'submitted',
'ee_unit_id': None}
bulk_ls.append(scene_dict)
try:
Scene.create(bulk_ls)
except SceneException as e:
logger.debug('Order creation failed on scene injection, '
'order: {}\nexception: {}'
.format(order.orderid, e.message))
with db_instance() as db:
db.execute('delete ordering_order where id = %s',
order.id)
db.commit()
raise OrderException(e)
return order
def __init__(self, *args, **kwargs):
super(LTASoapService, self).__init__(*args, **kwargs)
logger.info("Building SoapClient for:{0}".format(self.url))
self.client = SoapClient(self.url, location=self.location, cache=self.build_object_cache())
def create(cls, params):
"""
Place a new order into the system
:param params: dict of required parameters to be used
params = {'product_opts': {dictionary object of the order received}
'orderid': id generated from generate_order_id
'user_id': EE user id
'order_type': typically 'level2_ondemand'
'status': 'ordered'
'note': user notes
'ee_order_id': earth explorer order id, or ''
'order_source': 'espa' or 'ee'
'order_date': date time string
'priority': legacy item, should be 'normal'
'email': user's contact email
'product_options': legacy column}
:return: order object
"""
opts = params['product_opts']
params['product_opts'] = json.dumps(params['product_opts'])
sql = ('INSERT INTO ordering_order '
'(orderid, user_id, order_type, status, note, '
'product_opts, ee_order_id, order_source, order_date, '
'priority, email, product_options) '
'VALUES (%(orderid)s, %(user_id)s, %(order_type)s, '
'%(status)s, %(note)s, %(product_opts)s, '
'%(ee_order_id)s, %(order_source)s, %(order_date)s, '
'%(priority)s, %(email)s, %(product_options)s)')
logger.info('Order creation parameters: {}'.format(params))
log_sql = ''
try:
with db_instance() as db:
log_sql = db.cursor.mogrify(sql, params)
logger.info('New order complete SQL: {}'.format(log_sql))
db.execute(sql, params)
db.commit()
except DBConnectException as e:
logger.debug('Error creating new order: {}\n'
'sql: {}'.format(e.message, log_sql))
raise OrderException(e)
order = Order.find(params['orderid'])
# Let the load_ee_order method handle the scene injection
# as there is special logic for interacting with LTA
if params['ee_order_id']:
return order
sensor_keys = sensor.SensorCONST.instances.keys()
bulk_ls = []
for key in opts:
if key in sensor_keys:
sensor_type = ''
item1 = opts[key]['inputs'][0]
if isinstance(sensor.instance(item1), sensor.Landsat):
sensor_type = 'landsat'
elif isinstance(sensor.instance(item1), sensor.Modis):
sensor_type = 'modis'
for s in opts[key]['inputs']:
scene_dict = {
'name': s,
'sensor_type': sensor_type,
'order_id': order.id,
'status': 'submitted',
'ee_unit_id': None
}
bulk_ls.append(scene_dict)
if 'plot_statistics' in opts and opts['plot_statistics']:
scene_dict = {
'name': 'plot',
'sensor_type': 'plot',
'order_id': order.id,
'status': 'submitted',
'ee_unit_id': None
}
bulk_ls.append(scene_dict)
try:
Scene.create(bulk_ls)
except SceneException as e:
logger.debug('Order creation failed on scene injection, '
'order: {}\nexception: {}'.format(
order.orderid, e.message))
with db_instance() as db:
db.execute('delete ordering_order where id = %s', order.id)
db.commit()
raise OrderException(e)
return order
def __init__(self, *args, **kwargs):
super(LTASoapService, self).__init__(*args, **kwargs)
logger.info('Building SoapClient for:{0}'.format(self.url))
self.client = SoapClient(self.url, cache=self.build_object_cache())
class OnlineCache(object):
""" Client code to interact with the LSRD online cache """
config = ConfigurationProvider()
__order_path_key = 'online_cache_orders_dir'
__host_key = 'landsatds.host'
__user_key = 'landsatds.username'
__pw_key = 'landsatds.password'
def __init__(self):
self.orderpath = self.config.get(self.__order_path_key)
if not self.orderpath:
msg = '{} not defined in configurations'.format(
self.__order_path_key)
logger.critical(msg)
raise OnlineCacheException(msg)
host, user, pw = self.config.get(
[self.__host_key, self.__user_key, self.__pw_key])
self.client = sshcmd.RemoteHost(host, user, pw, timeout=5)
try:
self.client.execute('ls')
except Exception as e:
logger.critical('No connection to OnlineCache host: {}'.format(e))
raise OnlineCacheException(e)
def exists(self, orderid, filename=None):
""" Check if an order [optional filename] exists on the onlinecache
:param orderid: associated order to check
:param filename: file to check inside of an order
:return: bool
"""
if filename:
path = os.path.join(self.orderpath, orderid, filename)
else:
path = os.path.join(self.orderpath, orderid)
try:
result = self.execute_command('ls -d {0}'.format(path),
silent=True)
ret = tuple(x.rstrip() for x in result['stdout'])
return ret[-1] == path
except OnlineCacheException as e:
return False
def delete(self, orderid, filename=None):
"""
Removes an order from physical online cache disk
:param filename: file to delete inside of an order
:param orderid: associated order to delete
"""
if not self.exists(orderid, filename):
msg = 'Invalid orderid {} or filename {}'.format(orderid, filename)
logger.critical(msg)
return False
if filename:
path = os.path.join(self.orderpath, orderid, filename)
else:
path = os.path.join(self.orderpath, orderid)
# this should be the dir where the order is held
logger.info('Deleting {} from online cache'.format(path))
# TODO: if storage system supports immutable options
# >>> sudo chattr -fR -i {0};rm -rf {0}
# However, nfs does not support this extended attributes
try:
cmd = 'chmod -R 744 {0};rm -rf {0}'.format(path)
self.execute_command(cmd)
except OnlineCacheException as exc:
logger.critical('Failed to remove files from output cache. '
'Command: {} Error: {}'.format(cmd, exc))
return False
return True
def list(self, orderid=None):
"""
List the orders currently stored on cache, or files listed
insed of a specific order
:param orderid: order name to look inside of
:return: list of folders/files
"""
if orderid:
path = os.path.join(self.orderpath, orderid)
else:
path = self.orderpath
cmd = 'ls {}'.format(path)
result = self.execute_command(cmd)
ret = tuple(x.rstrip() for x in result['stdout'])
return ret
def capacity(self):
"""
Returns the capacity of the online cache
:return: dict
"""
cmd = 'df -mhP {}'.format(self.orderpath)
result = self.execute_command(cmd)
line = result['stdout'][1].split(' ')
clean = [l for l in line if len(l) > 0]
results = {
'capacity': clean[1],
'used': clean[2],
'available': clean[3],
'percent_used': clean[4]
}
return results
def execute_command(self, cmd, silent=False):
"""
Execute the given command on the cache
:param cmd: cmd string to execute
:return: results of the command
"""
try:
result = self.client.execute(cmd)
except Exception, exception:
if not silent:
logger.critical('Error executing command: {} '
'Raised exception: {}'.format(cmd, exception))
raise OnlineCacheException(exception)
if 'stderr' in result and result['stderr']:
if not silent:
logger.critical('Error executing command: {} '
'stderror returned: {}'.format(
cmd, result['stderr']))
raise OnlineCacheException(result['stderr'])
logger.info('call to {} returned {}'.format(cmd, result))
return result
class OnlineCache(object):
""" Client code to interact with the LSRD online cache """
config = ConfigurationProvider()
__order_path_key = 'online_cache_orders_dir'
__host_key = 'landsatds.host'
__user_key = 'landsatds.username'
__pw_key = 'landsatds.password'
def __init__(self):
self.orderpath = self.config.get(self.__order_path_key)
if not self.orderpath:
msg = '{} not defined in configurations'.format(self.__order_path_key)
logger.debug(msg)
raise OnlineCacheException(msg)
host, user, pw = self.config.get([self.__host_key,
self.__user_key,
self.__pw_key])
self.client = sshcmd.RemoteHost(host, user, pw, timeout=5)
try:
self.client.execute('ls')
except Exception as e:
logger.debug('No connection to OnlineCache host: {}'.format(e))
raise OnlineCacheException(e)
def delete(self, orderid, filename=None):
"""
Removes an order from physical online cache disk
:param filename: file to delete inside of an order
:param orderid: associated order to delete
"""
if filename:
path = os.path.join(self.orderpath, orderid, filename)
else:
path = os.path.join(self.orderpath, orderid)
# this should be the dir where the order is held
logger.info('Deleting {} from online cache'.format(path))
try:
self.execute_command('sudo chattr -fR -i {0};rm -rf {0}'.format(path))
except OnlineCacheException:
# in the event /lustre is mounted to an NFS system
logger.info("onlinecache delete, chattr error, attempting chmod instead...")
self.execute_command('chmod -R 644 {0};rm -rf {0}'.format(path))
return True
def list(self, orderid=None):
"""
List the orders currently stored on cache, or files listed
insed of a specific order
:param orderid: order name to look inside of
:return: list of folders/files
"""
if orderid:
path = os.path.join(self.orderpath, orderid)
else:
path = self.orderpath
cmd = 'ls {}'.format(path)
result = self.execute_command(cmd)
ret = tuple(x.rstrip() for x in result['stdout'])
return ret
def capacity(self):
"""
Returns the capacity of the online cache
:return: dict
"""
cmd = 'df -mhP {}'.format(self.orderpath)
result = self.execute_command(cmd)
line = result['stdout'][1].split(' ')
clean = [l for l in line if len(l) > 0]
results = {'capacity': clean[1],
'used': clean[2],
'available': clean[3],
'percent_free': clean[4]}
return results
def execute_command(self, cmd):
"""
Execute the given command on the cache
:param cmd: cmd string to execute
:return: results of the command
"""
try:
result = self.client.execute(cmd)
except Exception, exception:
logger.debug('Error executing command: {} '
'Raised exception: {}'.format(cmd, exception))
raise OnlineCacheException(exception)
if 'stderr' in result and result['stderr']:
logger.debug('Error executing command: {} '
'stderror returned: {}'.format(cmd, result['stderr']))
raise OnlineCacheException(result['stderr'])
logger.info('call to {} returned {}'.format(cmd, result))
return result
