def update_product_details(self, action, params):
"""Update product details
Args:
action (str): name of the action to peform. valid values include:
update_status, set_product_error,
set_product_unavailable,
mark_product_complete
params (dict): args for the action. valid keys: name, orderid,
processing_loc, status, error, note,
completed_file_location, cksum_file_location,
log_file_contents
Returns:
True if successful
"""
try:
response = self.production.update_product(action, **params)
except:
logger.critical(
"ERR version0 update_product_details, params: {0}\ntrace: {1}\n"
.format(params, traceback.format_exc()))
response = default_error_message
return response
def update_system_status(params):
if set(params) != {
'system_message_title', 'system_message_body',
'display_system_message'
}:
return {
'msg':
'Only 3 params are valid, and they must be present:'
'system_message_title, system_message_body,'
'display_system_message'
}
sql = '''update ordering_configuration set value = %s where key = 'msg.system_message_title';
update ordering_configuration set value = %s where key = 'msg.system_message_body';
update ordering_configuration set value = %s where key = 'system.display_system_message'; '''
sql_vals = (params['system_message_title'],
params['system_message_body'],
params['display_system_message'])
try:
with db_instance() as db:
db.execute(sql, sql_vals)
db.commit()
except DBConnectException as e:
logger.critical("error updating system status: {}".format(e))
return {'msg': "error updating database: {}".format(e.message)}
return True
def update(self, att, val):
"""
Update a specified column value for this Order object
:param att: column to update
:param val: new value
:return: updated value from self
"""
sql = 'update ordering_order set %s = %s where id = %s'
log_sql = ''
try:
with db_instance() as db:
log_sql = db.cursor.mogrify(sql,
(db_extns.AsIs(att), val, self.id))
logger.info(log_sql)
db.execute(sql, (db_extns.AsIs(att), val, self.id))
db.commit()
except DBConnectException as e:
logger.critical('Error updating order: {}\nSQL: {}'.format(
e.message, log_sql))
self.__setattr__(att, val)
return self.__getattribute__(att)
def order_attr(self, col):
"""
Select the column value from the ordering_order table for this
specific scene
:param col: column to select on
:return: value
"""
sql = ('SELECT %s '
'FROM ordering_scene JOIN ordering_order '
'ON ordering_order.id = ordering_scene.order_id '
'WHERE ordering_scene.id = %s')
log_sql = ''
try:
with db_instance() as db:
log_sql = db.cursor.mogrify(sql, (db_extns.AsIs(col), self.id))
db.select(sql, (db_extns.AsIs(col), self.id))
ret = db[0][col]
except DBConnectException as e:
logger.critical('Error retrieving order_attr: {}\n'
'sql: {} \n'.format(e.message, log_sql))
raise SceneException(e)
except KeyError as e:
logger.critical('Error order_attr returned no results\n'
'sql: {}'.format(log_sql))
raise SceneException('Key Error: {}'.format(e.message))
return ret
def delete(self, orderid, filename=None):
"""
Removes an order from physical online cache disk
:param filename: file to delete inside of an order
:param orderid: associated order to delete
"""
if not self.exists(orderid, filename):
msg = 'Invalid orderid {} or filename {}'.format(orderid, filename)
logger.critical(msg)
return False
if filename:
path = os.path.join(self.orderpath, orderid, filename)
else:
path = os.path.join(self.orderpath, orderid)
# this should be the dir where the order is held
logger.info('Deleting {} from online cache'.format(path))
try:
self.execute_command(
'sudo chattr -fR -i {0};rm -rf {0}'.format(path))
except OnlineCacheException:
# in the event /lustre is mounted to an NFS system
logger.info(
"onlinecache delete, chattr error, attempting chmod instead..."
)
self.execute_command('chmod -R 644 {0};rm -rf {0}'.format(path))
return True
def delete(self, orderid, filename=None):
"""
Removes an order from physical online cache disk
:param filename: file to delete inside of an order
:param orderid: associated order to delete
"""
if not self.exists(orderid, filename):
msg = 'Invalid orderid {} or filename {}'.format(orderid, filename)
logger.critical(msg)
return False
if filename:
path = os.path.join(self.orderpath, orderid, filename)
else:
path = os.path.join(self.orderpath, orderid)
# this should be the dir where the order is held
logger.info('Deleting {} from online cache'.format(path))
# TODO: if storage system supports immutable options
# >>> sudo chattr -fR -i {0};rm -rf {0}
# However, nfs does not support this extended attributes
try:
cmd = 'chmod -R 744 {0};rm -rf {0}'.format(path)
self.execute_command(cmd)
except OnlineCacheException as exc:
logger.critical('Failed to remove files from output cache. '
'Command: {} Error: {}'.format(cmd, exc))
return False
return True
def error_to(self, orderid, state):
order = Order.find(orderid)
err_scenes = order.scenes({'status': 'error'})
try:
if state == 'submitted':
Scene.bulk_update(
[s.id for s in err_scenes], {
'status': state,
'orphaned': None,
'reported_orphan': None,
'log_file_contents': '',
'note': '',
'retry_count': 0
})
order.status = 'ordered'
order.completion_email_sent = None
order.save()
else:
Scene.bulk_update([s.id for s in err_scenes],
{'status': state})
return True
except SceneException as e:
logger.critical('ERR admin provider error_to\ntrace: {}'.format(
e.message))
raise AdministrationProviderException('ERR updating with error_to')
def find_or_create_user(self):
""" check if user exists in our DB, if not create them
returns what should be assigned to self.id
"""
(username, email, first_name, last_name,
contactid) = (self.username, self.email, self.first_name,
self.last_name, self.contactid)
user_id = None
nownow = time.strftime('%Y-%m-%d %H:%M:%S')
insert_stmt = "insert into auth_user (username, " \
"email, first_name, last_name, password, " \
"is_staff, is_active, is_superuser, " \
"last_login, date_joined, contactid) values " \
"(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) " \
"on conflict (username) " \
"do update set (email, contactid, last_login) = (%s, %s, %s) " \
"where auth_user.username = %s" \
"returning id"
arg_tup = (username, email, first_name, last_name, 'pass', 'f', 't',
'f', nownow, nownow, contactid, email, contactid, nownow,
username)
with db_instance() as db:
try:
db.execute(insert_stmt, arg_tup)
db.commit()
user_id = db.fetcharr[0]['id']
except:
exc_type, exc_val, exc_trace = sys.exc_info()
logger.critical("ERR user find_or_create args {0} {1} " \
"{2} {3}\n trace: {4}".format(username, email, first_name,
last_name, traceback.format_exc()))
raise exc_type, exc_val, exc_trace
return user_id
def where(cls, params):
"""
Query for particular users
:param params: dictionary of column: value parameters
:return: list of matching User objects
"""
if not isinstance(params, dict):
raise UserException(
'Where arguments must be passed as a dictionary')
sql, values = format_sql_params(cls.base_sql, params)
ret = []
log_sql = ''
try:
with db_instance() as db:
log_sql = db.cursor.mogrify(sql, values)
logger.info('user.py where sql: {}'.format(log_sql))
db.select(sql, values)
for i in db:
obj = User(i["username"], i["email"], i["first_name"],
i["last_name"], i["contactid"])
ret.append(obj)
except DBConnectException as e:
logger.critical('Error querying for users: {}\n'
'sql: {}'.format(e.message, log_sql))
raise UserException(e)
return ret
def download_available(self, entity_ids, dataset):
"""
Iterates over a downloadoptions request for the provided
entity_ids, determining download url availability
"""
status_dict = dict()
m2m_ids = entity_ids.values()
dload_options = self.download_options(
m2m_ids,
dataset) # list of dicts, keys 'entityId', 'downloadOptions'
for option in dload_options:
# In some instances the entityId returned by downloadOptions is an integer
# so we want to convert it to a string to match with other instances of entityId.
# This prevents a KeyError from occurring in verify_scenes()
try:
entity_id = unicode(str(option['entityId']), "utf-8")
except Exception as e:
msg = 'Error converting entityID {0} to unicode string format - {1}'.format(
option['entityId'], e.message)
logger.critical(msg)
raise LTAError(msg)
standard_product = [
p for p in option['downloadOptions']
if p['downloadCode'] == 'STANDARD'
][0]
available = standard_product['available'] # True or False
status_dict[entity_id] = available
return status_dict
def put(version, ordernum=None):
user = flask.g.user
remote_addr = user_ip_address()
body = request.get_json(force=True)
if body is None or (isinstance(body, dict) and body.get('orderid') is None):
message = MessagesResponse(errors=['No orderid supplied'],
code=400)
return message()
elif isinstance(body, dict) and body.get('status') != 'cancelled':
message = MessagesResponse(errors=['Invalid status supplied'],
code=400)
return message()
else:
orderid, status = body.get('orderid'), body.get('status')
orders = espa.fetch_order(orderid)
if orders[0].user_id != user.id and not user.is_staff():
msg = ('User {} is not allowed to cancel order {}'
.format(user.username, orderid))
logger.critical(msg + '\nOrigin: {}'.format(remote_addr))
message = MessagesResponse(errors=[msg], code=403)
return message()
if orders[0].status != 'ordered':
msg = ('Order {} is already in a "{}" state'
.format(orderid, orders[0].status))
message = MessagesResponse(errors=[msg], code=400)
return message()
order = espa.cancel_order(orders[0].id, remote_addr)
message = OrderResponse(**order.as_dict())
message.limit = ('orderid', 'status')
message.code = 202
return message()
def get_default_ee_options(item_ls):
"""
Factory method to return default ESPA order options for orders
originating through Earth Explorer
:param item_ls: list of scenes received from EE for an order
structure: list({sceneid:, unit_num:})
:return: dictionary representation of the EE order
"""
ee_order = {'format': 'gtiff'}
for item in item_ls:
sceneid = item['orderingId']
try:
scene_info = sensor.instance(sceneid)
except sensor.ProductNotImplemented:
log_msg = (
'Received unsupported product via EE: {}'.format(sceneid))
logger.critical(log_msg)
continue
short = scene_info.shortname
if short in ee_order:
ee_order[short]['inputs'].append(sceneid)
else:
if isinstance(scene_info, sensor.Landsat):
ee_order[short] = {'inputs': [sceneid], 'products': ['sr']}
elif isinstance(scene_info, sensor.Modis):
ee_order[short] = {'inputs': [sceneid], 'products': ['l1']}
elif isinstance(scene_info, sensor.Viirs):
ee_order[short] = {'inputs': [sceneid], 'products': ['l1']}
return ee_order
def where(cls, params):
"""
Query for a particular row in the ordering_scene table
:param params: dictionary of column: value parameter to select on
:return: list of matching Scene objects
"""
if not isinstance(params, dict):
raise SceneException('Where arguments must be '
'passed as a dictionary')
sql, values = format_sql_params(cls.base_sql, params)
ret = []
log_sql = ''
try:
with db_instance() as db:
log_sql = db.cursor.mogrify(sql, values)
logger.info('scene.py where sql: {}'.format(log_sql))
db.select(sql, values)
for i in db:
sd = dict(i)
obj = Scene(**sd)
ret.append(obj)
except DBConnectException as e:
logger.critical('Error retrieving scenes: {}\n'
'sql: {}'.format(e.message, log_sql))
raise SceneException(e)
return ret
def check_dmid(prod_ls, contactid=None):
try:
token = inventory.get_cached_session()
return inventory.check_valid(token, prod_ls)
except Exception as e:
msg = 'Could not connect to EarthExplorer source'
logger.critical(msg + str(e))
raise InventoryConnectionException(msg)
def backup_configuration(self, filepath=None):
try:
response = self.admin.backup_configuration(filepath)
except:
logger.critical('ERR version1 backup_configuration: '
'{}\ntrace: {}\n'.format(filepath,
traceback.format_exc()))
response = default_error_message
return response
def verify_user(username, password):
if (username is None) or (not (str(username).strip())):
logger.warning('Invalid username supplied: %s', username)
flask.g.error_reason = 'auth'
return False
try:
# usernames with spaces are valid in EE, though they can't be used for cache keys
cache_key = '{}-credentials'.format(
username.replace(' ', '_espa_cred_insert_'))
cache_entry = cache.get(cache_key)
if cache_entry:
# Need to be encrypted?
if cache_entry['password'] == password:
user_entry = cache_entry['user_entry']
# User may have changed their password while it was still cached
else:
user_entry = User.get(username, password)
else:
user_entry = User.get(username, password)
cache_entry = {'password': password, 'user_entry': user_entry}
cache.set(cache_key, cache_entry, 7200)
user = User(*user_entry)
flask.g.user = user # Replace usage with cached version
except UserException as e:
logger.info('Invalid login attempt, username: {}, {}'.format(
username, e))
flask.g.error_reason = 'unknown'
return False
except ERSApiAuthFailedException as e:
logger.info('Invalid login attempt, username: {}, {}'.format(
username, e))
flask.g.error_reason = 'auth'
return False
except ERSApiErrorException as e:
logger.info('ERS lookup failed, username: {}, {}'.format(username, e))
flask.g.error_reason = 'unknown'
return False
except ERSApiConnectionException as e:
logger.info('ERS is down {}'.format(e))
flask.g.error_reason = 'conn'
return False
except DBConnectException as e:
logger.critical('! Database reported a problem: {}'.format(e))
flask.g.error_reasons = 'db'
return False
except Exception:
logger.info('Invalid login attempt, username: {}'.format(username))
flask.g.error_reason = 'unknown'
return False
return True
def catch_orphaned_scenes(self):
"""
Handler for marking queued scenes with no corresponding job in hadoop
:return: true
"""
try:
response = self.production.catch_orphaned_scenes()
except:
logger.critical("ERR handling orphaned scenes\ntrace: {}".format(
traceback.format_exc()))
response = default_error_message
return response
def get_system_config(self):
"""
retrieve system configuration variables
"""
try:
return self.admin.get_system_config()
except:
exc_type, exc_val, exc_trace = sys.exc_info()
logger.critical(
"ERR retrieving system config: exception {0}".format(
traceback.format_exc()))
raise exc_type, exc_val, exc_trace
def get_production_whitelist(self):
"""
Returns list of ip addresses in hadoop cluster
:return: list of strings
"""
try:
response = self.production.production_whitelist()
except:
logger.critical(
"ERR failure to generate production whitelist\ntrace: {}".
format(traceback.format_exc()))
response = default_error_message
return response
def reset_processing_status(self):
"""
Handler for resetting queued/processing scenes status to a 'submitted' state
:return: true
"""
try:
response = self.production.reset_processing_status()
except:
logger.critical(
"ERR handling queued/processing scenes\ntrace: {}".format(
traceback.format_exc()))
response = default_error_message
return response
def get(cls, username, password):
if username == 'espa_admin':
cp = ConfigurationProvider()
if pbkdf2_sha256.verify(password, cp.espa256):
return username, cp.get('apiemailreceive'), 'espa', 'admin', ''
else:
msg = "ERR validating espa_admin, invalid password "
logger.critical(msg)
raise UserException(msg)
else:
eu = ers.get_user_info(username, password)
return eu['username'], eu['email'], eu['firstName'], eu[
'lastName'], eu['contact_id']
def get_multistat(self, name):
"""
retrieve requested statistic value
:return: long
"""
try:
response = self.reporting.get_multistat(name)
except:
logger.critical(
"ERR version1 get_stat name: {0}, traceback: {1}".format(
name, traceback.format_exc()))
response = default_error_message
return response
def get_admin_whitelist(self):
"""
Returns list of ip addresses for whitelist hosts accessing stats
:return: list of strings
"""
try:
response = self.admin.admin_whitelist()
except:
logger.critical(
"ERR failure to generate web whitelist\ntrace:{}".format(
traceback.format_exc()))
response = default_error_message
return response
def get_stat_whitelist(self):
"""
Returns list of ip addresses for xymon monitoring application accessing stats
:return: list of strings
"""
try:
response = self.admin.stat_whitelist()
except:
logger.critical(
"ERR failure to generate statistics whitelist\ntrace:{}".
format(traceback.format_exc()))
response = default_error_message
return response
def get_backlog(self, user=None):
"""
retrive the global backlog scene count
:return: str
"""
try:
# TODO: Allow getting user-specific backlog?
response = self.reporting.get_stat('stat_backlog_depth')
except:
logger.critical("ERR version1 get_backlog, traceback: {0}".format(
traceback.format_exc()))
raise
return response
def get_system_status(self):
"""
retrieve the system status message
:return: str
"""
try:
response = self.ordering.get_system_status()
except:
logger.critical(
"ERR version1 get_system_status. traceback {0}".format(
traceback.format_exc()))
response = default_error_message
return response
def cancel_order(self, orderid, request_address):
"""
:param orderid: Primary Key for Order
:param request_address: Remote IP Address
:return:
"""
try:
response = self.ordering.cancel_order(orderid, request_address)
except:
logger.critical("ERR version1 cancel_order, traceback: {0}".format(
traceback.format_exc()))
raise
return response
def execute_command(self, cmd, silent=False):
"""
Execute the given command on the cache
:param cmd: cmd string to execute
:return: results of the command
"""
try:
result = self.client.execute(cmd)
except Exception, exception:
if not silent:
logger.critical('Error executing command: {} '
'Raised exception: {}'.format(cmd, exception))
raise OnlineCacheException(exception)
def update_system_status(self, params):
"""
update system status attributes
"""
try:
response = self.admin.update_system_status(params)
except:
exc_type, exc_val, exc_trace = sys.exc_info()
logger.critical(
"ERR updating system status params: {0}\n exception {1}".
format(params, traceback.format_exc()))
raise exc_type, exc_val, exc_trace
return response
def available_reports(self):
"""
returns list of available reports
:return: List
"""
try:
response = self.reporting.listing()
except:
logger.critical(
"ERR version1 available_reports traceback {0}".format(
traceback.format_exc()))
response = default_error_message
return response
