def update_product_details(self, action, params): """Update product details Args: action (str): name of the action to peform. valid values include: update_status, set_product_error, set_product_unavailable, mark_product_complete params (dict): args for the action. valid keys: name, orderid, processing_loc, status, error, note, completed_file_location, cksum_file_location, log_file_contents Returns: True if successful """ try: response = self.production.update_product(action, **params) except: logger.critical( "ERR version0 update_product_details, params: {0}\ntrace: {1}\n" .format(params, traceback.format_exc())) response = default_error_message return response
def update_system_status(params): if set(params) != { 'system_message_title', 'system_message_body', 'display_system_message' }: return { 'msg': 'Only 3 params are valid, and they must be present:' 'system_message_title, system_message_body,' 'display_system_message' } sql = '''update ordering_configuration set value = %s where key = 'msg.system_message_title'; update ordering_configuration set value = %s where key = 'msg.system_message_body'; update ordering_configuration set value = %s where key = 'system.display_system_message'; ''' sql_vals = (params['system_message_title'], params['system_message_body'], params['display_system_message']) try: with db_instance() as db: db.execute(sql, sql_vals) db.commit() except DBConnectException as e: logger.critical("error updating system status: {}".format(e)) return {'msg': "error updating database: {}".format(e.message)} return True
def update(self, att, val): """ Update a specified column value for this Order object :param att: column to update :param val: new value :return: updated value from self """ sql = 'update ordering_order set %s = %s where id = %s' log_sql = '' try: with db_instance() as db: log_sql = db.cursor.mogrify(sql, (db_extns.AsIs(att), val, self.id)) logger.info(log_sql) db.execute(sql, (db_extns.AsIs(att), val, self.id)) db.commit() except DBConnectException as e: logger.critical('Error updating order: {}\nSQL: {}'.format( e.message, log_sql)) self.__setattr__(att, val) return self.__getattribute__(att)
def order_attr(self, col): """ Select the column value from the ordering_order table for this specific scene :param col: column to select on :return: value """ sql = ('SELECT %s ' 'FROM ordering_scene JOIN ordering_order ' 'ON ordering_order.id = ordering_scene.order_id ' 'WHERE ordering_scene.id = %s') log_sql = '' try: with db_instance() as db: log_sql = db.cursor.mogrify(sql, (db_extns.AsIs(col), self.id)) db.select(sql, (db_extns.AsIs(col), self.id)) ret = db[0][col] except DBConnectException as e: logger.critical('Error retrieving order_attr: {}\n' 'sql: {} \n'.format(e.message, log_sql)) raise SceneException(e) except KeyError as e: logger.critical('Error order_attr returned no results\n' 'sql: {}'.format(log_sql)) raise SceneException('Key Error: {}'.format(e.message)) return ret
def delete(self, orderid, filename=None): """ Removes an order from physical online cache disk :param filename: file to delete inside of an order :param orderid: associated order to delete """ if not self.exists(orderid, filename): msg = 'Invalid orderid {} or filename {}'.format(orderid, filename) logger.critical(msg) return False if filename: path = os.path.join(self.orderpath, orderid, filename) else: path = os.path.join(self.orderpath, orderid) # this should be the dir where the order is held logger.info('Deleting {} from online cache'.format(path)) try: self.execute_command( 'sudo chattr -fR -i {0};rm -rf {0}'.format(path)) except OnlineCacheException: # in the event /lustre is mounted to an NFS system logger.info( "onlinecache delete, chattr error, attempting chmod instead..." ) self.execute_command('chmod -R 644 {0};rm -rf {0}'.format(path)) return True
def delete(self, orderid, filename=None): """ Removes an order from physical online cache disk :param filename: file to delete inside of an order :param orderid: associated order to delete """ if not self.exists(orderid, filename): msg = 'Invalid orderid {} or filename {}'.format(orderid, filename) logger.critical(msg) return False if filename: path = os.path.join(self.orderpath, orderid, filename) else: path = os.path.join(self.orderpath, orderid) # this should be the dir where the order is held logger.info('Deleting {} from online cache'.format(path)) # TODO: if storage system supports immutable options # >>> sudo chattr -fR -i {0};rm -rf {0} # However, nfs does not support this extended attributes try: cmd = 'chmod -R 744 {0};rm -rf {0}'.format(path) self.execute_command(cmd) except OnlineCacheException as exc: logger.critical('Failed to remove files from output cache. ' 'Command: {} Error: {}'.format(cmd, exc)) return False return True
def error_to(self, orderid, state): order = Order.find(orderid) err_scenes = order.scenes({'status': 'error'}) try: if state == 'submitted': Scene.bulk_update( [s.id for s in err_scenes], { 'status': state, 'orphaned': None, 'reported_orphan': None, 'log_file_contents': '', 'note': '', 'retry_count': 0 }) order.status = 'ordered' order.completion_email_sent = None order.save() else: Scene.bulk_update([s.id for s in err_scenes], {'status': state}) return True except SceneException as e: logger.critical('ERR admin provider error_to\ntrace: {}'.format( e.message)) raise AdministrationProviderException('ERR updating with error_to')
def find_or_create_user(self): """ check if user exists in our DB, if not create them returns what should be assigned to self.id """ (username, email, first_name, last_name, contactid) = (self.username, self.email, self.first_name, self.last_name, self.contactid) user_id = None nownow = time.strftime('%Y-%m-%d %H:%M:%S') insert_stmt = "insert into auth_user (username, " \ "email, first_name, last_name, password, " \ "is_staff, is_active, is_superuser, " \ "last_login, date_joined, contactid) values " \ "(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) " \ "on conflict (username) " \ "do update set (email, contactid, last_login) = (%s, %s, %s) " \ "where auth_user.username = %s" \ "returning id" arg_tup = (username, email, first_name, last_name, 'pass', 'f', 't', 'f', nownow, nownow, contactid, email, contactid, nownow, username) with db_instance() as db: try: db.execute(insert_stmt, arg_tup) db.commit() user_id = db.fetcharr[0]['id'] except: exc_type, exc_val, exc_trace = sys.exc_info() logger.critical("ERR user find_or_create args {0} {1} " \ "{2} {3}\n trace: {4}".format(username, email, first_name, last_name, traceback.format_exc())) raise exc_type, exc_val, exc_trace return user_id
def where(cls, params): """ Query for particular users :param params: dictionary of column: value parameters :return: list of matching User objects """ if not isinstance(params, dict): raise UserException( 'Where arguments must be passed as a dictionary') sql, values = format_sql_params(cls.base_sql, params) ret = [] log_sql = '' try: with db_instance() as db: log_sql = db.cursor.mogrify(sql, values) logger.info('user.py where sql: {}'.format(log_sql)) db.select(sql, values) for i in db: obj = User(i["username"], i["email"], i["first_name"], i["last_name"], i["contactid"]) ret.append(obj) except DBConnectException as e: logger.critical('Error querying for users: {}\n' 'sql: {}'.format(e.message, log_sql)) raise UserException(e) return ret
def download_available(self, entity_ids, dataset): """ Iterates over a downloadoptions request for the provided entity_ids, determining download url availability """ status_dict = dict() m2m_ids = entity_ids.values() dload_options = self.download_options( m2m_ids, dataset) # list of dicts, keys 'entityId', 'downloadOptions' for option in dload_options: # In some instances the entityId returned by downloadOptions is an integer # so we want to convert it to a string to match with other instances of entityId. # This prevents a KeyError from occurring in verify_scenes() try: entity_id = unicode(str(option['entityId']), "utf-8") except Exception as e: msg = 'Error converting entityID {0} to unicode string format - {1}'.format( option['entityId'], e.message) logger.critical(msg) raise LTAError(msg) standard_product = [ p for p in option['downloadOptions'] if p['downloadCode'] == 'STANDARD' ][0] available = standard_product['available'] # True or False status_dict[entity_id] = available return status_dict
def put(version, ordernum=None): user = flask.g.user remote_addr = user_ip_address() body = request.get_json(force=True) if body is None or (isinstance(body, dict) and body.get('orderid') is None): message = MessagesResponse(errors=['No orderid supplied'], code=400) return message() elif isinstance(body, dict) and body.get('status') != 'cancelled': message = MessagesResponse(errors=['Invalid status supplied'], code=400) return message() else: orderid, status = body.get('orderid'), body.get('status') orders = espa.fetch_order(orderid) if orders[0].user_id != user.id and not user.is_staff(): msg = ('User {} is not allowed to cancel order {}' .format(user.username, orderid)) logger.critical(msg + '\nOrigin: {}'.format(remote_addr)) message = MessagesResponse(errors=[msg], code=403) return message() if orders[0].status != 'ordered': msg = ('Order {} is already in a "{}" state' .format(orderid, orders[0].status)) message = MessagesResponse(errors=[msg], code=400) return message() order = espa.cancel_order(orders[0].id, remote_addr) message = OrderResponse(**order.as_dict()) message.limit = ('orderid', 'status') message.code = 202 return message()
def get_default_ee_options(item_ls): """ Factory method to return default ESPA order options for orders originating through Earth Explorer :param item_ls: list of scenes received from EE for an order structure: list({sceneid:, unit_num:}) :return: dictionary representation of the EE order """ ee_order = {'format': 'gtiff'} for item in item_ls: sceneid = item['orderingId'] try: scene_info = sensor.instance(sceneid) except sensor.ProductNotImplemented: log_msg = ( 'Received unsupported product via EE: {}'.format(sceneid)) logger.critical(log_msg) continue short = scene_info.shortname if short in ee_order: ee_order[short]['inputs'].append(sceneid) else: if isinstance(scene_info, sensor.Landsat): ee_order[short] = {'inputs': [sceneid], 'products': ['sr']} elif isinstance(scene_info, sensor.Modis): ee_order[short] = {'inputs': [sceneid], 'products': ['l1']} elif isinstance(scene_info, sensor.Viirs): ee_order[short] = {'inputs': [sceneid], 'products': ['l1']} return ee_order
def where(cls, params): """ Query for a particular row in the ordering_scene table :param params: dictionary of column: value parameter to select on :return: list of matching Scene objects """ if not isinstance(params, dict): raise SceneException('Where arguments must be ' 'passed as a dictionary') sql, values = format_sql_params(cls.base_sql, params) ret = [] log_sql = '' try: with db_instance() as db: log_sql = db.cursor.mogrify(sql, values) logger.info('scene.py where sql: {}'.format(log_sql)) db.select(sql, values) for i in db: sd = dict(i) obj = Scene(**sd) ret.append(obj) except DBConnectException as e: logger.critical('Error retrieving scenes: {}\n' 'sql: {}'.format(e.message, log_sql)) raise SceneException(e) return ret
def check_dmid(prod_ls, contactid=None): try: token = inventory.get_cached_session() return inventory.check_valid(token, prod_ls) except Exception as e: msg = 'Could not connect to EarthExplorer source' logger.critical(msg + str(e)) raise InventoryConnectionException(msg)
def backup_configuration(self, filepath=None): try: response = self.admin.backup_configuration(filepath) except: logger.critical('ERR version1 backup_configuration: ' '{}\ntrace: {}\n'.format(filepath, traceback.format_exc())) response = default_error_message return response
def verify_user(username, password): if (username is None) or (not (str(username).strip())): logger.warning('Invalid username supplied: %s', username) flask.g.error_reason = 'auth' return False try: # usernames with spaces are valid in EE, though they can't be used for cache keys cache_key = '{}-credentials'.format( username.replace(' ', '_espa_cred_insert_')) cache_entry = cache.get(cache_key) if cache_entry: # Need to be encrypted? if cache_entry['password'] == password: user_entry = cache_entry['user_entry'] # User may have changed their password while it was still cached else: user_entry = User.get(username, password) else: user_entry = User.get(username, password) cache_entry = {'password': password, 'user_entry': user_entry} cache.set(cache_key, cache_entry, 7200) user = User(*user_entry) flask.g.user = user # Replace usage with cached version except UserException as e: logger.info('Invalid login attempt, username: {}, {}'.format( username, e)) flask.g.error_reason = 'unknown' return False except ERSApiAuthFailedException as e: logger.info('Invalid login attempt, username: {}, {}'.format( username, e)) flask.g.error_reason = 'auth' return False except ERSApiErrorException as e: logger.info('ERS lookup failed, username: {}, {}'.format(username, e)) flask.g.error_reason = 'unknown' return False except ERSApiConnectionException as e: logger.info('ERS is down {}'.format(e)) flask.g.error_reason = 'conn' return False except DBConnectException as e: logger.critical('! Database reported a problem: {}'.format(e)) flask.g.error_reasons = 'db' return False except Exception: logger.info('Invalid login attempt, username: {}'.format(username)) flask.g.error_reason = 'unknown' return False return True
def catch_orphaned_scenes(self): """ Handler for marking queued scenes with no corresponding job in hadoop :return: true """ try: response = self.production.catch_orphaned_scenes() except: logger.critical("ERR handling orphaned scenes\ntrace: {}".format( traceback.format_exc())) response = default_error_message return response
def get_system_config(self): """ retrieve system configuration variables """ try: return self.admin.get_system_config() except: exc_type, exc_val, exc_trace = sys.exc_info() logger.critical( "ERR retrieving system config: exception {0}".format( traceback.format_exc())) raise exc_type, exc_val, exc_trace
def get_production_whitelist(self): """ Returns list of ip addresses in hadoop cluster :return: list of strings """ try: response = self.production.production_whitelist() except: logger.critical( "ERR failure to generate production whitelist\ntrace: {}". format(traceback.format_exc())) response = default_error_message return response
def reset_processing_status(self): """ Handler for resetting queued/processing scenes status to a 'submitted' state :return: true """ try: response = self.production.reset_processing_status() except: logger.critical( "ERR handling queued/processing scenes\ntrace: {}".format( traceback.format_exc())) response = default_error_message return response
def get(cls, username, password): if username == 'espa_admin': cp = ConfigurationProvider() if pbkdf2_sha256.verify(password, cp.espa256): return username, cp.get('apiemailreceive'), 'espa', 'admin', '' else: msg = "ERR validating espa_admin, invalid password " logger.critical(msg) raise UserException(msg) else: eu = ers.get_user_info(username, password) return eu['username'], eu['email'], eu['firstName'], eu[ 'lastName'], eu['contact_id']
def get_multistat(self, name): """ retrieve requested statistic value :return: long """ try: response = self.reporting.get_multistat(name) except: logger.critical( "ERR version1 get_stat name: {0}, traceback: {1}".format( name, traceback.format_exc())) response = default_error_message return response
def get_admin_whitelist(self): """ Returns list of ip addresses for whitelist hosts accessing stats :return: list of strings """ try: response = self.admin.admin_whitelist() except: logger.critical( "ERR failure to generate web whitelist\ntrace:{}".format( traceback.format_exc())) response = default_error_message return response
def get_stat_whitelist(self): """ Returns list of ip addresses for xymon monitoring application accessing stats :return: list of strings """ try: response = self.admin.stat_whitelist() except: logger.critical( "ERR failure to generate statistics whitelist\ntrace:{}". format(traceback.format_exc())) response = default_error_message return response
def get_backlog(self, user=None): """ retrive the global backlog scene count :return: str """ try: # TODO: Allow getting user-specific backlog? response = self.reporting.get_stat('stat_backlog_depth') except: logger.critical("ERR version1 get_backlog, traceback: {0}".format( traceback.format_exc())) raise return response
def get_system_status(self): """ retrieve the system status message :return: str """ try: response = self.ordering.get_system_status() except: logger.critical( "ERR version1 get_system_status. traceback {0}".format( traceback.format_exc())) response = default_error_message return response
def cancel_order(self, orderid, request_address): """ :param orderid: Primary Key for Order :param request_address: Remote IP Address :return: """ try: response = self.ordering.cancel_order(orderid, request_address) except: logger.critical("ERR version1 cancel_order, traceback: {0}".format( traceback.format_exc())) raise return response
def execute_command(self, cmd, silent=False): """ Execute the given command on the cache :param cmd: cmd string to execute :return: results of the command """ try: result = self.client.execute(cmd) except Exception, exception: if not silent: logger.critical('Error executing command: {} ' 'Raised exception: {}'.format(cmd, exception)) raise OnlineCacheException(exception)
def update_system_status(self, params): """ update system status attributes """ try: response = self.admin.update_system_status(params) except: exc_type, exc_val, exc_trace = sys.exc_info() logger.critical( "ERR updating system status params: {0}\n exception {1}". format(params, traceback.format_exc())) raise exc_type, exc_val, exc_trace return response
def available_reports(self): """ returns list of available reports :return: List """ try: response = self.reporting.listing() except: logger.critical( "ERR version1 available_reports traceback {0}".format( traceback.format_exc())) response = default_error_message return response