def update(self, att, val): """ Update a specifed column value for this Scene object with a new value :param att: column to update :param val: new value :return: updated value from self """ sql = 'update ordering_scene set %s = %s where id = %s' log_sql = '' try: with db_instance() as db: log_sql = db.cursor.mogrify(sql, (db_extns.AsIs(att), val, self.id)) logger.info("\n*** Updating scene: \n" + log_sql + '\n***\n"') db.execute(sql, (db_extns.AsIs(att), val, self.id)) db.commit() except DBConnectException as e: raise SceneException('Error updating scene: {}\nSQL: {}'.format( e.message, log_sql)) self.__setattr__(att, val) return self.__getattribute__(att)
def delete(self, orderid, filename=None): """ Removes an order from physical online cache disk :param filename: file to delete inside of an order :param orderid: associated order to delete """ if not self.exists(orderid, filename): msg = 'Invalid orderid {} or filename {}'.format(orderid, filename) logger.critical(msg) return False if filename: path = os.path.join(self.orderpath, orderid, filename) else: path = os.path.join(self.orderpath, orderid) # this should be the dir where the order is held logger.info('Deleting {} from online cache'.format(path)) # TODO: if storage system supports immutable options # >>> sudo chattr -fR -i {0};rm -rf {0} # However, nfs does not support this extended attributes try: cmd = 'chmod -R 744 {0};rm -rf {0}'.format(path) self.execute_command(cmd) except OnlineCacheException as exc: logger.critical('Failed to remove files from output cache. ' 'Command: {} Error: {}'.format(cmd, exc)) return False return True
def post(version): try: user = flask.g.user order = request.get_json(force=True) if not order: message = {"status": 400, "msg": "Unable to parse json data. Please ensure your order follows json " "conventions and your http call is correct. If you believe this " "message is in error please email customer service"} else: try: order = lowercase_all(order) orderid = espa.place_order(order, user) if isinstance(orderid, str) and "@" in orderid: # if order submission was successful, orderid is returned as a string # which includes the submitters email address message = {"status": 200, "orderid": orderid} else: # there was a problem, and orderid is a dict with the problem details logger.info("problem with user submitted order. user: {0}\n details: {1}".format(user.username, orderid)) message = {"status": 400, "message": orderid} except Exception as e: logger.debug("exception posting order: {0}\nuser: {1}\n msg: {2}".format(order, user.username, e.message)) message = {"status": 500, "msg": "the system experienced an exception. the admins have been notified"} except BadRequest as e: # request.get_json throws a BadRequest logger.debug("BadRequest, could not parse request into json {}\nuser: {}\nform data {}\n".format(e.description, user.username, request.form)) message = {"status": 400, "msg": "Could not parse the request into json"} except Exception as e: logger.debug("ERR posting order. user: {0}\n error: {1}".format(user.username, e)) message = {"status": 500, "msg": "the system has experienced an exception. the admins have been notified."} response = jsonify(message) response.status_code = message['status'] return response
def update(self, att, val): """ Update a specified column value for this Order object :param att: column to update :param val: new value :return: updated value from self """ sql = 'update ordering_order set %s = %s where id = %s' log_sql = '' try: with db_instance() as db: log_sql = db.cursor.mogrify(sql, (db_extns.AsIs(att), val, self.id)) logger.info(log_sql) db.execute(sql, (db_extns.AsIs(att), val, self.id)) db.commit() except DBConnectException as e: logger.debug('Error updating order: {}\nSQL: {}' .format(e.message, log_sql)) self.__setattr__(att, val) return self.__getattribute__(att)
def __init__(self, *args, **kwargs): super(LTASoapService, self).__init__(*args, **kwargs) if config.mode in ('dev', 'tst'): logger.info('Building SoapClient for:{0}'.format(self.url)) self.client = SoapClient(self.url, location=self.location, cache=self.build_object_cache())
def delete(self, orderid, filename=None): """ Removes an order from physical online cache disk :param filename: file to delete inside of an order :param orderid: associated order to delete """ if not self.exists(orderid, filename): msg = 'Invalid orderid {} or filename {}'.format(orderid, filename) logger.critical(msg) return False if filename: path = os.path.join(self.orderpath, orderid, filename) else: path = os.path.join(self.orderpath, orderid) # this should be the dir where the order is held logger.info('Deleting {} from online cache'.format(path)) try: self.execute_command( 'sudo chattr -fR -i {0};rm -rf {0}'.format(path)) except OnlineCacheException: # in the event /lustre is mounted to an NFS system logger.info( "onlinecache delete, chattr error, attempting chmod instead..." ) self.execute_command('chmod -R 644 {0};rm -rf {0}'.format(path)) return True
def update(self, att, val): """ Update a specified column value for this Order object :param att: column to update :param val: new value :return: updated value from self """ sql = 'update ordering_order set %s = %s where id = %s' log_sql = '' try: with db_instance() as db: log_sql = db.cursor.mogrify(sql, (db_extns.AsIs(att), val, self.id)) logger.info(log_sql) db.execute(sql, (db_extns.AsIs(att), val, self.id)) db.commit() except DBConnectException as e: logger.debug('Error updating order: {}\nSQL: {}'.format( e.message, log_sql)) self.__setattr__(att, val) return self.__getattribute__(att)
def where(cls, params): """ Query for particular users :param params: dictionary of column: value parameters :return: list of matching User objects """ if not isinstance(params, dict): raise UserException( 'Where arguments must be passed as a dictionary') sql, values = format_sql_params(cls.base_sql, params) ret = [] log_sql = '' try: with db_instance() as db: log_sql = db.cursor.mogrify(sql, values) logger.info('user.py where sql: {}'.format(log_sql)) db.select(sql, values) for i in db: obj = User(i["username"], i["email"], i["first_name"], i["last_name"], i["contactid"]) ret.append(obj) except DBConnectException as e: logger.debug('Error querying for users: {}\n' 'sql: {}'.format(e.message, log_sql)) raise UserException(e) return ret
def verify_scenes(self, product_ids, dataset): """ Check if supplied IDs successfully map to M2M entity IDs :param product_ids: Landsat Collection IDs ['LC08_..', ...] :type product_ids: list :return: dict """ entity_ids = self.id_lookup(product_ids, dataset) # gather M2M entity ids # M2M entity ids do not correlate to product availability though. And product download url # requests will Fail a request if any of the products requested are not actually # available (as of M2M api version 1.4.1). So we need to make a downloadOptions request # to the M2M Api to ensure said product is actually available availability = self.download_available( entity_ids, dataset) # dict of {entityid:availability, ...} response = dict() for k, v in entity_ids.items(): if v: response[k] = availability[v] else: logger.info("id: {} is not available".format(k)) response[k] = False return response
def update(self, att, val): """ Update a specifed column value for this Scene object with a new value :param att: column to update :param val: new value :return: updated value from self """ sql = 'update ordering_scene set %s = %s where id = %s' log_sql = '' try: with db_instance() as db: log_sql = db.cursor.mogrify(sql, (db_extns.AsIs(att), val, self.id)) logger.info("\n*** Updating scene: \n" + log_sql + '\n***\n"') db.execute(sql, (db_extns.AsIs(att), val, self.id)) db.commit() except DBConnectException as e: raise SceneException('Error updating scene: {}\nSQL: {}' .format(e.message, log_sql)) self.__setattr__(att, val) return self.__getattribute__(att)
def where(cls, params): """ Query for a particular row in the ordering_scene table :param params: dictionary of column: value parameter to select on :return: list of matching Scene objects """ if not isinstance(params, dict): raise SceneException('Where arguments must be ' 'passed as a dictionary') sql, values = format_sql_params(cls.base_sql, params) ret = [] log_sql = '' try: with db_instance() as db: log_sql = db.cursor.mogrify(sql, values) logger.info('scene.py where sql: {}'.format(log_sql)) db.select(sql, values) for i in db: sd = dict(i) obj = Scene(**sd) ret.append(obj) except DBConnectException as e: logger.critical('Error retrieving scenes: {}\n' 'sql: {}'.format(e.message, log_sql)) raise SceneException(e) return ret
def verify_user(username, password): try: # usernames with spaces are valid in EE, though they can't be used for cache keys cache_key = '{}-credentials'.format(username.replace(' ', '_espa_cred_insert_')) cache_entry = cache.get(cache_key) if cache_entry: # Need to be encrypted? if cache_entry['password'] == password: user_entry = cache_entry['user_entry'] # User may have changed their password while it was still cached else: user_entry = User.get(username, password) else: user_entry = User.get(username, password) cache_entry = {'password': password, 'user_entry': user_entry} cache.set(cache_key, cache_entry, 7200) user = User(*user_entry) flask.g.user = user # Replace usage with cached version except Exception: logger.info('Invalid login attempt, username: {}'.format(username)) return False return True
def where(cls, params): """ Query for a particular row in the ordering_oder table :param params: dictionary of column: value parameter to select on :return: list of matching Order objects """ if not isinstance(params, dict): raise OrderException('Where arguments must be ' 'passed as a dictionary') sql, values = format_sql_params(cls.base_sql, params) ret = [] log_sql = '' try: with db_instance() as db: log_sql = db.cursor.mogrify(sql, values) logger.info('order.py where sql: {}'.format(log_sql)) db.select(sql, values) for i in db: od = dict(i) obj = Order(**od) ret.append(obj) except DBConnectException as e: logger.debug('Error order where: {}\n' 'sql: {}'.format(e.message, log_sql)) raise OrderException(e) return ret
def check_LTA(prod_ls): not_avail = nlaps.products_are_nlaps(prod_ls) if not_avail: raise InventoryException(not_avail) logger.info('@@ VERIFY: FETCH via OrderWrapperService') results = lta.verify_scenes(prod_ls) logger.info('@@ VERIFY: COMPLETE via OrderWrapperService') return results
def check_dmid(prod_ls, contactid=None): # find all the submitted products that are nlaps and reject them not_avail = nlaps.products_are_nlaps(prod_ls) if not_avail: raise InventoryException(not_avail) logger.info('@@ VERIFY: FETCH via MACHINE-TO-MACHINE') token = inventory.get_cached_session() results = inventory.get_cached_verify_scenes(token, prod_ls) logger.info('@@ VERIFY: COMPLETE via MACHINE-TO-MACHINE') return results
def verify_user(username, password): if (username is None) or (not (str(username).strip())): logger.warning('Invalid username supplied: %s', username) flask.g.error_reason = 'auth' return False try: # usernames with spaces are valid in EE, though they can't be used for cache keys cache_key = '{}-credentials'.format( username.replace(' ', '_espa_cred_insert_')) cache_entry = cache.get(cache_key) if cache_entry: # Need to be encrypted? if cache_entry['password'] == password: user_entry = cache_entry['user_entry'] # User may have changed their password while it was still cached else: user_entry = User.get(username, password) else: user_entry = User.get(username, password) cache_entry = {'password': password, 'user_entry': user_entry} cache.set(cache_key, cache_entry, 7200) user = User(*user_entry) flask.g.user = user # Replace usage with cached version except UserException as e: logger.info('Invalid login attempt, username: {}, {}'.format( username, e)) flask.g.error_reason = 'unknown' return False except ERSApiAuthFailedException as e: logger.info('Invalid login attempt, username: {}, {}'.format( username, e)) flask.g.error_reason = 'auth' return False except ERSApiErrorException as e: logger.info('ERS lookup failed, username: {}, {}'.format(username, e)) flask.g.error_reason = 'unknown' return False except ERSApiConnectionException as e: logger.info('ERS is down {}'.format(e)) flask.g.error_reason = 'conn' return False except DBConnectException as e: logger.critical('! Database reported a problem: {}'.format(e)) flask.g.error_reasons = 'db' return False except Exception: logger.info('Invalid login attempt, username: {}'.format(username)) flask.g.error_reason = 'unknown' return False return True
def get(cls, col_name, scene_name, orderid): """ Retrieve a value for a particular column based on the long name of the order :param col_name: column value to retrieve :param scene_name: scene/collection id :param orderid: long name for the related order, [email protected] :return: column value """ sql = ('select %s ' 'from ordering_scene ' 'join ordering_order ' 'on ordering_order.id = ordering_scene.order_id ' 'where ordering_scene.name = %s ' 'and ordering_order.orderid = %s') if '.' in col_name: _, col = col_name.split('.') else: col = col_name log_sql = '' try: with db_instance() as db: log_sql = (db.cursor. mogrify(sql, (db_extns.AsIs(col_name), scene_name, orderid))) logger.info(log_sql) db.select(sql, (db_extns.AsIs(col_name), scene_name, orderid)) ret = db[0][col] except DBConnectException as e: logger.debug('Error scene get\n' 'msg: {0}\n' 'sql: {1}'.format(e.message, log_sql)) raise SceneException(e.message) except KeyError as e: logger.debug('Scene.get returned no results\n' 'sql: {}'.format(log_sql)) raise SceneException('Key Error: {}' .format(e.message)) return ret
def save(self): """ Upsert self to the database """ sql = ('INSERT INTO ordering_order %s VALUES %s ' 'ON CONFLICT (orderid) ' 'DO UPDATE ' 'SET %s = %s') attr_tup = ('orderid', 'status', 'order_source', 'product_options', 'product_opts', 'order_type', 'initial_email_sent', 'completion_email_sent', 'note', 'completion_date', 'order_date', 'user_id', 'ee_order_id', 'email', 'priority') vals = tuple(self.__getattribute__(v) if v != 'product_opts' else json.dumps(self.__getattribute__(v)) for v in attr_tup) cols = '({})'.format(','.join(attr_tup)) log_sql = '' try: with db_instance() as db: log_sql = db.cursor.mogrify(sql, (db_extns.AsIs(cols), vals, db_extns.AsIs(cols), vals)) db.execute(sql, (db_extns.AsIs(cols), vals, db_extns.AsIs(cols), vals)) db.commit() logger.info('Saved updates to order id: {}\n' 'order.id: {}\nsql: {}\nargs: {}' .format(self.orderid, self.id, log_sql, zip(attr_tup, vals))) except DBConnectException as e: logger.debug('Error saving order: {}\nsql: {}' .format(e.message, log_sql)) raise OrderException(e) new = Order.find(self.id) for att in attr_tup: self.__setattr__(att, new.__getattribute__(att))
def create(cls, params): """ Create a new scene entry in the ordering_scene table Also supports a bulk insert for large sets of scenes to insert dict{'name': , 'order_id': , 'status': , 'sensor_type': , 'ee_unit_id': } :param params: dictionary representation of a scene to insert into the system or a list of dictionary objects """ if isinstance(params, (list, tuple)): template = ','.join(['%s'] * len(params)) args = [(s['name'], s['order_id'], s['status'], s['sensor_type'], s['ee_unit_id'], '', '', '', '', '') for s in params] else: template = '%s' args = [(params['name'], params['order_id'], params['status'], params['sensor_type'], params['ee_unit_id'], '', '', '', '', '')] sql = ('INSERT INTO ordering_scene ' '(name, order_id, status, sensor_type, ee_unit_id, ' 'product_distro_location, product_dload_url, ' 'cksum_distro_location, cksum_download_url, ' 'processing_location) VALUES {}'.format(template)) log_sql = '' try: with db_instance() as db: log_sql = db.cursor.mogrify(sql, args) logger.info('scene creation sql: {}' .format(log_sql)) db.execute(sql, args) db.commit() except DBConnectException as e: logger.debug('error creating new scene(s): {}\n' 'sql: {}\n' .format(e.message, log_sql)) raise SceneException(e.message)
def save(self): """ Upsert self to the database """ sql = ('INSERT INTO ordering_order %s VALUES %s ' 'ON CONFLICT (orderid) ' 'DO UPDATE ' 'SET %s = %s') attr_tup = ('orderid', 'status', 'order_source', 'product_options', 'product_opts', 'order_type', 'initial_email_sent', 'completion_email_sent', 'note', 'completion_date', 'order_date', 'user_id', 'ee_order_id', 'email', 'priority') vals = tuple( self.__getattribute__(v) if v != 'product_opts' else json. dumps(self.__getattribute__(v)) for v in attr_tup) cols = '({})'.format(','.join(attr_tup)) log_sql = '' try: with db_instance() as db: log_sql = db.cursor.mogrify( sql, (db_extns.AsIs(cols), vals, db_extns.AsIs(cols), vals)) db.execute( sql, (db_extns.AsIs(cols), vals, db_extns.AsIs(cols), vals)) db.commit() logger.info('Saved updates to order id: {}\n' 'order.id: {}\nsql: {}\nargs: {}'.format( self.orderid, self.id, log_sql, zip(attr_tup, vals))) except DBConnectException as e: logger.debug('Error saving order: {}\nsql: {}'.format( e.message, log_sql)) raise OrderException(e) new = Order.find(self.id) for att in attr_tup: self.__setattr__(att, new.__getattribute__(att))
def place_order(self, order, user): """Enters a new order into the system. Args: :keyword order (api.domain.order.Order): The order to be entered into the system Returns: Order: The generated order Raises: api.api_exceptions.ValidationException: Error occurred validating params api.api_exceptions.InventoryException: Items were not found/unavailable """ try: # perform validation, raises ValidationException order = self.validation.validate(order, user.username) # perform check on open scene limit self.ordering.check_open_scenes(order=order, user_id=user.id, filters={ 'status': ('submitted', 'oncache', 'onorder', 'queued', 'processing') }) # performs inventory check, raises InventoryException self.inventory.check(order, user.contactid) # track metrics self.metrics.collect(order) # capture the order response = self.ordering.place_order(order, user) except (InventoryException, ValidationException, InventoryConnectionException, OpenSceneLimitException) as e: logger.info('Bad order submission: User %s Order %s\nexception %s', user.username, order, traceback.format_exc()) raise except: logger.critical( "ERR version1 place_order arg: %s: %s\n" "exception %s", user.username, order, traceback.format_exc()) response = default_error_message return response
def cancel_order(self, orderid, request_ip_address): """ Cancels an order, and all scenes contained within it :return: """ order = Order.where({'id': orderid}) if len(order) != 1: raise OrderingProviderException('Order not found') else: order = order.pop() logger.info('Received request to cancel {} from {}'.format( orderid, request_ip_address)) killable_scene_states = ('submitted', 'oncache', 'onorder', 'queued', 'retry', 'error', 'unavailable', 'complete') scenes = order.scenes(sql_dict={'status': killable_scene_states}) if len(scenes) > 0: Scene.bulk_update([s.id for s in scenes], Scene.cancel_opts()) else: logger.info('No scenes to cancel for order {}'.format( orderid, request_ip_address)) order.status = 'cancelled' order.save() logger.info('Request to cancel {} from {} successful.'.format( orderid, request_ip_address)) return order
def save(self): """ Save the current configuration of the scene object to the DB """ sql = 'UPDATE ordering_scene SET %s = %s WHERE id = %s' attr_tup = ('status', 'cksum_download_url', 'log_file_contents', 'processing_location', 'retry_after', 'job_name', 'note', 'retry_count', 'sensor_type', 'product_dload_url', 'tram_order_id', 'completion_date', 'ee_unit_id', 'retry_limit', 'cksum_distro_location', 'product_distro_location', 'reported_orphan', 'orphaned', 'failed_lta_status_update', 'download_size') vals = tuple(self.__getattribute__(v) for v in attr_tup) cols = '({})'.format(','.join(attr_tup)) log_sql = '' try: with db_instance() as db: log_sql = db.cursor.mogrify(sql, (db_extns.AsIs(cols), vals, self.id)) db.execute(sql, (db_extns.AsIs(cols), vals, self.id)) db.commit() logger.info('\n*** Saved updates to scene id: {}, name:{}\n' 'sql: {}\n args: {}\n***' .format(self.id, self.name, log_sql, zip(attr_tup, vals))) except DBConnectException as e: logger.debug("Error saving scene: {}\n" "sql: {}".format(e.message, log_sql)) raise SceneException(e) new = Scene.where({'id': self.id})[0] for att in attr_tup: self.__setattr__(att, new.__getattribute__(att))
def delete(self, orderid, filename=None): """ Removes an order from physical online cache disk :param filename: file to delete inside of an order :param orderid: associated order to delete """ if filename: path = os.path.join(self.orderpath, orderid, filename) else: path = os.path.join(self.orderpath, orderid) # this should be the dir where the order is held logger.info('Deleting {} from online cache'.format(path)) try: self.execute_command('sudo chattr -fR -i {0};rm -rf {0}'.format(path)) except OnlineCacheException: # in the event /lustre is mounted to an NFS system logger.info("onlinecache delete, chattr error, attempting chmod instead...") self.execute_command('chmod -R 644 {0};rm -rf {0}'.format(path)) return True
def place_order(self, order, user): """Enters a new order into the system. Args: :keyword order (api.domain.order.Order): The order to be entered into the system Returns: str: The generated order id Raises: api.api_exceptions.ValidationException: Error occurred validating params api.api_exceptions.InventoryException: Items were not found/unavailable """ try: # perform validation, raises ValidationException order = self.validation.validate(order, user.username) # performs inventory check, raises InventoryException self.inventory.check(order) # track metrics self.metrics.collect(order) # capture the order response = self.ordering.place_order(order, user) except ValidationException as e: logger.info('Invalid order received: {0}\nresponse {1}'.format( order, e.response)) # Need to format the string repr of the exception for end user consumption response = e.response except InventoryException as e: logger.info( 'Requested inputs not available: {0}\nresponse {1}'.format( order, e.response)) response = e.response except: logger.debug( "ERR version1 place_order arg: {0}\nexception {1}".format( order, traceback.format_exc())) response = default_error_message return response
def save(self): """ Save the current configuration of the scene object to the DB """ sql = 'UPDATE ordering_scene SET %s = %s WHERE id = %s' attr_tup = ('status', 'cksum_download_url', 'log_file_contents', 'processing_location', 'retry_after', 'job_name', 'note', 'retry_count', 'sensor_type', 'product_dload_url', 'tram_order_id', 'completion_date', 'ee_unit_id', 'retry_limit', 'cksum_distro_location', 'product_distro_location', 'reported_orphan', 'orphaned', 'failed_lta_status_update', 'download_size', 'status_modified') vals = tuple(self.__getattribute__(v) for v in attr_tup) cols = '({})'.format(','.join(attr_tup)) log_sql = '' try: with db_instance() as db: log_sql = db.cursor.mogrify( sql, (db_extns.AsIs(cols), vals, self.id)) db.execute(sql, (db_extns.AsIs(cols), vals, self.id)) db.commit() logger.info('\n*** Saved updates to scene id: {}, name:{}\n' 'sql: {}\n args: {}\n***'.format( self.id, self.name, log_sql, zip(attr_tup, vals))) except DBConnectException as e: logger.critical("Error saving scene: {}\n" "sql: {}".format(e.message, log_sql)) raise SceneException(e) new = Scene.where({'id': self.id})[0] for att in attr_tup: self.__setattr__(att, new.__getattribute__(att))
def user_email(self): """ Retrieve the email address associated with this order :return: email address """ sql = 'select email from auth_user where id = %s' ret = None log_sql = '' try: with db_instance() as db: log_sql = db.cursor.mogrify(sql, (self.user_id, )) logger.info('order.py user_email: {}'.format(log_sql)) db.select(sql, self.user_id) ret = db[0]['email'] except DBConnectException as e: logger.debug('Error retrieving user_email: {}'.format(log_sql)) raise OrderException(e) return ret
def bulk_update(cls, ids=None, updates=None): """ Update a list of scenes with :param ids: ids of scenes to update :param updates: attributes to update :return: True """ if not isinstance(ids, (list, tuple)): raise TypeError('Scene.bulk_update ids should be a list') if not isinstance(updates, dict): raise TypeError('Scene.bulk_update updates should be a dict') sql = 'UPDATE ordering_scene SET %s = %s WHERE id in %s' fields = '({})'.format(','.join(updates.keys())) vals = tuple(updates.values()) ids = tuple(ids) if ",)" in sql: sql = sql.replace(",)", ")") log_sql = '' try: with db_instance() as db: log_sql = db.cursor.mogrify(sql, (db_extns.AsIs(fields), vals, ids)) logger.info('\n*** Bulk Updating scenes: \n' + log_sql + "\n\***\n") db.execute(sql, (db_extns.AsIs(fields), vals, ids)) db.commit() except DBConnectException as e: logger.critical('Error scene bulk_update: {}\nSQL: {}'.format( e.message, log_sql)) raise SceneException(e) return True
def bulk_update(cls, ids=None, updates=None): """ Update a list of scenes with :param ids: ids of scenes to update :param updates: attributes to update :return: True """ if not isinstance(ids, (list, tuple)): raise TypeError('Scene.bulk_update ids should be a list') if not isinstance(updates, dict): raise TypeError('Scene.bulk_update updates should be a dict') sql = 'UPDATE ordering_scene SET %s = %s WHERE id in %s' fields = '({})'.format(','.join(updates.keys())) vals = tuple(updates.values()) ids = tuple(ids) if ",)" in sql: sql = sql.replace(",)", ")") log_sql = '' try: with db_instance() as db: log_sql = db.cursor.mogrify(sql, (db_extns.AsIs(fields), vals, ids)) logger.info('\n*** Bulk Updating scenes: \n' + log_sql + "\n\***\n") db.execute(sql, (db_extns.AsIs(fields), vals, ids)) db.commit() except DBConnectException as e: logger.debug('Error scene bulk_update: {}\nSQL: {}' .format(e.message, log_sql)) raise SceneException(e) return True
def user_email(self): """ Retrieve the email address associated with this order :return: email address """ sql = 'select email from auth_user where id = %s' ret = None log_sql = '' try: with db_instance() as db: log_sql = db.cursor.mogrify(sql, (self.user_id,)) logger.info('order.py user_email: {}'.format(log_sql)) db.select(sql, self.user_id) ret = db[0]['email'] except DBConnectException as e: logger.debug('Error retrieving user_email: {}' .format(log_sql)) raise OrderException(e) return ret
def create(cls, params): """ Place a new order into the system :param params: dict of required parameters to be used params = {'product_opts': {dictionary object of the order received} 'orderid': id generated from generate_order_id 'user_id': EE user id 'order_type': typically 'level2_ondemand' 'status': 'ordered' 'note': user notes 'ee_order_id': earth explorer order id, or '' 'order_source': 'espa' or 'ee' 'order_date': date time string 'priority': legacy item, should be 'normal' 'email': user's contact email 'product_options': legacy column} :return: order object """ opts = params['product_opts'] params['product_opts'] = json.dumps(params['product_opts']) sql = ('INSERT INTO ordering_order ' '(orderid, user_id, order_type, status, note, ' 'product_opts, ee_order_id, order_source, order_date, ' 'priority, email, product_options) ' 'VALUES (%(orderid)s, %(user_id)s, %(order_type)s, ' '%(status)s, %(note)s, %(product_opts)s, ' '%(ee_order_id)s, %(order_source)s, %(order_date)s, ' '%(priority)s, %(email)s, %(product_options)s)') logger.info('Order creation parameters: {}'.format(params)) log_sql = '' try: with db_instance() as db: log_sql = db.cursor.mogrify(sql, params) logger.info('New order complete SQL: {}' .format(log_sql)) db.execute(sql, params) db.commit() except DBConnectException as e: logger.debug('Error creating new order: {}\n' 'sql: {}'.format(e.message, log_sql)) raise OrderException(e) order = Order.find(params['orderid']) # Let the load_ee_order method handle the scene injection # as there is special logic for interacting with LTA if params['ee_order_id']: return order sensor_keys = sensor.SensorCONST.instances.keys() bulk_ls = [] for key in opts: if key in sensor_keys: sensor_type = '' item1 = opts[key]['inputs'][0] if isinstance(sensor.instance(item1), sensor.Landsat): sensor_type = 'landsat' elif isinstance(sensor.instance(item1), sensor.Modis): sensor_type = 'modis' for s in opts[key]['inputs']: scene_dict = {'name': s, 'sensor_type': sensor_type, 'order_id': order.id, 'status': 'submitted', 'ee_unit_id': None} bulk_ls.append(scene_dict) if 'plot_statistics' in opts and opts['plot_statistics']: scene_dict = {'name': 'plot', 'sensor_type': 'plot', 'order_id': order.id, 'status': 'submitted', 'ee_unit_id': None} bulk_ls.append(scene_dict) try: Scene.create(bulk_ls) except SceneException as e: logger.debug('Order creation failed on scene injection, ' 'order: {}\nexception: {}' .format(order.orderid, e.message)) with db_instance() as db: db.execute('delete ordering_order where id = %s', order.id) db.commit() raise OrderException(e) return order
def __init__(self, *args, **kwargs): super(LTASoapService, self).__init__(*args, **kwargs) logger.info("Building SoapClient for:{0}".format(self.url)) self.client = SoapClient(self.url, location=self.location, cache=self.build_object_cache())
def create(cls, params): """ Place a new order into the system :param params: dict of required parameters to be used params = {'product_opts': {dictionary object of the order received} 'orderid': id generated from generate_order_id 'user_id': EE user id 'order_type': typically 'level2_ondemand' 'status': 'ordered' 'note': user notes 'ee_order_id': earth explorer order id, or '' 'order_source': 'espa' or 'ee' 'order_date': date time string 'priority': legacy item, should be 'normal' 'email': user's contact email 'product_options': legacy column} :return: order object """ opts = params['product_opts'] params['product_opts'] = json.dumps(params['product_opts']) sql = ('INSERT INTO ordering_order ' '(orderid, user_id, order_type, status, note, ' 'product_opts, ee_order_id, order_source, order_date, ' 'priority, email, product_options) ' 'VALUES (%(orderid)s, %(user_id)s, %(order_type)s, ' '%(status)s, %(note)s, %(product_opts)s, ' '%(ee_order_id)s, %(order_source)s, %(order_date)s, ' '%(priority)s, %(email)s, %(product_options)s)') logger.info('Order creation parameters: {}'.format(params)) log_sql = '' try: with db_instance() as db: log_sql = db.cursor.mogrify(sql, params) logger.info('New order complete SQL: {}'.format(log_sql)) db.execute(sql, params) db.commit() except DBConnectException as e: logger.debug('Error creating new order: {}\n' 'sql: {}'.format(e.message, log_sql)) raise OrderException(e) order = Order.find(params['orderid']) # Let the load_ee_order method handle the scene injection # as there is special logic for interacting with LTA if params['ee_order_id']: return order sensor_keys = sensor.SensorCONST.instances.keys() bulk_ls = [] for key in opts: if key in sensor_keys: sensor_type = '' item1 = opts[key]['inputs'][0] if isinstance(sensor.instance(item1), sensor.Landsat): sensor_type = 'landsat' elif isinstance(sensor.instance(item1), sensor.Modis): sensor_type = 'modis' for s in opts[key]['inputs']: scene_dict = { 'name': s, 'sensor_type': sensor_type, 'order_id': order.id, 'status': 'submitted', 'ee_unit_id': None } bulk_ls.append(scene_dict) if 'plot_statistics' in opts and opts['plot_statistics']: scene_dict = { 'name': 'plot', 'sensor_type': 'plot', 'order_id': order.id, 'status': 'submitted', 'ee_unit_id': None } bulk_ls.append(scene_dict) try: Scene.create(bulk_ls) except SceneException as e: logger.debug('Order creation failed on scene injection, ' 'order: {}\nexception: {}'.format( order.orderid, e.message)) with db_instance() as db: db.execute('delete ordering_order where id = %s', order.id) db.commit() raise OrderException(e) return order
def __init__(self, *args, **kwargs): super(LTASoapService, self).__init__(*args, **kwargs) logger.info('Building SoapClient for:{0}'.format(self.url)) self.client = SoapClient(self.url, cache=self.build_object_cache())
class OnlineCache(object): """ Client code to interact with the LSRD online cache """ config = ConfigurationProvider() __order_path_key = 'online_cache_orders_dir' __host_key = 'landsatds.host' __user_key = 'landsatds.username' __pw_key = 'landsatds.password' def __init__(self): self.orderpath = self.config.get(self.__order_path_key) if not self.orderpath: msg = '{} not defined in configurations'.format( self.__order_path_key) logger.critical(msg) raise OnlineCacheException(msg) host, user, pw = self.config.get( [self.__host_key, self.__user_key, self.__pw_key]) self.client = sshcmd.RemoteHost(host, user, pw, timeout=5) try: self.client.execute('ls') except Exception as e: logger.critical('No connection to OnlineCache host: {}'.format(e)) raise OnlineCacheException(e) def exists(self, orderid, filename=None): """ Check if an order [optional filename] exists on the onlinecache :param orderid: associated order to check :param filename: file to check inside of an order :return: bool """ if filename: path = os.path.join(self.orderpath, orderid, filename) else: path = os.path.join(self.orderpath, orderid) try: result = self.execute_command('ls -d {0}'.format(path), silent=True) ret = tuple(x.rstrip() for x in result['stdout']) return ret[-1] == path except OnlineCacheException as e: return False def delete(self, orderid, filename=None): """ Removes an order from physical online cache disk :param filename: file to delete inside of an order :param orderid: associated order to delete """ if not self.exists(orderid, filename): msg = 'Invalid orderid {} or filename {}'.format(orderid, filename) logger.critical(msg) return False if filename: path = os.path.join(self.orderpath, orderid, filename) else: path = os.path.join(self.orderpath, orderid) # this should be the dir where the order is held logger.info('Deleting {} from online cache'.format(path)) # TODO: if storage system supports immutable options # >>> sudo chattr -fR -i {0};rm -rf {0} # However, nfs does not support this extended attributes try: cmd = 'chmod -R 744 {0};rm -rf {0}'.format(path) self.execute_command(cmd) except OnlineCacheException as exc: logger.critical('Failed to remove files from output cache. ' 'Command: {} Error: {}'.format(cmd, exc)) return False return True def list(self, orderid=None): """ List the orders currently stored on cache, or files listed insed of a specific order :param orderid: order name to look inside of :return: list of folders/files """ if orderid: path = os.path.join(self.orderpath, orderid) else: path = self.orderpath cmd = 'ls {}'.format(path) result = self.execute_command(cmd) ret = tuple(x.rstrip() for x in result['stdout']) return ret def capacity(self): """ Returns the capacity of the online cache :return: dict """ cmd = 'df -mhP {}'.format(self.orderpath) result = self.execute_command(cmd) line = result['stdout'][1].split(' ') clean = [l for l in line if len(l) > 0] results = { 'capacity': clean[1], 'used': clean[2], 'available': clean[3], 'percent_used': clean[4] } return results def execute_command(self, cmd, silent=False): """ Execute the given command on the cache :param cmd: cmd string to execute :return: results of the command """ try: result = self.client.execute(cmd) except Exception, exception: if not silent: logger.critical('Error executing command: {} ' 'Raised exception: {}'.format(cmd, exception)) raise OnlineCacheException(exception) if 'stderr' in result and result['stderr']: if not silent: logger.critical('Error executing command: {} ' 'stderror returned: {}'.format( cmd, result['stderr'])) raise OnlineCacheException(result['stderr']) logger.info('call to {} returned {}'.format(cmd, result)) return result
class OnlineCache(object): """ Client code to interact with the LSRD online cache """ config = ConfigurationProvider() __order_path_key = 'online_cache_orders_dir' __host_key = 'landsatds.host' __user_key = 'landsatds.username' __pw_key = 'landsatds.password' def __init__(self): self.orderpath = self.config.get(self.__order_path_key) if not self.orderpath: msg = '{} not defined in configurations'.format(self.__order_path_key) logger.debug(msg) raise OnlineCacheException(msg) host, user, pw = self.config.get([self.__host_key, self.__user_key, self.__pw_key]) self.client = sshcmd.RemoteHost(host, user, pw, timeout=5) try: self.client.execute('ls') except Exception as e: logger.debug('No connection to OnlineCache host: {}'.format(e)) raise OnlineCacheException(e) def delete(self, orderid, filename=None): """ Removes an order from physical online cache disk :param filename: file to delete inside of an order :param orderid: associated order to delete """ if filename: path = os.path.join(self.orderpath, orderid, filename) else: path = os.path.join(self.orderpath, orderid) # this should be the dir where the order is held logger.info('Deleting {} from online cache'.format(path)) try: self.execute_command('sudo chattr -fR -i {0};rm -rf {0}'.format(path)) except OnlineCacheException: # in the event /lustre is mounted to an NFS system logger.info("onlinecache delete, chattr error, attempting chmod instead...") self.execute_command('chmod -R 644 {0};rm -rf {0}'.format(path)) return True def list(self, orderid=None): """ List the orders currently stored on cache, or files listed insed of a specific order :param orderid: order name to look inside of :return: list of folders/files """ if orderid: path = os.path.join(self.orderpath, orderid) else: path = self.orderpath cmd = 'ls {}'.format(path) result = self.execute_command(cmd) ret = tuple(x.rstrip() for x in result['stdout']) return ret def capacity(self): """ Returns the capacity of the online cache :return: dict """ cmd = 'df -mhP {}'.format(self.orderpath) result = self.execute_command(cmd) line = result['stdout'][1].split(' ') clean = [l for l in line if len(l) > 0] results = {'capacity': clean[1], 'used': clean[2], 'available': clean[3], 'percent_free': clean[4]} return results def execute_command(self, cmd): """ Execute the given command on the cache :param cmd: cmd string to execute :return: results of the command """ try: result = self.client.execute(cmd) except Exception, exception: logger.debug('Error executing command: {} ' 'Raised exception: {}'.format(cmd, exception)) raise OnlineCacheException(exception) if 'stderr' in result and result['stderr']: logger.debug('Error executing command: {} ' 'stderror returned: {}'.format(cmd, result['stderr'])) raise OnlineCacheException(result['stderr']) logger.info('call to {} returned {}'.format(cmd, result)) return result