Пример #1
0
    def update(self, att, val):
        """
        Update a specifed column value for this Scene object
        with a new value

        :param att: column to update
        :param val: new value
        :return: updated value from self
        """
        sql = 'update ordering_scene set %s = %s where id = %s'

        log_sql = ''
        try:
            with db_instance() as db:
                log_sql = db.cursor.mogrify(sql,
                                            (db_extns.AsIs(att), val, self.id))
                logger.info("\n*** Updating scene: \n" + log_sql + '\n***\n"')
                db.execute(sql, (db_extns.AsIs(att), val, self.id))
                db.commit()
        except DBConnectException as e:
            raise SceneException('Error updating scene: {}\nSQL: {}'.format(
                e.message, log_sql))

        self.__setattr__(att, val)

        return self.__getattribute__(att)
Пример #2
0
    def delete(self, orderid, filename=None):
        """
        Removes an order from physical online cache disk

        :param filename: file to delete inside of an order
        :param orderid: associated order to delete
        """
        if not self.exists(orderid, filename):
            msg = 'Invalid orderid {} or filename {}'.format(orderid, filename)
            logger.critical(msg)
            return False

        if filename:
            path = os.path.join(self.orderpath, orderid, filename)
        else:
            path = os.path.join(self.orderpath, orderid)

        # this should be the dir where the order is held
        logger.info('Deleting {} from online cache'.format(path))
        # TODO: if storage system supports immutable options
        # >>> sudo chattr -fR -i {0};rm -rf {0}
        # However, nfs does not support this extended attributes
        try:
            cmd = 'chmod -R 744 {0};rm -rf {0}'.format(path)
            self.execute_command(cmd)
        except OnlineCacheException as exc:
            logger.critical('Failed to remove files from output cache. '
                            'Command: {} Error: {}'.format(cmd, exc))
            return False
        return True
Пример #3
0
    def post(version):
        try:
            user = flask.g.user
            order = request.get_json(force=True)
            if not order:
                message = {"status": 400, "msg": "Unable to parse json data. Please ensure your order follows json "
                                                 "conventions and your http call is correct. If you believe this "
                                                 "message is in error please email customer service"}
            else:
                try:
                    order = lowercase_all(order)
                    orderid = espa.place_order(order, user)
                    if isinstance(orderid, str) and "@" in orderid:
                        # if order submission was successful, orderid is returned as a string
                        # which includes the submitters email address
                        message = {"status": 200, "orderid": orderid}
                    else:
                        # there was a problem, and orderid is a dict with the problem details
                        logger.info("problem with user submitted order. user: {0}\n details: {1}".format(user.username, orderid))
                        message = {"status": 400, "message": orderid}
                except Exception as e:
                    logger.debug("exception posting order: {0}\nuser: {1}\n msg: {2}".format(order, user.username, e.message))
                    message = {"status": 500, "msg": "the system experienced an exception. the admins have been notified"}
        except BadRequest as e:
            # request.get_json throws a BadRequest
            logger.debug("BadRequest, could not parse request into json {}\nuser: {}\nform data {}\n".format(e.description, user.username, request.form))
            message = {"status": 400, "msg": "Could not parse the request into json"}
        except Exception as e:
            logger.debug("ERR posting order. user: {0}\n error: {1}".format(user.username, e))
            message = {"status": 500, "msg": "the system has experienced an exception. the admins have been notified."}

        response = jsonify(message)
        response.status_code = message['status']
        return response
Пример #4
0
    def update(self, att, val):
        """
        Update a specified column value for this Order object

        :param att: column to update
        :param val: new value
        :return: updated value from self
        """
        sql = 'update ordering_order set %s = %s where id = %s'

        log_sql = ''
        try:
            with db_instance() as db:
                log_sql = db.cursor.mogrify(sql, (db_extns.AsIs(att),
                                                  val, self.id))
                logger.info(log_sql)
                db.execute(sql, (db_extns.AsIs(att), val, self.id))
                db.commit()
        except DBConnectException as e:
            logger.debug('Error updating order: {}\nSQL: {}'
                         .format(e.message, log_sql))

        self.__setattr__(att, val)

        return self.__getattribute__(att)
Пример #5
0
 def __init__(self, *args, **kwargs):
     super(LTASoapService, self).__init__(*args, **kwargs)
     if config.mode in ('dev', 'tst'):
         logger.info('Building SoapClient for:{0}'.format(self.url))
     self.client = SoapClient(self.url,
                              location=self.location,
                              cache=self.build_object_cache())
Пример #6
0
    def delete(self, orderid, filename=None):
        """
        Removes an order from physical online cache disk

        :param filename: file to delete inside of an order
        :param orderid: associated order to delete
        """
        if not self.exists(orderid, filename):
            msg = 'Invalid orderid {} or filename {}'.format(orderid, filename)
            logger.critical(msg)
            return False

        if filename:
            path = os.path.join(self.orderpath, orderid, filename)
        else:
            path = os.path.join(self.orderpath, orderid)

        # this should be the dir where the order is held
        logger.info('Deleting {} from online cache'.format(path))
        try:
            self.execute_command(
                'sudo chattr -fR -i {0};rm -rf {0}'.format(path))
        except OnlineCacheException:
            # in the event /lustre is mounted to an NFS system
            logger.info(
                "onlinecache delete, chattr error, attempting chmod instead..."
            )
            self.execute_command('chmod -R 644 {0};rm -rf {0}'.format(path))

        return True
Пример #7
0
    def update(self, att, val):
        """
        Update a specified column value for this Order object

        :param att: column to update
        :param val: new value
        :return: updated value from self
        """
        sql = 'update ordering_order set %s = %s where id = %s'

        log_sql = ''
        try:
            with db_instance() as db:
                log_sql = db.cursor.mogrify(sql,
                                            (db_extns.AsIs(att), val, self.id))
                logger.info(log_sql)
                db.execute(sql, (db_extns.AsIs(att), val, self.id))
                db.commit()
        except DBConnectException as e:
            logger.debug('Error updating order: {}\nSQL: {}'.format(
                e.message, log_sql))

        self.__setattr__(att, val)

        return self.__getattribute__(att)
Пример #8
0
    def where(cls, params):
        """
        Query for particular users

        :param params: dictionary of column: value parameters
        :return: list of matching User objects
        """
        if not isinstance(params, dict):
            raise UserException(
                'Where arguments must be passed as a dictionary')

        sql, values = format_sql_params(cls.base_sql, params)

        ret = []
        log_sql = ''
        try:
            with db_instance() as db:
                log_sql = db.cursor.mogrify(sql, values)
                logger.info('user.py where sql: {}'.format(log_sql))
                db.select(sql, values)
                for i in db:
                    obj = User(i["username"], i["email"], i["first_name"],
                               i["last_name"], i["contactid"])
                    ret.append(obj)
        except DBConnectException as e:
            logger.debug('Error querying for users: {}\n'
                         'sql: {}'.format(e.message, log_sql))
            raise UserException(e)
        return ret
Пример #9
0
    def verify_scenes(self, product_ids, dataset):
        """
        Check if supplied IDs successfully map to M2M entity IDs

        :param product_ids: Landsat Collection IDs ['LC08_..', ...]
        :type product_ids: list
        :return: dict
        """
        entity_ids = self.id_lookup(product_ids,
                                    dataset)  # gather M2M entity ids
        # M2M entity ids do not correlate to product availability though. And product download url
        # requests will Fail a request if any of the products requested are not actually
        # available (as of M2M api version 1.4.1). So we need to make a downloadOptions request
        # to the M2M Api to ensure said product is actually available
        availability = self.download_available(
            entity_ids, dataset)  # dict of {entityid:availability, ...}

        response = dict()
        for k, v in entity_ids.items():
            if v:
                response[k] = availability[v]
            else:
                logger.info("id: {} is not available".format(k))
                response[k] = False

        return response
Пример #10
0
    def update(self, att, val):
        """
        Update a specifed column value for this Scene object
        with a new value

        :param att: column to update
        :param val: new value
        :return: updated value from self
        """
        sql = 'update ordering_scene set %s = %s where id = %s'

        log_sql = ''
        try:
            with db_instance() as db:
                log_sql = db.cursor.mogrify(sql, (db_extns.AsIs(att),
                                                  val, self.id))
                logger.info("\n*** Updating scene: \n" + log_sql + '\n***\n"')
                db.execute(sql, (db_extns.AsIs(att), val, self.id))
                db.commit()
        except DBConnectException as e:
            raise SceneException('Error updating scene: {}\nSQL: {}'
                                 .format(e.message, log_sql))

        self.__setattr__(att, val)

        return self.__getattribute__(att)
Пример #11
0
    def where(cls, params):
        """
        Query for a particular row in the ordering_scene table

        :param params: dictionary of column: value parameter to select on
        :return: list of matching Scene objects
        """
        if not isinstance(params, dict):
            raise SceneException('Where arguments must be '
                                 'passed as a dictionary')

        sql, values = format_sql_params(cls.base_sql, params)

        ret = []
        log_sql = ''
        try:
            with db_instance() as db:
                log_sql = db.cursor.mogrify(sql, values)
                logger.info('scene.py where sql: {}'.format(log_sql))
                db.select(sql, values)
                for i in db:
                    sd = dict(i)
                    obj = Scene(**sd)
                    ret.append(obj)
        except DBConnectException as e:
            logger.critical('Error retrieving scenes: {}\n'
                            'sql: {}'.format(e.message, log_sql))
            raise SceneException(e)

        return ret
Пример #12
0
def verify_user(username, password):
    try:
        # usernames with spaces are valid in EE, though they can't be used for cache keys
        cache_key = '{}-credentials'.format(username.replace(' ', '_espa_cred_insert_'))
        cache_entry = cache.get(cache_key)

        if cache_entry:
            # Need to be encrypted?
            if cache_entry['password'] == password:
                user_entry = cache_entry['user_entry']

            # User may have changed their password while it was still cached
            else:
                user_entry = User.get(username, password)
        else:
            user_entry = User.get(username, password)

        cache_entry = {'password': password,
                       'user_entry': user_entry}
        cache.set(cache_key, cache_entry, 7200)

        user = User(*user_entry)
        flask.g.user = user  # Replace usage with cached version
    except Exception:
        logger.info('Invalid login attempt, username: {}'.format(username))
        return False

    return True
Пример #13
0
    def where(cls, params):
        """
        Query for a particular row in the ordering_oder table

        :param params: dictionary of column: value parameter to select on
        :return: list of matching Order objects
        """
        if not isinstance(params, dict):
            raise OrderException('Where arguments must be '
                                 'passed as a dictionary')

        sql, values = format_sql_params(cls.base_sql, params)

        ret = []
        log_sql = ''
        try:
            with db_instance() as db:
                log_sql = db.cursor.mogrify(sql, values)
                logger.info('order.py where sql: {}'.format(log_sql))

                db.select(sql, values)

                for i in db:
                    od = dict(i)
                    obj = Order(**od)
                    ret.append(obj)
        except DBConnectException as e:
            logger.debug('Error order where: {}\n'
                         'sql: {}'.format(e.message, log_sql))
            raise OrderException(e)

        return ret
Пример #14
0
 def check_LTA(prod_ls):
     not_avail = nlaps.products_are_nlaps(prod_ls)
     if not_avail:
         raise InventoryException(not_avail)
     logger.info('@@ VERIFY: FETCH via OrderWrapperService')
     results = lta.verify_scenes(prod_ls)
     logger.info('@@ VERIFY: COMPLETE via OrderWrapperService')
     return results
Пример #15
0
 def check_dmid(prod_ls, contactid=None):
     # find all the submitted products that are nlaps and reject them
     not_avail = nlaps.products_are_nlaps(prod_ls)
     if not_avail:
         raise InventoryException(not_avail)
     logger.info('@@ VERIFY: FETCH via MACHINE-TO-MACHINE')
     token = inventory.get_cached_session()
     results = inventory.get_cached_verify_scenes(token, prod_ls)
     logger.info('@@ VERIFY: COMPLETE via MACHINE-TO-MACHINE')
     return results
Пример #16
0
def verify_user(username, password):
    if (username is None) or (not (str(username).strip())):
        logger.warning('Invalid username supplied: %s', username)
        flask.g.error_reason = 'auth'
        return False
    try:
        # usernames with spaces are valid in EE, though they can't be used for cache keys
        cache_key = '{}-credentials'.format(
            username.replace(' ', '_espa_cred_insert_'))
        cache_entry = cache.get(cache_key)

        if cache_entry:
            # Need to be encrypted?
            if cache_entry['password'] == password:
                user_entry = cache_entry['user_entry']

            # User may have changed their password while it was still cached
            else:
                user_entry = User.get(username, password)
        else:
            user_entry = User.get(username, password)

        cache_entry = {'password': password, 'user_entry': user_entry}
        cache.set(cache_key, cache_entry, 7200)

        user = User(*user_entry)
        flask.g.user = user  # Replace usage with cached version
    except UserException as e:
        logger.info('Invalid login attempt, username: {}, {}'.format(
            username, e))
        flask.g.error_reason = 'unknown'
        return False
    except ERSApiAuthFailedException as e:
        logger.info('Invalid login attempt, username: {}, {}'.format(
            username, e))
        flask.g.error_reason = 'auth'
        return False
    except ERSApiErrorException as e:
        logger.info('ERS lookup failed, username: {}, {}'.format(username, e))
        flask.g.error_reason = 'unknown'
        return False
    except ERSApiConnectionException as e:
        logger.info('ERS is down {}'.format(e))
        flask.g.error_reason = 'conn'
        return False
    except DBConnectException as e:
        logger.critical('! Database reported a problem: {}'.format(e))
        flask.g.error_reasons = 'db'
        return False
    except Exception:
        logger.info('Invalid login attempt, username: {}'.format(username))
        flask.g.error_reason = 'unknown'
        return False

    return True
Пример #17
0
    def get(cls, col_name, scene_name, orderid):
        """
        Retrieve a value for a particular column based on
        the long name of the order

        :param col_name: column value to retrieve
        :param scene_name: scene/collection id
        :param orderid: long name for the related order,
         [email protected]
        :return: column value
        """
        sql = ('select %s '
               'from ordering_scene '
               'join ordering_order '
               'on ordering_order.id = ordering_scene.order_id '
               'where ordering_scene.name = %s '
               'and ordering_order.orderid = %s')

        if '.' in col_name:
            _, col = col_name.split('.')
        else:
            col = col_name

        log_sql = ''
        try:
            with db_instance() as db:
                log_sql = (db.cursor.
                           mogrify(sql, (db_extns.AsIs(col_name),
                                         scene_name, orderid)))
                logger.info(log_sql)
                db.select(sql, (db_extns.AsIs(col_name),
                                scene_name, orderid))
                ret = db[0][col]

        except DBConnectException as e:
            logger.debug('Error scene get\n'
                         'msg: {0}\n'
                         'sql: {1}'.format(e.message, log_sql))

            raise SceneException(e.message)

        except KeyError as e:
            logger.debug('Scene.get returned no results\n'
                         'sql: {}'.format(log_sql))

            raise SceneException('Key Error: {}'
                                 .format(e.message))

        return ret
Пример #18
0
    def get(cls, col_name, scene_name, orderid):
        """
        Retrieve a value for a particular column based on
        the long name of the order

        :param col_name: column value to retrieve
        :param scene_name: scene/collection id
        :param orderid: long name for the related order,
         [email protected]
        :return: column value
        """
        sql = ('select %s '
               'from ordering_scene '
               'join ordering_order '
               'on ordering_order.id = ordering_scene.order_id '
               'where ordering_scene.name = %s '
               'and ordering_order.orderid = %s')

        if '.' in col_name:
            _, col = col_name.split('.')
        else:
            col = col_name

        log_sql = ''
        try:
            with db_instance() as db:
                log_sql = (db.cursor.
                           mogrify(sql, (db_extns.AsIs(col_name),
                                         scene_name, orderid)))
                logger.info(log_sql)
                db.select(sql, (db_extns.AsIs(col_name),
                                scene_name, orderid))
                ret = db[0][col]

        except DBConnectException as e:
            logger.debug('Error scene get\n'
                         'msg: {0}\n'
                         'sql: {1}'.format(e.message, log_sql))

            raise SceneException(e.message)

        except KeyError as e:
            logger.debug('Scene.get returned no results\n'
                         'sql: {}'.format(log_sql))

            raise SceneException('Key Error: {}'
                                 .format(e.message))

        return ret
Пример #19
0
    def save(self):
        """
        Upsert self to the database
        """
        sql = ('INSERT INTO ordering_order %s VALUES %s '
               'ON CONFLICT (orderid) '
               'DO UPDATE '
               'SET %s = %s')

        attr_tup = ('orderid', 'status', 'order_source',
                    'product_options', 'product_opts', 'order_type',
                    'initial_email_sent', 'completion_email_sent',
                    'note', 'completion_date', 'order_date', 'user_id',
                    'ee_order_id', 'email', 'priority')

        vals = tuple(self.__getattribute__(v)
                     if v != 'product_opts'
                     else json.dumps(self.__getattribute__(v))
                     for v in attr_tup)

        cols = '({})'.format(','.join(attr_tup))

        log_sql = ''
        try:
            with db_instance() as db:
                log_sql = db.cursor.mogrify(sql, (db_extns.AsIs(cols),
                                                  vals,
                                                  db_extns.AsIs(cols),
                                                  vals))
                db.execute(sql, (db_extns.AsIs(cols), vals,
                                 db_extns.AsIs(cols), vals))
                db.commit()

                logger.info('Saved updates to order id: {}\n'
                            'order.id: {}\nsql: {}\nargs: {}'
                            .format(self.orderid, self.id, log_sql,
                                    zip(attr_tup, vals)))
        except DBConnectException as e:
            logger.debug('Error saving order: {}\nsql: {}'
                         .format(e.message, log_sql))

            raise OrderException(e)

        new = Order.find(self.id)

        for att in attr_tup:
            self.__setattr__(att, new.__getattribute__(att))
Пример #20
0
    def create(cls, params):
        """
        Create a new scene entry in the ordering_scene table
        Also supports a bulk insert for large sets of scenes to insert

        dict{'name': ,
             'order_id': ,
             'status': ,
             'sensor_type': ,
             'ee_unit_id': }

        :param params: dictionary representation of a scene to insert
         into the system or a list of dictionary objects
        """
        if isinstance(params, (list, tuple)):
            template = ','.join(['%s'] * len(params))
            args = [(s['name'], s['order_id'],
                     s['status'], s['sensor_type'],
                     s['ee_unit_id'], '', '', '', '', '')
                    for s in params]
        else:
            template = '%s'
            args = [(params['name'], params['order_id'],
                     params['status'], params['sensor_type'],
                     params['ee_unit_id'], '', '', '', '', '')]

        sql = ('INSERT INTO ordering_scene '
               '(name, order_id, status, sensor_type, ee_unit_id, '
               'product_distro_location, product_dload_url, '
               'cksum_distro_location, cksum_download_url, '
               'processing_location) VALUES {}'.format(template))

        log_sql = ''
        try:
            with db_instance() as db:
                log_sql = db.cursor.mogrify(sql, args)
                logger.info('scene creation sql: {}'
                            .format(log_sql))
                db.execute(sql, args)
                db.commit()

        except DBConnectException as e:
            logger.debug('error creating new scene(s): {}\n'
                         'sql: {}\n'
                         .format(e.message, log_sql))
            raise SceneException(e.message)
Пример #21
0
    def save(self):
        """
        Upsert self to the database
        """
        sql = ('INSERT INTO ordering_order %s VALUES %s '
               'ON CONFLICT (orderid) '
               'DO UPDATE '
               'SET %s = %s')

        attr_tup = ('orderid', 'status', 'order_source', 'product_options',
                    'product_opts', 'order_type', 'initial_email_sent',
                    'completion_email_sent', 'note', 'completion_date',
                    'order_date', 'user_id', 'ee_order_id', 'email',
                    'priority')

        vals = tuple(
            self.__getattribute__(v) if v != 'product_opts' else json.
            dumps(self.__getattribute__(v)) for v in attr_tup)

        cols = '({})'.format(','.join(attr_tup))

        log_sql = ''
        try:
            with db_instance() as db:
                log_sql = db.cursor.mogrify(
                    sql,
                    (db_extns.AsIs(cols), vals, db_extns.AsIs(cols), vals))
                db.execute(
                    sql,
                    (db_extns.AsIs(cols), vals, db_extns.AsIs(cols), vals))
                db.commit()

                logger.info('Saved updates to order id: {}\n'
                            'order.id: {}\nsql: {}\nargs: {}'.format(
                                self.orderid, self.id, log_sql,
                                zip(attr_tup, vals)))
        except DBConnectException as e:
            logger.debug('Error saving order: {}\nsql: {}'.format(
                e.message, log_sql))

            raise OrderException(e)

        new = Order.find(self.id)

        for att in attr_tup:
            self.__setattr__(att, new.__getattribute__(att))
Пример #22
0
    def create(cls, params):
        """
        Create a new scene entry in the ordering_scene table
        Also supports a bulk insert for large sets of scenes to insert

        dict{'name': ,
             'order_id': ,
             'status': ,
             'sensor_type': ,
             'ee_unit_id': }

        :param params: dictionary representation of a scene to insert
         into the system or a list of dictionary objects
        """
        if isinstance(params, (list, tuple)):
            template = ','.join(['%s'] * len(params))
            args = [(s['name'], s['order_id'],
                     s['status'], s['sensor_type'],
                     s['ee_unit_id'], '', '', '', '', '')
                    for s in params]
        else:
            template = '%s'
            args = [(params['name'], params['order_id'],
                     params['status'], params['sensor_type'],
                     params['ee_unit_id'], '', '', '', '', '')]

        sql = ('INSERT INTO ordering_scene '
               '(name, order_id, status, sensor_type, ee_unit_id, '
               'product_distro_location, product_dload_url, '
               'cksum_distro_location, cksum_download_url, '
               'processing_location) VALUES {}'.format(template))

        log_sql = ''
        try:
            with db_instance() as db:
                log_sql = db.cursor.mogrify(sql, args)
                logger.info('scene creation sql: {}'
                            .format(log_sql))
                db.execute(sql, args)
                db.commit()

        except DBConnectException as e:
            logger.debug('error creating new scene(s): {}\n'
                         'sql: {}\n'
                         .format(e.message, log_sql))
            raise SceneException(e.message)
Пример #23
0
    def place_order(self, order, user):
        """Enters a new order into the system.

        Args:
            :keyword order (api.domain.order.Order): The order to be entered into the system

        Returns:
            Order: The generated order

        Raises:
            api.api_exceptions.ValidationException: Error occurred validating params
            api.api_exceptions.InventoryException: Items were not found/unavailable
        """
        try:
            # perform validation, raises ValidationException
            order = self.validation.validate(order, user.username)
            # perform check on open scene limit
            self.ordering.check_open_scenes(order=order,
                                            user_id=user.id,
                                            filters={
                                                'status':
                                                ('submitted', 'oncache',
                                                 'onorder', 'queued',
                                                 'processing')
                                            })
            # performs inventory check, raises InventoryException
            self.inventory.check(order, user.contactid)
            # track metrics
            self.metrics.collect(order)
            # capture the order
            response = self.ordering.place_order(order, user)
        except (InventoryException, ValidationException,
                InventoryConnectionException, OpenSceneLimitException) as e:
            logger.info('Bad order submission: User %s Order %s\nexception %s',
                        user.username, order, traceback.format_exc())
            raise
        except:
            logger.critical(
                "ERR version1 place_order arg: %s: %s\n"
                "exception %s", user.username, order, traceback.format_exc())
            response = default_error_message

        return response
Пример #24
0
    def cancel_order(self, orderid, request_ip_address):
        """
        Cancels an order, and all scenes contained within it

        :return:
        """
        order = Order.where({'id': orderid})
        if len(order) != 1:
            raise OrderingProviderException('Order not found')
        else:
            order = order.pop()

        logger.info('Received request to cancel {} from {}'.format(
            orderid, request_ip_address))
        killable_scene_states = ('submitted', 'oncache', 'onorder', 'queued',
                                 'retry', 'error', 'unavailable', 'complete')
        scenes = order.scenes(sql_dict={'status': killable_scene_states})
        if len(scenes) > 0:
            Scene.bulk_update([s.id for s in scenes], Scene.cancel_opts())
        else:
            logger.info('No scenes to cancel for order {}'.format(
                orderid, request_ip_address))

        order.status = 'cancelled'
        order.save()
        logger.info('Request to cancel {} from {} successful.'.format(
            orderid, request_ip_address))
        return order
Пример #25
0
    def save(self):
        """
        Save the current configuration of the scene object to the DB
        """
        sql = 'UPDATE ordering_scene SET %s = %s WHERE id = %s'

        attr_tup = ('status', 'cksum_download_url', 'log_file_contents',
                    'processing_location', 'retry_after', 'job_name',
                    'note', 'retry_count', 'sensor_type',
                    'product_dload_url', 'tram_order_id',
                    'completion_date', 'ee_unit_id', 'retry_limit',
                    'cksum_distro_location', 'product_distro_location',
                    'reported_orphan', 'orphaned', 'failed_lta_status_update',
                    'download_size')

        vals = tuple(self.__getattribute__(v) for v in attr_tup)
        cols = '({})'.format(','.join(attr_tup))

        log_sql = ''
        try:
            with db_instance() as db:
                log_sql = db.cursor.mogrify(sql, (db_extns.AsIs(cols),
                                                  vals, self.id))

                db.execute(sql, (db_extns.AsIs(cols), vals, self.id))
                db.commit()
                logger.info('\n*** Saved updates to scene id: {}, name:{}\n'
                            'sql: {}\n args: {}\n***'
                            .format(self.id, self.name,
                                    log_sql, zip(attr_tup, vals)))
        except DBConnectException as e:
            logger.debug("Error saving scene: {}\n"
                         "sql: {}".format(e.message, log_sql))
            raise SceneException(e)

        new = Scene.where({'id': self.id})[0]

        for att in attr_tup:
            self.__setattr__(att, new.__getattribute__(att))
Пример #26
0
    def delete(self, orderid, filename=None):
        """
        Removes an order from physical online cache disk

        :param filename: file to delete inside of an order
        :param orderid: associated order to delete
        """
        if filename:
            path = os.path.join(self.orderpath, orderid, filename)
        else:
            path = os.path.join(self.orderpath, orderid)

        # this should be the dir where the order is held
        logger.info('Deleting {} from online cache'.format(path))
        try:
            self.execute_command('sudo chattr -fR -i {0};rm -rf {0}'.format(path))
        except OnlineCacheException:
            # in the event /lustre is mounted to an NFS system
            logger.info("onlinecache delete, chattr error, attempting chmod instead...")
            self.execute_command('chmod -R 644 {0};rm -rf {0}'.format(path))

        return True
Пример #27
0
    def place_order(self, order, user):
        """Enters a new order into the system.

        Args:
            :keyword order (api.domain.order.Order): The order to be entered into the system

        Returns:
            str: The generated order id

        Raises:
            api.api_exceptions.ValidationException: Error occurred validating params
            api.api_exceptions.InventoryException: Items were not found/unavailable
        """
        try:
            # perform validation, raises ValidationException
            order = self.validation.validate(order, user.username)
            # performs inventory check, raises InventoryException
            self.inventory.check(order)
            # track metrics
            self.metrics.collect(order)
            # capture the order
            response = self.ordering.place_order(order, user)
        except ValidationException as e:
            logger.info('Invalid order received: {0}\nresponse {1}'.format(
                order, e.response))
            # Need to format the string repr of the exception for end user consumption
            response = e.response
        except InventoryException as e:
            logger.info(
                'Requested inputs not available: {0}\nresponse {1}'.format(
                    order, e.response))
            response = e.response
        except:
            logger.debug(
                "ERR version1 place_order arg: {0}\nexception {1}".format(
                    order, traceback.format_exc()))
            response = default_error_message

        return response
Пример #28
0
    def save(self):
        """
        Save the current configuration of the scene object to the DB
        """
        sql = 'UPDATE ordering_scene SET %s = %s WHERE id = %s'

        attr_tup = ('status', 'cksum_download_url', 'log_file_contents',
                    'processing_location', 'retry_after', 'job_name', 'note',
                    'retry_count', 'sensor_type', 'product_dload_url',
                    'tram_order_id', 'completion_date', 'ee_unit_id',
                    'retry_limit', 'cksum_distro_location',
                    'product_distro_location', 'reported_orphan', 'orphaned',
                    'failed_lta_status_update', 'download_size',
                    'status_modified')

        vals = tuple(self.__getattribute__(v) for v in attr_tup)
        cols = '({})'.format(','.join(attr_tup))

        log_sql = ''
        try:
            with db_instance() as db:
                log_sql = db.cursor.mogrify(
                    sql, (db_extns.AsIs(cols), vals, self.id))

                db.execute(sql, (db_extns.AsIs(cols), vals, self.id))
                db.commit()
                logger.info('\n*** Saved updates to scene id: {}, name:{}\n'
                            'sql: {}\n args: {}\n***'.format(
                                self.id, self.name, log_sql,
                                zip(attr_tup, vals)))
        except DBConnectException as e:
            logger.critical("Error saving scene: {}\n"
                            "sql: {}".format(e.message, log_sql))
            raise SceneException(e)

        new = Scene.where({'id': self.id})[0]

        for att in attr_tup:
            self.__setattr__(att, new.__getattribute__(att))
Пример #29
0
    def user_email(self):
        """
        Retrieve the email address associated with this order

        :return: email address
        """
        sql = 'select email from auth_user where id = %s'

        ret = None
        log_sql = ''
        try:
            with db_instance() as db:
                log_sql = db.cursor.mogrify(sql, (self.user_id, ))
                logger.info('order.py user_email: {}'.format(log_sql))

                db.select(sql, self.user_id)

                ret = db[0]['email']
        except DBConnectException as e:
            logger.debug('Error retrieving user_email: {}'.format(log_sql))
            raise OrderException(e)

        return ret
Пример #30
0
    def bulk_update(cls, ids=None, updates=None):
        """
        Update a list of scenes with

        :param ids: ids of scenes to update
        :param updates: attributes to update
        :return: True
        """
        if not isinstance(ids, (list, tuple)):
            raise TypeError('Scene.bulk_update ids should be a list')
        if not isinstance(updates, dict):
            raise TypeError('Scene.bulk_update updates should be a dict')

        sql = 'UPDATE ordering_scene SET %s = %s WHERE id in %s'

        fields = '({})'.format(','.join(updates.keys()))
        vals = tuple(updates.values())
        ids = tuple(ids)

        if ",)" in sql:
            sql = sql.replace(",)", ")")

        log_sql = ''
        try:
            with db_instance() as db:
                log_sql = db.cursor.mogrify(sql,
                                            (db_extns.AsIs(fields), vals, ids))
                logger.info('\n*** Bulk Updating scenes: \n' + log_sql +
                            "\n\***\n")
                db.execute(sql, (db_extns.AsIs(fields), vals, ids))
                db.commit()
        except DBConnectException as e:
            logger.critical('Error scene bulk_update: {}\nSQL: {}'.format(
                e.message, log_sql))
            raise SceneException(e)

        return True
Пример #31
0
    def bulk_update(cls, ids=None, updates=None):
        """
        Update a list of scenes with

        :param ids: ids of scenes to update
        :param updates: attributes to update
        :return: True
        """
        if not isinstance(ids, (list, tuple)):
            raise TypeError('Scene.bulk_update ids should be a list')
        if not isinstance(updates, dict):
            raise TypeError('Scene.bulk_update updates should be a dict')

        sql = 'UPDATE ordering_scene SET %s = %s WHERE id in %s'

        fields = '({})'.format(','.join(updates.keys()))
        vals = tuple(updates.values())
        ids = tuple(ids)

        if ",)" in sql:
            sql = sql.replace(",)", ")")

        log_sql = ''
        try:
            with db_instance() as db:
                log_sql = db.cursor.mogrify(sql, (db_extns.AsIs(fields),
                                                  vals, ids))
                logger.info('\n*** Bulk Updating scenes: \n' + log_sql + "\n\***\n")
                db.execute(sql, (db_extns.AsIs(fields), vals, ids))
                db.commit()
        except DBConnectException as e:
            logger.debug('Error scene bulk_update: {}\nSQL: {}'
                         .format(e.message, log_sql))
            raise SceneException(e)

        return True
Пример #32
0
    def user_email(self):
        """
        Retrieve the email address associated with this order

        :return: email address
        """
        sql = 'select email from auth_user where id = %s'

        ret = None
        log_sql = ''
        try:
            with db_instance() as db:
                log_sql = db.cursor.mogrify(sql, (self.user_id,))
                logger.info('order.py user_email: {}'.format(log_sql))

                db.select(sql, self.user_id)

                ret = db[0]['email']
        except DBConnectException as e:
            logger.debug('Error retrieving user_email: {}'
                         .format(log_sql))
            raise OrderException(e)

        return ret
Пример #33
0
    def create(cls, params):
        """
        Place a new order into the system

        :param params: dict of required parameters to be used
            params = {'product_opts': {dictionary object of the order received}
                      'orderid': id generated from generate_order_id
                      'user_id': EE user id
                      'order_type': typically 'level2_ondemand'
                      'status': 'ordered'
                      'note': user notes
                      'ee_order_id': earth explorer order id, or ''
                      'order_source': 'espa' or 'ee'
                      'order_date': date time string
                      'priority': legacy item, should be 'normal'
                      'email': user's contact email
                      'product_options': legacy column}
        :return: order object
        """
        opts = params['product_opts']

        params['product_opts'] = json.dumps(params['product_opts'])

        sql = ('INSERT INTO ordering_order '
               '(orderid, user_id, order_type, status, note, '
               'product_opts, ee_order_id, order_source, order_date, '
               'priority, email, product_options) '
               'VALUES (%(orderid)s, %(user_id)s, %(order_type)s, '
               '%(status)s, %(note)s, %(product_opts)s, '
               '%(ee_order_id)s, %(order_source)s, %(order_date)s, '
               '%(priority)s, %(email)s, %(product_options)s)')

        logger.info('Order creation parameters: {}'.format(params))

        log_sql = ''
        try:
            with db_instance() as db:
                log_sql = db.cursor.mogrify(sql, params)
                logger.info('New order complete SQL: {}'
                            .format(log_sql))
                db.execute(sql, params)
                db.commit()
        except DBConnectException as e:
            logger.debug('Error creating new order: {}\n'
                         'sql: {}'.format(e.message, log_sql))
            raise OrderException(e)

        order = Order.find(params['orderid'])

        # Let the load_ee_order method handle the scene injection
        # as there is special logic for interacting with LTA
        if params['ee_order_id']:
            return order

        sensor_keys = sensor.SensorCONST.instances.keys()

        bulk_ls = []
        for key in opts:
            if key in sensor_keys:
                sensor_type = ''
                item1 = opts[key]['inputs'][0]

                if isinstance(sensor.instance(item1), sensor.Landsat):
                    sensor_type = 'landsat'
                elif isinstance(sensor.instance(item1), sensor.Modis):
                    sensor_type = 'modis'

                for s in opts[key]['inputs']:
                    scene_dict = {'name': s,
                                  'sensor_type': sensor_type,
                                  'order_id': order.id,
                                  'status': 'submitted',
                                  'ee_unit_id': None}

                    bulk_ls.append(scene_dict)

        if 'plot_statistics' in opts and opts['plot_statistics']:
            scene_dict = {'name': 'plot',
                          'sensor_type': 'plot',
                          'order_id': order.id,
                          'status': 'submitted',
                          'ee_unit_id': None}

            bulk_ls.append(scene_dict)

        try:
            Scene.create(bulk_ls)
        except SceneException as e:
            logger.debug('Order creation failed on scene injection, '
                         'order: {}\nexception: {}'
                         .format(order.orderid, e.message))

            with db_instance() as db:
                db.execute('delete ordering_order where id = %s',
                           order.id)
                db.commit()

            raise OrderException(e)

        return order
Пример #34
0
 def __init__(self, *args, **kwargs):
     super(LTASoapService, self).__init__(*args, **kwargs)
     logger.info("Building SoapClient for:{0}".format(self.url))
     self.client = SoapClient(self.url, location=self.location, cache=self.build_object_cache())
Пример #35
0
    def create(cls, params):
        """
        Place a new order into the system

        :param params: dict of required parameters to be used
            params = {'product_opts': {dictionary object of the order received}
                      'orderid': id generated from generate_order_id
                      'user_id': EE user id
                      'order_type': typically 'level2_ondemand'
                      'status': 'ordered'
                      'note': user notes
                      'ee_order_id': earth explorer order id, or ''
                      'order_source': 'espa' or 'ee'
                      'order_date': date time string
                      'priority': legacy item, should be 'normal'
                      'email': user's contact email
                      'product_options': legacy column}
        :return: order object
        """
        opts = params['product_opts']

        params['product_opts'] = json.dumps(params['product_opts'])

        sql = ('INSERT INTO ordering_order '
               '(orderid, user_id, order_type, status, note, '
               'product_opts, ee_order_id, order_source, order_date, '
               'priority, email, product_options) '
               'VALUES (%(orderid)s, %(user_id)s, %(order_type)s, '
               '%(status)s, %(note)s, %(product_opts)s, '
               '%(ee_order_id)s, %(order_source)s, %(order_date)s, '
               '%(priority)s, %(email)s, %(product_options)s)')

        logger.info('Order creation parameters: {}'.format(params))

        log_sql = ''
        try:
            with db_instance() as db:
                log_sql = db.cursor.mogrify(sql, params)
                logger.info('New order complete SQL: {}'.format(log_sql))
                db.execute(sql, params)
                db.commit()
        except DBConnectException as e:
            logger.debug('Error creating new order: {}\n'
                         'sql: {}'.format(e.message, log_sql))
            raise OrderException(e)

        order = Order.find(params['orderid'])

        # Let the load_ee_order method handle the scene injection
        # as there is special logic for interacting with LTA
        if params['ee_order_id']:
            return order

        sensor_keys = sensor.SensorCONST.instances.keys()

        bulk_ls = []
        for key in opts:
            if key in sensor_keys:
                sensor_type = ''
                item1 = opts[key]['inputs'][0]

                if isinstance(sensor.instance(item1), sensor.Landsat):
                    sensor_type = 'landsat'
                elif isinstance(sensor.instance(item1), sensor.Modis):
                    sensor_type = 'modis'

                for s in opts[key]['inputs']:
                    scene_dict = {
                        'name': s,
                        'sensor_type': sensor_type,
                        'order_id': order.id,
                        'status': 'submitted',
                        'ee_unit_id': None
                    }

                    bulk_ls.append(scene_dict)

        if 'plot_statistics' in opts and opts['plot_statistics']:
            scene_dict = {
                'name': 'plot',
                'sensor_type': 'plot',
                'order_id': order.id,
                'status': 'submitted',
                'ee_unit_id': None
            }

            bulk_ls.append(scene_dict)

        try:
            Scene.create(bulk_ls)
        except SceneException as e:
            logger.debug('Order creation failed on scene injection, '
                         'order: {}\nexception: {}'.format(
                             order.orderid, e.message))

            with db_instance() as db:
                db.execute('delete ordering_order where id = %s', order.id)
                db.commit()

            raise OrderException(e)

        return order
Пример #36
0
 def __init__(self, *args, **kwargs):
     super(LTASoapService, self).__init__(*args, **kwargs)
     logger.info('Building SoapClient for:{0}'.format(self.url))
     self.client = SoapClient(self.url, cache=self.build_object_cache())
Пример #37
0
class OnlineCache(object):
    """ Client code to interact with the LSRD online cache """

    config = ConfigurationProvider()

    __order_path_key = 'online_cache_orders_dir'
    __host_key = 'landsatds.host'
    __user_key = 'landsatds.username'
    __pw_key = 'landsatds.password'

    def __init__(self):
        self.orderpath = self.config.get(self.__order_path_key)

        if not self.orderpath:
            msg = '{} not defined in configurations'.format(
                self.__order_path_key)
            logger.critical(msg)
            raise OnlineCacheException(msg)

        host, user, pw = self.config.get(
            [self.__host_key, self.__user_key, self.__pw_key])

        self.client = sshcmd.RemoteHost(host, user, pw, timeout=5)

        try:
            self.client.execute('ls')
        except Exception as e:
            logger.critical('No connection to OnlineCache host: {}'.format(e))
            raise OnlineCacheException(e)

    def exists(self, orderid, filename=None):
        """ Check if an order [optional filename] exists on the onlinecache

        :param orderid:  associated order to check
        :param filename: file to check inside of an order
        :return: bool
        """
        if filename:
            path = os.path.join(self.orderpath, orderid, filename)
        else:
            path = os.path.join(self.orderpath, orderid)

        try:
            result = self.execute_command('ls -d {0}'.format(path),
                                          silent=True)
            ret = tuple(x.rstrip() for x in result['stdout'])
            return ret[-1] == path
        except OnlineCacheException as e:
            return False

    def delete(self, orderid, filename=None):
        """
        Removes an order from physical online cache disk

        :param filename: file to delete inside of an order
        :param orderid: associated order to delete
        """
        if not self.exists(orderid, filename):
            msg = 'Invalid orderid {} or filename {}'.format(orderid, filename)
            logger.critical(msg)
            return False

        if filename:
            path = os.path.join(self.orderpath, orderid, filename)
        else:
            path = os.path.join(self.orderpath, orderid)

        # this should be the dir where the order is held
        logger.info('Deleting {} from online cache'.format(path))
        # TODO: if storage system supports immutable options
        # >>> sudo chattr -fR -i {0};rm -rf {0}
        # However, nfs does not support this extended attributes
        try:
            cmd = 'chmod -R 744 {0};rm -rf {0}'.format(path)
            self.execute_command(cmd)
        except OnlineCacheException as exc:
            logger.critical('Failed to remove files from output cache. '
                            'Command: {} Error: {}'.format(cmd, exc))
            return False
        return True

    def list(self, orderid=None):
        """
        List the orders currently stored on cache, or files listed
        insed of a specific order

        :param orderid: order name to look inside of
        :return: list of folders/files
        """
        if orderid:
            path = os.path.join(self.orderpath, orderid)
        else:
            path = self.orderpath

        cmd = 'ls {}'.format(path)

        result = self.execute_command(cmd)
        ret = tuple(x.rstrip() for x in result['stdout'])

        return ret

    def capacity(self):
        """
        Returns the capacity of the online cache

        :return: dict
        """

        cmd = 'df -mhP {}'.format(self.orderpath)

        result = self.execute_command(cmd)

        line = result['stdout'][1].split(' ')

        clean = [l for l in line if len(l) > 0]

        results = {
            'capacity': clean[1],
            'used': clean[2],
            'available': clean[3],
            'percent_used': clean[4]
        }

        return results

    def execute_command(self, cmd, silent=False):
        """
        Execute the given command on the cache

        :param cmd: cmd string to execute
        :return: results of the command
        """
        try:
            result = self.client.execute(cmd)
        except Exception, exception:
            if not silent:
                logger.critical('Error executing command: {} '
                                'Raised exception: {}'.format(cmd, exception))
            raise OnlineCacheException(exception)

        if 'stderr' in result and result['stderr']:
            if not silent:
                logger.critical('Error executing command: {} '
                                'stderror returned: {}'.format(
                                    cmd, result['stderr']))

            raise OnlineCacheException(result['stderr'])

        logger.info('call to {} returned {}'.format(cmd, result))

        return result
Пример #38
0
class OnlineCache(object):
    """ Client code to interact with the LSRD online cache """

    config = ConfigurationProvider()

    __order_path_key = 'online_cache_orders_dir'
    __host_key = 'landsatds.host'
    __user_key = 'landsatds.username'
    __pw_key = 'landsatds.password'

    def __init__(self):
        self.orderpath = self.config.get(self.__order_path_key)

        if not self.orderpath:
            msg = '{} not defined in configurations'.format(self.__order_path_key)
            logger.debug(msg)
            raise OnlineCacheException(msg)

        host, user, pw = self.config.get([self.__host_key,
                                          self.__user_key,
                                          self.__pw_key])

        self.client = sshcmd.RemoteHost(host, user, pw, timeout=5)

        try:
            self.client.execute('ls')
        except Exception as e:
            logger.debug('No connection to OnlineCache host: {}'.format(e))
            raise OnlineCacheException(e)

    def delete(self, orderid, filename=None):
        """
        Removes an order from physical online cache disk

        :param filename: file to delete inside of an order
        :param orderid: associated order to delete
        """
        if filename:
            path = os.path.join(self.orderpath, orderid, filename)
        else:
            path = os.path.join(self.orderpath, orderid)

        # this should be the dir where the order is held
        logger.info('Deleting {} from online cache'.format(path))
        try:
            self.execute_command('sudo chattr -fR -i {0};rm -rf {0}'.format(path))
        except OnlineCacheException:
            # in the event /lustre is mounted to an NFS system
            logger.info("onlinecache delete, chattr error, attempting chmod instead...")
            self.execute_command('chmod -R 644 {0};rm -rf {0}'.format(path))

        return True

    def list(self, orderid=None):
        """
        List the orders currently stored on cache, or files listed
        insed of a specific order

        :param orderid: order name to look inside of
        :return: list of folders/files
        """
        if orderid:
            path = os.path.join(self.orderpath, orderid)
        else:
            path = self.orderpath

        cmd = 'ls {}'.format(path)

        result = self.execute_command(cmd)
        ret = tuple(x.rstrip() for x in result['stdout'])

        return ret

    def capacity(self):
        """
        Returns the capacity of the online cache

        :return: dict
        """

        cmd = 'df -mhP {}'.format(self.orderpath)

        result = self.execute_command(cmd)

        line = result['stdout'][1].split(' ')

        clean = [l for l in line if len(l) > 0]

        results = {'capacity': clean[1],
                   'used': clean[2],
                   'available': clean[3],
                   'percent_free': clean[4]}

        return results

    def execute_command(self, cmd):
        """
        Execute the given command on the cache

        :param cmd: cmd string to execute
        :return: results of the command
        """
        try:
            result = self.client.execute(cmd)
        except Exception, exception:
            logger.debug('Error executing command: {} '
                         'Raised exception: {}'.format(cmd, exception))
            raise OnlineCacheException(exception)

        if 'stderr' in result and result['stderr']:
            logger.debug('Error executing command: {} '
                         'stderror returned: {}'.format(cmd, result['stderr']))

            raise OnlineCacheException(result['stderr'])

        logger.info('call to {} returned {}'.format(cmd, result))

        return result