Esempio n. 1
0
def process(proc_cfg, developer_sleep_mode=False):
    """Read all lines from STDIN and process them

    Each line is converted to a JSON dictionary of the parameters for
    processing.  Validation is performed on the JSON dictionary to test if
    valid for this mapper.  After validation the generation of the products
    is performed.
    """

    # Initially set to the base logger
    logger = EspaLogging.get_logger('base')

    processing_location = socket.gethostname()

    # Process each line from stdin
    for line in sys.stdin:
        if not line or len(line) < 1 or not line.strip().find('{') > -1:
            # this is how the nlineinputformat is supplying values:
            # 341104        {"orderid":
            # logger.info('BAD LINE:{}##'.format(line))
            continue
        else:
            # take the entry starting at the first opening parenth to the end
            line = line[line.find('{'):]
            line = line.strip()

        # Reset these for each line
        (server, order_id, product_id) = (None, None, None)

        start_time = datetime.datetime.now()

        # Initialize so that we don't sleep
        dont_sleep = True

        try:
            line = line.replace('#', '')
            parms = json.loads(line)

            if not parameters.test_for_parameter(parms, 'options'):
                raise ValueError('Error missing JSON [options] record')

            # TODO scene will be replaced with product_id someday
            (order_id, product_id, product_type, options) = \
                (parms['orderid'], parms['scene'], parms['product_type'],
                 parms['options'])

            if product_id != 'plot':
                # Developer mode is always false unless you are a developer
                # so sleeping will always occur for none plotting requests
                # Override with the developer mode
                dont_sleep = developer_sleep_mode

            # Fix the orderid in-case it contains any single quotes
            # The processors can not handle single quotes in the email
            # portion due to usage in command lines.
            parms['orderid'] = order_id.replace("'", '')

            # If it is missing due to above TODO, then add it
            if not parameters.test_for_parameter(parms, 'product_id'):
                parms['product_id'] = product_id

            # Figure out if debug level logging was requested
            debug = False
            if parameters.test_for_parameter(options, 'debug'):
                debug = options['debug']

            # Configure and get the logger for this order request
            EspaLogging.configure(settings.PROCESSING_LOGGER, order=order_id,
                                  product=product_id, debug=debug)
            logger = EspaLogging.get_logger(settings.PROCESSING_LOGGER)

            logger.info('Processing {}:{}'.format(order_id, product_id))

            # Update the status in the database
            if parameters.test_for_parameter(parms, 'espa_api'):
                if parms['espa_api'] != 'skip_api':
                    server = api_interface.api_connect(parms['espa_api'])
                    if server is not None:
                        status = server.update_status(product_id, order_id,
                                                      processing_location,
                                                      'processing')
                        if not status:
                            logger.warning('Failed processing API call'
                                           ' to update_status to processing')

            if product_id != 'plot':
                # Make sure we can process the sensor
                tmp_info = sensor.info(product_id)
                del tmp_info

                # Make sure we have a valid output format
                if not parameters.test_for_parameter(options, 'output_format'):
                    logger.warning('[output_format] parameter missing'
                                   ' defaulting to envi')
                    options['output_format'] = 'envi'

                if (options['output_format']
                        not in parameters.VALID_OUTPUT_FORMATS):

                    raise ValueError('Invalid Output format {}'
                                     .format(options['output_format']))

            # ----------------------------------------------------------------
            # NOTE: The first thing the product processor does during
            #       initialization is validate the input parameters.
            # ----------------------------------------------------------------

            destination_product_file = 'ERROR'
            destination_cksum_file = 'ERROR'
            pp = None
            try:
                # All processors are implemented in the processor module
                pp = processor.get_instance(proc_cfg, parms)
                (destination_product_file, destination_cksum_file) = \
                    pp.process()

            finally:
                # Free disk space to be nice to the whole system.
                if pp is not None:
                    pp.remove_product_directory()

            # Sleep the number of seconds for minimum request duration
            sleep(get_sleep_duration(proc_cfg, start_time, dont_sleep))

            archive_log_files(order_id, product_id)

            # Everything was successfull so mark the scene complete
            if server is not None:
                status = server.mark_scene_complete(product_id, order_id,
                                                    processing_location,
                                                    destination_product_file,
                                                    destination_cksum_file,
                                                    '')
                if not status:
                    logger.warning('Failed processing API call to'
                                   ' mark_scene_complete')

        except Exception as excep:

            # First log the exception
            logger.exception('Exception encountered stacktrace follows')

            # Sleep the number of seconds for minimum request duration
            sleep(get_sleep_duration(proc_cfg, start_time, dont_sleep))

            archive_log_files(order_id, product_id)

            if server is not None:
                try:
                    status = set_product_error(server,
                                               order_id,
                                               product_id,
                                               processing_location)
                except Exception:
                    logger.exception('Exception encountered stacktrace'
                                     ' follows')
        finally:
            # Reset back to the base logger
            logger = EspaLogging.get_logger('base')
def main():
    """Configures an order from the command line input and calls the
       processing code using the order
    """

    args = parse_command_line()
    proc_cfg = config.retrieve_cfg(PROC_CFG_FILENAME)

    proc_cfg = override_config(args, proc_cfg)

    # Configure the base logger for this request
    EspaLogging.configure_base_logger(filename=cli_log_filename(args))
    # Configure the processing logger for this request
    EspaLogging.configure(settings.PROCESSING_LOGGER,
                          order=args.order_id,
                          product=args.product_id,
                          debug=args.debug)

    # CLI will use the base logger
    logger = EspaLogging.get_logger('base')

    logger.info('*** Begin ESPA Processing on host [{}] ***'.format(
        socket.gethostname()))

    # Set to error condition
    proc_status = False

    try:
        # Extra command line validation
        if args.pixel_size is not None and args.pixel_size_units is None:
            raise CliError('Must specify --pixel-size-units if specifying'
                           ' --pixel-size')

        export_environment_variables(proc_cfg)

        template = load_template(filename=TEMPLATE_FILENAME)

        order = update_template(args=args, template=template)

        # Change to the processing directory
        current_directory = os.getcwd()
        os.chdir(proc_cfg.get('processing', 'espa_work_dir'))

        try:
            # All processors are implemented in the processor module
            pp = processor.get_instance(proc_cfg, order)
            (destination_product_file, destination_cksum_file) = pp.process()

            # Set to success condition
            proc_status = True

        finally:
            # Change back to the previous directory
            os.chdir(current_directory)

    except Exception:
        logger.exception('*** Errors during processing ***')
        sys.exit(1)

    finally:
        logger.info('*** ESPA Processing Terminated ***')

        if not args.bridge_mode:
            archive_log_files(args, proc_cfg, proc_status)
Esempio n. 3
0
def work(cfg, params, developer_sleep_mode=False):
    """
    Take the environment configuration, order parameters and initiate order processing.
    Note: Much of this code was taken from the ondemand_mapper.py script in espa-processing.

    Args:
        cfg (dict): Configuration params given by config.config() and by the worker environment
        params (dict): JSON response from the API for a single granule or scene

    Returns:
        None, Products are generated, packaged, and distributed if processing was successful

    """
    # This will be the Mesos node hostname
    processing_location = socket.gethostname()

    # Use the base_logger initially, if an exception occurs before the processing logger is configured
    # the base_logger will handle log it
    logger = base_logger

    if not parameters.test_for_parameter(params, 'options'):
        raise ValueError('Error missing JSON [options] record')

    start_time = datetime.datetime.now()

    # Initialize so that we don't sleep
    dont_sleep = True

    # Note that the API response "scene" value is what we use for product_id
    try:
        (order_id, product_id, product_type, options) = \
            (params['orderid'], params['scene'], params['product_type'],
             params['options'])

        if product_id != 'plot':
            # Developer mode is always false unless you are a developer
            # so sleeping will always occur for non-plotting requests
            # Override with the developer mode
            dont_sleep = developer_sleep_mode

        # Fix the orderid in-case it contains any single quotes
        # The processors can not handle single quotes in the email
        # portion due to usage in command lines.
        params['orderid'] = order_id.replace("'", '')

        # product_id is not part of the API response - we add it here
        if not parameters.test_for_parameter(params, 'product_id'):
            params['product_id'] = product_id

        # Figure out if debug level logging was requested
        debug = False
        if parameters.test_for_parameter(options, 'debug'):
            debug = options['debug']

        # Configure and get the logger for this order request
        EspaLogging.configure(settings.PROCESSING_LOGGER, order=order_id,
                              product=product_id, debug=debug)

        # Replace the base_logger with the processing_logger
        logger = EspaLogging.get_logger(settings.PROCESSING_LOGGER)

        # add our stdout/stderr log streams
        logger.addHandler(get_stdout_handler())
        logger.addHandler(get_stderr_handler())

        logger.info('Processing {}:{}'.format(order_id, product_id))
        logger.info('Attempting connection to {0}'.format(cfg['espa_api']))

        # will throw an exception on init if unable to get a 200 response
        server = APIServer(cfg['espa_api'])

        # will throw an exception if does not receive a 200 response
        status = server.update_status(product_id, order_id, processing_location, 'processing')

        if product_id != 'plot':
            # Make sure we can process the sensor
            tmp_info = sensor.info(product_id)
            del tmp_info

            # Make sure we have a valid output format
            if not parameters.test_for_parameter(options, 'output_format'):
                logger.warning('[output_format] parameter missing defaulting to envi')

                options['output_format'] = 'envi'

            if (options['output_format'] not in parameters.VALID_OUTPUT_FORMATS):
                raise ValueError('Invalid Output format {}'.format(options['output_format']))

                # ----------------------------------------------------------------
                # NOTE: The first thing the product processor does during
                #       initialization is validate the input parameters.
                # ----------------------------------------------------------------

        destination_product_file = 'ERROR'
        destination_cksum_file = 'ERROR'

        pp = None

        try:
            # All processors are implemented in the processor module
            pp = processor.get_instance(cfg, params)

            (destination_product_file, destination_cksum_file) = pp.process()

        finally:
            # Free disk space to be nice to the whole system.
            if pp is not None:
                pp.remove_product_directory()

        # Sleep the number of seconds for minimum request duration
        sleep(utilities.get_sleep_duration(cfg, start_time, dont_sleep))

        log_items = archive_log_files(order_id, product_id)
        for item in log_items:
            utilities.change_ownership(item, cfg.get('espa_user'), cfg.get('espa_group'))

        # Everything was successful so mark the scene complete
        server.mark_scene_complete(product_id, order_id,
                                   processing_location,
                                   destination_product_file,
                                   destination_cksum_file,
                                   '') # sets log_file_contents to empty string ''
        return True

    except Exception as e:
        # First log the exception
        logger.exception('Exception encountered in processing.main.work:\nexception: {}'.format(e))

        try:
            # Sleep the number of seconds for minimum request duration
            logger.debug('Attempting to archive log files for order_id: {}\nproduct_id: {}'.format(order_id, product_id))
            sleep(utilities.get_sleep_duration(cfg, start_time, dont_sleep))
            log_items = archive_log_files(order_id, product_id)
            for item in log_items:
                utilities.change_ownership(item, cfg.get('espa_user'), cfg.get('espa_group'))

        except Exception as e2:
            logger.exception('Problem archiving log files. error: {}'.format(e2))

        try:
            logger.debug('Attempting to set product error, order_id: {}\nproduct_id: {}'.format(order_id, product_id))
            logged_contents = EspaLogging.read_logger_file(settings.PROCESSING_LOGGER)
            error_log = "Processing Log: {}\n\nException: {}".format(logged_contents, e)
            server.set_scene_error(product_id, order_id, processing_location, error_log)
        except Exception as e3:
            logger.exception('Unable to reach ESPA API and set product error for order_id: {}\nproduct_id: {}\nerror: {}'.format(order_id, product_id, e3))
            raise e3

        return False
Esempio n. 4
0
    finally:
        # Change back to the previous directory
        os.chdir(current_directory)


if __name__ == '__main__':
    '''
    Description:
      This is test code only used during proto-typing.
      It only provides stats for landsat and modis data.
    '''

    # Configure logging
    EspaLogging.configure(settings.PROCESSING_LOGGER,
                          order='test',
                          product='statistics')
    logger = EspaLogging.get_logger(settings.PROCESSING_LOGGER)

    # Hold the wild card strings in a type based dictionary
    files_to_search_for = dict()

    # Landsat files
    files_to_search_for['SR'] = ['*_sr_band[0-9].img']
    files_to_search_for['TOA'] = ['*_toa_band[0-9].img']
    files_to_search_for['INDEX'] = [
        '*_nbr.img', '*_nbr2.img', '*_ndmi.img', '*_ndvi.img', '*_evi.img',
        '*_savi.img', '*_msavi.img'
    ]

    # MODIS files
Esempio n. 5
0
def main():
    """Configures an order from the command line input and calls the
       processing code using the order
    """

    args = parse_command_line()
    proc_cfg = config.retrieve_cfg(PROC_CFG_FILENAME)

    proc_cfg = override_config(args, proc_cfg)

    # Configure the base logger for this request
    EspaLogging.configure_base_logger(filename=cli_log_filename(args))
    # Configure the processing logger for this request
    EspaLogging.configure(settings.PROCESSING_LOGGER,
                          order=args.order_id,
                          product=args.product_id,
                          debug=args.debug)

    # CLI will use the base logger
    logger = EspaLogging.get_logger('base')

    logger.info('*** Begin ESPA Processing on host [{}] ***'
                .format(socket.gethostname()))

    # Set to error condition
    proc_status = False

    try:
        # Extra command line validation
        if args.pixel_size is not None and args.pixel_size_units is None:
            raise CliError('Must specify --pixel-size-units if specifying'
                           ' --pixel-size')

        export_environment_variables(proc_cfg)

        template = load_template(filename=TEMPLATE_FILENAME)

        order = update_template(args=args, template=template)

        # Change to the processing directory
        current_directory = os.getcwd()
        os.chdir(proc_cfg.get('processing', 'espa_work_dir'))

        try:
            # All processors are implemented in the processor module
            pp = processor.get_instance(proc_cfg, order)
            (destination_product_file, destination_cksum_file) = pp.process()

            # Set to success condition
            proc_status = True

        finally:
            # Change back to the previous directory
            os.chdir(current_directory)

    except Exception:
        logger.exception('*** Errors during processing ***')
        sys.exit(1)

    finally:
        logger.info('*** ESPA Processing Terminated ***')

        if not args.bridge_mode:
            archive_log_files(args, proc_cfg, proc_status)