def process(proc_cfg, developer_sleep_mode=False):
"""Read all lines from STDIN and process them
Each line is converted to a JSON dictionary of the parameters for
processing. Validation is performed on the JSON dictionary to test if
valid for this mapper. After validation the generation of the products
is performed.
"""
# Initially set to the base logger
logger = EspaLogging.get_logger('base')
processing_location = socket.gethostname()
# Process each line from stdin
for line in sys.stdin:
if not line or len(line) < 1 or not line.strip().find('{') > -1:
# this is how the nlineinputformat is supplying values:
# 341104 {"orderid":
# logger.info('BAD LINE:{}##'.format(line))
continue
else:
# take the entry starting at the first opening parenth to the end
line = line[line.find('{'):]
line = line.strip()
# Reset these for each line
(server, order_id, product_id) = (None, None, None)
start_time = datetime.datetime.now()
# Initialize so that we don't sleep
dont_sleep = True
try:
line = line.replace('#', '')
parms = json.loads(line)
if not parameters.test_for_parameter(parms, 'options'):
raise ValueError('Error missing JSON [options] record')
# TODO scene will be replaced with product_id someday
(order_id, product_id, product_type, options) = \
(parms['orderid'], parms['scene'], parms['product_type'],
parms['options'])
if product_id != 'plot':
# Developer mode is always false unless you are a developer
# so sleeping will always occur for none plotting requests
# Override with the developer mode
dont_sleep = developer_sleep_mode
# Fix the orderid in-case it contains any single quotes
# The processors can not handle single quotes in the email
# portion due to usage in command lines.
parms['orderid'] = order_id.replace("'", '')
# If it is missing due to above TODO, then add it
if not parameters.test_for_parameter(parms, 'product_id'):
parms['product_id'] = product_id
# Figure out if debug level logging was requested
debug = False
if parameters.test_for_parameter(options, 'debug'):
debug = options['debug']
# Configure and get the logger for this order request
EspaLogging.configure(settings.PROCESSING_LOGGER, order=order_id,
product=product_id, debug=debug)
logger = EspaLogging.get_logger(settings.PROCESSING_LOGGER)
logger.info('Processing {}:{}'.format(order_id, product_id))
# Update the status in the database
if parameters.test_for_parameter(parms, 'espa_api'):
if parms['espa_api'] != 'skip_api':
server = api_interface.api_connect(parms['espa_api'])
if server is not None:
status = server.update_status(product_id, order_id,
processing_location,
'processing')
if not status:
logger.warning('Failed processing API call'
' to update_status to processing')
if product_id != 'plot':
# Make sure we can process the sensor
tmp_info = sensor.info(product_id)
del tmp_info
# Make sure we have a valid output format
if not parameters.test_for_parameter(options, 'output_format'):
logger.warning('[output_format] parameter missing'
' defaulting to envi')
options['output_format'] = 'envi'
if (options['output_format']
not in parameters.VALID_OUTPUT_FORMATS):
raise ValueError('Invalid Output format {}'
.format(options['output_format']))
# ----------------------------------------------------------------
# NOTE: The first thing the product processor does during
# initialization is validate the input parameters.
# ----------------------------------------------------------------
destination_product_file = 'ERROR'
destination_cksum_file = 'ERROR'
pp = None
try:
# All processors are implemented in the processor module
pp = processor.get_instance(proc_cfg, parms)
(destination_product_file, destination_cksum_file) = \
pp.process()
finally:
# Free disk space to be nice to the whole system.
if pp is not None:
pp.remove_product_directory()
# Sleep the number of seconds for minimum request duration
sleep(get_sleep_duration(proc_cfg, start_time, dont_sleep))
archive_log_files(order_id, product_id)
# Everything was successfull so mark the scene complete
if server is not None:
status = server.mark_scene_complete(product_id, order_id,
processing_location,
destination_product_file,
destination_cksum_file,
'')
if not status:
logger.warning('Failed processing API call to'
' mark_scene_complete')
except Exception as excep:
# First log the exception
logger.exception('Exception encountered stacktrace follows')
# Sleep the number of seconds for minimum request duration
sleep(get_sleep_duration(proc_cfg, start_time, dont_sleep))
archive_log_files(order_id, product_id)
if server is not None:
try:
status = set_product_error(server,
order_id,
product_id,
processing_location)
except Exception:
logger.exception('Exception encountered stacktrace'
' follows')
finally:
# Reset back to the base logger
logger = EspaLogging.get_logger('base')
def main():
"""Configures an order from the command line input and calls the
processing code using the order
"""
args = parse_command_line()
proc_cfg = config.retrieve_cfg(PROC_CFG_FILENAME)
proc_cfg = override_config(args, proc_cfg)
# Configure the base logger for this request
EspaLogging.configure_base_logger(filename=cli_log_filename(args))
# Configure the processing logger for this request
EspaLogging.configure(settings.PROCESSING_LOGGER,
order=args.order_id,
product=args.product_id,
debug=args.debug)
# CLI will use the base logger
logger = EspaLogging.get_logger('base')
logger.info('*** Begin ESPA Processing on host [{}] ***'.format(
socket.gethostname()))
# Set to error condition
proc_status = False
try:
# Extra command line validation
if args.pixel_size is not None and args.pixel_size_units is None:
raise CliError('Must specify --pixel-size-units if specifying'
' --pixel-size')
export_environment_variables(proc_cfg)
template = load_template(filename=TEMPLATE_FILENAME)
order = update_template(args=args, template=template)
# Change to the processing directory
current_directory = os.getcwd()
os.chdir(proc_cfg.get('processing', 'espa_work_dir'))
try:
# All processors are implemented in the processor module
pp = processor.get_instance(proc_cfg, order)
(destination_product_file, destination_cksum_file) = pp.process()
# Set to success condition
proc_status = True
finally:
# Change back to the previous directory
os.chdir(current_directory)
except Exception:
logger.exception('*** Errors during processing ***')
sys.exit(1)
finally:
logger.info('*** ESPA Processing Terminated ***')
if not args.bridge_mode:
archive_log_files(args, proc_cfg, proc_status)
def work(cfg, params, developer_sleep_mode=False):
"""
Take the environment configuration, order parameters and initiate order processing.
Note: Much of this code was taken from the ondemand_mapper.py script in espa-processing.
Args:
cfg (dict): Configuration params given by config.config() and by the worker environment
params (dict): JSON response from the API for a single granule or scene
Returns:
None, Products are generated, packaged, and distributed if processing was successful
"""
# This will be the Mesos node hostname
processing_location = socket.gethostname()
# Use the base_logger initially, if an exception occurs before the processing logger is configured
# the base_logger will handle log it
logger = base_logger
if not parameters.test_for_parameter(params, 'options'):
raise ValueError('Error missing JSON [options] record')
start_time = datetime.datetime.now()
# Initialize so that we don't sleep
dont_sleep = True
# Note that the API response "scene" value is what we use for product_id
try:
(order_id, product_id, product_type, options) = \
(params['orderid'], params['scene'], params['product_type'],
params['options'])
if product_id != 'plot':
# Developer mode is always false unless you are a developer
# so sleeping will always occur for non-plotting requests
# Override with the developer mode
dont_sleep = developer_sleep_mode
# Fix the orderid in-case it contains any single quotes
# The processors can not handle single quotes in the email
# portion due to usage in command lines.
params['orderid'] = order_id.replace("'", '')
# product_id is not part of the API response - we add it here
if not parameters.test_for_parameter(params, 'product_id'):
params['product_id'] = product_id
# Figure out if debug level logging was requested
debug = False
if parameters.test_for_parameter(options, 'debug'):
debug = options['debug']
# Configure and get the logger for this order request
EspaLogging.configure(settings.PROCESSING_LOGGER, order=order_id,
product=product_id, debug=debug)
# Replace the base_logger with the processing_logger
logger = EspaLogging.get_logger(settings.PROCESSING_LOGGER)
# add our stdout/stderr log streams
logger.addHandler(get_stdout_handler())
logger.addHandler(get_stderr_handler())
logger.info('Processing {}:{}'.format(order_id, product_id))
logger.info('Attempting connection to {0}'.format(cfg['espa_api']))
# will throw an exception on init if unable to get a 200 response
server = APIServer(cfg['espa_api'])
# will throw an exception if does not receive a 200 response
status = server.update_status(product_id, order_id, processing_location, 'processing')
if product_id != 'plot':
# Make sure we can process the sensor
tmp_info = sensor.info(product_id)
del tmp_info
# Make sure we have a valid output format
if not parameters.test_for_parameter(options, 'output_format'):
logger.warning('[output_format] parameter missing defaulting to envi')
options['output_format'] = 'envi'
if (options['output_format'] not in parameters.VALID_OUTPUT_FORMATS):
raise ValueError('Invalid Output format {}'.format(options['output_format']))
# ----------------------------------------------------------------
# NOTE: The first thing the product processor does during
# initialization is validate the input parameters.
# ----------------------------------------------------------------
destination_product_file = 'ERROR'
destination_cksum_file = 'ERROR'
pp = None
try:
# All processors are implemented in the processor module
pp = processor.get_instance(cfg, params)
(destination_product_file, destination_cksum_file) = pp.process()
finally:
# Free disk space to be nice to the whole system.
if pp is not None:
pp.remove_product_directory()
# Sleep the number of seconds for minimum request duration
sleep(utilities.get_sleep_duration(cfg, start_time, dont_sleep))
log_items = archive_log_files(order_id, product_id)
for item in log_items:
utilities.change_ownership(item, cfg.get('espa_user'), cfg.get('espa_group'))
# Everything was successful so mark the scene complete
server.mark_scene_complete(product_id, order_id,
processing_location,
destination_product_file,
destination_cksum_file,
'') # sets log_file_contents to empty string ''
return True
except Exception as e:
# First log the exception
logger.exception('Exception encountered in processing.main.work:\nexception: {}'.format(e))
try:
# Sleep the number of seconds for minimum request duration
logger.debug('Attempting to archive log files for order_id: {}\nproduct_id: {}'.format(order_id, product_id))
sleep(utilities.get_sleep_duration(cfg, start_time, dont_sleep))
log_items = archive_log_files(order_id, product_id)
for item in log_items:
utilities.change_ownership(item, cfg.get('espa_user'), cfg.get('espa_group'))
except Exception as e2:
logger.exception('Problem archiving log files. error: {}'.format(e2))
try:
logger.debug('Attempting to set product error, order_id: {}\nproduct_id: {}'.format(order_id, product_id))
logged_contents = EspaLogging.read_logger_file(settings.PROCESSING_LOGGER)
error_log = "Processing Log: {}\n\nException: {}".format(logged_contents, e)
server.set_scene_error(product_id, order_id, processing_location, error_log)
except Exception as e3:
logger.exception('Unable to reach ESPA API and set product error for order_id: {}\nproduct_id: {}\nerror: {}'.format(order_id, product_id, e3))
raise e3
return False
finally:
# Change back to the previous directory
os.chdir(current_directory)
if __name__ == '__main__':
'''
Description:
This is test code only used during proto-typing.
It only provides stats for landsat and modis data.
'''
# Configure logging
EspaLogging.configure(settings.PROCESSING_LOGGER,
order='test',
product='statistics')
logger = EspaLogging.get_logger(settings.PROCESSING_LOGGER)
# Hold the wild card strings in a type based dictionary
files_to_search_for = dict()
# Landsat files
files_to_search_for['SR'] = ['*_sr_band[0-9].img']
files_to_search_for['TOA'] = ['*_toa_band[0-9].img']
files_to_search_for['INDEX'] = [
'*_nbr.img', '*_nbr2.img', '*_ndmi.img', '*_ndvi.img', '*_evi.img',
'*_savi.img', '*_msavi.img'
]
# MODIS files
def main():
"""Configures an order from the command line input and calls the
processing code using the order
"""
args = parse_command_line()
proc_cfg = config.retrieve_cfg(PROC_CFG_FILENAME)
proc_cfg = override_config(args, proc_cfg)
# Configure the base logger for this request
EspaLogging.configure_base_logger(filename=cli_log_filename(args))
# Configure the processing logger for this request
EspaLogging.configure(settings.PROCESSING_LOGGER,
order=args.order_id,
product=args.product_id,
debug=args.debug)
# CLI will use the base logger
logger = EspaLogging.get_logger('base')
logger.info('*** Begin ESPA Processing on host [{}] ***'
.format(socket.gethostname()))
# Set to error condition
proc_status = False
try:
# Extra command line validation
if args.pixel_size is not None and args.pixel_size_units is None:
raise CliError('Must specify --pixel-size-units if specifying'
' --pixel-size')
export_environment_variables(proc_cfg)
template = load_template(filename=TEMPLATE_FILENAME)
order = update_template(args=args, template=template)
# Change to the processing directory
current_directory = os.getcwd()
os.chdir(proc_cfg.get('processing', 'espa_work_dir'))
try:
# All processors are implemented in the processor module
pp = processor.get_instance(proc_cfg, order)
(destination_product_file, destination_cksum_file) = pp.process()
# Set to success condition
proc_status = True
finally:
# Change back to the previous directory
os.chdir(current_directory)
except Exception:
logger.exception('*** Errors during processing ***')
sys.exit(1)
finally:
logger.info('*** ESPA Processing Terminated ***')
if not args.bridge_mode:
archive_log_files(args, proc_cfg, proc_status)