def process(args): ''' Description: Read all lines from STDIN and process them. Each line is converted to a JSON dictionary of the parameters for processing. Validation is performed on the JSON dictionary to test if valid for this mapper. After validation the generation of the products is performed. ''' # Initially set to the base logger logger = EspaLogging.get_logger('base') processing_location = socket.gethostname() # Process each line from stdin for line in sys.stdin: if not line or len(line) < 1 or not line.strip().find('{') > -1: # this is how the nlineinputformat is supplying values: # 341104 {"orderid": # logger.info("BAD LINE:%s##" % line) continue else: # take the entry starting at the first opening parenth to the end line = line[line.find("{"):] line = line.strip() # Reset these for each line (server, order_id, product_id) = (None, None, None) # Default to the command line value mapper_keep_log = args.keep_log start_time = datetime.datetime.now() try: line = line.replace('#', '') parms = json.loads(line) if not parameters.test_for_parameter(parms, 'options'): raise ValueError("Error missing JSON 'options' record") # TODO scene will be replaced with product_id someday (order_id, product_id, product_type, options) = \ (parms['orderid'], parms['scene'], parms['product_type'], parms['options']) # Fix the orderid in-case it contains any single quotes # The processors can not handle single quotes in the email # portion due to usage in command lines. parms['orderid'] = order_id.replace("'", '') # If it is missing due to above TODO, then add it if not parameters.test_for_parameter(parms, 'product_id'): parms['product_id'] = product_id # Figure out if debug level logging was requested debug = False if parameters.test_for_parameter(options, 'debug'): debug = options['debug'] # Configure and get the logger for this order request EspaLogging.configure(settings.PROCESSING_LOGGER, order=order_id, product=product_id, debug=debug) logger = EspaLogging.get_logger(settings.PROCESSING_LOGGER) # If the command line option is True don't use the scene option if not mapper_keep_log: if not parameters.test_for_parameter(options, 'keep_log'): options['keep_log'] = False mapper_keep_log = options['keep_log'] logger.info("Processing %s:%s" % (order_id, product_id)) # Update the status in the database if parameters.test_for_parameter(parms, 'xmlrpcurl'): if parms['xmlrpcurl'] != 'skip_xmlrpc': server = xmlrpclib.ServerProxy(parms['xmlrpcurl'], allow_none=True) if server is not None: status = server.update_status(product_id, order_id, processing_location, 'processing') if not status: logger.warning("Failed processing xmlrpc call" " to update_status to processing") if product_id != 'plot': # Make sure we can process the sensor tmp_inst = sensor.instance(product_id) del tmp_inst # Make sure we have a valid output format if not parameters.test_for_parameter(options, 'output_format'): logger.warning("'output_format' parameter missing" " defaulting to envi") options['output_format'] = 'envi' if (options['output_format'] not in parameters.valid_output_formats): raise ValueError("Invalid Output format %s" % options['output_format']) # ---------------------------------------------------------------- # NOTE: The first thing the product processor does during # initialization is validate the input parameters. # ---------------------------------------------------------------- destination_product_file = 'ERROR' destination_cksum_file = 'ERROR' pp = None try: # All processors are implemented in the processor module pp = processor.get_instance(parms) (destination_product_file, destination_cksum_file) = \ pp.process() finally: # Free disk space to be nice to the whole system. if not mapper_keep_log and pp is not None: pp.remove_product_directory() # Sleep the number of seconds for minimum request duration sleep(get_sleep_duration(start_time)) archive_log_files(order_id, product_id) # Everything was successfull so mark the scene complete if server is not None: status = server.mark_scene_complete(product_id, order_id, processing_location, destination_product_file, destination_cksum_file, "") if not status: logger.warning("Failed processing xmlrpc call to" " mark_scene_complete") except Exception as excep: # First log the exception if hasattr(excep, 'output'): logger.error("Output [%s]" % excep.output) logger.exception("Exception encountered stacktrace follows") # Sleep the number of seconds for minimum request duration sleep(get_sleep_duration(start_time)) archive_log_files(order_id, product_id) if server is not None: try: status = set_product_error(server, order_id, product_id, processing_location) except Exception: logger.exception("Exception encountered stacktrace" " follows") finally: # Reset back to the base logger logger = EspaLogging.get_logger('base')
def process_test_order(request_file, products_file, env_vars, keep_log, plot, pre, post): ''' Description: Process the test order file. ''' logger = logging.getLogger(__name__) template_file = 'template.json' template_dict = None tmp_order = 'tmp-test-order' order_id = (request_file.split('.json')[0]).replace("'", '') if pre: order_id = ''.join([order_id, '-PRE']) if post: order_id = ''.join([order_id, '-POST']) have_error = False status = True error_msg = '' products = list() if not plot: with open(products_file, 'r') as scenes_fd: while (1): product = scenes_fd.readline().strip() if not product: break products.append(product) else: products = ['plot'] logger.info("Processing Products [%s]" % ', '.join(products)) with open(template_file, 'r') as template_fd: template_contents = template_fd.read() if not template_contents: raise Exception("Template file [%s] is empty" % template_file) template_dict = json.loads(template_contents) if template_dict is None: logger.error("Loading template.json") for product in products: logger.info("Processing Product [%s]" % product) with open(request_file, 'r') as request_fd: request_contents = request_fd.read() if not request_contents: raise Exception("Order file [%s] is empty" % request_file) request_dict = json.loads(request_contents) if request_dict is None: logger.error("Loading [%s]" % request_file) # Merge the requested options with the template options, to create # a new dict with the requested options overriding the template. new_dict = template_dict.copy() new_dict.update(request_dict) new_dict['options'] = template_dict['options'].copy() new_dict['options'].update(request_dict['options']) # Turn it into a string for follow-on processing order_contents = json.dumps(new_dict, indent=4, sort_keys=True) logger.info("Processing Request File [%s]" % request_file) with open(tmp_order, 'w') as tmp_fd: logger.info("Creating [%s]" % tmp_order) tmp_line = order_contents # Update the order for the developer tmp = product[:3] download_url = 'null' is_modis = False if tmp == 'MOD' or tmp == 'MYD': is_modis = True # for plots if not is_modis and not plot: product_path = ('%s/%s/%s%s' % (env_vars['dev_data_dir']['value'], product[:3], product, '.tar.gz')) logger.info("Using Product Path [%s]" % product_path) if not os.path.isfile(product_path): error_msg = ("Missing product data (%s)" % product_path) have_error = True break download_url = ('file://%s' % product_path) elif not plot: if tmp == 'MOD': base_source_path = settings.TERRA_BASE_SOURCE_PATH else: base_source_path = settings.AQUA_BASE_SOURCE_PATH short_name = sensor.instance(product).short_name version = sensor.instance(product).version archive_date = utilities.date_from_doy( sensor.instance(product).year, sensor.instance(product).doy) xxx = '%s.%s.%s' % (str(archive_date.year).zfill(4), str(archive_date.month).zfill(2), str(archive_date.day).zfill(2)) product_path = ('%s/%s.%s/%s' % (base_source_path, short_name, version, xxx)) if tmp == 'MOD' or tmp == 'MYD': download_url = ('http://%s/%s/%s.hdf' % (settings.MODIS_INPUT_CHECK_HOST, product_path, product)) sensor_name = 'plot' if not plot: sensor_name = sensor.instance(product).sensor_name logger.info("Processing Sensor [%s]" % sensor_name) else: logger.info("Processing Plot Request") tmp_line = tmp_line.replace('\n', '') tmp_line = tmp_line.replace("ORDER_ID", order_id) tmp_line = tmp_line.replace("SCENE_ID", product) if sensor_name in ['tm', 'etm', 'olitirs']: tmp_line = tmp_line.replace("PRODUCT_TYPE", 'landsat') elif sensor_name in ['terra', 'aqua']: tmp_line = tmp_line.replace("PRODUCT_TYPE", 'modis') else: tmp_line = tmp_line.replace("PRODUCT_TYPE", 'plot') tmp_line = tmp_line.replace("DOWNLOAD_URL", download_url) tmp_fd.write(tmp_line) # Validate again, since we modified it parms = json.loads(tmp_line) # parms = parameters.instance(json.loads(tmp_line)) print(json.dumps(parms, indent=4, sort_keys=True)) # END - with tmp_order # END - with request_file if have_error: logger.error(error_msg) return False keep_log_str = '' if keep_log: keep_log_str = '--keep-log' cmd = ("cd ..; cat test-orders/%s | ./ondemand_mapper.py %s" % (tmp_order, keep_log_str)) output = '' try: logger.info("Processing [%s]" % cmd) output = utilities.execute_cmd(cmd) if len(output) > 0: print output except Exception, e: logger.exception("Processing failed") status = False
def process(args): ''' Description: Read all lines from STDIN and process them. Each line is converted to a JSON dictionary of the parameters for processing. Validation is performed on the JSON dictionary to test if valid for this mapper. After validation the generation of the products is performed. ''' # Initially set to the base logger logger = EspaLogging.get_logger('base') processing_location = socket.gethostname() # Process each line from stdin for line in sys.stdin: if not line or len(line) < 1 or not line.strip().find('{') > -1: # this is how the nlineinputformat is supplying values: # 341104 {"orderid": # logger.info("BAD LINE:%s##" % line) continue else: # take the entry starting at the first opening parenth to the end line = line[line.find("{"):] line = line.strip() # Reset these for each line (server, order_id, product_id) = (None, None, None) # Default to the command line value mapper_keep_log = args.keep_log try: line = line.replace('#', '') parms = json.loads(line) if not parameters.test_for_parameter(parms, 'options'): raise ValueError("Error missing JSON 'options' record") # TODO scene will be replaced with product_id someday (order_id, product_id, product_type, options) = \ (parms['orderid'], parms['scene'], parms['product_type'], parms['options']) # Fix the orderid in-case it contains any single quotes # The processors can not handle single quotes in the email # portion due to usage in command lines. parms['orderid'] = order_id.replace("'", '') # If it is missing due to above TODO, then add it if not parameters.test_for_parameter(parms, 'product_id'): parms['product_id'] = product_id # Figure out if debug level logging was requested debug = False if parameters.test_for_parameter(options, 'debug'): debug = options['debug'] # Configure and get the logger for this order request EspaLogging.configure(settings.PROCESSING_LOGGER, order=order_id, product=product_id, debug=debug) logger = EspaLogging.get_logger(settings.PROCESSING_LOGGER) # If the command line option is True don't use the scene option if not mapper_keep_log: if not parameters.test_for_parameter(options, 'keep_log'): options['keep_log'] = False mapper_keep_log = options['keep_log'] logger.info("Processing %s:%s" % (order_id, product_id)) # Update the status in the database if parameters.test_for_parameter(parms, 'xmlrpcurl'): if parms['xmlrpcurl'] != 'skip_xmlrpc': server = xmlrpclib.ServerProxy(parms['xmlrpcurl'], allow_none=True) if server is not None: status = server.update_status(product_id, order_id, processing_location, 'processing') if not status: logger.warning("Failed processing xmlrpc call" " to update_status to processing") if product_id != 'plot': # Make sure we can process the sensor sensor_name = sensor.instance(product_id).sensor_name if sensor_name not in parameters.valid_sensors: raise ValueError("Invalid Sensor %s" % sensor_name) # Make sure we have a valid output format if not parameters.test_for_parameter(options, 'output_format'): logger.warning("'output_format' parameter missing" " defaulting to envi") options['output_format'] = 'envi' if (options['output_format'] not in parameters.valid_output_formats): raise ValueError("Invalid Output format %s" % options['output_format']) # ---------------------------------------------------------------- # NOTE: The first thing the product processor does during # initialization is validate the input parameters. # ---------------------------------------------------------------- destination_product_file = 'ERROR' destination_cksum_file = 'ERROR' pp = None try: # All processors are implemented in the processor module pp = processor.get_instance(parms) (destination_product_file, destination_cksum_file) = \ pp.process() finally: # Free disk space to be nice to the whole system. if not mapper_keep_log and pp is not None: pp.remove_product_directory() # Everything was successfull so mark the scene complete if server is not None: status = server.mark_scene_complete(product_id, order_id, processing_location, destination_product_file, destination_cksum_file, "") if not status: logger.warning("Failed processing xmlrpc call to" " mark_scene_complete") # Cleanup the log file if not mapper_keep_log: EspaLogging.delete_logger_file(settings.PROCESSING_LOGGER) # Reset back to the base logger logger = EspaLogging.get_logger('base') except ee.ESPAException, e: # First log the exception if hasattr(e, 'output'): logger.error("Code [%s]" % str(e.error_code)) if hasattr(e, 'output'): logger.error("Output [%s]" % e.output) logger.exception("Exception encountered and follows") # Log the error information to the server # Depending on the error_code do something different # TODO - Today we are failing everything, but some things could be # made recovereable in the future. # So this code seems a bit ridiculous. status = False if server is not None: try: if (e.error_code == ee.ErrorCodes.creating_stage_dir or (e.error_code == ee.ErrorCodes.creating_work_dir) or (e.error_code == ee.ErrorCodes.creating_output_dir)): status = set_product_error(server, order_id, product_id, processing_location) elif (e.error_code == ee.ErrorCodes.staging_data or e.error_code == ee.ErrorCodes.unpacking): status = set_product_error(server, order_id, product_id, processing_location) elif (e.error_code == ee.ErrorCodes.metadata or e.error_code == ee.ErrorCodes.surface_reflectance or e.error_code == ee.ErrorCodes.browse or e.error_code == ee.ErrorCodes.spectral_indices or e.error_code == ee.ErrorCodes.create_dem or e.error_code == ee.ErrorCodes.solr or e.error_code == ee.ErrorCodes.cloud_masking or e.error_code == ee.ErrorCodes.dswe or e.error_code == ee.ErrorCodes.land_surface_temperature or e.error_code == ee.ErrorCodes.cleanup_work_dir or e.error_code == ee.ErrorCodes.remove_products): status = set_product_error(server, order_id, product_id, processing_location) elif e.error_code == ee.ErrorCodes.warping: status = set_product_error(server, order_id, product_id, processing_location) elif e.error_code == ee.ErrorCodes.reformat: status = set_product_error(server, order_id, product_id, processing_location) elif e.error_code == ee.ErrorCodes.statistics: status = set_product_error(server, order_id, product_id, processing_location) elif (e.error_code == ee.ErrorCodes.packaging_product or (e.error_code == ee.ErrorCodes.distributing_product) or (e.error_code == ee.ErrorCodes.verifying_checksum)): status = set_product_error(server, order_id, product_id, processing_location) else: # Catch all remaining errors status = set_product_error(server, order_id, product_id, processing_location) if status and not mapper_keep_log: try: # Cleanup the log file EspaLogging. \ delete_logger_file(settings.PROCESSING_LOGGER) except Exception: logger.exception("Exception encountered" " stacktrace follows") except Exception: logger.exception("Exception encountered and follows") # END - if server is not None except Exception, e: # First log the exception if hasattr(e, 'output'): logger.error("Output [%s]" % e.output) logger.exception("Exception encountered stacktrace follows") if server is not None: try: status = set_product_error(server, order_id, product_id, processing_location) if status and not mapper_keep_log: try: # Cleanup the log file EspaLogging. \ delete_logger_file(settings.PROCESSING_LOGGER) except Exception: logger.exception("Exception encountered" " stacktrace follows") except Exception: logger.exception("Exception encountered stacktrace" " follows")
def validate_reprojection_parameters(parms, scene, projections, ns_values, pixel_size_units, image_extents_units, resample_methods, datum_values): ''' Description: Perform a check on the possible reprojection parameters Note: We blindly convert values to float or int without checking them. It is assumed that the web tier has validated them. ''' logger = EspaLogging.get_logger(settings.PROCESSING_LOGGER) # Create this and set to None if not present if not test_for_parameter(parms, 'projection'): logger.warning("'projection' parameter missing defaulting to None") parms['projection'] = None # Create this and set to 'near' if not present if not test_for_parameter(parms, 'resample_method'): logger.warning("'resample_method' parameter missing defaulting to" " near") parms['resample_method'] = 'near' # Make sure these have at least a False value required_parameters = ['reproject', 'image_extents', 'resize'] for parameter in required_parameters: if not test_for_parameter(parms, parameter): logger.warning("'%s' parameter missing defaulting to False" % parameter) parms[parameter] = False if parms['reproject']: if not test_for_parameter(parms, 'target_projection'): raise RuntimeError("Missing target_projection parameter") else: # Convert to lower case target_projection = parms['target_projection'].lower() parms['target_projection'] = target_projection # Verify a valid projection if target_projection not in projections: raise ValueError("Invalid target_projection [%s]:" " Argument must be one of (%s)" % (target_projection, ', '.join(projections))) # ................................................................ if target_projection == "sinu": if not test_for_parameter(parms, 'central_meridian'): raise RuntimeError("Missing central_meridian parameter") else: parms['central_meridian'] = \ float(parms['central_meridian']) if not test_for_parameter(parms, 'false_easting'): raise RuntimeError("Missing false_easting parameter") else: parms['false_easting'] = float(parms['false_easting']) if not test_for_parameter(parms, 'false_northing'): raise RuntimeError("Missing false_northing parameter") else: parms['false_northing'] = float(parms['false_northing']) if not test_for_parameter(parms, 'datum'): parms['datum'] = None # ................................................................ if target_projection == 'aea': if not test_for_parameter(parms, 'std_parallel_1'): raise RuntimeError("Missing std_parallel_1 parameter") else: parms['std_parallel_1'] = float(parms['std_parallel_1']) if not test_for_parameter(parms, 'std_parallel_2'): raise RuntimeError("Missing std_parallel_2 parameter") else: parms['std_parallel_2'] = float(parms['std_parallel_2']) if not test_for_parameter(parms, 'origin_lat'): raise RuntimeError("Missing origin_lat parameter") else: parms['origin_lat'] = float(parms['origin_lat']) if not test_for_parameter(parms, 'central_meridian'): raise RuntimeError("Missing central_meridian parameter") else: parms['central_meridian'] = \ float(parms['central_meridian']) if not test_for_parameter(parms, 'false_easting'): raise RuntimeError("Missing false_easting parameter") else: parms['false_easting'] = float(parms['false_easting']) if not test_for_parameter(parms, 'false_northing'): raise RuntimeError("Missing false_northing parameter") else: parms['false_northing'] = float(parms['false_northing']) # The datum must be in uppercase for the processing code to # work so if it is present here, we force it if not test_for_parameter(parms, 'datum'): raise RuntimeError("Missing datum parameter") else: parms['datum'] = parms['datum'].upper() if parms['datum'] not in datum_values: raise ValueError("Invalid datum [%s]:" " Argument must be one of (%s)" % (parms['datum'], ', '.join(datum_values))) # ................................................................ if target_projection == 'utm': if not test_for_parameter(parms, 'utm_zone'): raise RuntimeError("Missing utm_zone parameter") else: zone = int(parms['utm_zone']) if zone < 0 or zone > 60: raise ValueError("Invalid utm_zone [%d]:" " Value must be 0-60" % zone) parms['utm_zone'] = zone if not test_for_parameter(parms, 'utm_north_south'): raise RuntimeError("Missing utm_north_south parameter") elif parms['utm_north_south'] not in ns_values: raise ValueError("Invalid utm_north_south [%s]:" " Argument must be one of (%s)" % (parms['utm_north_south'], ', '.join(ns_values))) if not test_for_parameter(parms, 'datum'): parms['datum'] = None # ................................................................ if target_projection == 'ps': if not test_for_parameter(parms, 'latitude_true_scale'): # Must be tested before origin_lat raise RuntimeError("Missing latitude_true_scale parameter") else: value = float(parms['latitude_true_scale']) if ((value < 60.0 and value > -60.0) or value > 90.0 or value < -90.0): raise ValueError("Invalid latitude_true_scale [%f]:" " Value must be between" " (-60.0 and -90.0) or" " (60.0 and 90.0)" % value) parms['latitude_true_scale'] = value if not test_for_parameter(parms, 'longitude_pole'): raise RuntimeError("Missing longitude_pole parameter") else: parms['longitude_pole'] = float(parms['longitude_pole']) if not test_for_parameter(parms, 'origin_lat'): # If the user did not specify the origin_lat value, then # set it based on the latitude true scale lat_ts = float(parms['latitude_true_scale']) if lat_ts < 0: parms['origin_lat'] = -90.0 else: parms['origin_lat'] = 90.0 else: value = float(parms['origin_lat']) if value != -90.0 and value != 90.0: raise ValueError("Invalid origin_lat [%f]:" " Value must be -90.0 or 90.0" % value) parms['origin_lat'] = value if not test_for_parameter(parms, 'false_easting'): raise RuntimeError("Missing false_easting parameter") else: parms['false_easting'] = float(parms['false_easting']) if not test_for_parameter(parms, 'false_northing'): raise RuntimeError("Missing false_northing parameter") else: parms['false_northing'] = float(parms['false_northing']) if not test_for_parameter(parms, 'datum'): parms['datum'] = None # ................................................................ if target_projection == 'lonlat': if not test_for_parameter(parms, 'datum'): parms['datum'] = None # ------------------------------------------------------------------------ if parms['resample_method'] not in resample_methods: raise ValueError("Invalid resample_method [%s]:" " Argument must be one of (%s)" % (parms['resample_method'], ', '.join(resample_methods))) # ------------------------------------------------------------------------ if parms['image_extents']: if not test_for_parameter(parms, 'image_extents_units'): raise RuntimeError("Missing image_extents_units parameter") else: if parms['image_extents_units'] not in image_extents_units: raise ValueError("Invalid image_extents_units [%s]:" " Argument must be one of (%s)" % (parms['image_extents_units'], ', '.join(image_extents_units))) if not test_for_parameter(parms, 'minx'): raise RuntimeError("Missing minx parameter") else: parms['minx'] = float(parms['minx']) if not test_for_parameter(parms, 'miny'): raise RuntimeError("Missing miny parameter") else: parms['miny'] = float(parms['miny']) if not test_for_parameter(parms, 'maxx'): raise RuntimeError("Missing maxx parameter") else: parms['maxx'] = float(parms['maxx']) if not test_for_parameter(parms, 'maxy'): raise RuntimeError("Missing maxy parameter") else: parms['maxy'] = float(parms['maxy']) else: # Default these parms['minx'] = None parms['miny'] = None parms['maxx'] = None parms['maxy'] = None parms['image_extents_units'] = None # ------------------------------------------------------------------------ if parms['resize']: if not test_for_parameter(parms, 'pixel_size'): raise RuntimeError("Missing pixel_size parameter") else: parms['pixel_size'] = float(parms['pixel_size']) if not test_for_parameter(parms, 'pixel_size_units'): raise RuntimeError("Missing pixel_size_units parameter") else: if parms['pixel_size_units'] not in pixel_size_units: raise ValueError("Invalid pixel_size_units [%s]:" " Argument must be one of (%s)" % (parms['pixel_size_units'], ', '.join(pixel_size_units))) else: # Default this parms['pixel_size'] = None parms['pixel_size_units'] = None # ------------------------------------------------------------------------ if ((parms['reproject'] or parms['image_extents']) and not parms['resize']): # Sombody asked for reproject or extents, but didn't specify a pixel # size units = 'meters' if parms['reproject'] and parms['target_projection'] == 'lonlat': units = 'dd' # Default to the sensor specific meters or dd equivalent parms['pixel_size'] = sensor.instance(scene).default_pixel_size[units] parms['pixel_size_units'] = units logger.warning("'resize' parameter not provided but required for" " reprojection or image extents" " (Defaulting pixel_size(%f) and pixel_size_units(%s)" % (parms['pixel_size'], parms['pixel_size_units']))