def __init__(self, product_id): super(Modis, self).__init__(product_id) parts = product_id.strip().split('.') self.short_name = parts[0] self.date_acquired = parts[1][1:] self.year = self.date_acquired[0:4] self.doy = self.date_acquired[4:8] # Now that we have the year and doy, we can get the month and day of # month date = utilities.date_from_doy(self.year, self.doy) self.month = date.month self.day = date.day __hv = parts[2] self.horizontal = __hv[1:3] self.vertical = __hv[4:6] self.version = parts[3] self.date_produced = parts[4] # set the default pixel sizes # this comes out to 09A1, 09GA, 13A1, etc _product_code = self.short_name.split(self.sensor_code)[1] _meters = DEFAULT_PIXEL_SIZE['meters'][_product_code] _dd = DEFAULT_PIXEL_SIZE['dd'][_product_code] self.default_pixel_size = {'meters': _meters, 'dd': _dd}
def __init__(self, product_id): product_id = product_id.strip() super(Landsat, self).__init__(product_id) self.path = utilities.strip_zeros(product_id[3:6]) self.row = utilities.strip_zeros(product_id[6:9]) self.year = product_id[9:13] self.doy = product_id[13:16] self.station = product_id[16:19] self.version = product_id[19:21] # Now that we have the year and doy, we can get the month and day of # month date = utilities.date_from_doy(self.year, self.doy) self.month = date.month self.day = date.day # set the default pixel sizes _pixels = DEFAULT_PIXEL_SIZE _meters = _pixels['meters'][self.sensor_code.upper()] _dd = _pixels['dd'][self.sensor_code.upper()] self.default_pixel_size = {'meters': _meters, 'dd': _dd}
def process_test_order(request_file, products_file, env_vars, keep_log, plot, pre, post): ''' Description: Process the test order file. ''' logger = logging.getLogger(__name__) template_file = 'template.json' template_dict = None tmp_order = 'tmp-test-order' order_id = (request_file.split('.json')[0]).replace("'", '') if pre: order_id = ''.join([order_id, '-PRE']) if post: order_id = ''.join([order_id, '-POST']) have_error = False status = True error_msg = '' products = list() if not plot: with open(products_file, 'r') as scenes_fd: while (1): product = scenes_fd.readline().strip() if not product: break products.append(product) else: products = ['plot'] logger.info("Processing Products [%s]" % ', '.join(products)) with open(template_file, 'r') as template_fd: template_contents = template_fd.read() if not template_contents: raise Exception("Template file [%s] is empty" % template_file) template_dict = json.loads(template_contents) if template_dict is None: logger.error("Loading template.json") for product in products: logger.info("Processing Product [%s]" % product) with open(request_file, 'r') as request_fd: request_contents = request_fd.read() if not request_contents: raise Exception("Order file [%s] is empty" % request_file) request_dict = json.loads(request_contents) if request_dict is None: logger.error("Loading [%s]" % request_file) # Merge the requested options with the template options, to create # a new dict with the requested options overriding the template. new_dict = template_dict.copy() new_dict.update(request_dict) new_dict['options'] = template_dict['options'].copy() new_dict['options'].update(request_dict['options']) # Turn it into a string for follow-on processing order_contents = json.dumps(new_dict, indent=4, sort_keys=True) logger.info("Processing Request File [%s]" % request_file) with open(tmp_order, 'w') as tmp_fd: logger.info("Creating [%s]" % tmp_order) tmp_line = order_contents # Update the order for the developer tmp = product[:3] download_url = 'null' is_modis = False if tmp == 'MOD' or tmp == 'MYD': is_modis = True # for plots if not is_modis and not plot: product_path = ('%s/%s/%s%s' % (env_vars['dev_data_dir']['value'], product[:3], product, '.tar.gz')) logger.info("Using Product Path [%s]" % product_path) if not os.path.isfile(product_path): error_msg = ("Missing product data (%s)" % product_path) have_error = True break download_url = ('file://%s' % product_path) elif not plot: if tmp == 'MOD': base_source_path = settings.TERRA_BASE_SOURCE_PATH else: base_source_path = settings.AQUA_BASE_SOURCE_PATH short_name = sensor.instance(product).short_name version = sensor.instance(product).version archive_date = utilities.date_from_doy( sensor.instance(product).year, sensor.instance(product).doy) xxx = '%s.%s.%s' % (str(archive_date.year).zfill(4), str(archive_date.month).zfill(2), str(archive_date.day).zfill(2)) product_path = ('%s/%s.%s/%s' % (base_source_path, short_name, version, xxx)) if tmp == 'MOD' or tmp == 'MYD': download_url = ('http://%s/%s/%s.hdf' % (settings.MODIS_INPUT_CHECK_HOST, product_path, product)) sensor_name = 'plot' if not plot: sensor_name = sensor.instance(product).sensor_name logger.info("Processing Sensor [%s]" % sensor_name) else: logger.info("Processing Plot Request") tmp_line = tmp_line.replace('\n', '') tmp_line = tmp_line.replace("ORDER_ID", order_id) tmp_line = tmp_line.replace("SCENE_ID", product) if sensor_name in ['tm', 'etm', 'olitirs']: tmp_line = tmp_line.replace("PRODUCT_TYPE", 'landsat') elif sensor_name in ['terra', 'aqua']: tmp_line = tmp_line.replace("PRODUCT_TYPE", 'modis') else: tmp_line = tmp_line.replace("PRODUCT_TYPE", 'plot') tmp_line = tmp_line.replace("DOWNLOAD_URL", download_url) tmp_fd.write(tmp_line) # Validate again, since we modified it parms = json.loads(tmp_line) # parms = parameters.instance(json.loads(tmp_line)) print(json.dumps(parms, indent=4, sort_keys=True)) # END - with tmp_order # END - with request_file if have_error: logger.error(error_msg) return False keep_log_str = '' if keep_log: keep_log_str = '--keep-log' cmd = ("cd ..; cat test-orders/%s | ./ondemand_mapper.py %s" % (tmp_order, keep_log_str)) output = '' try: logger.info("Processing [%s]" % cmd) output = utilities.execute_cmd(cmd) if len(output) > 0: print output except Exception, e: logger.exception("Processing failed") status = False