def __init__(self, product_id):
super(Modis, self).__init__(product_id)
parts = product_id.strip().split('.')
self.short_name = parts[0]
self.date_acquired = parts[1][1:]
self.year = self.date_acquired[0:4]
self.doy = self.date_acquired[4:8]
# Now that we have the year and doy, we can get the month and day of
# month
date = utilities.date_from_doy(self.year, self.doy)
self.month = date.month
self.day = date.day
__hv = parts[2]
self.horizontal = __hv[1:3]
self.vertical = __hv[4:6]
self.version = parts[3]
self.date_produced = parts[4]
# set the default pixel sizes
# this comes out to 09A1, 09GA, 13A1, etc
_product_code = self.short_name.split(self.sensor_code)[1]
_meters = DEFAULT_PIXEL_SIZE['meters'][_product_code]
_dd = DEFAULT_PIXEL_SIZE['dd'][_product_code]
self.default_pixel_size = {'meters': _meters, 'dd': _dd}
def __init__(self, product_id):
product_id = product_id.strip()
super(Landsat, self).__init__(product_id)
self.path = utilities.strip_zeros(product_id[3:6])
self.row = utilities.strip_zeros(product_id[6:9])
self.year = product_id[9:13]
self.doy = product_id[13:16]
self.station = product_id[16:19]
self.version = product_id[19:21]
# Now that we have the year and doy, we can get the month and day of
# month
date = utilities.date_from_doy(self.year, self.doy)
self.month = date.month
self.day = date.day
# set the default pixel sizes
_pixels = DEFAULT_PIXEL_SIZE
_meters = _pixels['meters'][self.sensor_code.upper()]
_dd = _pixels['dd'][self.sensor_code.upper()]
self.default_pixel_size = {'meters': _meters, 'dd': _dd}
def process_test_order(request_file, products_file, env_vars,
keep_log, plot, pre, post):
'''
Description:
Process the test order file.
'''
logger = logging.getLogger(__name__)
template_file = 'template.json'
template_dict = None
tmp_order = 'tmp-test-order'
order_id = (request_file.split('.json')[0]).replace("'", '')
if pre:
order_id = ''.join([order_id, '-PRE'])
if post:
order_id = ''.join([order_id, '-POST'])
have_error = False
status = True
error_msg = ''
products = list()
if not plot:
with open(products_file, 'r') as scenes_fd:
while (1):
product = scenes_fd.readline().strip()
if not product:
break
products.append(product)
else:
products = ['plot']
logger.info("Processing Products [%s]" % ', '.join(products))
with open(template_file, 'r') as template_fd:
template_contents = template_fd.read()
if not template_contents:
raise Exception("Template file [%s] is empty" % template_file)
template_dict = json.loads(template_contents)
if template_dict is None:
logger.error("Loading template.json")
for product in products:
logger.info("Processing Product [%s]" % product)
with open(request_file, 'r') as request_fd:
request_contents = request_fd.read()
if not request_contents:
raise Exception("Order file [%s] is empty" % request_file)
request_dict = json.loads(request_contents)
if request_dict is None:
logger.error("Loading [%s]" % request_file)
# Merge the requested options with the template options, to create
# a new dict with the requested options overriding the template.
new_dict = template_dict.copy()
new_dict.update(request_dict)
new_dict['options'] = template_dict['options'].copy()
new_dict['options'].update(request_dict['options'])
# Turn it into a string for follow-on processing
order_contents = json.dumps(new_dict, indent=4, sort_keys=True)
logger.info("Processing Request File [%s]" % request_file)
with open(tmp_order, 'w') as tmp_fd:
logger.info("Creating [%s]" % tmp_order)
tmp_line = order_contents
# Update the order for the developer
tmp = product[:3]
download_url = 'null'
is_modis = False
if tmp == 'MOD' or tmp == 'MYD':
is_modis = True
# for plots
if not is_modis and not plot:
product_path = ('%s/%s/%s%s'
% (env_vars['dev_data_dir']['value'],
product[:3], product, '.tar.gz'))
logger.info("Using Product Path [%s]" % product_path)
if not os.path.isfile(product_path):
error_msg = ("Missing product data (%s)"
% product_path)
have_error = True
break
download_url = ('file://%s' % product_path)
elif not plot:
if tmp == 'MOD':
base_source_path = settings.TERRA_BASE_SOURCE_PATH
else:
base_source_path = settings.AQUA_BASE_SOURCE_PATH
short_name = sensor.instance(product).short_name
version = sensor.instance(product).version
archive_date = utilities.date_from_doy(
sensor.instance(product).year,
sensor.instance(product).doy)
xxx = '%s.%s.%s' % (str(archive_date.year).zfill(4),
str(archive_date.month).zfill(2),
str(archive_date.day).zfill(2))
product_path = ('%s/%s.%s/%s' % (base_source_path,
short_name,
version,
xxx))
if tmp == 'MOD' or tmp == 'MYD':
download_url = ('http://%s/%s/%s.hdf'
% (settings.MODIS_INPUT_CHECK_HOST,
product_path,
product))
sensor_name = 'plot'
if not plot:
sensor_name = sensor.instance(product).sensor_name
logger.info("Processing Sensor [%s]" % sensor_name)
else:
logger.info("Processing Plot Request")
tmp_line = tmp_line.replace('\n', '')
tmp_line = tmp_line.replace("ORDER_ID", order_id)
tmp_line = tmp_line.replace("SCENE_ID", product)
if sensor_name in ['tm', 'etm', 'olitirs']:
tmp_line = tmp_line.replace("PRODUCT_TYPE", 'landsat')
elif sensor_name in ['terra', 'aqua']:
tmp_line = tmp_line.replace("PRODUCT_TYPE", 'modis')
else:
tmp_line = tmp_line.replace("PRODUCT_TYPE", 'plot')
tmp_line = tmp_line.replace("DOWNLOAD_URL", download_url)
tmp_fd.write(tmp_line)
# Validate again, since we modified it
parms = json.loads(tmp_line)
# parms = parameters.instance(json.loads(tmp_line))
print(json.dumps(parms, indent=4, sort_keys=True))
# END - with tmp_order
# END - with request_file
if have_error:
logger.error(error_msg)
return False
keep_log_str = ''
if keep_log:
keep_log_str = '--keep-log'
cmd = ("cd ..; cat test-orders/%s | ./ondemand_mapper.py %s"
% (tmp_order, keep_log_str))
output = ''
try:
logger.info("Processing [%s]" % cmd)
output = utilities.execute_cmd(cmd)
if len(output) > 0:
print output
except Exception, e:
logger.exception("Processing failed")
status = False
