def _convert_new_to_old(cls, opts, scene):
"""
Basically need to make a flat data structure
and change the names used
:param opts: order options in the new format
:return: order options in the old format
"""
ret = Order.get_default_options()
opts = copy.deepcopy(opts)
short = sensor.instance(scene).shortname
sen_keys = sensor.SensorCONST.instances.keys()
for sen in sen_keys:
if sen in opts and sen != short:
opts.pop(sen)
if 'resampling_method' in opts.keys():
for resample_opts in cls.resample_map:
if opts['resampling_method'] == resample_opts[1]:
ret['resample_method'] = resample_opts[0]
break
ret.update(cls._flatten(opts, cls.keywords_map))
return ret
def build_request(contact_id, products):
# build the request body
sb = StringIO()
sb.write(self.xml_header)
head = (
"<downloadSceneList "
"xmlns="
"'https://earthexplorer.usgs.gov/schema/downloadSceneList' "
"xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' "
"xsi:schemaLocation="
"'https://earthexplorer.usgs.gov/schema/downloadSceneList "
"https://earthexplorer.usgs.gov/EE/downloadSceneList.xsd'>"
)
sb.write(head)
sb.write("<contactId>{0}</contactId>".format(contact_id))
for p in products:
try:
product = sensor.instance(p)
except sensor.ProductNotImplemented:
logger.warn("{0} not implemented, skipping".format(p))
else:
sb.write("<scene>")
sb.write("<sceneId>{0}</sceneId>".format(product.product_id))
sb.write("<sensor>{0}</sensor>".format(product.lta_name))
sb.write("</scene>")
sb.write("</downloadSceneList>")
request_body = sb.getvalue()
return request_body
def check(self, order):
ids = sensor.SensorCONST.instances.keys()
lta_ls = []
lpdaac_ls = []
results = {}
for key in order:
l1 = ''
if key in ids:
inst = sensor.instance(order[key]['inputs'][0])
l1 = inst.l1_provider
if l1 == 'lta':
lta_ls.extend(order[key]['inputs'])
elif l1 == 'lpdaac':
lpdaac_ls.extend(order[key]['inputs'])
if lta_ls:
results.update(self.check_LTA(lta_ls))
if lpdaac_ls:
results.update(self.check_LPDAAC(lpdaac_ls))
not_avail = []
for key, val in results.items():
if not val:
not_avail.append(key)
if not_avail:
raise InventoryException(not_avail)
def get_default_ee_options(item_ls):
"""
Factory method to return default ESPA order options for orders
originating through Earth Explorer
:param item_ls: list of scenes received from EE for an order
structure: list({sceneid:, unit_num:})
:return: dictionary representation of the EE order
"""
ee_order = {'format': 'gtiff'}
for item in item_ls:
try:
scene_info = sensor.instance(item['sceneid'])
except sensor.ProductNotImplemented:
log_msg = ('Received unsupported product via EE: {}'
.format(item['sceneid']))
logger.debug(log_msg)
continue
short = scene_info.shortname
if short in ee_order:
ee_order[short]['inputs'].append(item['sceneid'])
else:
if isinstance(scene_info, sensor.Landsat):
ee_order[short] = {'inputs': [item['sceneid']],
'products': ['sr']}
elif isinstance(scene_info, sensor.Modis):
ee_order[short] = {'inputs': [item['sceneid']],
'products': ['l1']}
return ee_order
def _build_nested_sensors(cls, prods, scenes):
"""
Build the nested sensor structures
:param prods: associated products
:param scenes: associated scenes
:return: nested sensor structures
"""
ret = {}
for scene in scenes:
if scene == 'plot':
ret.update({'plot_statistics': True})
continue
try:
short = sensor.instance(scene).shortname
except: # Invalid scene identifier
short = 'invalid'
if short in ret:
ret[short]['inputs'].append(scene)
else:
ret[short] = {'inputs': [scene],
'products': prods}
return ret
def build_request(contact_id, priority, product_list):
# build the request body
sb = StringIO()
sb.write(self.xml_header)
head = (
"<orderParameters "
"xmlns="
"'https://earthexplorer.usgs.gov/schema/orderParameters' "
"xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' "
"xsi:schemaLocation="
"'https://earthexplorer.usgs.gov/schema/orderParameters "
"https://earthexplorer.usgs.gov/EE/orderParameters.xsd'>"
)
sb.write(head)
sb.write("<contactId>{0}</contactId>".format(contact_id))
sb.write("<requestor>ESPA</requestor>")
# 1111111 is a dummy value.
sb.write("<externalReferenceNumber>{0}</externalReferenceNumber>".format(1111111))
sb.write("<priority>{0}</priority>".format(priority))
product_info = self.get_download_urls(product_list, contact_id)
for p in product_info.keys():
try:
sensor.instance(p)
except sensor.ProductNotImplemented, pne:
raise pne
else:
sb.write("<scene>")
sb.write("<sceneId>{0}</sceneId>".format(p))
sb.write("<prodCode>{0}</prodCode>".format(product_info[p]["lta_code"]))
sb.write("<sensor>{0}</sensor>".format(product_info[p]["sensor"]))
sb.write("</scene>")
def verify_products(self, products):
response = {}
if isinstance(products, str):
products = [products]
for product in products:
if isinstance(product, basestring):
product = sensor.instance(product)
response[product.product_id] = self.input_exists(product)
return response
def gzip_errors_online_cache(self, error_message):
''' products on cache are corrupted '''
keys = ['gzip: stdin: invalid compressed data--format violated']
status = 'retry'
reason = 'Input gzip corrupt'
extras = self.__add_retry('gzip_errors')
resolution = self.__find_error(error_message,
keys,
status,
reason,
extras)
is_landsat = False
if self.product_name is not None:
is_landsat = isinstance(sensor.instance(self.product_name),
sensor.Landsat)
if resolution is not None and is_landsat:
logger.debug("err api/errors.py gzip_errors_online_cache\n"\
"product_name: {0}\nerror_message: {1}".format(self.product_name, error_message))
emails.Emails().send_gzip_error_email(self.product_name)
return resolution
def massage_formatting(order):
"""
To avoid complications down the line, we need to ensure proper case formatting
on the order, while still being somewhat case agnostic
We also need to add 'stats' product to all the sensors if 'plot_statistics'
was set to True
:param order: incoming order after validation
:return: order with the inputs reformatted
"""
prod_keys = sn.SensorCONST.instances.keys()
stats = False
if 'plot_statistics' in order and order['plot_statistics']:
stats = True
for key in order:
if key in prod_keys:
item1 = order[key]['inputs'][0]
prod = sn.instance(item1)
if isinstance(prod, sn.Landsat):
order[key]['inputs'] = [s.upper() for s in order[key]['inputs']]
elif isinstance(prod, sn.Modis):
order[key]['inputs'] = ['.'.join([p[0].upper(),
p[1].upper(),
p[2].lower(),
p[3],
p[4]]) for p in [s.split('.') for s in order[key]['inputs']]]
if stats:
if 'stats' not in order[key]['products']:
order[key]['products'].append('stats')
return order
def input_exists(self, product):
'''Determines if a LPDAAC product is available for download
Keyword args:
product - The name of the product
Returns:
True/False
'''
if isinstance(product, str) or isinstance(product, unicode):
product = sensor.instance(product)
result = False
try:
url = self.get_download_url(product)
if 'download_url' in url[product.product_id]:
url = url[product.product_id]['download_url']
response = None
try:
response = requests.head(url)
if response.ok is True:
result = True
except Exception, e:
logger.exception('Exception checking modis input {0}\n '
'Exception:{1}'
.format(url, e))
return result
finally:
if response is not None:
response.close()
response = None
def get_download_url(self, product):
url = {}
#be nice and accept a string
if isinstance(product, str) or isinstance(product, unicode):
product = sensor.instance(product)
#also be nice and accept a sensor.Modis object
if isinstance(product, sensor.Modis):
path = self._build_modis_input_file_path(product)
product_url = ''.join([self.datapool, path])
if not product_url.lower().startswith("http"):
product_url = ''.join(['http://', product_url])
if not product.product_id in url:
url[product.product_id] = {}
url[product.product_id]['download_url'] = product_url
return url
def massage_formatting(order):
"""
To avoid complications down the line, we need to ensure proper case formatting
on the order, while still being somewhat case agnostic
We also need to add 'stats' product to all the sensors if 'plot_statistics'
was set to True
:param order: incoming order after validation
:return: order with the inputs reformatted
"""
prod_keys = sn.SensorCONST.instances.keys()
stats = False
if "plot_statistics" in order and order["plot_statistics"]:
stats = True
for key in order:
if key in prod_keys:
item1 = order[key]["inputs"][0]
prod = sn.instance(item1)
if isinstance(prod, sn.Landsat):
order[key]["inputs"] = [s.upper() for s in order[key]["inputs"]]
elif isinstance(prod, sn.Modis):
order[key]["inputs"] = [
".".join([p[0].upper(), p[1].upper(), p[2].lower(), p[3], p[4]])
for p in [s.split(".") for s in order[key]["inputs"]]
]
if stats:
if "stats" not in order[key]["products"]:
order[key]["products"].append("stats")
return order
def get_default_ee_options(item_ls):
"""
Factory method to return default ESPA order options for orders
originating through Earth Explorer
:param item_ls: list of scenes received from EE for an order
structure: list({sceneid:, unit_num:})
:return: dictionary representation of the EE order
"""
ee_order = {'format': 'gtiff'}
for item in item_ls:
try:
scene_info = sensor.instance(item['sceneid'])
except sensor.ProductNotImplemented:
log_msg = ('Received unsupported product via EE: {}'.format(
item['sceneid']))
logger.debug(log_msg)
continue
short = scene_info.shortname
if short in ee_order:
ee_order[short]['inputs'].append(item['sceneid'])
else:
if isinstance(scene_info, sensor.Landsat):
ee_order[short] = {
'inputs': [item['sceneid']],
'products': ['sr']
}
elif isinstance(scene_info, sensor.Modis):
ee_order[short] = {
'inputs': [item['sceneid']],
'products': ['l1']
}
return ee_order
def _build_modis_input_file_path(self, product):
if isinstance(product, str) or isinstance(product, unicode):
product = sensor.instance(product)
if isinstance(product, sensor.Aqua):
base_path = config.get('path.aqua_base_source')
elif isinstance(product, sensor.Terra):
base_path = config.get('path.terra_base_source')
else:
msg = "Cant build input file path for unknown LPDAAC product:%s"
raise Exception(msg % product.product_id)
date = utils.date_from_doy(product.year, product.doy)
path_date = "%s.%s.%s" % (date.year,
str(date.month).zfill(2),
str(date.day).zfill(2))
input_extension = config.get('file.extension.modis.input.filename')
parts = product.product_id.split('.')
prod_id = '.'.join([parts[0].upper(),
parts[1].upper(),
parts[2].lower(),
parts[3],
parts[4]])
input_file_name = "%s%s" % (prod_id, input_extension)
path = os.path.join(base_path,
'.'.join([product.short_name.upper(),
product.version.upper()]),
path_date.upper(), input_file_name)
return path
def verify_scenes(self, scene_list):
""" Checks to make sure the scene list is valid, where valid means
the scene ids supplied exist in the Landsat inventory and are orderable
Keyword args:
scene_list A list of scenes to be verified
Returns:
A dictionary with keys matching the scene list and values are 'true'
if valid, and 'false' if not.
Return value example:
dictionary = dict()
dictionary['LT51490212007234IKR00'] = True
dictionary['asdf'] = False
...
...
...
"""
# build the service + operation url
request_url = "{0}/verifyScenes".format(self.url)
# build the request body
sb = StringIO()
sb.write(self.xml_header)
head = (
"<sceneList "
"xmlns='https://earthexplorer.usgs.gov/schema/sceneList' "
"xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' "
"xsi:schemaLocation="
"'https://earthexplorer.usgs.gov/schema/sceneList "
"https://earthexplorer.usgs.gov/EE/sceneList.xsd'>"
)
sb.write(head)
for s in scene_list:
product = sensor.instance(s)
sb.write("<sceneId sensor='{0}'>{1}</sceneId>".format(product.lta_name, s.upper()))
sb.write("</sceneList>")
request_body = sb.getvalue()
# set the required headers
headers = dict()
headers["Content-Type"] = "application/xml"
headers["Content-Length"] = len(request_body)
# send the request and check return status
# print "*** request_url: ", request_url
# print "*** request_body: ", request_body
# print "*** headers: ", headers
__response = requests.post(request_url, data=request_body, headers=headers)
response = None
if __response.ok:
response = __response.content
else:
msg = StringIO()
msg.write("Error in lta.OrderWrapperServiceClient.verify_scenes\n")
msg.write("Non 200 response code from service\n")
msg.write("Response code was:{0}".format(__response.status_code))
msg.write("Reason:{0}".format(__response.reason))
# Return the code and reason as an exception
raise Exception(msg.getvalue())
__response.close()
# parse, transform and return response
retval = dict()
response = response.replace("&", "&")
response = response.replace("\n", "")
root = xml.fromstring(response)
scenes = root.getchildren()
for s in list(scenes):
if s.attrib["valid"] == "true":
status = True
else:
status = False
retval[s.text] = status
return retval
def validate_restricted(self, x, fieldname, schema, path, restricted):
"""Validate that the requested products are available by date or role"""
if not restricted:
return
# Like extents, we need to do some initial validation of the input up front,
# and let those individual validators output the errors
if 'inputs' not in x:
return
if not self.validate_type_array(x['inputs']):
return
req_prods = x.get(fieldname)
if not req_prods:
return
try:
req_scene = x.get('inputs')[0]
except IndexError:
return
inst = sn.instance(req_scene)
avail_prods = (ordering.OrderingProvider().available_products(
x['inputs'], self.username))
not_implemented = avail_prods.pop('not_implemented', None)
date_restricted = avail_prods.pop('date_restricted', None)
ordering_restricted = avail_prods.pop('ordering_restricted', None)
# Check for to make sure there is only one sensor type in there
if len(avail_prods) > 1:
return
if not_implemented:
self._errors.append(
"Requested IDs are not recognized. Remove: {}".format(
not_implemented))
if date_restricted:
restr_prods = date_restricted.keys()
for key in restr_prods:
if key not in req_prods:
date_restricted.pop(key, None)
if date_restricted:
for product_type in date_restricted:
msg = ("Requested {} products are restricted by date. "
"Remove {} scenes: {}".format(
product_type,
path.split('.products')[0], [
x.upper()
for x in date_restricted[product_type]
]))
self._errors.append(msg)
if ordering_restricted:
restr_sensors = ordering_restricted.keys()
for sensor in restr_sensors:
msg = ("Requested sensor is restricted from ordering. "
"Remove: {}".format(sensor))
self._errors.append(msg)
prods = []
for key in avail_prods:
prods = [_ for _ in avail_prods[key]['products']]
if not prods:
return
dif = list(set(req_prods) - set(prods))
if date_restricted:
for d in dif:
if d in date_restricted:
dif.remove(d)
if dif:
for d in dif:
if type(x) == dict:
scene_ids = [s.upper() for s in x['inputs']]
msg = ("Requested {} products are not available. "
"Remove {} scenes: {}".format(
d,
path.split('.products')[0], scene_ids))
self._errors.append(msg)
# Enforce non-customized l1 ordering restriction for Landsat
restr_source = self.restricted['source']
landsat_sensors = restr_source['sensors']
sensors = [
s for s in self.data_source.keys()
if s in sn.SensorCONST.instances.keys() and s in landsat_sensors
]
parse_customize = lambda c: (
(c in self.data_source) and
(self.data_source.get(c) != restr_source.get(c)))
if sensors and 'LANDSAT' in inst.lta_json_name:
if not set(req_prods) - set(restr_source['products']):
if not any(map(parse_customize, restr_source['custom'])):
msg = restr_source['message'].strip()
if msg not in self._errors:
self._errors.append(msg)
# Enforce non-customized l1 ordering restriction for MODIS and VIIRS
restr_modis_viirs = self.restricted['source_daac']
modis_viirs_sensors = restr_modis_viirs['sensors']
sensors_mv = [
s for s in self.data_source.keys() if
s in sn.SensorCONST.instances.keys() and s in modis_viirs_sensors
]
parse_modis_customize = lambda c: (
(c in self.data_source) and
(self.data_source.get(c) != restr_modis_viirs.get(c)))
if sensors_mv and ('MODIS' in inst.lta_json_name
or 'VIIRS' in inst.lta_json_name):
if not set(req_prods) - set(restr_modis_viirs['products']):
if not any(
map(parse_modis_customize,
restr_modis_viirs['custom'])):
msg = restr_modis_viirs['message'].strip()
if msg not in self._errors:
self._errors.append(msg)
# Enforce restricted product ordering for MODIS NDVI
if 'MODIS' in inst.lta_json_name:
restr_modis_ndvi = self.restricted['source_modis_ndvi']
modis_sensors = restr_modis_ndvi['modis_sensors']
req_ndvi_sensors = [
s for s in self.data_source.keys()
if s in sn.SensorCONST.instances.keys() and s in modis_sensors
]
invalid_req = set(req_ndvi_sensors) - set(
restr_modis_ndvi['ndvi_sensors'])
if invalid_req:
if 'modis_ndvi' in req_prods:
msg = restr_modis_ndvi['message'].strip()
if msg not in self._errors:
self._errors.append(msg)
# Enforce sensor-restricted product ordering for LaORCA
if 'orca' in req_prods:
restr_orca_info = self.restricted['source_orca_sensors']
orca_sensors = restr_orca_info['sensors']
req_orca_sensors = [
s for s in self.data_source.keys()
if s in sn.SensorCONST.instances.keys() and s in orca_sensors
]
invalid_orca_req = set(req_orca_sensors) - set(
restr_orca_info['orca_sensors'])
if invalid_orca_req:
msg = restr_orca_info['message'].strip()
if msg not in self._errors:
self._errors.append(msg)
# Make sure that all required st options are included
st = 'st'
stalg_single_channel = 'stalg_single_channel'
stalg_split_window = 'stalg_split_window'
reanalysis_data = [
'reanalsrc_narr', 'reanalsrc_merra2', 'reanalsrc_fp',
'reanalsrc_fpit'
]
if st in req_prods:
if stalg_single_channel not in req_prods and stalg_split_window not in req_prods:
msg = "Missing surface temperature algorithm - " \
"please choose from ['{0}' (olitirs only), '{1}']".format(stalg_split_window,
stalg_single_channel)
if msg not in self._errors:
self._errors.append(msg)
if stalg_single_channel in req_prods and not any(
[r for r in reanalysis_data if r in req_prods]):
msg = "Missing reanalysis data source for single channel algorithm - " \
"please choose from {}".format(reanalysis_data)
if msg not in self._errors:
self._errors.append(msg)
all_st_options = list()
all_st_options.append(stalg_split_window)
all_st_options.append(stalg_single_channel)
all_st_options.extend(reanalysis_data)
if any([x for x in all_st_options if x in req_prods
]) and st not in req_prods:
msg = "Must include 'st' in products if specifying surface temperature options"
if msg not in self._errors:
self._errors.append(msg)
def create(cls, params):
"""
Place a new order into the system
:param params: dict of required parameters to be used
params = {'product_opts': {dictionary object of the order received}
'orderid': id generated from generate_order_id
'user_id': EE user id
'order_type': typically 'level2_ondemand'
'status': 'ordered'
'note': user notes
'ee_order_id': earth explorer order id, or ''
'order_source': 'espa' or 'ee'
'order_date': date time string
'priority': legacy item, should be 'normal'
'email': user's contact email
'product_options': legacy column}
:return: order object
"""
opts = params['product_opts']
params['product_opts'] = json.dumps(params['product_opts'])
sql = ('INSERT INTO ordering_order '
'(orderid, user_id, order_type, status, note, '
'product_opts, ee_order_id, order_source, order_date, '
'priority, email, product_options) '
'VALUES (%(orderid)s, %(user_id)s, %(order_type)s, '
'%(status)s, %(note)s, %(product_opts)s, '
'%(ee_order_id)s, %(order_source)s, %(order_date)s, '
'%(priority)s, %(email)s, %(product_options)s)')
logger.info('Order creation parameters: {}'.format(params))
log_sql = ''
try:
with db_instance() as db:
log_sql = db.cursor.mogrify(sql, params)
logger.info('New order complete SQL: {}'.format(log_sql))
db.execute(sql, params)
db.commit()
except DBConnectException as e:
logger.debug('Error creating new order: {}\n'
'sql: {}'.format(e.message, log_sql))
raise OrderException(e)
order = Order.find(params['orderid'])
# Let the load_ee_order method handle the scene injection
# as there is special logic for interacting with LTA
if params['ee_order_id']:
return order
sensor_keys = sensor.SensorCONST.instances.keys()
bulk_ls = []
for key in opts:
if key in sensor_keys:
sensor_type = ''
item1 = opts[key]['inputs'][0]
if isinstance(sensor.instance(item1), sensor.Landsat):
sensor_type = 'landsat'
elif isinstance(sensor.instance(item1), sensor.Modis):
sensor_type = 'modis'
for s in opts[key]['inputs']:
scene_dict = {
'name': s,
'sensor_type': sensor_type,
'order_id': order.id,
'status': 'submitted',
'ee_unit_id': None
}
bulk_ls.append(scene_dict)
if 'plot_statistics' in opts and opts['plot_statistics']:
scene_dict = {
'name': 'plot',
'sensor_type': 'plot',
'order_id': order.id,
'status': 'submitted',
'ee_unit_id': None
}
bulk_ls.append(scene_dict)
try:
Scene.create(bulk_ls)
except SceneException as e:
logger.debug('Order creation failed on scene injection, '
'order: {}\nexception: {}'.format(
order.orderid, e.message))
with db_instance() as db:
db.execute('delete ordering_order where id = %s', order.id)
db.commit()
raise OrderException(e)
return order
def build_base_order():
"""
Builds the following dictionary (with the products filled out from sensor.py):
base = {'MOD09A1': {'inputs': 'MOD09A1.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MOD09GA': {'inputs': 'MOD09GA.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MOD09GQ': {'inputs': 'MOD09GQ.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MOD09Q1': {'inputs': 'MOD09Q1.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MYD09A1': {'inputs': 'MYD09A1.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MYD09GA': {'inputs': 'MYD09GA.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MYD09GQ': {'inputs': 'MYD09GQ.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MYD09Q1': {'inputs': 'MYD09Q1.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MOD13A1': {'inputs': 'MOD13A1.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MOD13A2': {'inputs': 'MOD13A2.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MOD13A3': {'inputs': 'MOD13A3.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MOD13Q1': {'inputs': 'MOD13Q1.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MYD13A1': {'inputs': 'MYD13A1.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MYD13A2': {'inputs': 'MYD13A2.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MYD13A3': {'inputs': 'MYD13A3.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MYD13Q1': {'inputs': 'MYD13Q1.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'tm4': {'inputs': 'LT42181092013069PFS00',
'products': ['l1']},
'tm5': {'inputs': 'LT52181092013069PFS00',
'products': ['l1']},
'etm7': {'inputs': 'LE72181092013069PFS00',
'products': ['l1']},
'oli8': {'inputs': 'LO82181092013069PFS00',
'products': ['l1']},
'olitirs8': {'inputs': 'LC82181092013069PFS00',
'products': ['l1']},
'projection': {'lonlat': None},
'image_extents': {'north': 0.0002695,
'south': 0,
'east': 0.0002695,
'west': 0,
'units': 'dd'},
'format': 'gtiff',
'resampling_method': 'cc',
'resize': {'pixel_size': 0.0002695,
'pixel_size_units': 'dd'},
'plot_statistics': True}"""
base = {'projection': {'lonlat': None},
'image_extents': {'north': 0.002695,
'south': 0,
'east': 0.002695,
'west': 0,
'units': 'dd'},
'format': 'gtiff',
'resampling_method': 'cc',
'resize': {'pixel_size': 0.0002695,
'pixel_size_units': 'dd'},
'plot_statistics': True}
sensor_acqids = {'.A2000072.h02v09.005.2008237032813': (['MOD09A1', 'MOD09Q1', 'MYD13A1', 'MYD13Q1'],
['mod09a1', 'mod09q1', 'myd13a1', 'myd13q1']),
'.A2016305.h11v04.006.2016314200836': (['MOD09GA', 'MOD09GQ', 'MYD13A2', 'MYD13A3'],
['mod09ga', 'mod09gq', 'myd13a2', 'myd13a3']),
'.A2000361.h24v03.006.2015111114747': (['MOD11A1', 'MYD11A1'],
['mod11a1', 'myd11a1']),
# TODO: REMOVE _collection from IDs
'L1TP_044030_19851028_20161004_01_T1': (['LT04_', 'LT05_', 'LE07_', 'LO08_', 'LC08_'],
['tm4_collection', 'tm5_collection', 'etm7_collection', 'oli8_collection', 'olitirs8_collection'])}
for acq in sensor_acqids:
for prefix, label in zip(sensor_acqids[acq][0], sensor_acqids[acq][1]):
base[label] = {'inputs': ['{}{}'.format(prefix, acq)],
'products': sn.instance('{}{}'.format(prefix, acq)).products}
return base
def available_products(self, product_id, username):
"""
Check to see what products are available to user based on
an input list of scenes
:param product_id: list of desired inputs
:param username: username
:return: dictionary
"""
user = User.by_username(username)
pub_prods = copy.deepcopy(OrderingProvider.sensor_products(product_id))
with open('api/domain/restricted.yaml') as f:
restricted = yaml.load(f.read())
role = False if user.is_staff() else True
restrict_all = restricted.get('all', {})
all_role = restrict_all.get('role', [])
all_by_date = restrict_all.get('by_date', {})
upd = {'date_restricted': {}}
for sensor_type, prods in pub_prods.items():
if sensor_type == 'not_implemented':
continue
sensor_restr = restricted.get(sensor_type, {})
role_restr = sensor_restr.get('role', []) + all_role
by_date_restr = sensor_restr.get('by_date', {})
# All overrides any sensor related dates
by_date_restr.update(all_by_date)
outs = pub_prods[sensor_type]['products']
ins = pub_prods[sensor_type]['inputs']
remove_me = []
if role:
for prod in role_restr:
try:
outs.remove(prod)
except ValueError:
continue
for prod in outs:
if prod in by_date_restr:
r = sensor_restr['by_date'][prod]
for sc_id in ins:
obj = sensor.instance(sc_id)
julian = '{}{}'.format(obj.year, obj.doy)
if not julian_date_check(julian, r):
remove_me.append(prod)
if prod in upd['date_restricted']:
upd['date_restricted'][prod].append(sc_id)
else:
upd['date_restricted'][prod] = [sc_id]
for rem in remove_me:
try:
outs.remove(rem)
except ValueError:
continue
if upd['date_restricted']:
pub_prods.update(upd)
return pub_prods
def verify_scenes(self, scene_list):
''' Checks to make sure the scene list is valid, where valid means
the scene ids supplied exist in the Landsat inventory and are orderable
Keyword args:
scene_list A list of scenes to be verified
Returns:
A dictionary with keys matching the scene list and values are 'true'
if valid, and 'false' if not.
Return value example:
dictionary = dict()
dictionary['LT51490212007234IKR00'] = True
dictionary['asdf'] = False
...
...
...
'''
#build the service + operation url
request_url = '{0}/verifyScenes'.format(self.url)
#build the request body
sb = StringIO()
sb.write(self.xml_header)
head = ("<sceneList "
"xmlns='https://earthexplorer.usgs.gov/schema/sceneList' "
"xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' "
"xsi:schemaLocation="
"'https://earthexplorer.usgs.gov/schema/sceneList "
"https://earthexplorer.usgs.gov/EE/sceneList.xsd'>")
sb.write(head)
for s in scene_list:
product = sensor.instance(s)
sb.write("<sceneId sensor='{0}'>{1}</sceneId>"
.format(product.lta_name, s.upper()))
sb.write("</sceneList>")
request_body = sb.getvalue()
#set the required headers
headers = dict()
headers['Content-Type'] = 'application/xml'
headers['Content-Length'] = len(request_body)
#send the request and check return status
#print "*** request_url: ", request_url
#print "*** request_body: ", request_body
#print "*** headers: ", headers
__response = requests.post(request_url,
data=request_body,
headers=headers)
response = None
if __response.ok:
response = __response.content
else:
msg = StringIO()
msg.write("Error in lta.OrderWrapperServiceClient.verify_scenes\n")
msg.write("Non 200 response code from service\n")
msg.write("Response code was:{0}".format( __response.status_code))
msg.write("Reason:{0}".format(__response.reason))
# Return the code and reason as an exception
raise Exception(msg.getvalue())
__response.close()
#parse, transform and return response
retval = dict()
response = response.replace('&', '&')
response = response.replace('\n', '')
root = xml.fromstring(response)
scenes = root.getchildren()
for s in list(scenes):
if s.attrib['valid'] == 'true':
status = True
else:
status = False
retval[s.text] = status
return retval
class BaseValidationSchema(object):
formats = ['gtiff', 'hdf-eos2', 'envi', 'netcdf']
resampling_methods = ['nn', 'bil', 'cc']
projections = {'aea': {'type': 'object',
'properties': {'standard_parallel_1': {'type': 'number',
'required': True,
'minimum': -90,
'maximum': 90},
'standard_parallel_2': {'type': 'number',
'required': True,
'minimum': -90,
'maximum': 90},
'central_meridian': {'type': 'number',
'required': True,
'minimum': -180,
'maximum': 180},
'latitude_of_origin': {'type': 'number',
'required': True,
'minimum': -90,
'maximum': 90},
'false_easting': {'type': 'number',
'required': True},
'false_northing': {'type': 'number',
'required': True},
'datum': {'type': 'string',
'required': True,
'enum': ['wgs84', 'nad27', 'nad83']}}},
'utm': {'type': 'object',
'properties': {'zone': {'type': 'integer',
'required': True,
'minimum': 1,
'maximum': 60},
'zone_ns': {'type': 'string',
'required': True,
'enum': ['north', 'south']}}},
'lonlat': {'type': 'null'},
'sinu': {'type': 'object',
'properties': {'central_meridian': {'type': 'number',
'required': True,
'minimum': -180,
'maximum': 180},
'false_easting': {'type': 'number',
'required': True},
'false_northing': {'type': 'number',
'required': True}}},
'ps': {'type': 'object',
'properties': {'longitudinal_pole': {'type': 'number',
'required': True,
'minimum': -180,
'maximum': 180},
'latitude_true_scale': {'type': 'number',
'required': True,
'abs_rng': (60, 90)},
'false_easting': {'type': 'number',
'required': True},
'false_northing': {'type': 'number',
'required': True}}}}
extents = {'north': {'type': 'number',
'required': True},
'south': {'type': 'number',
'required': True},
'east': {'type': 'number',
'required': True},
'west': {'type': 'number',
'required': True},
'units': {'type': 'string',
'required': True,
'enum': ['dd', 'meters']}}
resize = {'pixel_size': {'type': 'number',
'required': True,
'ps_dd_rng': (0.0002695, 0.0449155),
'ps_meter_rng': (30, 5000)},
'pixel_size_units': {'type': 'string',
'required': True,
'enum': ['dd', 'meters']}}
request_schema = {'type': 'object',
'set_ItemCount': ('inputs', 5000),
'extents': 200000000,
'properties': {'projection': {'properties': projections,
'type': 'object',
# 'enum_keys': self.projections.keys(),
'single_obj': True},
'image_extents': {'type': 'object',
'properties': extents,
# 'enum_keys': self.extents.keys(),
'dependencies': ['projection']},
'format': {'type': 'string',
'required': True,
'enum': formats},
'resize': {'type': 'object',
'properties': resize},
'resampling_method': {'type': 'string',
'enum': resampling_methods},
'plot_statistics': {'type': 'boolean'},
'note': {'type': 'string',
'required': False,
'blank': True}}}
_sensor_reg = sn.SensorCONST.instances
sensor_schema = {}
for key in _sensor_reg:
sensor_schema[key] = {'type': 'object',
'properties': {'inputs': {'type': 'array',
'required': True,
'ItemCount': 'inputs',
'uniqueItems': True,
'minItems': 1,
'items': {'type': 'string',
'pattern': _sensor_reg[key][0]}},
'products': {'type': 'array',
'uniqueItems': True,
'required': True,
'restricted': True,
'stats': True,
'minItems': 1,
'items': {'type': 'string',
'enum': sn.instance(
_sensor_reg[key][2]).products}}}}
request_schema['properties'].update(sensor_schema)
request_schema['oneormoreobjects'] = sensor_schema.keys()
valid_params = {'formats': {'formats': formats},
'resampling_methods': {'resampling_methods': resampling_methods},
'projections': projections}
class BaseValidationSchema(object):
formats = {'gtiff': 'GeoTiff',
'envi': 'ENVI',
'hdf-eos2': 'HDF-EOS2',
'netcdf': 'NetCDF'}
resampling_methods = {'nn': 'Nearest Neighbor',
'bil': 'Bilinear Interpolation',
'cc': 'Cubic Convolution'}
projections = {'aea': {'type': 'object',
'title': 'Albers Equal Area',
'pixel_units': ('meters', 'dd'),
'display_rank': 0,
'properties': {'standard_parallel_1': {'type': 'number',
'title': '1st Standard Parallel',
'display_rank': 2,
'required': True,
'minimum': -90,
'maximum': 90},
'standard_parallel_2': {'type': 'number',
'title': '2nd Standard Parallel',
'display_rank': 3,
'required': True,
'minimum': -90,
'maximum': 90},
'central_meridian': {'type': 'number',
'title': 'Central Meridian',
'display_rank': 1,
'required': True,
'minimum': -180,
'maximum': 180},
'latitude_of_origin': {'type': 'number',
'title': 'Latitude of Origin',
'display_rank': 0,
'required': True,
'minimum': -90,
'maximum': 90},
'false_easting': {'type': 'number',
'title': 'False Easting (meters)',
'display_rank': 4,
'required': True},
'false_northing': {'type': 'number',
'title': 'False Northing (meters)',
'display_rank': 5,
'required': True},
'datum': {'type': 'string',
'title': 'Datum',
'required': True,
'display_rank': 6,
'enum': {'wgs84': 'World Geodetic System 1984',
'nad27': 'North American Datum 1927',
'nad83': 'North American Datum 1983'}}}},
'utm': {'type': 'object',
'pixel_units': ('meters', 'dd'),
'display_rank': 1,
'title': 'Universal Transverse Mercator',
'properties': {'zone': {'type': 'integer',
'title': 'UTM Grid Zone Number',
'display_rank': 0,
'required': True,
'minimum': 1,
'maximum': 60},
'zone_ns': {'type': 'string',
'title': 'UTM Hemisphere',
'display_rank': 1,
'required': True,
'enum': {'north': 'North', 'south': 'South'}}}},
'lonlat': {'type': 'null',
'pixel_units': ('dd',),
'title': 'Geographic',
'display_rank': 2},
'sinu': {'type': 'object',
'title': 'Sinusoidal',
'pixel_units': ('meters', 'dd'),
'display_rank': 3,
'properties': {'central_meridian': {'type': 'number',
'title': 'Central Meridian',
'display_rank': 0,
'required': True,
'minimum': -180,
'maximum': 180},
'false_easting': {'type': 'number',
'title': 'False Easting (meters)',
'display_rank': 1,
'required': True},
'false_northing': {'type': 'number',
'title': 'False Northing (meters)',
'display_rank': 2,
'required': True}}},
'ps': {'type': 'object',
'title': 'Polar Stereographic',
'pixel_units': ('meters', 'dd'),
'display_rank': 4,
'properties': {'longitudinal_pole': {'type': 'number',
'title': 'Longitudinal Pole',
'display_rank': 0,
'required': True,
'minimum': -180,
'maximum': 180},
'latitude_true_scale': {'type': 'number',
'title': 'Latitude True Scale',
'display_rank': 1,
'required': True,
'abs_rng': (60, 90)},
'false_easting': {'type': 'number',
'title': 'False Easting (meters)',
'display_rank': 2,
'required': True},
'false_northing': {'type': 'number',
'title': 'False Northing (meters)',
'display_rank': 3,
'required': True}}}}
extents = {'north': {'type': 'number',
'title': 'Upper left Y coordinate',
'display_rank': 2,
'required': True},
'south': {'type': 'number',
'title': 'Lower right Y coordinate',
'display_rank': 4,
'required': True},
'east': {'type': 'number',
'title': 'Lower right X coordinate',
'display_rank': 3,
'required': True},
'west': {'type': 'number',
'title': 'Upper left X coordinate',
'display_rank': 1,
'required': True},
'units': {'type': 'string',
'title': 'Coordinate system units',
'display_rank': 0,
'required': True,
'enum': {'dd': 'Decimal Degrees', 'meters': 'Meters'}}}
resize = {'pixel_size': {'type': 'number',
'title': 'Pixel Size',
'display_rank': 0,
'required': True,
'ps_dd_rng': (0.0002695, 0.0449155),
'ps_meters_rng': (30, 5000)},
'pixel_size_units': {'type': 'string',
'title': 'Pixel Size Units',
'display_rank': 1,
'required': True,
'enum': {'dd': 'Decimal Degrees', 'meters': 'Meters'}}}
request_schema = {'type': 'object',
'set_ItemCount': ('inputs', 5000),
'extents': 200000000,
'properties': {'projection': {'properties': projections,
'type': 'object',
'title': 'Reproject Products',
'display_rank': 1,
'single_obj': True},
'image_extents': {'type': 'object',
'title': 'Modify Image Extents',
'display_rank': 2,
'properties': extents,
'dependencies': ['projection']},
'format': {'type': 'string',
'title': 'Output Format',
'display_rank': 0,
'required': True,
'enum': formats},
'resize': {'type': 'object',
'title': 'Pixel Resizing',
'display_rank': 3,
'properties': resize},
'resampling_method': {'type': 'string',
'title': 'Resample Method',
'display_rank': 4,
'enum': resampling_methods},
'plot_statistics': {'type': 'boolean',
'title': 'Plot Output Product Statistics'},
'note': {'type': 'string',
'title': 'Order Description (optional)',
'required': False,
'blank': True}}}
_sensor_reg = sn.SensorCONST.instances
sensor_schema = {}
for key in _sensor_reg:
sensor_schema[key] = {'type': 'object',
'properties': {'inputs': {'type': 'array',
'required': True,
'ItemCount': 'inputs',
'uniqueItems': True,
'minItems': 1,
'items': {'type': 'string',
'pattern': _sensor_reg[key][0]}},
'products': {'type': 'array',
'uniqueItems': True,
'required': True,
'restricted': True,
'stats': True,
'minItems': 1,
'items': {'type': 'string',
'enum': sn.instance(
_sensor_reg[key][2]).products}}}}
request_schema['properties'].update(sensor_schema)
request_schema['oneormoreobjects'] = sensor_schema.keys()
valid_params = {'formats': {'formats': formats},
'resampling_methods': {'resampling_methods': resampling_methods},
'projections': projections}
def build_base_order():
"""
Builds the following dictionary (with the products filled out from sensor.py):
base = {'MOD09A1': {'inputs': 'MOD09A1.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MOD09GA': {'inputs': 'MOD09GA.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MOD09GQ': {'inputs': 'MOD09GQ.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MOD09Q1': {'inputs': 'MOD09Q1.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MYD09A1': {'inputs': 'MYD09A1.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MYD09GA': {'inputs': 'MYD09GA.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MYD09GQ': {'inputs': 'MYD09GQ.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MYD09Q1': {'inputs': 'MYD09Q1.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MOD13A1': {'inputs': 'MOD13A1.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MOD13A2': {'inputs': 'MOD13A2.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MOD13A3': {'inputs': 'MOD13A3.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MOD13Q1': {'inputs': 'MOD13Q1.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MYD13A1': {'inputs': 'MYD13A1.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MYD13A2': {'inputs': 'MYD13A2.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MYD13A3': {'inputs': 'MYD13A3.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MYD13Q1': {'inputs': 'MYD13Q1.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'tm4': {'inputs': 'LT42181092013069PFS00',
'products': ['l1']},
'tm5': {'inputs': 'LT52181092013069PFS00',
'products': ['l1']},
'etm7': {'inputs': 'LE72181092013069PFS00',
'products': ['l1']},
'oli8': {'inputs': 'LO82181092013069PFS00',
'products': ['l1']},
'olitirs8': {'inputs': 'LC82181092013069PFS00',
'products': ['l1']},
'projection': {'lonlat': None},
'image_extents': {'north': 0.0002695,
'south': 0,
'east': 0.0002695,
'west': 0,
'units': 'dd'},
'format': 'gtiff',
'resampling_method': 'cc',
'resize': {'pixel_size': 0.0002695,
'pixel_size_units': 'dd'},
'plot_statistics': True}"""
base = {
'projection': {
'lonlat': None
},
'image_extents': {
'north': 0.002695,
'south': 0,
'east': 0.002695,
'west': 0,
'units': 'dd'
},
'format': 'gtiff',
'resampling_method': 'cc',
'resize': {
'pixel_size': 0.0002695,
'pixel_size_units': 'dd'
},
'plot_statistics': True
}
sensor_acqids = {
'.A2000072.h02v09.005.2008237032813': ([
'MOD09A1', 'MOD09GA', 'MOD09GQ', 'MOD09Q1', 'MYD09A1', 'MYD09GA',
'MYD09GQ', 'MYD09Q1', 'MOD13A1', 'MOD13A2', 'MOD13A3', 'MOD13Q1',
'MYD13A1', 'MYD13A2', 'MYD13A3', 'MYD13Q1'
], [
'mod09a1', 'mod09ga', 'mod09gq', 'mod09q1', 'myd09a1', 'myd09ga',
'myd09gq', 'myd09q1', 'mod13a1', 'mod13a2', 'mod13a3', 'mod13q1',
'myd13a1', 'myd13a2', 'myd13a3', 'myd13q1'
]),
'2181092013069PFS00':
(['LT4', 'LT5', 'LE7', 'LO8',
'LC8'], ['tm4', 'tm5', 'etm7', 'oli8', 'olitirs8'])
}
for acq in sensor_acqids:
for prefix, label in zip(sensor_acqids[acq][0], sensor_acqids[acq][1]):
base[label] = {
'inputs': ['{}{}'.format(prefix, acq)],
'products': sn.instance('{}{}'.format(prefix, acq)).products
}
return base
def available_products(self, product_id, username):
"""
Check to see what products are available to user based on
an input list of scenes
:param product_id: list of desired inputs
:param username: username
:return: dictionary
"""
user = User.by_username(username)
pub_prods = copy.deepcopy(OrderingProvider.sensor_products(product_id))
with open(os.path.join(__location__, 'domain/restricted.yaml')) as f:
restricted = yaml.safe_load(f.read())
role = False if user.is_staff() else True
restrict_all = restricted.get('all', {})
all_role = restrict_all.get('role', [])
all_by_date = restrict_all.get('by_date', {})
all_ordering_rsctd = restrict_all.get('ordering', [])
upd = {'date_restricted': {}, 'ordering_restricted': {}}
for sensor_type, prods in pub_prods.items():
if sensor_type == 'not_implemented':
continue
stype = sensor_type.replace(
'_collection',
'') if '_collection' in sensor_type else sensor_type
sensor_restr = restricted.get(stype, {})
role_restr = sensor_restr.get('role', []) + all_role
by_date_restr = sensor_restr.get('by_date', {})
# All overrides any sensor related dates
by_date_restr.update(all_by_date)
outs = pub_prods[sensor_type]['products']
ins = pub_prods[sensor_type]['inputs']
if sensor_type in all_ordering_rsctd:
for sc_id in ins:
if sensor_type in upd['ordering_restricted']:
upd['ordering_restricted'][sensor_type].append(sc_id)
else:
upd['ordering_restricted'][sensor_type] = [sc_id]
pub_prods.pop(sensor_type)
continue
remove_me = []
if role:
for prod in role_restr:
try:
outs.remove(prod)
except ValueError:
continue
for prod in outs:
if prod in by_date_restr:
r = sensor_restr['by_date'][prod]
for sc_id in ins:
obj = sensor.instance(sc_id)
julian = '{}{}'.format(obj.year, obj.doy)
if not julian_date_check(julian, r):
remove_me.append(prod)
if prod in upd['date_restricted']:
upd['date_restricted'][prod].append(sc_id)
else:
upd['date_restricted'][prod] = [sc_id]
for rem in remove_me:
try:
outs.remove(rem)
except ValueError:
continue
if upd['date_restricted']:
pub_prods.update(date_restricted=upd['date_restricted'])
if upd['ordering_restricted']:
pub_prods.update(ordering_restricted=upd['ordering_restricted'])
return pub_prods
def build_base_order():
"""
Builds the following dictionary (with the products filled out from sensor.py):
base = {'MOD09A1': {'inputs': 'MOD09A1.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MOD09GA': {'inputs': 'MOD09GA.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MOD09GQ': {'inputs': 'MOD09GQ.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MOD09Q1': {'inputs': 'MOD09Q1.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MYD09A1': {'inputs': 'MYD09A1.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MYD09GA': {'inputs': 'MYD09GA.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MYD09GQ': {'inputs': 'MYD09GQ.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MYD09Q1': {'inputs': 'MYD09Q1.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MOD13A1': {'inputs': 'MOD13A1.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MOD13A2': {'inputs': 'MOD13A2.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MOD13A3': {'inputs': 'MOD13A3.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MOD13Q1': {'inputs': 'MOD13Q1.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MYD13A1': {'inputs': 'MYD13A1.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MYD13A2': {'inputs': 'MYD13A2.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MYD13A3': {'inputs': 'MYD13A3.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MYD13Q1': {'inputs': 'MYD13Q1.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'vnp09ga': {'inputs': ['VNP09GA.A2014245.h10v04.001.2017043103958',
'VNP09GA.A2014245.h10v04.001.2015012103931'],
'products': ['l1']},
'tm4': {'inputs': 'LT42181092013069PFS00',
'products': ['l1']},
'tm5': {'inputs': 'LT52181092013069PFS00',
'products': ['l1']},
'etm7': {'inputs': 'LE72181092013069PFS00',
'products': ['l1']},
'oli8': {'inputs': 'LO82181092013069PFS00',
'products': ['l1']},
'olitirs8': {'inputs': 'LC82181092013069PFS00',
'products': ['l1']},
'sentinel': {'inputs': ['L1C_T14TPP_A022031_20190910T172721',
'S2A_OPER_MSI_L1C_TL_SGS__20160130T184417_20160130T203840_A003170_T13VCC_N02_01_01'],
'products': ['s2_sr', 's2_ndvi', 's2_msavi', 's2_evi', 's2_savi',
's2_nbr', 's2_nbr2', 's2_ndmi']
'projection': {'lonlat': None},
'image_extents': {'north': 0.0002695,
'south': 0,
'east': 0.0002695,
'west': 0,
'units': 'dd'},
'format': 'gtiff',
'resampling_method': 'cc',
'resize': {'pixel_size': 0.0002695,
'pixel_size_units': 'dd'},
'plot_statistics': True}"""
base = {
'projection': {
'lonlat': None
},
'image_extents': {
'north': 0.002695,
'south': 0,
'east': 0.002695,
'west': 0,
'units': 'dd'
},
'format': 'gtiff',
'resampling_method': 'cc',
'resize': {
'pixel_size': 0.0002695,
'pixel_size_units': 'dd'
},
'plot_statistics': True
}
# This will give the base order 15 scenes
sensor_acqids = {
'.A2000072.h02v09.005.2008237032813':
(['MOD09A1', 'MOD09Q1', 'MYD13A1',
'MYD13Q1'], ['mod09a1', 'mod09q1', 'myd13a1', 'myd13q1']),
'.A2016305.h11v04.006.2016314200836':
(['MOD09GA', 'MOD09GQ', 'MYD13A2',
'MYD13A3'], ['mod09ga', 'mod09gq', 'myd13a2', 'myd13a3']),
'.A2000361.h24v03.006.2015111114747':
(['MOD11A1', 'MYD11A1'], ['mod11a1', 'myd11a1']),
# TODO: REMOVE _collection from IDs
'L1TP_044030_19851028_20161004_01_T1':
(['LT04_', 'LT05_', 'LE07_', 'LO08_', 'LC08_'], [
'tm4_collection', 'tm5_collection', 'etm7_collection',
'oli8_collection', 'olitirs8_collection'
])
}
for acq in sensor_acqids:
for prefix, label in zip(sensor_acqids[acq][0], sensor_acqids[acq][1]):
# We need to avoid any requests for modis_ndvi - this will raise an error since we have
# non-NDVI available MODIS products in the base order. Testing of modis_ndvi
# and viirs_ndvi orders is conducted separately in test_api.py
if label.startswith('mod') or label.startswith('myd'):
base[label] = {
'inputs': ['{}{}'.format(prefix, acq)],
'products': ['l1']
}
else:
base[label] = {
'inputs': ['{}{}'.format(prefix, acq)],
# don't include orca in our test base order since it has sensor restrictions
'products': [
p for p in sn.instance('{}{}'.format(
prefix, acq)).products if p != 'orca'
]
}
# We need to have at least 2 scenes for viirs-related tests to work
base['vnp09ga'] = {
'inputs': [
'VNP09GA.A2014245.h10v04.001.2017043103958',
'VNP09GA.A2014245.h10v04.001.2015012103931'
],
'products': ['l1']
}
# add sentinel-2 scene IDs to the order
base['sentinel'] = {
'inputs': [
'L1C_T14TPP_A022031_20190910T172721',
'S2A_OPER_MSI_L1C_TL_SGS__20160130T184417_20160130T203840_A003170_T13VCC_N02_01_01'
],
'products': [
's2_sr', 's2_ndvi', 's2_msavi', 's2_savi', 's2_evi', 's2_ndmi',
's2_nbr', 's2_nbr2'
]
}
return base
def create(cls, params):
"""
Place a new order into the system
:param params: dict of required parameters to be used
params = {'product_opts': {dictionary object of the order received}
'orderid': id generated from generate_order_id
'user_id': EE user id
'order_type': typically 'level2_ondemand'
'status': 'ordered'
'note': user notes
'ee_order_id': earth explorer order id, or ''
'order_source': 'espa' or 'ee'
'order_date': date time string
'priority': legacy item, should be 'normal'
'email': user's contact email
'product_options': legacy column}
:return: order object
"""
opts = params['product_opts']
params['product_opts'] = json.dumps(params['product_opts'])
sql = ('INSERT INTO ordering_order '
'(orderid, user_id, order_type, status, note, '
'product_opts, ee_order_id, order_source, order_date, '
'priority, email, product_options) '
'VALUES (%(orderid)s, %(user_id)s, %(order_type)s, '
'%(status)s, %(note)s, %(product_opts)s, '
'%(ee_order_id)s, %(order_source)s, %(order_date)s, '
'%(priority)s, %(email)s, %(product_options)s)')
logger.info('Order creation parameters: {}'.format(params))
log_sql = ''
try:
with db_instance() as db:
log_sql = db.cursor.mogrify(sql, params)
logger.info('New order complete SQL: {}'
.format(log_sql))
db.execute(sql, params)
db.commit()
except DBConnectException as e:
logger.debug('Error creating new order: {}\n'
'sql: {}'.format(e.message, log_sql))
raise OrderException(e)
order = Order.find(params['orderid'])
# Let the load_ee_order method handle the scene injection
# as there is special logic for interacting with LTA
if params['ee_order_id']:
return order
sensor_keys = sensor.SensorCONST.instances.keys()
bulk_ls = []
for key in opts:
if key in sensor_keys:
sensor_type = ''
item1 = opts[key]['inputs'][0]
if isinstance(sensor.instance(item1), sensor.Landsat):
sensor_type = 'landsat'
elif isinstance(sensor.instance(item1), sensor.Modis):
sensor_type = 'modis'
for s in opts[key]['inputs']:
scene_dict = {'name': s,
'sensor_type': sensor_type,
'order_id': order.id,
'status': 'submitted',
'ee_unit_id': None}
bulk_ls.append(scene_dict)
if 'plot_statistics' in opts and opts['plot_statistics']:
scene_dict = {'name': 'plot',
'sensor_type': 'plot',
'order_id': order.id,
'status': 'submitted',
'ee_unit_id': None}
bulk_ls.append(scene_dict)
try:
Scene.create(bulk_ls)
except SceneException as e:
logger.debug('Order creation failed on scene injection, '
'order: {}\nexception: {}'
.format(order.orderid, e.message))
with db_instance() as db:
db.execute('delete ordering_order where id = %s',
order.id)
db.commit()
raise OrderException(e)
return order
def build_base_order():
"""
Builds the following dictionary (with the products filled out from sensor.py):
base = {'MOD09A1': {'inputs': 'MOD09A1.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MOD09GA': {'inputs': 'MOD09GA.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MOD09GQ': {'inputs': 'MOD09GQ.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MOD09Q1': {'inputs': 'MOD09Q1.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MYD09A1': {'inputs': 'MYD09A1.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MYD09GA': {'inputs': 'MYD09GA.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MYD09GQ': {'inputs': 'MYD09GQ.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MYD09Q1': {'inputs': 'MYD09Q1.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MOD13A1': {'inputs': 'MOD13A1.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MOD13A2': {'inputs': 'MOD13A2.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MOD13A3': {'inputs': 'MOD13A3.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MOD13Q1': {'inputs': 'MOD13Q1.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MYD13A1': {'inputs': 'MYD13A1.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MYD13A2': {'inputs': 'MYD13A2.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MYD13A3': {'inputs': 'MYD13A3.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'MYD13Q1': {'inputs': 'MYD13Q1.A2000072.h02v09.005.2008237032813',
'products': ['l1']},
'tm4': {'inputs': 'LT42181092013069PFS00',
'products': ['l1']},
'tm5': {'inputs': 'LT52181092013069PFS00',
'products': ['l1']},
'etm7': {'inputs': 'LE72181092013069PFS00',
'products': ['l1']},
'oli8': {'inputs': 'LO82181092013069PFS00',
'products': ['l1']},
'olitirs8': {'inputs': 'LC82181092013069PFS00',
'products': ['l1']},
'projection': {'lonlat': None},
'image_extents': {'north': 0.0002695,
'south': 0,
'east': 0.0002695,
'west': 0,
'units': 'dd'},
'format': 'gtiff',
'resampling_method': 'cc',
'resize': {'pixel_size': 0.0002695,
'pixel_size_units': 'dd'},
'plot_statistics': True}"""
base = {'projection': {'lonlat': None},
'image_extents': {'north': 0.002695,
'south': 0,
'east': 0.002695,
'west': 0,
'units': 'dd'},
'format': 'gtiff',
'resampling_method': 'cc',
'resize': {'pixel_size': 0.0002695,
'pixel_size_units': 'dd'},
'plot_statistics': True}
sensor_acqids = {'.A2000072.h02v09.005.2008237032813': (['MOD09A1', 'MOD09GA', 'MOD09GQ', 'MOD09Q1',
'MYD09A1', 'MYD09GA', 'MYD09GQ', 'MYD09Q1',
'MOD13A1', 'MOD13A2', 'MOD13A3', 'MOD13Q1',
'MYD13A1', 'MYD13A2', 'MYD13A3', 'MYD13Q1'],
['mod09a1', 'mod09ga', 'mod09gq', 'mod09q1',
'myd09a1', 'myd09ga', 'myd09gq', 'myd09q1',
'mod13a1', 'mod13a2', 'mod13a3', 'mod13q1',
'myd13a1', 'myd13a2', 'myd13a3', 'myd13q1']),
'2181092013069PFS00': (['LT4', 'LT5', 'LE7', 'LO8', 'LC8'],
['tm4', 'tm5', 'etm7', 'oli8', 'olitirs8'])}
for acq in sensor_acqids:
for prefix, label in zip(sensor_acqids[acq][0], sensor_acqids[acq][1]):
base[label] = {'inputs': ['{}{}'.format(prefix, acq)],
'products': sn.instance('{}{}'.format(prefix, acq)).products}
return base
