def _convert_new_to_old(cls, opts, scene): """ Basically need to make a flat data structure and change the names used :param opts: order options in the new format :return: order options in the old format """ ret = Order.get_default_options() opts = copy.deepcopy(opts) short = sensor.instance(scene).shortname sen_keys = sensor.SensorCONST.instances.keys() for sen in sen_keys: if sen in opts and sen != short: opts.pop(sen) if 'resampling_method' in opts.keys(): for resample_opts in cls.resample_map: if opts['resampling_method'] == resample_opts[1]: ret['resample_method'] = resample_opts[0] break ret.update(cls._flatten(opts, cls.keywords_map)) return ret
def build_request(contact_id, products): # build the request body sb = StringIO() sb.write(self.xml_header) head = ( "<downloadSceneList " "xmlns=" "'https://earthexplorer.usgs.gov/schema/downloadSceneList' " "xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' " "xsi:schemaLocation=" "'https://earthexplorer.usgs.gov/schema/downloadSceneList " "https://earthexplorer.usgs.gov/EE/downloadSceneList.xsd'>" ) sb.write(head) sb.write("<contactId>{0}</contactId>".format(contact_id)) for p in products: try: product = sensor.instance(p) except sensor.ProductNotImplemented: logger.warn("{0} not implemented, skipping".format(p)) else: sb.write("<scene>") sb.write("<sceneId>{0}</sceneId>".format(product.product_id)) sb.write("<sensor>{0}</sensor>".format(product.lta_name)) sb.write("</scene>") sb.write("</downloadSceneList>") request_body = sb.getvalue() return request_body
def check(self, order): ids = sensor.SensorCONST.instances.keys() lta_ls = [] lpdaac_ls = [] results = {} for key in order: l1 = '' if key in ids: inst = sensor.instance(order[key]['inputs'][0]) l1 = inst.l1_provider if l1 == 'lta': lta_ls.extend(order[key]['inputs']) elif l1 == 'lpdaac': lpdaac_ls.extend(order[key]['inputs']) if lta_ls: results.update(self.check_LTA(lta_ls)) if lpdaac_ls: results.update(self.check_LPDAAC(lpdaac_ls)) not_avail = [] for key, val in results.items(): if not val: not_avail.append(key) if not_avail: raise InventoryException(not_avail)
def get_default_ee_options(item_ls): """ Factory method to return default ESPA order options for orders originating through Earth Explorer :param item_ls: list of scenes received from EE for an order structure: list({sceneid:, unit_num:}) :return: dictionary representation of the EE order """ ee_order = {'format': 'gtiff'} for item in item_ls: try: scene_info = sensor.instance(item['sceneid']) except sensor.ProductNotImplemented: log_msg = ('Received unsupported product via EE: {}' .format(item['sceneid'])) logger.debug(log_msg) continue short = scene_info.shortname if short in ee_order: ee_order[short]['inputs'].append(item['sceneid']) else: if isinstance(scene_info, sensor.Landsat): ee_order[short] = {'inputs': [item['sceneid']], 'products': ['sr']} elif isinstance(scene_info, sensor.Modis): ee_order[short] = {'inputs': [item['sceneid']], 'products': ['l1']} return ee_order
def _build_nested_sensors(cls, prods, scenes): """ Build the nested sensor structures :param prods: associated products :param scenes: associated scenes :return: nested sensor structures """ ret = {} for scene in scenes: if scene == 'plot': ret.update({'plot_statistics': True}) continue try: short = sensor.instance(scene).shortname except: # Invalid scene identifier short = 'invalid' if short in ret: ret[short]['inputs'].append(scene) else: ret[short] = {'inputs': [scene], 'products': prods} return ret
def build_request(contact_id, priority, product_list): # build the request body sb = StringIO() sb.write(self.xml_header) head = ( "<orderParameters " "xmlns=" "'https://earthexplorer.usgs.gov/schema/orderParameters' " "xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' " "xsi:schemaLocation=" "'https://earthexplorer.usgs.gov/schema/orderParameters " "https://earthexplorer.usgs.gov/EE/orderParameters.xsd'>" ) sb.write(head) sb.write("<contactId>{0}</contactId>".format(contact_id)) sb.write("<requestor>ESPA</requestor>") # 1111111 is a dummy value. sb.write("<externalReferenceNumber>{0}</externalReferenceNumber>".format(1111111)) sb.write("<priority>{0}</priority>".format(priority)) product_info = self.get_download_urls(product_list, contact_id) for p in product_info.keys(): try: sensor.instance(p) except sensor.ProductNotImplemented, pne: raise pne else: sb.write("<scene>") sb.write("<sceneId>{0}</sceneId>".format(p)) sb.write("<prodCode>{0}</prodCode>".format(product_info[p]["lta_code"])) sb.write("<sensor>{0}</sensor>".format(product_info[p]["sensor"])) sb.write("</scene>")
def verify_products(self, products): response = {} if isinstance(products, str): products = [products] for product in products: if isinstance(product, basestring): product = sensor.instance(product) response[product.product_id] = self.input_exists(product) return response
def gzip_errors_online_cache(self, error_message): ''' products on cache are corrupted ''' keys = ['gzip: stdin: invalid compressed data--format violated'] status = 'retry' reason = 'Input gzip corrupt' extras = self.__add_retry('gzip_errors') resolution = self.__find_error(error_message, keys, status, reason, extras) is_landsat = False if self.product_name is not None: is_landsat = isinstance(sensor.instance(self.product_name), sensor.Landsat) if resolution is not None and is_landsat: logger.debug("err api/errors.py gzip_errors_online_cache\n"\ "product_name: {0}\nerror_message: {1}".format(self.product_name, error_message)) emails.Emails().send_gzip_error_email(self.product_name) return resolution
def massage_formatting(order): """ To avoid complications down the line, we need to ensure proper case formatting on the order, while still being somewhat case agnostic We also need to add 'stats' product to all the sensors if 'plot_statistics' was set to True :param order: incoming order after validation :return: order with the inputs reformatted """ prod_keys = sn.SensorCONST.instances.keys() stats = False if 'plot_statistics' in order and order['plot_statistics']: stats = True for key in order: if key in prod_keys: item1 = order[key]['inputs'][0] prod = sn.instance(item1) if isinstance(prod, sn.Landsat): order[key]['inputs'] = [s.upper() for s in order[key]['inputs']] elif isinstance(prod, sn.Modis): order[key]['inputs'] = ['.'.join([p[0].upper(), p[1].upper(), p[2].lower(), p[3], p[4]]) for p in [s.split('.') for s in order[key]['inputs']]] if stats: if 'stats' not in order[key]['products']: order[key]['products'].append('stats') return order
def input_exists(self, product): '''Determines if a LPDAAC product is available for download Keyword args: product - The name of the product Returns: True/False ''' if isinstance(product, str) or isinstance(product, unicode): product = sensor.instance(product) result = False try: url = self.get_download_url(product) if 'download_url' in url[product.product_id]: url = url[product.product_id]['download_url'] response = None try: response = requests.head(url) if response.ok is True: result = True except Exception, e: logger.exception('Exception checking modis input {0}\n ' 'Exception:{1}' .format(url, e)) return result finally: if response is not None: response.close() response = None
def get_download_url(self, product): url = {} #be nice and accept a string if isinstance(product, str) or isinstance(product, unicode): product = sensor.instance(product) #also be nice and accept a sensor.Modis object if isinstance(product, sensor.Modis): path = self._build_modis_input_file_path(product) product_url = ''.join([self.datapool, path]) if not product_url.lower().startswith("http"): product_url = ''.join(['http://', product_url]) if not product.product_id in url: url[product.product_id] = {} url[product.product_id]['download_url'] = product_url return url
def massage_formatting(order): """ To avoid complications down the line, we need to ensure proper case formatting on the order, while still being somewhat case agnostic We also need to add 'stats' product to all the sensors if 'plot_statistics' was set to True :param order: incoming order after validation :return: order with the inputs reformatted """ prod_keys = sn.SensorCONST.instances.keys() stats = False if "plot_statistics" in order and order["plot_statistics"]: stats = True for key in order: if key in prod_keys: item1 = order[key]["inputs"][0] prod = sn.instance(item1) if isinstance(prod, sn.Landsat): order[key]["inputs"] = [s.upper() for s in order[key]["inputs"]] elif isinstance(prod, sn.Modis): order[key]["inputs"] = [ ".".join([p[0].upper(), p[1].upper(), p[2].lower(), p[3], p[4]]) for p in [s.split(".") for s in order[key]["inputs"]] ] if stats: if "stats" not in order[key]["products"]: order[key]["products"].append("stats") return order
def get_default_ee_options(item_ls): """ Factory method to return default ESPA order options for orders originating through Earth Explorer :param item_ls: list of scenes received from EE for an order structure: list({sceneid:, unit_num:}) :return: dictionary representation of the EE order """ ee_order = {'format': 'gtiff'} for item in item_ls: try: scene_info = sensor.instance(item['sceneid']) except sensor.ProductNotImplemented: log_msg = ('Received unsupported product via EE: {}'.format( item['sceneid'])) logger.debug(log_msg) continue short = scene_info.shortname if short in ee_order: ee_order[short]['inputs'].append(item['sceneid']) else: if isinstance(scene_info, sensor.Landsat): ee_order[short] = { 'inputs': [item['sceneid']], 'products': ['sr'] } elif isinstance(scene_info, sensor.Modis): ee_order[short] = { 'inputs': [item['sceneid']], 'products': ['l1'] } return ee_order
def _build_modis_input_file_path(self, product): if isinstance(product, str) or isinstance(product, unicode): product = sensor.instance(product) if isinstance(product, sensor.Aqua): base_path = config.get('path.aqua_base_source') elif isinstance(product, sensor.Terra): base_path = config.get('path.terra_base_source') else: msg = "Cant build input file path for unknown LPDAAC product:%s" raise Exception(msg % product.product_id) date = utils.date_from_doy(product.year, product.doy) path_date = "%s.%s.%s" % (date.year, str(date.month).zfill(2), str(date.day).zfill(2)) input_extension = config.get('file.extension.modis.input.filename') parts = product.product_id.split('.') prod_id = '.'.join([parts[0].upper(), parts[1].upper(), parts[2].lower(), parts[3], parts[4]]) input_file_name = "%s%s" % (prod_id, input_extension) path = os.path.join(base_path, '.'.join([product.short_name.upper(), product.version.upper()]), path_date.upper(), input_file_name) return path
def verify_scenes(self, scene_list): """ Checks to make sure the scene list is valid, where valid means the scene ids supplied exist in the Landsat inventory and are orderable Keyword args: scene_list A list of scenes to be verified Returns: A dictionary with keys matching the scene list and values are 'true' if valid, and 'false' if not. Return value example: dictionary = dict() dictionary['LT51490212007234IKR00'] = True dictionary['asdf'] = False ... ... ... """ # build the service + operation url request_url = "{0}/verifyScenes".format(self.url) # build the request body sb = StringIO() sb.write(self.xml_header) head = ( "<sceneList " "xmlns='https://earthexplorer.usgs.gov/schema/sceneList' " "xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' " "xsi:schemaLocation=" "'https://earthexplorer.usgs.gov/schema/sceneList " "https://earthexplorer.usgs.gov/EE/sceneList.xsd'>" ) sb.write(head) for s in scene_list: product = sensor.instance(s) sb.write("<sceneId sensor='{0}'>{1}</sceneId>".format(product.lta_name, s.upper())) sb.write("</sceneList>") request_body = sb.getvalue() # set the required headers headers = dict() headers["Content-Type"] = "application/xml" headers["Content-Length"] = len(request_body) # send the request and check return status # print "*** request_url: ", request_url # print "*** request_body: ", request_body # print "*** headers: ", headers __response = requests.post(request_url, data=request_body, headers=headers) response = None if __response.ok: response = __response.content else: msg = StringIO() msg.write("Error in lta.OrderWrapperServiceClient.verify_scenes\n") msg.write("Non 200 response code from service\n") msg.write("Response code was:{0}".format(__response.status_code)) msg.write("Reason:{0}".format(__response.reason)) # Return the code and reason as an exception raise Exception(msg.getvalue()) __response.close() # parse, transform and return response retval = dict() response = response.replace("&", "&") response = response.replace("\n", "") root = xml.fromstring(response) scenes = root.getchildren() for s in list(scenes): if s.attrib["valid"] == "true": status = True else: status = False retval[s.text] = status return retval
def validate_restricted(self, x, fieldname, schema, path, restricted): """Validate that the requested products are available by date or role""" if not restricted: return # Like extents, we need to do some initial validation of the input up front, # and let those individual validators output the errors if 'inputs' not in x: return if not self.validate_type_array(x['inputs']): return req_prods = x.get(fieldname) if not req_prods: return try: req_scene = x.get('inputs')[0] except IndexError: return inst = sn.instance(req_scene) avail_prods = (ordering.OrderingProvider().available_products( x['inputs'], self.username)) not_implemented = avail_prods.pop('not_implemented', None) date_restricted = avail_prods.pop('date_restricted', None) ordering_restricted = avail_prods.pop('ordering_restricted', None) # Check for to make sure there is only one sensor type in there if len(avail_prods) > 1: return if not_implemented: self._errors.append( "Requested IDs are not recognized. Remove: {}".format( not_implemented)) if date_restricted: restr_prods = date_restricted.keys() for key in restr_prods: if key not in req_prods: date_restricted.pop(key, None) if date_restricted: for product_type in date_restricted: msg = ("Requested {} products are restricted by date. " "Remove {} scenes: {}".format( product_type, path.split('.products')[0], [ x.upper() for x in date_restricted[product_type] ])) self._errors.append(msg) if ordering_restricted: restr_sensors = ordering_restricted.keys() for sensor in restr_sensors: msg = ("Requested sensor is restricted from ordering. " "Remove: {}".format(sensor)) self._errors.append(msg) prods = [] for key in avail_prods: prods = [_ for _ in avail_prods[key]['products']] if not prods: return dif = list(set(req_prods) - set(prods)) if date_restricted: for d in dif: if d in date_restricted: dif.remove(d) if dif: for d in dif: if type(x) == dict: scene_ids = [s.upper() for s in x['inputs']] msg = ("Requested {} products are not available. " "Remove {} scenes: {}".format( d, path.split('.products')[0], scene_ids)) self._errors.append(msg) # Enforce non-customized l1 ordering restriction for Landsat restr_source = self.restricted['source'] landsat_sensors = restr_source['sensors'] sensors = [ s for s in self.data_source.keys() if s in sn.SensorCONST.instances.keys() and s in landsat_sensors ] parse_customize = lambda c: ( (c in self.data_source) and (self.data_source.get(c) != restr_source.get(c))) if sensors and 'LANDSAT' in inst.lta_json_name: if not set(req_prods) - set(restr_source['products']): if not any(map(parse_customize, restr_source['custom'])): msg = restr_source['message'].strip() if msg not in self._errors: self._errors.append(msg) # Enforce non-customized l1 ordering restriction for MODIS and VIIRS restr_modis_viirs = self.restricted['source_daac'] modis_viirs_sensors = restr_modis_viirs['sensors'] sensors_mv = [ s for s in self.data_source.keys() if s in sn.SensorCONST.instances.keys() and s in modis_viirs_sensors ] parse_modis_customize = lambda c: ( (c in self.data_source) and (self.data_source.get(c) != restr_modis_viirs.get(c))) if sensors_mv and ('MODIS' in inst.lta_json_name or 'VIIRS' in inst.lta_json_name): if not set(req_prods) - set(restr_modis_viirs['products']): if not any( map(parse_modis_customize, restr_modis_viirs['custom'])): msg = restr_modis_viirs['message'].strip() if msg not in self._errors: self._errors.append(msg) # Enforce restricted product ordering for MODIS NDVI if 'MODIS' in inst.lta_json_name: restr_modis_ndvi = self.restricted['source_modis_ndvi'] modis_sensors = restr_modis_ndvi['modis_sensors'] req_ndvi_sensors = [ s for s in self.data_source.keys() if s in sn.SensorCONST.instances.keys() and s in modis_sensors ] invalid_req = set(req_ndvi_sensors) - set( restr_modis_ndvi['ndvi_sensors']) if invalid_req: if 'modis_ndvi' in req_prods: msg = restr_modis_ndvi['message'].strip() if msg not in self._errors: self._errors.append(msg) # Enforce sensor-restricted product ordering for LaORCA if 'orca' in req_prods: restr_orca_info = self.restricted['source_orca_sensors'] orca_sensors = restr_orca_info['sensors'] req_orca_sensors = [ s for s in self.data_source.keys() if s in sn.SensorCONST.instances.keys() and s in orca_sensors ] invalid_orca_req = set(req_orca_sensors) - set( restr_orca_info['orca_sensors']) if invalid_orca_req: msg = restr_orca_info['message'].strip() if msg not in self._errors: self._errors.append(msg) # Make sure that all required st options are included st = 'st' stalg_single_channel = 'stalg_single_channel' stalg_split_window = 'stalg_split_window' reanalysis_data = [ 'reanalsrc_narr', 'reanalsrc_merra2', 'reanalsrc_fp', 'reanalsrc_fpit' ] if st in req_prods: if stalg_single_channel not in req_prods and stalg_split_window not in req_prods: msg = "Missing surface temperature algorithm - " \ "please choose from ['{0}' (olitirs only), '{1}']".format(stalg_split_window, stalg_single_channel) if msg not in self._errors: self._errors.append(msg) if stalg_single_channel in req_prods and not any( [r for r in reanalysis_data if r in req_prods]): msg = "Missing reanalysis data source for single channel algorithm - " \ "please choose from {}".format(reanalysis_data) if msg not in self._errors: self._errors.append(msg) all_st_options = list() all_st_options.append(stalg_split_window) all_st_options.append(stalg_single_channel) all_st_options.extend(reanalysis_data) if any([x for x in all_st_options if x in req_prods ]) and st not in req_prods: msg = "Must include 'st' in products if specifying surface temperature options" if msg not in self._errors: self._errors.append(msg)
def create(cls, params): """ Place a new order into the system :param params: dict of required parameters to be used params = {'product_opts': {dictionary object of the order received} 'orderid': id generated from generate_order_id 'user_id': EE user id 'order_type': typically 'level2_ondemand' 'status': 'ordered' 'note': user notes 'ee_order_id': earth explorer order id, or '' 'order_source': 'espa' or 'ee' 'order_date': date time string 'priority': legacy item, should be 'normal' 'email': user's contact email 'product_options': legacy column} :return: order object """ opts = params['product_opts'] params['product_opts'] = json.dumps(params['product_opts']) sql = ('INSERT INTO ordering_order ' '(orderid, user_id, order_type, status, note, ' 'product_opts, ee_order_id, order_source, order_date, ' 'priority, email, product_options) ' 'VALUES (%(orderid)s, %(user_id)s, %(order_type)s, ' '%(status)s, %(note)s, %(product_opts)s, ' '%(ee_order_id)s, %(order_source)s, %(order_date)s, ' '%(priority)s, %(email)s, %(product_options)s)') logger.info('Order creation parameters: {}'.format(params)) log_sql = '' try: with db_instance() as db: log_sql = db.cursor.mogrify(sql, params) logger.info('New order complete SQL: {}'.format(log_sql)) db.execute(sql, params) db.commit() except DBConnectException as e: logger.debug('Error creating new order: {}\n' 'sql: {}'.format(e.message, log_sql)) raise OrderException(e) order = Order.find(params['orderid']) # Let the load_ee_order method handle the scene injection # as there is special logic for interacting with LTA if params['ee_order_id']: return order sensor_keys = sensor.SensorCONST.instances.keys() bulk_ls = [] for key in opts: if key in sensor_keys: sensor_type = '' item1 = opts[key]['inputs'][0] if isinstance(sensor.instance(item1), sensor.Landsat): sensor_type = 'landsat' elif isinstance(sensor.instance(item1), sensor.Modis): sensor_type = 'modis' for s in opts[key]['inputs']: scene_dict = { 'name': s, 'sensor_type': sensor_type, 'order_id': order.id, 'status': 'submitted', 'ee_unit_id': None } bulk_ls.append(scene_dict) if 'plot_statistics' in opts and opts['plot_statistics']: scene_dict = { 'name': 'plot', 'sensor_type': 'plot', 'order_id': order.id, 'status': 'submitted', 'ee_unit_id': None } bulk_ls.append(scene_dict) try: Scene.create(bulk_ls) except SceneException as e: logger.debug('Order creation failed on scene injection, ' 'order: {}\nexception: {}'.format( order.orderid, e.message)) with db_instance() as db: db.execute('delete ordering_order where id = %s', order.id) db.commit() raise OrderException(e) return order
def build_base_order(): """ Builds the following dictionary (with the products filled out from sensor.py): base = {'MOD09A1': {'inputs': 'MOD09A1.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MOD09GA': {'inputs': 'MOD09GA.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MOD09GQ': {'inputs': 'MOD09GQ.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MOD09Q1': {'inputs': 'MOD09Q1.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MYD09A1': {'inputs': 'MYD09A1.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MYD09GA': {'inputs': 'MYD09GA.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MYD09GQ': {'inputs': 'MYD09GQ.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MYD09Q1': {'inputs': 'MYD09Q1.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MOD13A1': {'inputs': 'MOD13A1.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MOD13A2': {'inputs': 'MOD13A2.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MOD13A3': {'inputs': 'MOD13A3.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MOD13Q1': {'inputs': 'MOD13Q1.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MYD13A1': {'inputs': 'MYD13A1.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MYD13A2': {'inputs': 'MYD13A2.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MYD13A3': {'inputs': 'MYD13A3.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MYD13Q1': {'inputs': 'MYD13Q1.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'tm4': {'inputs': 'LT42181092013069PFS00', 'products': ['l1']}, 'tm5': {'inputs': 'LT52181092013069PFS00', 'products': ['l1']}, 'etm7': {'inputs': 'LE72181092013069PFS00', 'products': ['l1']}, 'oli8': {'inputs': 'LO82181092013069PFS00', 'products': ['l1']}, 'olitirs8': {'inputs': 'LC82181092013069PFS00', 'products': ['l1']}, 'projection': {'lonlat': None}, 'image_extents': {'north': 0.0002695, 'south': 0, 'east': 0.0002695, 'west': 0, 'units': 'dd'}, 'format': 'gtiff', 'resampling_method': 'cc', 'resize': {'pixel_size': 0.0002695, 'pixel_size_units': 'dd'}, 'plot_statistics': True}""" base = {'projection': {'lonlat': None}, 'image_extents': {'north': 0.002695, 'south': 0, 'east': 0.002695, 'west': 0, 'units': 'dd'}, 'format': 'gtiff', 'resampling_method': 'cc', 'resize': {'pixel_size': 0.0002695, 'pixel_size_units': 'dd'}, 'plot_statistics': True} sensor_acqids = {'.A2000072.h02v09.005.2008237032813': (['MOD09A1', 'MOD09Q1', 'MYD13A1', 'MYD13Q1'], ['mod09a1', 'mod09q1', 'myd13a1', 'myd13q1']), '.A2016305.h11v04.006.2016314200836': (['MOD09GA', 'MOD09GQ', 'MYD13A2', 'MYD13A3'], ['mod09ga', 'mod09gq', 'myd13a2', 'myd13a3']), '.A2000361.h24v03.006.2015111114747': (['MOD11A1', 'MYD11A1'], ['mod11a1', 'myd11a1']), # TODO: REMOVE _collection from IDs 'L1TP_044030_19851028_20161004_01_T1': (['LT04_', 'LT05_', 'LE07_', 'LO08_', 'LC08_'], ['tm4_collection', 'tm5_collection', 'etm7_collection', 'oli8_collection', 'olitirs8_collection'])} for acq in sensor_acqids: for prefix, label in zip(sensor_acqids[acq][0], sensor_acqids[acq][1]): base[label] = {'inputs': ['{}{}'.format(prefix, acq)], 'products': sn.instance('{}{}'.format(prefix, acq)).products} return base
def available_products(self, product_id, username): """ Check to see what products are available to user based on an input list of scenes :param product_id: list of desired inputs :param username: username :return: dictionary """ user = User.by_username(username) pub_prods = copy.deepcopy(OrderingProvider.sensor_products(product_id)) with open('api/domain/restricted.yaml') as f: restricted = yaml.load(f.read()) role = False if user.is_staff() else True restrict_all = restricted.get('all', {}) all_role = restrict_all.get('role', []) all_by_date = restrict_all.get('by_date', {}) upd = {'date_restricted': {}} for sensor_type, prods in pub_prods.items(): if sensor_type == 'not_implemented': continue sensor_restr = restricted.get(sensor_type, {}) role_restr = sensor_restr.get('role', []) + all_role by_date_restr = sensor_restr.get('by_date', {}) # All overrides any sensor related dates by_date_restr.update(all_by_date) outs = pub_prods[sensor_type]['products'] ins = pub_prods[sensor_type]['inputs'] remove_me = [] if role: for prod in role_restr: try: outs.remove(prod) except ValueError: continue for prod in outs: if prod in by_date_restr: r = sensor_restr['by_date'][prod] for sc_id in ins: obj = sensor.instance(sc_id) julian = '{}{}'.format(obj.year, obj.doy) if not julian_date_check(julian, r): remove_me.append(prod) if prod in upd['date_restricted']: upd['date_restricted'][prod].append(sc_id) else: upd['date_restricted'][prod] = [sc_id] for rem in remove_me: try: outs.remove(rem) except ValueError: continue if upd['date_restricted']: pub_prods.update(upd) return pub_prods
def verify_scenes(self, scene_list): ''' Checks to make sure the scene list is valid, where valid means the scene ids supplied exist in the Landsat inventory and are orderable Keyword args: scene_list A list of scenes to be verified Returns: A dictionary with keys matching the scene list and values are 'true' if valid, and 'false' if not. Return value example: dictionary = dict() dictionary['LT51490212007234IKR00'] = True dictionary['asdf'] = False ... ... ... ''' #build the service + operation url request_url = '{0}/verifyScenes'.format(self.url) #build the request body sb = StringIO() sb.write(self.xml_header) head = ("<sceneList " "xmlns='https://earthexplorer.usgs.gov/schema/sceneList' " "xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' " "xsi:schemaLocation=" "'https://earthexplorer.usgs.gov/schema/sceneList " "https://earthexplorer.usgs.gov/EE/sceneList.xsd'>") sb.write(head) for s in scene_list: product = sensor.instance(s) sb.write("<sceneId sensor='{0}'>{1}</sceneId>" .format(product.lta_name, s.upper())) sb.write("</sceneList>") request_body = sb.getvalue() #set the required headers headers = dict() headers['Content-Type'] = 'application/xml' headers['Content-Length'] = len(request_body) #send the request and check return status #print "*** request_url: ", request_url #print "*** request_body: ", request_body #print "*** headers: ", headers __response = requests.post(request_url, data=request_body, headers=headers) response = None if __response.ok: response = __response.content else: msg = StringIO() msg.write("Error in lta.OrderWrapperServiceClient.verify_scenes\n") msg.write("Non 200 response code from service\n") msg.write("Response code was:{0}".format( __response.status_code)) msg.write("Reason:{0}".format(__response.reason)) # Return the code and reason as an exception raise Exception(msg.getvalue()) __response.close() #parse, transform and return response retval = dict() response = response.replace('&', '&') response = response.replace('\n', '') root = xml.fromstring(response) scenes = root.getchildren() for s in list(scenes): if s.attrib['valid'] == 'true': status = True else: status = False retval[s.text] = status return retval
class BaseValidationSchema(object): formats = ['gtiff', 'hdf-eos2', 'envi', 'netcdf'] resampling_methods = ['nn', 'bil', 'cc'] projections = {'aea': {'type': 'object', 'properties': {'standard_parallel_1': {'type': 'number', 'required': True, 'minimum': -90, 'maximum': 90}, 'standard_parallel_2': {'type': 'number', 'required': True, 'minimum': -90, 'maximum': 90}, 'central_meridian': {'type': 'number', 'required': True, 'minimum': -180, 'maximum': 180}, 'latitude_of_origin': {'type': 'number', 'required': True, 'minimum': -90, 'maximum': 90}, 'false_easting': {'type': 'number', 'required': True}, 'false_northing': {'type': 'number', 'required': True}, 'datum': {'type': 'string', 'required': True, 'enum': ['wgs84', 'nad27', 'nad83']}}}, 'utm': {'type': 'object', 'properties': {'zone': {'type': 'integer', 'required': True, 'minimum': 1, 'maximum': 60}, 'zone_ns': {'type': 'string', 'required': True, 'enum': ['north', 'south']}}}, 'lonlat': {'type': 'null'}, 'sinu': {'type': 'object', 'properties': {'central_meridian': {'type': 'number', 'required': True, 'minimum': -180, 'maximum': 180}, 'false_easting': {'type': 'number', 'required': True}, 'false_northing': {'type': 'number', 'required': True}}}, 'ps': {'type': 'object', 'properties': {'longitudinal_pole': {'type': 'number', 'required': True, 'minimum': -180, 'maximum': 180}, 'latitude_true_scale': {'type': 'number', 'required': True, 'abs_rng': (60, 90)}, 'false_easting': {'type': 'number', 'required': True}, 'false_northing': {'type': 'number', 'required': True}}}} extents = {'north': {'type': 'number', 'required': True}, 'south': {'type': 'number', 'required': True}, 'east': {'type': 'number', 'required': True}, 'west': {'type': 'number', 'required': True}, 'units': {'type': 'string', 'required': True, 'enum': ['dd', 'meters']}} resize = {'pixel_size': {'type': 'number', 'required': True, 'ps_dd_rng': (0.0002695, 0.0449155), 'ps_meter_rng': (30, 5000)}, 'pixel_size_units': {'type': 'string', 'required': True, 'enum': ['dd', 'meters']}} request_schema = {'type': 'object', 'set_ItemCount': ('inputs', 5000), 'extents': 200000000, 'properties': {'projection': {'properties': projections, 'type': 'object', # 'enum_keys': self.projections.keys(), 'single_obj': True}, 'image_extents': {'type': 'object', 'properties': extents, # 'enum_keys': self.extents.keys(), 'dependencies': ['projection']}, 'format': {'type': 'string', 'required': True, 'enum': formats}, 'resize': {'type': 'object', 'properties': resize}, 'resampling_method': {'type': 'string', 'enum': resampling_methods}, 'plot_statistics': {'type': 'boolean'}, 'note': {'type': 'string', 'required': False, 'blank': True}}} _sensor_reg = sn.SensorCONST.instances sensor_schema = {} for key in _sensor_reg: sensor_schema[key] = {'type': 'object', 'properties': {'inputs': {'type': 'array', 'required': True, 'ItemCount': 'inputs', 'uniqueItems': True, 'minItems': 1, 'items': {'type': 'string', 'pattern': _sensor_reg[key][0]}}, 'products': {'type': 'array', 'uniqueItems': True, 'required': True, 'restricted': True, 'stats': True, 'minItems': 1, 'items': {'type': 'string', 'enum': sn.instance( _sensor_reg[key][2]).products}}}} request_schema['properties'].update(sensor_schema) request_schema['oneormoreobjects'] = sensor_schema.keys() valid_params = {'formats': {'formats': formats}, 'resampling_methods': {'resampling_methods': resampling_methods}, 'projections': projections}
class BaseValidationSchema(object): formats = {'gtiff': 'GeoTiff', 'envi': 'ENVI', 'hdf-eos2': 'HDF-EOS2', 'netcdf': 'NetCDF'} resampling_methods = {'nn': 'Nearest Neighbor', 'bil': 'Bilinear Interpolation', 'cc': 'Cubic Convolution'} projections = {'aea': {'type': 'object', 'title': 'Albers Equal Area', 'pixel_units': ('meters', 'dd'), 'display_rank': 0, 'properties': {'standard_parallel_1': {'type': 'number', 'title': '1st Standard Parallel', 'display_rank': 2, 'required': True, 'minimum': -90, 'maximum': 90}, 'standard_parallel_2': {'type': 'number', 'title': '2nd Standard Parallel', 'display_rank': 3, 'required': True, 'minimum': -90, 'maximum': 90}, 'central_meridian': {'type': 'number', 'title': 'Central Meridian', 'display_rank': 1, 'required': True, 'minimum': -180, 'maximum': 180}, 'latitude_of_origin': {'type': 'number', 'title': 'Latitude of Origin', 'display_rank': 0, 'required': True, 'minimum': -90, 'maximum': 90}, 'false_easting': {'type': 'number', 'title': 'False Easting (meters)', 'display_rank': 4, 'required': True}, 'false_northing': {'type': 'number', 'title': 'False Northing (meters)', 'display_rank': 5, 'required': True}, 'datum': {'type': 'string', 'title': 'Datum', 'required': True, 'display_rank': 6, 'enum': {'wgs84': 'World Geodetic System 1984', 'nad27': 'North American Datum 1927', 'nad83': 'North American Datum 1983'}}}}, 'utm': {'type': 'object', 'pixel_units': ('meters', 'dd'), 'display_rank': 1, 'title': 'Universal Transverse Mercator', 'properties': {'zone': {'type': 'integer', 'title': 'UTM Grid Zone Number', 'display_rank': 0, 'required': True, 'minimum': 1, 'maximum': 60}, 'zone_ns': {'type': 'string', 'title': 'UTM Hemisphere', 'display_rank': 1, 'required': True, 'enum': {'north': 'North', 'south': 'South'}}}}, 'lonlat': {'type': 'null', 'pixel_units': ('dd',), 'title': 'Geographic', 'display_rank': 2}, 'sinu': {'type': 'object', 'title': 'Sinusoidal', 'pixel_units': ('meters', 'dd'), 'display_rank': 3, 'properties': {'central_meridian': {'type': 'number', 'title': 'Central Meridian', 'display_rank': 0, 'required': True, 'minimum': -180, 'maximum': 180}, 'false_easting': {'type': 'number', 'title': 'False Easting (meters)', 'display_rank': 1, 'required': True}, 'false_northing': {'type': 'number', 'title': 'False Northing (meters)', 'display_rank': 2, 'required': True}}}, 'ps': {'type': 'object', 'title': 'Polar Stereographic', 'pixel_units': ('meters', 'dd'), 'display_rank': 4, 'properties': {'longitudinal_pole': {'type': 'number', 'title': 'Longitudinal Pole', 'display_rank': 0, 'required': True, 'minimum': -180, 'maximum': 180}, 'latitude_true_scale': {'type': 'number', 'title': 'Latitude True Scale', 'display_rank': 1, 'required': True, 'abs_rng': (60, 90)}, 'false_easting': {'type': 'number', 'title': 'False Easting (meters)', 'display_rank': 2, 'required': True}, 'false_northing': {'type': 'number', 'title': 'False Northing (meters)', 'display_rank': 3, 'required': True}}}} extents = {'north': {'type': 'number', 'title': 'Upper left Y coordinate', 'display_rank': 2, 'required': True}, 'south': {'type': 'number', 'title': 'Lower right Y coordinate', 'display_rank': 4, 'required': True}, 'east': {'type': 'number', 'title': 'Lower right X coordinate', 'display_rank': 3, 'required': True}, 'west': {'type': 'number', 'title': 'Upper left X coordinate', 'display_rank': 1, 'required': True}, 'units': {'type': 'string', 'title': 'Coordinate system units', 'display_rank': 0, 'required': True, 'enum': {'dd': 'Decimal Degrees', 'meters': 'Meters'}}} resize = {'pixel_size': {'type': 'number', 'title': 'Pixel Size', 'display_rank': 0, 'required': True, 'ps_dd_rng': (0.0002695, 0.0449155), 'ps_meters_rng': (30, 5000)}, 'pixel_size_units': {'type': 'string', 'title': 'Pixel Size Units', 'display_rank': 1, 'required': True, 'enum': {'dd': 'Decimal Degrees', 'meters': 'Meters'}}} request_schema = {'type': 'object', 'set_ItemCount': ('inputs', 5000), 'extents': 200000000, 'properties': {'projection': {'properties': projections, 'type': 'object', 'title': 'Reproject Products', 'display_rank': 1, 'single_obj': True}, 'image_extents': {'type': 'object', 'title': 'Modify Image Extents', 'display_rank': 2, 'properties': extents, 'dependencies': ['projection']}, 'format': {'type': 'string', 'title': 'Output Format', 'display_rank': 0, 'required': True, 'enum': formats}, 'resize': {'type': 'object', 'title': 'Pixel Resizing', 'display_rank': 3, 'properties': resize}, 'resampling_method': {'type': 'string', 'title': 'Resample Method', 'display_rank': 4, 'enum': resampling_methods}, 'plot_statistics': {'type': 'boolean', 'title': 'Plot Output Product Statistics'}, 'note': {'type': 'string', 'title': 'Order Description (optional)', 'required': False, 'blank': True}}} _sensor_reg = sn.SensorCONST.instances sensor_schema = {} for key in _sensor_reg: sensor_schema[key] = {'type': 'object', 'properties': {'inputs': {'type': 'array', 'required': True, 'ItemCount': 'inputs', 'uniqueItems': True, 'minItems': 1, 'items': {'type': 'string', 'pattern': _sensor_reg[key][0]}}, 'products': {'type': 'array', 'uniqueItems': True, 'required': True, 'restricted': True, 'stats': True, 'minItems': 1, 'items': {'type': 'string', 'enum': sn.instance( _sensor_reg[key][2]).products}}}} request_schema['properties'].update(sensor_schema) request_schema['oneormoreobjects'] = sensor_schema.keys() valid_params = {'formats': {'formats': formats}, 'resampling_methods': {'resampling_methods': resampling_methods}, 'projections': projections}
def build_base_order(): """ Builds the following dictionary (with the products filled out from sensor.py): base = {'MOD09A1': {'inputs': 'MOD09A1.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MOD09GA': {'inputs': 'MOD09GA.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MOD09GQ': {'inputs': 'MOD09GQ.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MOD09Q1': {'inputs': 'MOD09Q1.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MYD09A1': {'inputs': 'MYD09A1.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MYD09GA': {'inputs': 'MYD09GA.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MYD09GQ': {'inputs': 'MYD09GQ.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MYD09Q1': {'inputs': 'MYD09Q1.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MOD13A1': {'inputs': 'MOD13A1.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MOD13A2': {'inputs': 'MOD13A2.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MOD13A3': {'inputs': 'MOD13A3.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MOD13Q1': {'inputs': 'MOD13Q1.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MYD13A1': {'inputs': 'MYD13A1.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MYD13A2': {'inputs': 'MYD13A2.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MYD13A3': {'inputs': 'MYD13A3.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MYD13Q1': {'inputs': 'MYD13Q1.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'tm4': {'inputs': 'LT42181092013069PFS00', 'products': ['l1']}, 'tm5': {'inputs': 'LT52181092013069PFS00', 'products': ['l1']}, 'etm7': {'inputs': 'LE72181092013069PFS00', 'products': ['l1']}, 'oli8': {'inputs': 'LO82181092013069PFS00', 'products': ['l1']}, 'olitirs8': {'inputs': 'LC82181092013069PFS00', 'products': ['l1']}, 'projection': {'lonlat': None}, 'image_extents': {'north': 0.0002695, 'south': 0, 'east': 0.0002695, 'west': 0, 'units': 'dd'}, 'format': 'gtiff', 'resampling_method': 'cc', 'resize': {'pixel_size': 0.0002695, 'pixel_size_units': 'dd'}, 'plot_statistics': True}""" base = { 'projection': { 'lonlat': None }, 'image_extents': { 'north': 0.002695, 'south': 0, 'east': 0.002695, 'west': 0, 'units': 'dd' }, 'format': 'gtiff', 'resampling_method': 'cc', 'resize': { 'pixel_size': 0.0002695, 'pixel_size_units': 'dd' }, 'plot_statistics': True } sensor_acqids = { '.A2000072.h02v09.005.2008237032813': ([ 'MOD09A1', 'MOD09GA', 'MOD09GQ', 'MOD09Q1', 'MYD09A1', 'MYD09GA', 'MYD09GQ', 'MYD09Q1', 'MOD13A1', 'MOD13A2', 'MOD13A3', 'MOD13Q1', 'MYD13A1', 'MYD13A2', 'MYD13A3', 'MYD13Q1' ], [ 'mod09a1', 'mod09ga', 'mod09gq', 'mod09q1', 'myd09a1', 'myd09ga', 'myd09gq', 'myd09q1', 'mod13a1', 'mod13a2', 'mod13a3', 'mod13q1', 'myd13a1', 'myd13a2', 'myd13a3', 'myd13q1' ]), '2181092013069PFS00': (['LT4', 'LT5', 'LE7', 'LO8', 'LC8'], ['tm4', 'tm5', 'etm7', 'oli8', 'olitirs8']) } for acq in sensor_acqids: for prefix, label in zip(sensor_acqids[acq][0], sensor_acqids[acq][1]): base[label] = { 'inputs': ['{}{}'.format(prefix, acq)], 'products': sn.instance('{}{}'.format(prefix, acq)).products } return base
def available_products(self, product_id, username): """ Check to see what products are available to user based on an input list of scenes :param product_id: list of desired inputs :param username: username :return: dictionary """ user = User.by_username(username) pub_prods = copy.deepcopy(OrderingProvider.sensor_products(product_id)) with open(os.path.join(__location__, 'domain/restricted.yaml')) as f: restricted = yaml.safe_load(f.read()) role = False if user.is_staff() else True restrict_all = restricted.get('all', {}) all_role = restrict_all.get('role', []) all_by_date = restrict_all.get('by_date', {}) all_ordering_rsctd = restrict_all.get('ordering', []) upd = {'date_restricted': {}, 'ordering_restricted': {}} for sensor_type, prods in pub_prods.items(): if sensor_type == 'not_implemented': continue stype = sensor_type.replace( '_collection', '') if '_collection' in sensor_type else sensor_type sensor_restr = restricted.get(stype, {}) role_restr = sensor_restr.get('role', []) + all_role by_date_restr = sensor_restr.get('by_date', {}) # All overrides any sensor related dates by_date_restr.update(all_by_date) outs = pub_prods[sensor_type]['products'] ins = pub_prods[sensor_type]['inputs'] if sensor_type in all_ordering_rsctd: for sc_id in ins: if sensor_type in upd['ordering_restricted']: upd['ordering_restricted'][sensor_type].append(sc_id) else: upd['ordering_restricted'][sensor_type] = [sc_id] pub_prods.pop(sensor_type) continue remove_me = [] if role: for prod in role_restr: try: outs.remove(prod) except ValueError: continue for prod in outs: if prod in by_date_restr: r = sensor_restr['by_date'][prod] for sc_id in ins: obj = sensor.instance(sc_id) julian = '{}{}'.format(obj.year, obj.doy) if not julian_date_check(julian, r): remove_me.append(prod) if prod in upd['date_restricted']: upd['date_restricted'][prod].append(sc_id) else: upd['date_restricted'][prod] = [sc_id] for rem in remove_me: try: outs.remove(rem) except ValueError: continue if upd['date_restricted']: pub_prods.update(date_restricted=upd['date_restricted']) if upd['ordering_restricted']: pub_prods.update(ordering_restricted=upd['ordering_restricted']) return pub_prods
def build_base_order(): """ Builds the following dictionary (with the products filled out from sensor.py): base = {'MOD09A1': {'inputs': 'MOD09A1.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MOD09GA': {'inputs': 'MOD09GA.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MOD09GQ': {'inputs': 'MOD09GQ.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MOD09Q1': {'inputs': 'MOD09Q1.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MYD09A1': {'inputs': 'MYD09A1.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MYD09GA': {'inputs': 'MYD09GA.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MYD09GQ': {'inputs': 'MYD09GQ.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MYD09Q1': {'inputs': 'MYD09Q1.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MOD13A1': {'inputs': 'MOD13A1.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MOD13A2': {'inputs': 'MOD13A2.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MOD13A3': {'inputs': 'MOD13A3.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MOD13Q1': {'inputs': 'MOD13Q1.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MYD13A1': {'inputs': 'MYD13A1.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MYD13A2': {'inputs': 'MYD13A2.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MYD13A3': {'inputs': 'MYD13A3.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MYD13Q1': {'inputs': 'MYD13Q1.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'vnp09ga': {'inputs': ['VNP09GA.A2014245.h10v04.001.2017043103958', 'VNP09GA.A2014245.h10v04.001.2015012103931'], 'products': ['l1']}, 'tm4': {'inputs': 'LT42181092013069PFS00', 'products': ['l1']}, 'tm5': {'inputs': 'LT52181092013069PFS00', 'products': ['l1']}, 'etm7': {'inputs': 'LE72181092013069PFS00', 'products': ['l1']}, 'oli8': {'inputs': 'LO82181092013069PFS00', 'products': ['l1']}, 'olitirs8': {'inputs': 'LC82181092013069PFS00', 'products': ['l1']}, 'sentinel': {'inputs': ['L1C_T14TPP_A022031_20190910T172721', 'S2A_OPER_MSI_L1C_TL_SGS__20160130T184417_20160130T203840_A003170_T13VCC_N02_01_01'], 'products': ['s2_sr', 's2_ndvi', 's2_msavi', 's2_evi', 's2_savi', 's2_nbr', 's2_nbr2', 's2_ndmi'] 'projection': {'lonlat': None}, 'image_extents': {'north': 0.0002695, 'south': 0, 'east': 0.0002695, 'west': 0, 'units': 'dd'}, 'format': 'gtiff', 'resampling_method': 'cc', 'resize': {'pixel_size': 0.0002695, 'pixel_size_units': 'dd'}, 'plot_statistics': True}""" base = { 'projection': { 'lonlat': None }, 'image_extents': { 'north': 0.002695, 'south': 0, 'east': 0.002695, 'west': 0, 'units': 'dd' }, 'format': 'gtiff', 'resampling_method': 'cc', 'resize': { 'pixel_size': 0.0002695, 'pixel_size_units': 'dd' }, 'plot_statistics': True } # This will give the base order 15 scenes sensor_acqids = { '.A2000072.h02v09.005.2008237032813': (['MOD09A1', 'MOD09Q1', 'MYD13A1', 'MYD13Q1'], ['mod09a1', 'mod09q1', 'myd13a1', 'myd13q1']), '.A2016305.h11v04.006.2016314200836': (['MOD09GA', 'MOD09GQ', 'MYD13A2', 'MYD13A3'], ['mod09ga', 'mod09gq', 'myd13a2', 'myd13a3']), '.A2000361.h24v03.006.2015111114747': (['MOD11A1', 'MYD11A1'], ['mod11a1', 'myd11a1']), # TODO: REMOVE _collection from IDs 'L1TP_044030_19851028_20161004_01_T1': (['LT04_', 'LT05_', 'LE07_', 'LO08_', 'LC08_'], [ 'tm4_collection', 'tm5_collection', 'etm7_collection', 'oli8_collection', 'olitirs8_collection' ]) } for acq in sensor_acqids: for prefix, label in zip(sensor_acqids[acq][0], sensor_acqids[acq][1]): # We need to avoid any requests for modis_ndvi - this will raise an error since we have # non-NDVI available MODIS products in the base order. Testing of modis_ndvi # and viirs_ndvi orders is conducted separately in test_api.py if label.startswith('mod') or label.startswith('myd'): base[label] = { 'inputs': ['{}{}'.format(prefix, acq)], 'products': ['l1'] } else: base[label] = { 'inputs': ['{}{}'.format(prefix, acq)], # don't include orca in our test base order since it has sensor restrictions 'products': [ p for p in sn.instance('{}{}'.format( prefix, acq)).products if p != 'orca' ] } # We need to have at least 2 scenes for viirs-related tests to work base['vnp09ga'] = { 'inputs': [ 'VNP09GA.A2014245.h10v04.001.2017043103958', 'VNP09GA.A2014245.h10v04.001.2015012103931' ], 'products': ['l1'] } # add sentinel-2 scene IDs to the order base['sentinel'] = { 'inputs': [ 'L1C_T14TPP_A022031_20190910T172721', 'S2A_OPER_MSI_L1C_TL_SGS__20160130T184417_20160130T203840_A003170_T13VCC_N02_01_01' ], 'products': [ 's2_sr', 's2_ndvi', 's2_msavi', 's2_savi', 's2_evi', 's2_ndmi', 's2_nbr', 's2_nbr2' ] } return base
def create(cls, params): """ Place a new order into the system :param params: dict of required parameters to be used params = {'product_opts': {dictionary object of the order received} 'orderid': id generated from generate_order_id 'user_id': EE user id 'order_type': typically 'level2_ondemand' 'status': 'ordered' 'note': user notes 'ee_order_id': earth explorer order id, or '' 'order_source': 'espa' or 'ee' 'order_date': date time string 'priority': legacy item, should be 'normal' 'email': user's contact email 'product_options': legacy column} :return: order object """ opts = params['product_opts'] params['product_opts'] = json.dumps(params['product_opts']) sql = ('INSERT INTO ordering_order ' '(orderid, user_id, order_type, status, note, ' 'product_opts, ee_order_id, order_source, order_date, ' 'priority, email, product_options) ' 'VALUES (%(orderid)s, %(user_id)s, %(order_type)s, ' '%(status)s, %(note)s, %(product_opts)s, ' '%(ee_order_id)s, %(order_source)s, %(order_date)s, ' '%(priority)s, %(email)s, %(product_options)s)') logger.info('Order creation parameters: {}'.format(params)) log_sql = '' try: with db_instance() as db: log_sql = db.cursor.mogrify(sql, params) logger.info('New order complete SQL: {}' .format(log_sql)) db.execute(sql, params) db.commit() except DBConnectException as e: logger.debug('Error creating new order: {}\n' 'sql: {}'.format(e.message, log_sql)) raise OrderException(e) order = Order.find(params['orderid']) # Let the load_ee_order method handle the scene injection # as there is special logic for interacting with LTA if params['ee_order_id']: return order sensor_keys = sensor.SensorCONST.instances.keys() bulk_ls = [] for key in opts: if key in sensor_keys: sensor_type = '' item1 = opts[key]['inputs'][0] if isinstance(sensor.instance(item1), sensor.Landsat): sensor_type = 'landsat' elif isinstance(sensor.instance(item1), sensor.Modis): sensor_type = 'modis' for s in opts[key]['inputs']: scene_dict = {'name': s, 'sensor_type': sensor_type, 'order_id': order.id, 'status': 'submitted', 'ee_unit_id': None} bulk_ls.append(scene_dict) if 'plot_statistics' in opts and opts['plot_statistics']: scene_dict = {'name': 'plot', 'sensor_type': 'plot', 'order_id': order.id, 'status': 'submitted', 'ee_unit_id': None} bulk_ls.append(scene_dict) try: Scene.create(bulk_ls) except SceneException as e: logger.debug('Order creation failed on scene injection, ' 'order: {}\nexception: {}' .format(order.orderid, e.message)) with db_instance() as db: db.execute('delete ordering_order where id = %s', order.id) db.commit() raise OrderException(e) return order
def build_base_order(): """ Builds the following dictionary (with the products filled out from sensor.py): base = {'MOD09A1': {'inputs': 'MOD09A1.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MOD09GA': {'inputs': 'MOD09GA.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MOD09GQ': {'inputs': 'MOD09GQ.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MOD09Q1': {'inputs': 'MOD09Q1.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MYD09A1': {'inputs': 'MYD09A1.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MYD09GA': {'inputs': 'MYD09GA.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MYD09GQ': {'inputs': 'MYD09GQ.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MYD09Q1': {'inputs': 'MYD09Q1.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MOD13A1': {'inputs': 'MOD13A1.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MOD13A2': {'inputs': 'MOD13A2.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MOD13A3': {'inputs': 'MOD13A3.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MOD13Q1': {'inputs': 'MOD13Q1.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MYD13A1': {'inputs': 'MYD13A1.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MYD13A2': {'inputs': 'MYD13A2.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MYD13A3': {'inputs': 'MYD13A3.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'MYD13Q1': {'inputs': 'MYD13Q1.A2000072.h02v09.005.2008237032813', 'products': ['l1']}, 'tm4': {'inputs': 'LT42181092013069PFS00', 'products': ['l1']}, 'tm5': {'inputs': 'LT52181092013069PFS00', 'products': ['l1']}, 'etm7': {'inputs': 'LE72181092013069PFS00', 'products': ['l1']}, 'oli8': {'inputs': 'LO82181092013069PFS00', 'products': ['l1']}, 'olitirs8': {'inputs': 'LC82181092013069PFS00', 'products': ['l1']}, 'projection': {'lonlat': None}, 'image_extents': {'north': 0.0002695, 'south': 0, 'east': 0.0002695, 'west': 0, 'units': 'dd'}, 'format': 'gtiff', 'resampling_method': 'cc', 'resize': {'pixel_size': 0.0002695, 'pixel_size_units': 'dd'}, 'plot_statistics': True}""" base = {'projection': {'lonlat': None}, 'image_extents': {'north': 0.002695, 'south': 0, 'east': 0.002695, 'west': 0, 'units': 'dd'}, 'format': 'gtiff', 'resampling_method': 'cc', 'resize': {'pixel_size': 0.0002695, 'pixel_size_units': 'dd'}, 'plot_statistics': True} sensor_acqids = {'.A2000072.h02v09.005.2008237032813': (['MOD09A1', 'MOD09GA', 'MOD09GQ', 'MOD09Q1', 'MYD09A1', 'MYD09GA', 'MYD09GQ', 'MYD09Q1', 'MOD13A1', 'MOD13A2', 'MOD13A3', 'MOD13Q1', 'MYD13A1', 'MYD13A2', 'MYD13A3', 'MYD13Q1'], ['mod09a1', 'mod09ga', 'mod09gq', 'mod09q1', 'myd09a1', 'myd09ga', 'myd09gq', 'myd09q1', 'mod13a1', 'mod13a2', 'mod13a3', 'mod13q1', 'myd13a1', 'myd13a2', 'myd13a3', 'myd13q1']), '2181092013069PFS00': (['LT4', 'LT5', 'LE7', 'LO8', 'LC8'], ['tm4', 'tm5', 'etm7', 'oli8', 'olitirs8'])} for acq in sensor_acqids: for prefix, label in zip(sensor_acqids[acq][0], sensor_acqids[acq][1]): base[label] = {'inputs': ['{}{}'.format(prefix, acq)], 'products': sn.instance('{}{}'.format(prefix, acq)).products} return base