def build_request(contact_id, products):
# build the request body
sb = StringIO()
sb.write(self.xml_header)
head = ("<downloadSceneList "
"xmlns="
"'https://earthexplorer.usgs.gov/schema/downloadSceneList' "
"xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' "
"xsi:schemaLocation="
"'https://earthexplorer.usgs.gov/schema/downloadSceneList "
"https://earthexplorer.usgs.gov/EE/downloadSceneList.xsd'>")
sb.write(head)
sb.write("<contactId>{0}</contactId>".format(contact_id))
for p in products:
try:
product = sensor.instance(p)
except sensor.ProductNotImplemented:
logger.warn("{0} not implemented, skipping".format(p))
else:
sb.write("<scene>")
sb.write("<sceneId>{0}</sceneId>"
.format(product.product_id))
sb.write("<sensor>{0}</sensor>".format(product.lta_name))
sb.write("</scene>")
sb.write("</downloadSceneList>")
request_body = sb.getvalue()
return request_body
def input_exists(self, product):
'''Determines if a LPDAAC product is available for download
Keyword args:
product - The name of the product
Returns:
True/False
'''
if isinstance(product, str) or isinstance(product, unicode):
product = sensor.instance(product)
result = False
try:
url = self.get_download_url(product)
if 'download_url' in url[product.product_id]:
url = url[product.product_id]['download_url']
try:
result = utils.connections.is_reachable(url, timeout=1)
except Exception, e:
logger.exception('Exception checking modis input {0}\n '
'Exception:{1}'.format(url, e))
return result
except sensor.ProductNotImplemented:
logger.warn(
'{0} is not an implemented LPDAAC product'.format(product))
return result
def build_request(contact_id, products):
# build the request body
sb = StringIO()
sb.write(self.xml_header)
head = (
"<downloadSceneList "
"xmlns="
"'https://earthexplorer.usgs.gov/schema/downloadSceneList' "
"xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' "
"xsi:schemaLocation="
"'https://earthexplorer.usgs.gov/schema/downloadSceneList "
"https://earthexplorer.usgs.gov/EE/downloadSceneList.xsd'>"
)
sb.write(head)
sb.write("<contactId>{0}</contactId>".format(contact_id))
for p in products:
try:
product = sensor.instance(p)
except sensor.ProductNotImplemented:
logger.warn("{0} not implemented, skipping".format(p))
else:
sb.write("<scene>")
sb.write("<sceneId>{0}</sceneId>".format(product.product_id))
sb.write("<sensor>{0}</sensor>".format(product.lta_name))
sb.write("</scene>")
sb.write("</downloadSceneList>")
request_body = sb.getvalue()
return request_body
def decorated(*args, **kwargs):
white_ls = espa.get_stat_whitelist()
denied_response = MessagesResponse(errors=['Access Denied'], code=403)
remote_addr = user_ip_address()
if ((remote_addr in white_ls or request.remote_addr in white_ls)
and remote_addr != 'untrackable'):
return func(*args, **kwargs)
else:
logger.warn('*** Not in whitelist ({1}): {0}'.format(
remote_addr, white_ls))
return denied_response()
def run(self, name):
if name not in REPORTS:
raise NotImplementedError
query = REPORTS[name]['query']
if query is not None and len(query) > 0:
with db_instance() as db:
db.select(query)
return db.dictfetchall
else:
logger.warn("Query was empty for {0}: {1}".format(name, query))
return {}
def get_download_urls(self,
entity_ids,
dataset,
products='STANDARD',
stage=True,
usage='[espa]:sr'):
"""
Fetch the download location for supplied IDs, replacing the public host
with an internal network host (to bypass public firewall routing)
:param product_ids: Landsat Collection IDs ['LC08_..', ...]
:type product_ids: list
:param products: download type to grab (STANDARD is for L1-GeoTIFF)
:type products: str
:param stage: If true, initiates a data stage command
:type stage: bool
:param usage: Identify higher level products this data is used to create
:type usage: str
:return: dict
"""
payload = dict(apiKey=self.token,
datasetName=dataset,
products=products,
entityIds=entity_ids,
stage=stage,
dataUse=usage)
resp = self._post('download', payload)
results = resp.get('data')
if results:
# Use the URL taken directly from the M2M JSON API if VIIRS
if 'VNP' in dataset:
return {i['entityId']: i['url'] for i in results}
# For now, let's keep the original M2M download url for sentinel
elif 'SENTINEL' in dataset:
return {i['entityId']: i['url'] for i in results}
# Otherwise use our internal network conversion
else:
return self.network_urls(
self.network_urls(
{i['entityId']: i['url']
for i in results}, 'landsat'), 'modis')
else:
logger.warn(
"inventory.get_download_urls - no data in POST response returned for entity_ids: {}, dataset: {}"
.format(entity_ids, dataset))
def check(self, order, contactid=None):
ids = sensor.SensorCONST.instances.keys()
lta_ls = []
lpdaac_ls = []
results = {}
for key in order:
l1 = ''
if key in ids:
inst = sensor.instance(order[key]['inputs'][0])
l1 = inst.l1_provider
if l1 == 'dmid':
lta_ls.extend(order[key]['inputs'])
elif l1 == 'lpdaac':
lpdaac_ls.extend(order[key]['inputs'])
if lta_ls:
if inventory.available():
results.update(self.check_dmid(lta_ls, contactid))
else:
msg = 'Could not connect to Landsat data source'
raise InventoryConnectionException(msg)
if lpdaac_ls:
if inventory.available():
logger.warn('Checking M2M Inventory for LP DAAC granules')
try:
results.update(self.check_dmid(lpdaac_ls, contactid))
except InventoryException:
logger.warn("Unable to verify inventory with DMID")
elif lpdaac.check_lpdaac_available():
try:
results.update(self.check_LPDAAC(lpdaac_ls))
except InventoryException:
logger.warn("Unable to verify inventory with LPDAAC")
else:
msg = "Could not connect to any data source to verify LPDAAC products"
raise InventoryConnectionException(msg)
not_avail = []
for key, val in results.items():
if not val:
not_avail.append(key)
if not_avail:
raise InventoryException(not_avail)
try:
response = requests.head(url)
if response.ok is True:
result = True
except Exception, e:
logger.exception('Exception checking modis input {0}\n '
'Exception:{1}'
.format(url, e))
return result
finally:
if response is not None:
response.close()
response = None
except sensor.ProductNotImplemented:
logger.warn('{0} is not an implemented LPDAAC product'
.format(product))
return result
def get_download_url(self, product):
url = {}
#be nice and accept a string
if isinstance(product, str) or isinstance(product, unicode):
product = sensor.instance(product)
#also be nice and accept a sensor.Modis object
if isinstance(product, sensor.Modis):
path = self._build_modis_input_file_path(product)
class OrderWrapperServiceClient(LTAService):
'''LTA's OrderWrapper Service is a business process service that handles
populating demographics and interacting with the inventory properly when
callers order data. It is implemented as a REST style service that passes
schema-bound XML as the payload.
This is the preferred method for ordering data from the LTA (instead of
calling TRAM services directly), as there are multiple service calls that
must be performed when placing orders, and only the LTA team really know
what those calls are. Their services are largely undocumented.
'''
service_name = 'orderservice'
def __init__(self, *args, **kwargs):
super(OrderWrapperServiceClient, self).__init__(*args, **kwargs)
def verify_scenes(self, scene_list):
''' Checks to make sure the scene list is valid, where valid means
the scene ids supplied exist in the Landsat inventory and are orderable
Keyword args:
scene_list A list of scenes to be verified
Returns:
A dictionary with keys matching the scene list and values are 'true'
if valid, and 'false' if not.
Return value example:
dictionary = dict()
dictionary['LT51490212007234IKR00'] = True
dictionary['asdf'] = False
...
...
...
'''
#build the service + operation url
request_url = '{0}/verifyScenes'.format(self.url)
#build the request body
sb = StringIO()
sb.write(self.xml_header)
head = ("<sceneList "
"xmlns='https://earthexplorer.usgs.gov/schema/sceneList' "
"xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' "
"xsi:schemaLocation="
"'https://earthexplorer.usgs.gov/schema/sceneList "
"https://earthexplorer.usgs.gov/EE/sceneList.xsd'>")
sb.write(head)
for s in scene_list:
product = sensor.instance(s)
sb.write("<sceneId sensor='{0}'>{1}</sceneId>"
.format(product.lta_name, s.upper()))
sb.write("</sceneList>")
request_body = sb.getvalue()
#set the required headers
headers = dict()
headers['Content-Type'] = 'application/xml'
headers['Content-Length'] = len(request_body)
#send the request and check return status
#print "*** request_url: ", request_url
#print "*** request_body: ", request_body
#print "*** headers: ", headers
__response = requests.post(request_url,
data=request_body,
headers=headers)
response = None
if __response.ok:
response = __response.content
else:
msg = StringIO()
msg.write("Error in lta.OrderWrapperServiceClient.verify_scenes\n")
msg.write("Non 200 response code from service\n")
msg.write("Response code was:{0}".format( __response.status_code))
msg.write("Reason:{0}".format(__response.reason))
# Return the code and reason as an exception
raise Exception(msg.getvalue())
__response.close()
#parse, transform and return response
retval = dict()
response = response.replace('&', '&')
response = response.replace('\n', '')
root = xml.fromstring(response)
scenes = root.getchildren()
for s in list(scenes):
if s.attrib['valid'] == 'true':
status = True
else:
status = False
retval[s.text] = status
return retval
def order_scenes(self, scene_list, contact_id, priority=5):
''' Orders scenes through OrderWrapperService
Keyword args:
scene_list A list of scene ids to order
contactId The EE user id that is ordering the data
priority The priority placed on the backend ordering system.
Landsat has asked us to set the priority to 5 for all ESPA
orders.
Returns:
A dictionary containing the lta_order_id and up to three lists of scene
ids, organized by their status. If there are no scenes in the
ordered status, the ordered list and the lta_order_id will not be
present. If there are no scenes in either the invalid or available
status, then those respective lists will not be present.
Example 1 (Scenes in each status):
{
'lta_order_id': 'abc123456',
'ordered': ['scene1', 'scene2', 'scene3'],
'invalid': ['scene4', 'scene5', 'scene6'],
'available': ['scene7', 'scene8', 'scene9']
}
Example 2 (No scenes ordered):
{
'invalid': ['scene1', 'scene2', 'scene3'],
'available': ['scene4', 'scene5', 'scene6']
}
Example 3 (No scenes available):
{
'lta_order_id': 'abc123456',
'ordered': ['scene1', 'scene2', 'scene3'],
'invalid': ['scene4', 'scene5', 'scene6']
}
'''
# build service url
request_url = '{0}/submitOrder'.format(self.url)
def build_request(contact_id, priority, product_list):
# build the request body
sb = StringIO()
sb.write(self.xml_header)
head = ("<orderParameters "
"xmlns="
"'https://earthexplorer.usgs.gov/schema/orderParameters' "
"xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' "
"xsi:schemaLocation="
"'https://earthexplorer.usgs.gov/schema/orderParameters "
"https://earthexplorer.usgs.gov/EE/orderParameters.xsd'>")
sb.write(head)
sb.write('<contactId>{0}</contactId>'.format(contact_id))
sb.write('<requestor>ESPA</requestor>')
# 1111111 is a dummy value.
sb.write('<externalReferenceNumber>{0}</externalReferenceNumber>'
.format(1111111))
sb.write('<priority>{0}</priority>'.format(priority))
product_info = self.get_download_urls(product_list, contact_id)
for p in product_info.keys():
try:
sensor.instance(p)
except sensor.ProductNotImplemented, pne:
raise pne
else:
sb.write('<scene>')
sb.write('<sceneId>{0}</sceneId>'.format(p))
sb.write('<prodCode>{0}</prodCode>'
.format(product_info[p]['lta_code']))
sb.write('<sensor>{0}</sensor>'
.format(product_info[p]['sensor']))
sb.write('</scene>')
sb.write('</orderParameters>')
request_body = sb.getvalue()
return request_body
payload = build_request(contact_id, priority, scene_list)
# set the required headers
headers = dict()
headers['Content-Type'] = 'application/xml'
headers['Content-Length'] = len(payload)
# send the request and check response
__response = requests.post(request_url, data=payload, headers=headers)
if __response.ok:
response = __response.content
else:
logger.critical('Non 200 response from lta.order_scenes, '
'Response:{0}, '
'Request:{1}'.format(__response.content, payload))
msg = StringIO()
msg.write('Error in lta.OrderWrapperServiceClient.order_scenes\n')
msg.write('Non 200 response code from service\n')
msg.write('Response code was:{0}'.format(__response.status_code))
msg.write('Reason:{0}'.format(__response.reason))
raise Exception(msg.getvalue())
__response.close()
# parse the response
'''
Example response for scenes
<?xml version="1.0" encoding="UTF-8"?>
<orderStatus xmlns="http://earthexplorer.usgs.gov/schema/orderStatus"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://host/schema/orderStatus
http://host/OrderWrapperServicedevsys/orderStatus.xsd">
<scene>
<sceneId>LT51490212007234IKR00</sceneId>
<prodCode>T273</prodCode>
<sensor>LANDSAT_TM</sensor>
<status>ordered</status>
<orderNumber>0621405213419</orderNumber>
</scene>
<scene>
<sceneId>LE70290302013153EDC00</sceneId>
<prodCode>T271</prodCode>
<sensor>LANDSAT_ETM_SLC_OFF</sensor>
<status>available</status>
<downloadURL>http://one_time_use.tar.gz</downloadURL>
</scene>
<scene>
<sceneId>LE70290302003142EDC00</sceneId>
<prodCode>T272</prodCode>
<sensor>LANDSAT_ETM_PLUS</sensor>
<status>invalid</status>
</scene>
</orderStatus>
'''
logger.warn('Ordering scenes SOAP response:{0}'.format(response))
# since the xml is namespaced there is a namespace prefix for every
# element we are looking for. Build those values to make the code
# a little more sane
schema = 'orderStatus'
# leave this here for now. We aren't using it yet but will when EE
# straightens out their urls + internal dowloading capability
#dload_url_elem = ''.join([ns_prefix, 'downloadURL'])
# escape the ampersands and get rid of newlines if they exist
# was having problems with the sax escape() function
response = response.replace('&', '&').replace('\n', '')
#this will get us the list of <scene></scene> elements
scene_elements = xml.fromstring(response).getchildren()
# the dictionary we will return as the response
# contains the lta_order_id at the top (if anything is ordered)
# and possibly three lists of scenes, one for each status
# retval['available'] = list()
# retval['invalid'] = list()
# retval['ordered'] = list()
retval = dict()
for scene in scene_elements:
name = self.get_xml_item(scene, schema, 'sceneId').text
status = self.get_xml_item(scene, schema, 'status').text
if status == 'available':
values = retval.get(status, []) + [name]
retval.update(available=values)
elif status == 'invalid':
retval.update(invalid=retval.get(status, []) + [name])
elif status == 'ordered':
retval.update(ordered=retval.get(status, []) + [name])
order_num = self.get_xml_item(scene, schema, 'orderNumber').text
values = retval.get('lta_order_id', ()) + (order_num,)
retval.update(lta_order_id=values)
return retval
class OrderDeliveryServiceClient(LTASoapService):
'''EE SOAP Service client to find orders for ESPA which originated in EE'''
service_name = 'orderdelivery'
def __init__(self, *args, **kwargs):
super(OrderDeliveryServiceClient, self).__init__(*args, **kwargs)
def get_available_orders(self):
''' Returns all the orders that were submitted for ESPA through EE
Returns:
A dictionary of lists that contain dictionaries
response[ordernumber, email, contactid] = [
{'sceneid':orderingId, 'unit_num':unitNbr},
{...}
]
'''
rtn = dict()
# resp = self.client.factory.create("getAvailableOrdersResponse")
try:
resp = self.client.service.getAvailableOrders("ESPA")
except Exception, e:
raise e
#if there were none jusgetAvailt return
if len(resp.units) == 0:
return rtn
#return these to the caller.
for u in resp.units.unit:
#ignore anything that is not for us
if str(u.productCode).lower() not in ('sr01', 'sr02', 'sr03', 'sr04', 'sr05'):
logger.warn('{0} is not an ESPA product. Order[{1}] Unit[{2}]'
'Product code[{3}]... ignoring'
.format(u.orderingId, u.orderNbr,
u.unitNbr, u.productCode))
# continue
# get the processing parameters
pp = u.processingParam
try:
email = pp[pp.index("<email>") + 7:pp.index("</email>")]
except:
logger.warn('Could not find an email address for '
'unit {0} in order {1] : rejecting'
.format(u.unitNbr,u.orderNbr))
# we didn't get an email... fail the order
resp = OrderUpdateServiceClient().update_order(u.orderNbr,
u.unitNbr,
"R")
# we didn't get a response from the service
if not resp.success:
raise Exception('Could not update order[{0}] unit[{1}] '
'to status:F. Error message:{2} '
'Error status code:{3}'
.format(u.orderNbr,
u.unitNbr,
resp.message,
resp.status))
else:
continue
try:
# get the contact id
cid = pp[pp.index("<contactid>") + 11:pp.index("</contactid>")]
except:
logger.warn('Could not find a contactid for unit {0} in '
'order {1}... rejecting'
.format(u.unitNbr, u.orderNbr))
# didn't get an email... fail the order
resp = OrderUpdateServiceClient().update_order(u.orderNbr,
u.unitNbr,
"R")
# didn't get a response from the service
if not resp.success:
raise Exception('Could not update unit {0} in order {1} '
'to status:F. Error message:{2} '
'Error status code:{3}'
.format(u.orderNbr,
u.unitNbr,
resp.message,
resp.status))
else:
continue
# This is a dictionary that contains a list of dictionaries
key = (str(u.orderNbr), str(email), str(cid))
if not key in rtn:
rtn[key] = list()
rtn[key].append({'sceneid': str(u.orderingId),
'unit_num': int(u.unitNbr)}
)
return rtn