def setUp(self):
logger.warning('Testing API started...')
os.environ['espa_api_testing'] = 'True'
# create a user
self.mock_user = MockUser()
self.mock_order = MockOrder()
user_id = self.mock_user.add_testing_user()
order_id = self.mock_order.generate_testing_order(user_id)
self.order = Order.find(order_id)
self.user = User.find(user_id)
self.product_id = 'LT05_L1TP_032028_20120425_20160830_01_T1'
self.sensor_id = 'tm5_collection'
self.staff_product_id = 'LE07_L1TP_010028_20050420_20160925_01_T1'
self.staff_sensor = 'etm7_collection'
self.global_product_id = 'LE70450302003206EDC01'
staff_user_id = self.mock_user.add_testing_user()
self.staff_user = User.find(staff_user_id)
self.staff_user.update('is_staff', True)
staff_order_id = self.mock_order.generate_testing_order(staff_user_id)
staff_order = Order.find(staff_order_id)
staff_scene = staff_order.scenes()[0]
staff_scene.update('name', self.staff_product_id)
user_scene = self.order.scenes()[0]
user_scene.update('name', self.staff_product_id)
with open(os.path.join(__location__, 'domain/restricted.yaml')) as f:
self.restricted = yaml.load(f.read())
self.restricted['all']['role'].remove('restricted_prod')
def test_production_load_ee_orders_partial(self):
order = Order.find(
self.mock_order.generate_ee_testing_order(self.user_id,
partial=True))
self.assertEqual(
order.product_opts, {
'format': 'gtiff',
'etm7_collection': {
'inputs': ['LE07_L1TP_026027_20170912_20171008_01_T1'],
'products': ['sr']
}
})
key = 'system.load_ee_orders_enabled'
self.assertEqual(api.get_production_key(key)[key], 'True')
production_provider.load_ee_orders()
reorder = Order.find(order.id)
self.assertEqual(
reorder.product_opts, {
'format': 'gtiff',
'etm7_collection': {
'inputs': ['LE07_L1TP_026027_20170912_20171008_01_T1'],
'products': ['sr']
},
'tm5_collection': {
'inputs': ['LT05_L1TP_025027_20110913_20160830_01_T1'],
'products': ['sr']
}
})
def setUp(self):
os.environ['espa_api_testing'] = 'True'
# create a user
self.mock_user = MockUser()
self.mock_order = MockOrder()
user_id = self.mock_user.add_testing_user()
order_id = self.mock_order.generate_testing_order(user_id)
self.order = Order.find(order_id)
self.user = User.find(user_id)
self.product_id = 'LT50150401987120XXX02'
self.staff_product_id = 'LE70450302003206EDC01'
staff_user_id = self.mock_user.add_testing_user()
self.staff_user = User.find(staff_user_id)
self.staff_user.update('is_staff', True)
staff_order_id = self.mock_order.generate_testing_order(staff_user_id)
staff_order = Order.find(staff_order_id)
staff_scene = staff_order.scenes()[0]
staff_scene.update('name', self.staff_product_id)
user_scene = self.order.scenes()[0]
user_scene.update('name', self.staff_product_id)
with open('api/domain/restricted.yaml') as f:
self.restricted = yaml.load(f.read())
self.restricted['all']['role'].remove('restricted_prod')
def setUp(self):
os.environ['espa_api_testing'] = 'True'
# create a user
self.mock_user = MockUser()
self.mock_order = MockOrder()
user_id = self.mock_user.add_testing_user()
order_id = self.mock_order.generate_testing_order(user_id)
self.order = Order.find(order_id)
self.user = User.find(user_id)
self.product_id = 'LT50150401987120XXX02'
self.staff_product_id = 'LE70450302003206EDC01'
staff_user_id = self.mock_user.add_testing_user()
self.staff_user = User.find(staff_user_id)
self.staff_user.update('is_staff', True)
staff_order_id = self.mock_order.generate_testing_order(staff_user_id)
staff_order = Order.find(staff_order_id)
staff_scene = staff_order.scenes()[0]
staff_scene.update('name', self.staff_product_id)
user_scene = self.order.scenes()[0]
user_scene.update('name', self.staff_product_id)
with open('api/domain/restricted.yaml') as f:
self.restricted = yaml.load(f.read())
self.restricted['all']['role'].remove('restricted_prod')
def test_production_load_ee_orders_partial(self):
order = Order.find(
self.mock_order.generate_ee_testing_order(self.user_id,
partial=True))
self.assertTrue(
order.product_opts == {
'format': 'gtiff',
'etm7': {
'inputs': ['LE70900652008327EDC00'],
'products': ['sr']
}
})
production_provider.load_ee_orders()
reorder = Order.find(order.id)
self.assertTrue(
reorder.product_opts == {
'format': 'gtiff',
'etm7': {
'inputs': ['LE70900652008327EDC00'],
'products': ['sr']
},
'tm5': {
'inputs': ['LT50900652008327EDC00'],
'products': ['sr']
}
})
def test_production_calc_scene_download_sizes(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scenes = order.scenes()
Scene.bulk_update([s.id for s in scenes], {'status': 'complete', 'download_size': 0})
scenes = Order.find(order.id).scenes({'status': 'complete', 'download_size': 0})
self.assertTrue(production_provider.calc_scene_download_sizes(scenes))
upscenes = Scene.where({'status': 'complete', 'download_size': 999})
self.assertEqual(len(upscenes), len(scenes))
def test_production_handle_failed_ee_updates(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.update('failed_lta_status_update', 'C')
scenes = Order.find(order.id).scenes({'failed_lta_status_update IS NOT': None})
production_provider.handle_failed_ee_updates(scenes)
scenes = Scene.where({'failed_lta_status_update IS NOT': None})
self.assertTrue(len(scenes) == 0)
def test_production_get_contactids_for_submitted_landsat_products(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes({'name !=': 'plot'}):
scene.status = 'submitted'
scene.sensor_type = 'landsat'
scene.save()
scenes = Order.find(order.id).scenes({'sensor_type': 'landsat'})
response = production_provider.get_contactids_for_submitted_landsat_products(scenes)
self.assertIsInstance(response, set)
self.assertTrue(len(response) > 0)
def test_update_product_details_mark_product_processing(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[0]
res = production_provider.update_product('update_status',
name=scene.name,
orderid=order.orderid,
processing_loc='L8SRLEXAMPLE',
status='processing')
order = Order.find(order.id)
scene = order.scenes({'id': scene.id})[0]
self.assertEqual('processing', scene.status)
def test_production_load_ee_orders_partial(self):
order = Order.find(self.mock_order.generate_ee_testing_order(self.user_id, partial=True))
self.assertTrue(order.product_opts == {'format': 'gtiff',
'etm7': {'inputs': ['LE70900652008327EDC00'],
'products': ['sr']}})
production_provider.load_ee_orders()
reorder = Order.find(order.id)
self.assertTrue(reorder.product_opts == {'format': 'gtiff',
'etm7': {'inputs': ['LE70900652008327EDC00'],
'products': ['sr']},
'tm5': {'inputs': ['LT50900652008327EDC00'],
'products': ['sr']}})
def test_fetch_production_products_plot(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
self.mock_order.update_scenes(order_id, ('landsat', 'modis'), 'status', ['complete'])
order = Order.find(order_id)
plot_scene = order.scenes({'name': 'plot'})[0]
plot_scene.name = 'plot'
plot_scene.sensor_type = 'plot'
plot_scene.status = 'submitted'
plot_scene.save()
scenes = Order.find(order_id).scenes({'name': 'plot', 'status': 'submitted'})
response = production_provider.handle_submitted_plot_products(scenes)
pscene = order.scenes({'status': 'oncache', 'sensor_type': 'plot'})
self.assertTrue(response is True)
self.assertEqual(len(pscene), 1)
def test_production_update_landsat_product_status(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.status = 'submitted'
scene.sensor_type = 'landsat'
scene.save()
self.assertTrue(production_provider.update_landsat_product_status(User.find(self.user_id).contactid))
def test_production_update_landsat_product_status(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.status = 'submitted'
scene.sensor_type = 'landsat'
scene.save()
self.assertTrue(production_provider.update_landsat_product_status(User.find(self.user_id).contactid))
def test_production_calc_scene_download_sizes(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scenes = order.scenes()
Scene.bulk_update([s.id for s in scenes], {'status': 'complete', 'download_size': 0})
self.assertTrue(production_provider.calc_scene_download_sizes())
upscenes = Scene.where({'status': 'complete', 'download_size': 999})
self.assertEqual(len(upscenes), len(scenes))
def test_production_mark_nlaps_unavailable(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.status = 'submitted'
scene.sensor_type = 'landsat'
scene.save()
self.assertTrue(production_provider.mark_nlaps_unavailable())
def names_tuple(self, number, user_id):
order_id = self.generate_testing_order(user_id)
order = Order.find(order_id)
scene_names = [i.name for i in order.scenes()]
scene_names = scene_names[0:number]
list_of_tuples = [(order.orderid, s) for s in scene_names]
return list_of_tuples
def test_production_handle_submitted_plot_products(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
order.status = 'ordered'
order.order_type = 'lpcs'
order.save()
plot_id = None
for idx, scene in enumerate(order.scenes()):
# at the moment, mock_order.generate_testing_order
# creates 21 products for the order. divvy those
# up between 'complete' and 'unavailable', setting
# one aside as the 'plot' product
if scene.sensor_type == 'plot':
# need to define a plot product
scene.update('status', 'submitted')
plot_id = scene.id
else:
if idx % 2 == 0:
scene.update('status', 'complete')
else:
scene.update('status', 'unavailable')
scenes = order.scenes()
self.assertTrue(
production_provider.handle_submitted_plot_products(scenes))
self.assertEqual(Scene.find(plot_id).status, "oncache")
def test_production_update_order_if_complete(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
Scene.bulk_update([s.id for s in order.scenes()], {'status': 'retry'})
order.order_source = 'espa'
order.completion_email_sent = None
order.save()
self.assertTrue(production_provider.update_order_if_complete(order))
def test_production_update_order_if_complete(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
Scene.bulk_update([s.id for s in order.scenes()], {'status': 'retry'})
order.order_source = 'espa'
order.completion_email_sent = None
order.save()
self.assertTrue(production_provider.update_order_if_complete(order))
def fetch_user_orders_feed(self, email):
orders = self.fetch_user_orders(email)
if 'orders' not in orders.keys():
return orders
cache_key = "{0}_feed".format(email)
outd = cache.get(cache_key) or {}
if not outd:
for orderid in orders['orders']:
order = Order.find(orderid)
scenes = order.scenes({"status": "complete"})
if scenes:
outd[order.orderid] = {'orderdate': str(order.order_date)}
scene_list = []
for scene in scenes:
scene_list.append({
'name': scene.name,
'url': scene.product_dload_url,
'status': scene.status
})
outd[order.orderid]['scenes'] = scene_list
cache.set(cache_key, outd)
return outd
def error_to(self, orderid, state):
order = Order.find(orderid)
err_scenes = order.scenes({'status': 'error'})
try:
if state == 'submitted':
Scene.bulk_update(
[s.id for s in err_scenes], {
'status': state,
'orphaned': None,
'reported_orphan': None,
'log_file_contents': '',
'note': '',
'retry_count': 0
})
order.status = 'ordered'
order.completion_email_sent = None
order.save()
else:
Scene.bulk_update([s.id for s in err_scenes],
{'status': state})
return True
except SceneException as e:
logger.critical('ERR admin provider error_to\ntrace: {}'.format(
e.message))
raise AdministrationProviderException('ERR updating with error_to')
def test_production_mark_nlaps_unavailable(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.status = 'submitted'
scene.sensor_type = 'landsat'
scene.save()
self.assertTrue(production_provider.mark_nlaps_unavailable())
def names_tuple(self, number, user_id):
order_id = self.generate_testing_order(user_id)
order = Order.find(order_id)
scene_names = [i.name for i in order.scenes()]
scene_names = scene_names[0:number]
list_of_tuples = [(order.orderid, s) for s in scene_names]
return list_of_tuples
def test_update_product_details_set_product_unavailable(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[0]
production_provider.update_product('set_product_unavailable',
name=scene.name, orderid=order.orderid,
processing_loc="L8SRLEXAMPLE",
error='include_dswe is an unavailable product option for OLITIRS')
self.assertTrue('unavailable' == Scene.get('ordering_scene.status', scene.name, order.orderid))
def test_update_product_details_set_product_unavailable(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[0]
production_provider.update_product('set_product_unavailable',
name=scene.name, orderid=order.orderid,
processing_loc="L8SRLEXAMPLE",
error='include_dswe is an unavailable product option for OLITIRS')
self.assertTrue('unavailable' == Scene.get('ordering_scene.status', scene.name, order.orderid))
def test_production_purge_orders(self):
new_completion_date = datetime.datetime.now() - datetime.timedelta(
days=12)
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
order.update('status', 'complete')
order.update('completion_date', new_completion_date)
self.assertTrue(production_provider.purge_orders())
def test_production_handle_retry_products(self):
prev = datetime.datetime.now() - datetime.timedelta(hours=1)
order_id = self.mock_order.generate_testing_order(self.user_id)
self.mock_order.update_scenes(order_id, 'landsat', 'status', ['retry'])
self.mock_order.update_scenes(order_id, 'landsat', 'retry_after', [prev])
scenes = Order.find(order_id).scenes({'status': 'retry'})
production_provider.handle_retry_products(scenes)
for s in Scene.where({'order_id': order_id, 'sensor_type': 'landsat'}):
self.assertTrue(s.status == 'submitted')
def test_production_get_contactids_for_submitted_landsat_products(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.status = 'submitted'
scene.sensor_type = 'landsat'
scene.save()
response = production_provider.get_contactids_for_submitted_landsat_products()
self.assertIsInstance(response, set)
self.assertTrue(len(response) > 0)
def generate_testing_order(self, user_id):
user = User.find(user_id)
# need to monkey with the email, otherwise we get collisions with each
# test creating a new scratch order with the same user
rand = str(random.randint(1, 99))
user.email = rand + user.email
orderid = self.ordering_provider.place_order(self.base_order, user)
order = Order.find(orderid)
return order.id
def test_production_handle_failed_ee_updates(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.update('failed_lta_status_update', 'C')
production_provider.handle_failed_ee_updates()
scenes = Scene.where({'failed_lta_status_update IS NOT': None})
self.assertTrue(len(scenes) == 0)
def send_completion(self, order):
if not isinstance(order, Order):
order = Order.find(order)
messages = {
'complete': [
'{orderid} is now complete and can be downloaded from {url}',
'', '',
'For large orders, the ESPA Bulk Downloader is available {bdurl}',
'', '',
'This order will remain available for 7 days. Any data not downloaded '
'will need to be reordered after this time.', '', '',
'Please contact Customer Services at 1-800-252-4547 or '
'email [email protected] with any questions.', '', '',
'Requested products',
'-------------------------------------------'
],
'unsuccessful': [
'{orderid} was unsuccessful, and the reason can be found at {url}',
'', '', 'This order will remain available for 7 days.', '', '',
'Please contact Customer Services at 1-800-252-4547 or '
'email [email protected] with any questions.', '', '',
'Requested products',
'-------------------------------------------'
]
}
scenes = order.scenes({'status': 'complete'})
status = 'complete' if len(scenes) > 0 else 'unsuccessful'
outmessage = messages[status]
email = order.user_email()
url = self.__order_status_url(order.orderid)
bdl_url = "https://github.com/USGS-EROS/espa-bulk-downloader"
scenes = order.scenes()
pbs = order.products_by_sensor()
for product in scenes:
if product.sensor_type == 'plot':
line = "plotting & statistics"
else:
if product.status == 'complete':
line = "{}: {}".format(product.name,
", ".join(pbs[product.name]))
else:
line = "{}: {}".format(product.name, product.note.strip())
outmessage.append(line)
outmessage.append(contact_footer)
body = '\n'.join(outmessage).format(url=url,
bdurl=bdl_url,
orderid=order.orderid)
subject = 'USGS ESPA Processing for {} {}'.format(
order.orderid, status)
return self.__send(recipient=email, subject=subject, body=body)
def test_production_handle_submitted_modis_products_input_missing(self):
# handle unavailable scenario
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.status = 'submitted'
scene.sensor_type = 'modis'
scene.save()
sid = scene.id
self.assertTrue(production_provider.handle_submitted_modis_products())
self.assertEquals(Scene.find(sid).status, "unavailable")
def test_production_handle_submitted_modis_products_input_missing(self):
# handle unavailable scenario
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.status = 'submitted'
scene.sensor_type = 'modis'
scene.save()
sid = scene.id
self.assertTrue(production_provider.handle_submitted_modis_products())
self.assertEquals(Scene.find(sid).status, "unavailable")
def test_production_handle_submitted_modis_products_input_exists(self):
# handle oncache scenario
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes({'name !=': 'plot'}):
scene.status = 'submitted'
scene.sensor_type = 'modis'
scene.save()
sid = scene.id
scenes = order.scenes({'sensor_type': 'modis'})
self.assertTrue(production_provider.handle_submitted_modis_products(scenes))
def test_production_set_product_error_retry(self):
"""
Move a scene status from error to retry based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[2]
production_provider.set_product_error(scene.name, order.orderid,
'somewhere',
'Verify the missing auxillary data products')
self.assertTrue('retry' == Scene.get('ordering_scene.status', scene.name, order.orderid))
def test_production_set_product_error_submitted(self):
"""
Move a scene status from error to submitted based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes({'name !=': 'plot'})[0]
production_provider.set_product_error(scene.name, order.orderid,
'get_products_to_process',
'BLOCK, COMING FROM LST AS WELL: No such file or directory')
self.assertTrue('submitted' == Scene.get('ordering_scene.status', scene.name, order.orderid))
def test_production_set_product_error_retry(self):
"""
Move a scene status from error to retry based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[2]
production_provider.set_product_error(scene.name, order.orderid,
'somewhere',
'Verify the missing auxillary data products')
self.assertTrue('retry' == Scene.get('ordering_scene.status', scene.name, order.orderid))
def test_production_set_product_error_submitted(self):
"""
Move a scene status from error to submitted based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes({'name !=': 'plot'})[0]
production_provider.set_product_error(scene.name, order.orderid,
'get_products_to_process',
'BLOCK, COMING FROM LST AS WELL: No such file or directory')
self.assertTrue('submitted' == Scene.get('ordering_scene.status', scene.name, order.orderid))
def test_production_set_product_error_unavailable(self):
"""
Move a scene status from error to unavailable based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes({'name !=': 'plot'})[0]
production_provider.set_product_error(scene.name, order.orderid,
'get_products_to_process',
'include_dswe is an unavailable product option for OLITIRS')
self.assertTrue('unavailable' == Scene.get('ordering_scene.status', scene.name, order.orderid))
def test_update_product_details_mark_cancelled_product_processing(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
order.status = 'cancelled'
order.save()
scene = order.scenes()[0]
res = production_provider.update_product('update_status',
name=scene.name,
orderid=order.orderid,
processing_loc='L8SRLEXAMPLE',
status='processing')
self.assertFalse(res)
def test_production_set_product_error_retry_lasrc_segfault(self):
"""
Move a scene status from error to retry based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes({'sensor_type': 'landsat'})[-1]
production_provider.set_product_error(scene.name, order.orderid,
'somewhere',
'runSr sh: line 1: 1010 Segmentation fault lasrc --xml=')
self.assertTrue('retry' == Scene.get('ordering_scene.status', scene.name, order.orderid))
def test_production_set_product_error_unavailable(self):
"""
Move a scene status from error to unavailable based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes({'name !=': 'plot'})[0]
production_provider.set_product_error(scene.name, order.orderid,
'get_products_to_process',
'include_dswe is an unavailable product option for OLITIRS')
self.assertTrue('unavailable' == Scene.get('ordering_scene.status', scene.name, order.orderid))
def test_update_product_details_set_product_error(self):
"""
Set a scene status to error
:return:
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[0]
production_provider.update_product('set_product_error',
name=scene.name, orderid=order.orderid,
processing_loc="L8SRLEXAMPLE",
error='problems yo')
self.assertTrue(Scene.find(scene.id).status == 'error')
def test_update_product_details_mark_product_complete(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[0]
production_provider.update_product('mark_product_complete',
name=scene.name,
orderid=order.orderid,
processing_loc='L8SRLEXAMPLE',
completed_file_location='/some/loc',
cksum_file_location='some checksum',
log_file_contents='some log')
self.assertTrue('complete' == Scene.get('ordering_scene.status', scene.name, order.orderid))
def test_update_product_details_mark_product_complete(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[0]
production_provider.update_product('mark_product_complete',
name=scene.name,
orderid=order.orderid,
processing_loc='L8SRLEXAMPLE',
completed_file_location='/some/loc',
cksum_file_location='some checksum',
log_file_contents='some log')
self.assertTrue('complete' == Scene.get('ordering_scene.status', scene.name, order.orderid))
def test_update_product_details_update_status(self):
"""
Set a scene status to Queued
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[0]
api.update_product_details('update_status',
{'name': scene.name,
'orderid': order.orderid,
'processing_loc': 'L8SRLEXAMPLE',
'status': 'Queued'})
self.assertTrue(Scene.get('ordering_scene.status', scene.name, order.orderid) == 'Queued')
def test_update_product_details_set_product_error(self):
"""
Set a scene status to error
:return:
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[0]
production_provider.update_product('set_product_error',
name=scene.name, orderid=order.orderid,
processing_loc="L8SRLEXAMPLE",
error='problems yo')
self.assertTrue(Scene.find(scene.id).status == 'error')
def test_update_product_details_update_status(self):
"""
Set a scene status to Queued
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[0]
api.update_product_details('update_status',
{'name': scene.name,
'orderid': order.orderid,
'processing_loc': 'L8SRLEXAMPLE',
'status': 'Queued'})
self.assertTrue(Scene.get('ordering_scene.status', scene.name, order.orderid) == 'Queued')
def test_fetch_production_products_plot(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
self.mock_order.update_scenes(order_id, 'status', ['complete'])
order = Order.find(order_id)
plot_scene = order.scenes()[0]
plot_scene.name = 'plot'
plot_scene.sensor_type = 'plot'
plot_scene.status = 'submitted'
plot_scene.save()
response = production_provider.handle_submitted_plot_products()
pscene = order.scenes({'status': 'oncache', 'sensor_type': 'plot'})
self.assertTrue(response is True)
self.assertEqual(len(pscene), 1)
def test_production_handle_onorder_landsat_products(self):
tram_order_ids = lta.sample_tram_order_ids()[0:3]
scene_names = lta.sample_scene_names()[0:3]
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scenes = order.scenes()[0:3]
for idx, scene in enumerate(scenes):
scene.tram_order_id = tram_order_ids[idx]
scene.status = 'onorder'
# save() doesn't let you update name,
# b/c updating a scene name is not acceptable
# outside of testing
scene.update('name', scene_names[idx])
scene.save()
self.assertTrue(production_provider.handle_onorder_landsat_products())
def test_update_product_details_fail_lta_mark_product_complete(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[1]
order.update('order_source', 'ee')
production_provider.update_product('mark_product_complete',
name=scene.name,
orderid=order.orderid,
processing_loc='L8SRLEXAMPLE',
completed_file_location='/some/loc',
cksum_file_location='some checksum',
log_file_contents='some log')
s = Scene.where({'name': scene.name, 'order_id': scene.order_id})[0]
self.assertTrue('C' == s.failed_lta_status_update)
def test_production_set_product_retry(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
order = Order.find(order_id)
scene = order.scenes()[3]
scene.update('retry_count', 4)
processing_loc = "get_products_to_process"
error = 'not available after EE call '
note = 'note this'
retry_after = datetime.datetime.now() + datetime.timedelta(hours=1)
retry_limit = 9
response = production_provider.set_product_retry(scene.name, order.orderid, processing_loc,
error, note, retry_after, retry_limit)
new = Scene.get('ordering_scene.status', scene.name, order.orderid)
self.assertTrue('retry' == new)
def error_to(self, orderid, state):
order = Order.find(orderid)
err_scenes = order.scenes({'status': 'error'})
try:
for scene in err_scenes:
scene.update('status', state)
if state == 'submitted':
order.status = 'ordered'
order.completion_email_sent = None
order.save()
return True
except SceneException as e:
logger.debug('ERR admin provider error_to\ntrace: {}'.format(e.message))
raise AdministrationProviderException('ERR updating with error_to')
def fetch_user_orders_ext(self, uid, filters={}):
orders = self.fetch_user_orders(uid, filters=filters)
if 'orders' not in orders.keys():
return orders
orders_d = orders['orders']
output = []
for orderid in orders_d:
order = Order.find(orderid)
products_complete = order.scene_status_count('complete')
products_error = order.scene_status_count('error')
products_ordered = order.scene_status_count()
out_d = {'orderid': orderid, 'products_ordered': products_ordered,
'products_complete': products_complete,
'products_error': products_error,
'order_status': order.status, 'order_note': order.note}
output.append(out_d)
return output
def send_initial(self, order_id):
if isinstance(order_id, Order):
order = order_id
else:
order = Order.find(order_id)
if not isinstance(order, Order):
msg = 'order must be str of orderid, int of pk or instance of Order'
raise TypeError(msg)
email = order.user_email()
url = self.__order_status_url(order.orderid)
m = list()
m.append("Thank you for your order.\n\n")
m.append("%s has been received and is currently " % order.orderid)
m.append("being processed. ")
m.append("Another email will be sent when this order is complete.\n\n")
m.append("You may view the status of your order and download ")
m.append("completed products directly from %s\n\n" % url)
m.append("Requested products\n")
m.append("-------------------------------------------\n")
scenes = order.scenes()
for product in scenes:
name = product.name
if name == 'plot':
name = "Plotting & Statistics"
m.append("%s\n" % name)
m.append(contact_footer)
email_msg = ''.join(m)
subject = 'Processing order %s received' % order.orderid
return self.__send(recipient=email, subject=subject, body=email_msg)
def fetch_user_orders_feed(self, email):
orders = self.fetch_user_orders(email)
if 'orders' not in orders.keys():
return orders
cache_key = "{0}_feed".format(email)
outd = cache.get(cache_key) or {}
if not outd:
for orderid in orders['orders']:
order = Order.find(orderid)
scenes = order.scenes({"status": "complete"})
if scenes:
outd[order.orderid] = {'orderdate': str(order.order_date)}
scene_list = []
for scene in scenes:
scene_list.append({'name': scene.name,
'url': scene.product_dload_url,
'status': scene.status})
outd[order.orderid]['scenes'] = scene_list
cache.set(cache_key, outd)
return outd
def test_production_handle_submitted_plot_products(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
order.status = 'ordered'
order.order_type = 'lpcs'
order.save()
plot_id = None
for idx, scene in enumerate(order.scenes()):
# at the moment, mock_order.generate_testing_order
# creates 21 products for the order. divvy those
# up between 'complete' and 'unavailable', setting
# one aside as the 'plot' product
if idx % 2 == 0:
if idx == 0:
# need to define a plot product
scene.update('status', 'submitted')
scene.update('sensor_type', 'plot')
plot_id = scene.id
else:
scene.update('status', 'complete')
else:
scene.update('status', 'unavailable')
self.assertTrue(production_provider.handle_submitted_plot_products())
self.assertEqual(Scene.find(plot_id).status, "oncache")
def send_completion(self, order):
if not isinstance(order, Order):
order = Order.find(order)
email = order.user_email()
url = self.__order_status_url(order.orderid)
bdl_url = "https://github.com/USGS-EROS/espa-bulk-downloader"
m = list()
m.append("%s is now complete and can be downloaded " % order.orderid)
m.append("from %s.\n\n" % url)
m.append("For large orders, the ESPA Bulk Downloader is available %s \n\n" % bdl_url)
m.append("This order will remain available for 10 days. ")
m.append("Any data not downloaded will need to be reordered ")
m.append("after this time.\n\n")
m.append("Please contact Customer Services at 1-800-252-4547 or ")
m.append("email [email protected] with any questions.\n\n")
m.append("Requested products\n")
m.append("-------------------------------------------\n")
scenes = order.scenes({"status": "complete"})
pbs = order.products_by_sensor()
for product in scenes:
if product.sensor_type == 'plot':
line = "plotting & statistics"
else:
line = "{}: {}".format(product.name, ", ".join(pbs[product.name]))
m.append("%s\n" % line)
m.append(contact_footer)
body = ''.join(m)
subject = 'Processing for %s complete.' % order.orderid
return self.__send(recipient=email, subject=subject, body=body)
def test_production_set_products_unavailable(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
self.assertTrue(production_provider.set_products_unavailable(order.scenes(), "you want a reason?"))
def test_production_finalize_orders(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
order.update('status', 'ordered')
self.assertTrue(production_provider.finalize_orders())