class TestProductionAPI(unittest.TestCase):
def setUp(self):
os.environ['espa_api_testing'] = 'True'
# create a user
self.mock_user = MockUser()
self.mock_order = MockOrder()
self.user_id = self.mock_user.add_testing_user()
def tearDown(self):
# clean up orders
self.mock_order.tear_down_testing_orders()
# clean up users
self.mock_user.cleanup()
os.environ['espa_api_testing'] = ''
@patch('api.external.inventory.available', lambda: True)
@patch(
'api.providers.production.production_provider.ProductionProvider.parse_urls_m2m',
lambda x, y: y)
def test_fetch_production_products_modis(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
# need scenes with statuses of 'processing'
self.mock_order.update_scenes(order_id, 'modis', 'status',
['processing', 'oncache'])
user = User.find(self.user_id)
params = {'product_types': ['modis']}
response = api.fetch_production_products(params)
self.assertTrue('bilbo' in response[0]['orderid'])
@patch('api.external.inventory.available', lambda: True)
@patch(
'api.providers.production.production_provider.ProductionProvider.parse_urls_m2m',
lambda x, y: y)
def test_fetch_production_products_landsat(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
# need scenes with statuses of 'processing'
self.mock_order.update_scenes(order_id, 'landsat', 'status',
['processing', 'oncache'])
user = User.find(self.user_id)
params = {'for_user': user.username, 'product_types': ['landsat']}
response = api.fetch_production_products(params)
self.assertTrue('bilbo' in response[0]['orderid'])
def test_fetch_production_products_plot(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
self.mock_order.update_scenes(order_id, ('landsat', 'modis'), 'status',
['complete'])
order = Order.find(order_id)
plot_scene = order.scenes({'name': 'plot'})[0]
plot_scene.name = 'plot'
plot_scene.sensor_type = 'plot'
plot_scene.status = 'submitted'
plot_scene.save()
scenes = Order.find(order_id).scenes({
'name': 'plot',
'status': 'submitted'
})
response = production_provider.handle_submitted_plot_products(scenes)
pscene = order.scenes({'status': 'oncache', 'sensor_type': 'plot'})
self.assertTrue(response is True)
self.assertEqual(len(pscene), 1)
def test_production_set_product_retry(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
order = Order.find(order_id)
scene = order.scenes()[3]
scene.update('retry_count', 4)
processing_loc = "get_products_to_process"
error = 'not available after EE call '
note = 'note this'
retry_after = datetime.datetime.now() + datetime.timedelta(hours=1)
retry_limit = 9
response = production_provider.set_product_retry(
scene.name, order.orderid, processing_loc, error, note,
retry_after, retry_limit)
new = Scene.get('ordering_scene.status', scene.name, order.orderid)
self.assertTrue('retry' == new)
@patch('api.external.lta.update_order_status',
mock_production_provider.respond_true)
def test_production_set_product_error_unavailable(self):
"""
Move a scene status from error to unavailable based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes({'name !=': 'plot'})[0]
production_provider.set_product_error(
scene.name, order.orderid, 'get_products_to_process',
'include_dswe is an unavailable product option for OLITIRS')
self.assertTrue('unavailable' == Scene.get('ordering_scene.status',
scene.name, order.orderid))
def test_production_set_product_error_submitted(self):
"""
Move a scene status from error to submitted based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes({'name !=': 'plot'})[0]
production_provider.set_product_error(
scene.name, order.orderid, 'get_products_to_process',
'BLOCK, COMING FROM LST AS WELL: No such file or directory')
self.assertTrue('submitted' == Scene.get('ordering_scene.status',
scene.name, order.orderid))
def test_production_set_product_error_retry(self):
"""
Move a scene status from error to retry based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes()[2]
production_provider.set_product_error(
scene.name, order.orderid, 'somewhere',
'Verify the missing auxillary data products')
self.assertTrue('retry' == Scene.get('ordering_scene.status',
scene.name, order.orderid))
def test_production_set_product_error_retry_lasrc_segfault(self):
"""
Move a scene status from error to retry based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes({'sensor_type': 'landsat'})[-1]
production_provider.set_product_error(
scene.name, order.orderid, 'somewhere',
'runSr sh: line 1: 1010 Segmentation fault lasrc --xml=')
self.assertTrue('retry' == Scene.get('ordering_scene.status',
scene.name, order.orderid))
def test_production_set_product_error_unavail_reproject(self):
"""
Move a scene status from error to retry based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes({'sensor_type': 'landsat'})[-1]
log_file_contents = ('BLAH BLAH BLAH WarpVerificationError: Failed to '
'compute statistics, no valid pixels found in '
'sampling BLAH BLAH BLAH')
production_provider.set_product_error(scene.name, order.orderid,
'somewhere', log_file_contents)
self.assertEqual(
'unavailable',
Scene.get('ordering_scene.status', scene.name, order.orderid))
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch(
'api.providers.production.production_provider.ProductionProvider.set_product_retry',
mock_production_provider.set_product_retry)
def test_update_product_details_update_status(self):
"""
Set a scene status to Queued
"""
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes()[0]
api.update_product_details(
'update_status', {
'name': scene.name,
'orderid': order.orderid,
'processing_loc': 'L8SRLEXAMPLE',
'status': 'Queued'
})
self.assertTrue(
Scene.get('ordering_scene.status', scene.name, order.orderid) ==
'Queued')
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch(
'api.providers.production.production_provider.ProductionProvider.set_product_retry',
mock_production_provider.set_product_retry)
@patch('api.external.onlinecache.capacity', onlinecache.capacity)
def test_update_product_details_set_product_error(self):
"""
Set a scene status to error
:return:
"""
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes()[0]
production_provider.update_product('set_product_error',
name=scene.name,
orderid=order.orderid,
processing_loc="L8SRLEXAMPLE",
error='problems yo')
self.assertTrue(Scene.find(scene.id).status == 'error')
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch(
'api.providers.production.production_provider.ProductionProvider.set_product_retry',
mock_production_provider.set_product_retry)
def test_update_product_details_set_product_unavailable(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes()[0]
production_provider.update_product(
'set_product_unavailable',
name=scene.name,
orderid=order.orderid,
processing_loc="L8SRLEXAMPLE",
error='include_dswe is an unavailable product option for OLITIRS')
self.assertTrue('unavailable' == Scene.get('ordering_scene.status',
scene.name, order.orderid))
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch(
'api.providers.production.production_provider.ProductionProvider.set_product_retry',
mock_production_provider.set_product_retry)
@patch('os.path.getsize', lambda y: 999)
def test_update_product_details_mark_product_complete(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes()[0]
production_provider.update_product('mark_product_complete',
name=scene.name,
orderid=order.orderid,
processing_loc='L8SRLEXAMPLE',
completed_file_location='/some/loc',
cksum_file_location='some checksum',
log_file_contents='some log')
self.assertTrue('complete' == Scene.get('ordering_scene.status',
scene.name, order.orderid))
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch(
'api.providers.production.production_provider.ProductionProvider.set_product_retry',
mock_production_provider.set_product_retry)
@patch('os.path.getsize', lambda y: 999)
def test_update_product_details_mark_product_processing(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes()[0]
res = production_provider.update_product('update_status',
name=scene.name,
orderid=order.orderid,
processing_loc='L8SRLEXAMPLE',
status='processing')
order = Order.find(order.id)
scene = order.scenes({'id': scene.id})[0]
self.assertEqual('processing', scene.status)
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch(
'api.providers.production.production_provider.ProductionProvider.set_product_retry',
mock_production_provider.set_product_retry)
@patch('os.path.getsize', lambda y: 999)
def test_update_product_details_mark_cancelled_product_processing(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
order.status = 'cancelled'
order.save()
scene = order.scenes()[0]
res = production_provider.update_product('update_status',
name=scene.name,
orderid=order.orderid,
processing_loc='L8SRLEXAMPLE',
status='processing')
self.assertFalse(res)
def test_production_set_product_error_unavailable_night(self):
"""
Move a scene status from error to unavailable based on the solar zenith (TOA)
error message
"""
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes({'name !=': 'plot'})[0]
production_provider.set_product_error(
name=scene.name,
orderid=order.orderid,
processing_loc='L8SRLEXAMPLE',
error='solar zenith angle out of range')
scene = Scene.by_name_orderid(name=scene.name, order_id=order.id)
self.assertTrue('unavailable' == scene.status)
self.assertTrue(
'Solar zenith angle out of range, cannot process night scene' in
scene.note)
def test_production_set_product_error_unavailable_almost_night(self):
"""
Move a scene status from error to unavailable based on the solar zenith (SR)
error message
"""
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes({'name !=': 'plot'})[0]
production_provider.set_product_error(
name=scene.name,
orderid=order.orderid,
processing_loc='L8SRLEXAMPLE',
error='solar zenith angle is too large')
scene = Scene.by_name_orderid(name=scene.name, order_id=order.id)
self.assertTrue('unavailable' == scene.status)
self.assertTrue(
'Solar zenith angle is too large, cannot process scene to SR' in
scene.note)
@patch('api.external.lta.update_order_status',
lta.update_order_status_fail)
@patch(
'api.providers.production.production_provider.ProductionProvider.set_product_retry',
mock_production_provider.set_product_retry)
@patch('os.path.getsize', lambda y: 999)
def test_update_product_details_fail_lta_mark_product_complete(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes()[1]
order.update('order_source', 'ee')
production_provider.update_product('mark_product_complete',
name=scene.name,
orderid=order.orderid,
processing_loc='L8SRLEXAMPLE',
completed_file_location='/some/loc',
cksum_file_location='some checksum',
log_file_contents='some log')
s = Scene.where({'name': scene.name, 'order_id': scene.order_id})[0]
self.assertTrue('C' == s.failed_lta_status_update)
@patch(
'api.providers.production.production_provider.ProductionProvider.send_initial_emails',
mock_production_provider.respond_true)
@patch(
'api.providers.production.production_provider.ProductionProvider.handle_onorder_landsat_products',
mock_production_provider.respond_true)
@patch(
'api.providers.production.production_provider.ProductionProvider.handle_retry_products',
mock_production_provider.respond_true)
@patch(
'api.providers.production.production_provider.ProductionProvider.load_ee_orders',
mock_production_provider.respond_true)
@patch(
'api.providers.production.production_provider.ProductionProvider.finalize_orders',
mock_production_provider.respond_true)
@patch(
'api.providers.production.production_provider.ProductionProvider.purge_orders',
mock_production_provider.respond_true)
def test_handle_orders_success(self):
_ = self.mock_order.generate_testing_order(self.user_id)
self.assertTrue(
api.handle_orders({'username': User.find(self.user_id)}))
@patch('api.external.onlinecache.delete',
mock_production_provider.respond_true)
@patch('api.notification.emails.send_purge_report',
mock_production_provider.respond_true)
@patch('api.external.onlinecache.capacity', onlinecache.mock_capacity)
@patch('api.external.onlinecache.exists', onlinecache.mock_exists)
@patch('api.external.onlinecache.delete', onlinecache.mock_delete)
def test_production_purge_orders(self):
new_completion_date = datetime.datetime.now() - datetime.timedelta(
days=12)
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
order.update('status', 'complete')
order.update('completion_date', new_completion_date)
self.assertTrue(production_provider.purge_orders())
# need to figure a test for emails.send_email
@patch('api.notification.emails.Emails.send_email',
mock_production_provider.respond_true)
def test_production_send_initial_emails(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
order.update('status', 'ordered')
self.assertTrue(emails.Emails().send_all_initial([order]))
@patch('api.external.lta.get_order_status', lta.get_order_status)
@patch('api.external.lta.update_order_status', lta.update_order_status)
def test_production_handle_onorder_landsat_products(self):
tram_order_ids = lta.sample_tram_order_ids()[0:3]
scene_names = lta.sample_scene_names()[0:3]
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scenes = order.scenes()[0:3]
for idx, scene in enumerate(scenes):
scene.tram_order_id = tram_order_ids[idx]
scene.status = 'onorder'
# save() doesn't let you update name,
# b/c updating a scene name is not acceptable
# outside of testing
scene.update('name', scene_names[idx])
scene.save()
self.assertTrue(
production_provider.handle_onorder_landsat_products(scenes))
def test_production_handle_retry_products(self):
prev = datetime.datetime.now() - datetime.timedelta(hours=1)
order_id = self.mock_order.generate_testing_order(self.user_id)
self.mock_order.update_scenes(order_id, 'landsat', 'status', ['retry'])
self.mock_order.update_scenes(order_id, 'landsat', 'retry_after',
[prev])
scenes = Order.find(order_id).scenes({'status': 'retry'})
production_provider.handle_retry_products(scenes)
for s in Scene.where({'order_id': order_id, 'sensor_type': 'landsat'}):
self.assertTrue(s.status == 'submitted')
@patch('api.external.lta.get_available_orders',
lta.get_available_orders_partial)
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch('api.external.inventory.get_user_name', inventory.get_user_name)
@patch('api.external.inventory.get_session', lambda: True)
def test_production_load_ee_orders_partial(self):
order = Order.find(
self.mock_order.generate_ee_testing_order(self.user_id,
partial=True))
self.assertEqual(
order.product_opts, {
'format': 'gtiff',
'etm7_collection': {
'inputs': ['LE07_L1TP_026027_20170912_20171008_01_T1'],
'products': ['sr']
}
})
key = 'system.load_ee_orders_enabled'
self.assertEqual(api.get_production_key(key)[key], 'True')
production_provider.load_ee_orders()
reorder = Order.find(order.id)
self.assertEqual(
reorder.product_opts, {
'format': 'gtiff',
'etm7_collection': {
'inputs': ['LE07_L1TP_026027_20170912_20171008_01_T1'],
'products': ['sr']
},
'tm5_collection': {
'inputs': ['LT05_L1TP_025027_20110913_20160830_01_T1'],
'products': ['sr']
}
})
@patch('api.external.lta.update_order_status', lta.update_order_status)
def test_production_handle_failed_ee_updates(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
for scene in order.scenes():
scene.update('failed_lta_status_update', 'C')
scenes = Order.find(order.id).scenes(
{'failed_lta_status_update IS NOT': None})
production_provider.handle_failed_ee_updates(scenes)
scenes = Scene.where({'failed_lta_status_update IS NOT': None})
self.assertTrue(len(scenes) == 0)
@patch(
'api.providers.production.production_provider.ProductionProvider.update_landsat_product_status',
mock_production_provider.respond_true)
@patch(
'api.providers.production.production_provider.ProductionProvider.get_contactids_for_submitted_landsat_products',
mock_production_provider.contact_ids_list)
@patch('api.external.inventory.available', lambda: True)
def test_production_handle_submitted_landsat_products(self):
orders = Order.find(
self.mock_order.generate_testing_order(self.user_id))
scenes = orders.scenes({'sensor_type': 'landsat'})
self.assertTrue(
production_provider.handle_submitted_landsat_products(scenes))
@patch('api.external.lta.update_order_status', lta.update_order_status)
def test_production_set_products_unavailable(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
self.assertTrue(
production_provider.set_products_unavailable(
order.scenes(), "you want a reason?"))
@patch(
'api.providers.production.production_provider.ProductionProvider.check_dependencies_for_products',
mock_production_provider.check_dependencies_for_products)
@patch('api.external.inventory.get_cached_session',
inventory.get_cached_session)
@patch('api.external.inventory.check_valid', inventory.check_valid_landsat)
def test_production_update_landsat_product_status(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
for scene in order.scenes({'name !=': 'plot'}):
scene.status = 'submitted'
scene.sensor_type = 'landsat'
user = User.find(self.user_id)
scene.save()
self.assertTrue(
production_provider.update_landsat_product_status(user.contactid))
def test_production_get_contactids_for_submitted_landsat_products(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
for scene in order.scenes({'name !=': 'plot'}):
scene.status = 'submitted'
scene.sensor_type = 'landsat'
scene.save()
scenes = Order.find(order.id).scenes({'sensor_type': 'landsat'})
response = production_provider.get_contactids_for_submitted_landsat_products(
scenes)
self.assertIsInstance(response, set)
self.assertTrue(len(response) > 0)
@patch('api.external.inventory.available', lambda: True)
@patch('api.external.inventory.get_cached_session',
inventory.get_cached_session)
@patch('api.external.inventory.check_valid', inventory.check_valid_modis)
def test_production_handle_submitted_modis_products_input_exists(self):
# handle oncache scenario
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
for scene in order.scenes({'name !=': 'plot'}):
scene.status = 'submitted'
scene.sensor_type = 'modis'
scene.save()
sid = scene.id
scenes = order.scenes({'sensor_type': 'modis'})
self.assertTrue(
production_provider.handle_submitted_modis_products(scenes))
#self.assertEquals(Scene.find(sid).status, "oncache")
@patch('api.external.inventory.available', lambda: True)
@patch('api.external.inventory.get_cached_session',
inventory.get_cached_session)
@patch('api.external.inventory.check_valid', inventory.check_valid_modis)
def test_production_handle_submitted_modis_products_input_missing(self):
# handle unavailable scenario
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
for scene in order.scenes({'name !=': 'plot'}):
scene.status = 'submitted'
scene.sensor_type = 'modis'
scene.save()
sid = scene.id
scenes = order.scenes({'sensor_type': 'modis'})
self.assertTrue(
production_provider.handle_submitted_modis_products(scenes))
self.assertEquals(Scene.find(sid).status, "unavailable")
def test_production_handle_submitted_plot_products(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
order.status = 'ordered'
order.order_type = 'lpcs'
order.save()
plot_id = None
for idx, scene in enumerate(order.scenes()):
# at the moment, mock_order.generate_testing_order
# creates 21 products for the order. divvy those
# up between 'complete' and 'unavailable', setting
# one aside as the 'plot' product
if scene.sensor_type == 'plot':
# need to define a plot product
scene.update('status', 'submitted')
plot_id = scene.id
else:
if idx % 2 == 0:
scene.update('status', 'complete')
else:
scene.update('status', 'unavailable')
scenes = order.scenes()
self.assertTrue(
production_provider.handle_submitted_plot_products(scenes))
self.assertEqual(Scene.find(plot_id).status, "oncache")
@patch('os.path.exists', lambda y: True)
@patch('os.path.getsize', lambda y: 999)
def test_production_calc_scene_download_sizes(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scenes = order.scenes()
Scene.bulk_update([s.id for s in scenes], {
'status': 'complete',
'download_size': 0
})
scenes = Order.find(order.id).scenes({
'status': 'complete',
'download_size': 0
})
self.assertTrue(production_provider.calc_scene_download_sizes(scenes))
upscenes = Scene.where({'status': 'complete', 'download_size': 999})
self.assertEqual(len(upscenes), len(scenes))
@patch(
'api.providers.production.production_provider.ProductionProvider.update_order_if_complete',
mock_production_provider.respond_true)
def test_production_finalize_orders(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
order.update('status', 'ordered')
self.assertTrue(production_provider.finalize_orders([order]))
@patch(
'api.providers.production.production_provider.ProductionProvider.send_completion_email',
mock_production_provider.respond_true)
def test_production_update_order_if_complete(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
Scene.bulk_update([s.id for s in order.scenes()], {'status': 'retry'})
order.order_source = 'espa'
order.completion_email_sent = None
order.save()
self.assertTrue(production_provider.update_order_if_complete(order))
def test_production_queue_products_success(self):
names_tuple = self.mock_order.names_tuple(3, self.user_id)
processing_loc = "get_products_to_process"
job_name = 'jobname49'
params = (names_tuple, processing_loc, job_name)
response = api.queue_products(*params)
self.assertTrue(response)
def test_production_get_key(self):
key = 'system_message_title'
response = api.get_production_key(key)
val = response[key]
self.assertIsInstance(val, str)
def test_get_production_key_invalid(self):
bad_key = 'foobar'
response = api.get_production_key(bad_key)
self.assertEqual(response.keys(), ['msg'])
@patch('api.external.hadoop.HadoopHandler.job_names_ids',
hadoop.jobs_names_ids)
def test_catch_orphaned_scenes(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
# need scenes with statuses of 'queued'
self.mock_order.update_scenes(order_id, ('landsat', 'modis', 'plot'),
'status', ['queued'])
response = production_provider.catch_orphaned_scenes()
self.assertTrue(response)
old_time = datetime.datetime.now() - datetime.timedelta(minutes=15)
for s in Scene.where({'order_id': order_id}):
self.assertTrue(s.reported_orphan is not None)
s.reported_orphan = old_time
s.save()
response = production_provider.catch_orphaned_scenes()
self.assertTrue(response)
for s in Scene.where({'order_id': order_id}):
self.assertTrue(s.orphaned)
@patch('api.external.hadoop.HadoopHandler.job_names_ids',
hadoop.jobs_names_ids)
def test_handle_stuck_jobs(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
# Make some really old jobs
self.mock_order.update_scenes(order_id, ('landsat', 'modis', 'plot'),
'status', ['processing'])
self.mock_order.update_scenes(order_id, ('landsat', 'modis', 'plot'),
'status_modified',
[datetime.datetime(1900, 1, 1)])
scenes = Scene.where({'status': 'processing', 'order_id': order_id})
n_scenes = len(scenes)
response = production_provider.handle_stuck_jobs(scenes)
self.assertTrue(response)
scenes = Scene.where({
'status': 'processing',
'order_id': order_id,
'reported_orphan is not': None
})
self.assertEqual(n_scenes, len(scenes))
self.mock_order.update_scenes(order_id, ('landsat', 'modis', 'plot'),
'reported_orphan',
[datetime.datetime(1900, 1, 1)])
response = production_provider.handle_stuck_jobs(scenes)
self.assertTrue(response)
scenes = Scene.where({'status': 'processing', 'order_id': order_id})
self.assertEqual(0, len(scenes))
def test_convert_product_options(self):
"""
Test the conversion procedure to make sure that the new format for orders converts
to the old format
"""
scenes = [
'LE07_L1TP_026027_20170912_20171008_01_T1',
'LC08_L1TP_025027_20160521_20170223_01_T1',
'LT05_L1TP_025027_20110913_20160830_01_T1'
]
includes = ['include_sr', 'include_sr_toa', 'include_sr_thermal']
new_format = {
u'etm7_collection': {
u'inputs': [u'LE07_L1TP_026027_20170912_20171008_01_T1'],
u'products': [u'sr']
},
u'olitirs8_collection': {
u'inputs': [u'LC08_L1TP_025027_20160521_20170223_01_T1'],
u'products': [u'toa']
},
u'tm5_collection': {
u'inputs': [u'LT05_L1TP_025027_20110913_20160830_01_T1'],
u'products': [u'bt']
},
u'format': u'gtiff',
u'image_extents': {
u'east': -2265585.0,
u'north': 3164805.0,
u'south': 3014805.0,
u'units': u'meters',
u'west': -2415585.0
},
u'note': u'CONUS_h1v1',
u'projection': {
u'aea': {
u'central_meridian': -96,
u'datum': u'nad83',
u'false_easting': 0,
u'false_northing': 0,
u'latitude_of_origin': 23,
u'standard_parallel_1': 29.5,
u'standard_parallel_2': 45.5
}
},
u'resampling_method': u'cc'
}
ruberic = {
'central_meridian': -96,
'datum': u'nad83',
'false_easting': 0,
'false_northing': 0,
'image_extents': True,
'image_extents_units': u'meters',
'include_customized_source_data': False,
'include_dswe': False,
'include_st': False,
'include_solr_index': False,
'include_source_data': False,
'include_source_metadata': False,
'include_sr': False,
'include_sr_browse': False,
'include_sr_evi': False,
'include_sr_msavi': False,
'include_sr_nbr': False,
'include_sr_nbr2': False,
'include_sr_ndmi': False,
'include_sr_ndvi': False,
'include_sr_savi': False,
'include_sr_thermal': False,
'include_sr_toa': False,
'include_statistics': False,
'latitude_true_scale': None,
'longitude_pole': None,
'maxx': -2265585.0,
'maxy': 3164805.0,
'minx': -2415585.0,
'miny': 3014805.0,
'origin_lat': 23,
'output_format': u'gtiff',
'pixel_size': None,
'pixel_size_units': None,
'reproject': True,
'resample_method': 'cubic',
'resize': False,
'std_parallel_1': 29.5,
'std_parallel_2': 45.5,
'target_projection': u'aea',
'utm_north_south': None,
'utm_zone': None
}
for scene, include in zip(scenes, includes):
ruberic[include] = True
old_format = OptionsConversion.convert(new=new_format,
scenes=[scene])
self.assertDictEqual(ruberic, old_format)
ruberic[include] = False
def test_status_modified(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
scene = Scene.where({'order_id': order_id}).pop()
scene.status = 'oncache'
scene.save()
old_time = scene.status_modified
scene.status = 'queued'
scene.save()
new_time = scene.status_modified
self.assertGreater(new_time, old_time)
@patch('api.external.hadoop.HadoopHandler.list_jobs',
hadoop.jobs_names_ids)
@patch('api.external.hadoop.HadoopHandler.kill_job', lambda x, y: True)
def test_hadoop_reset_status(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
scenes = Scene.where({'order_id': order_id})
Scene.bulk_update([s.id for s in scenes], {'status': 'processing'})
self.assertTrue(production_provider.reset_processing_status())
scenes = Scene.where({'order_id': order_id})
self.assertEqual({'submitted'}, set([s.status for s in scenes]))
class TestProductionAPI(unittest.TestCase):
def setUp(self):
os.environ['espa_api_testing'] = 'True'
# create a user
self.mock_user = MockUser()
self.mock_order = MockOrder()
self.user_id = self.mock_user.add_testing_user()
def tearDown(self):
# clean up orders
self.mock_order.tear_down_testing_orders()
# clean up users
self.mock_user.cleanup()
os.environ['espa_api_testing'] = ''
@patch('api.external.lpdaac.get_download_urls', lpdaac.get_download_urls)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry',
mock_production_provider.set_product_retry)
def test_fetch_production_products_modis(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
# need scenes with statuses of 'processing' and 'ordered'
self.mock_order.update_scenes(order_id, 'status', ['processing', 'ordered', 'oncache'])
user = User.find(self.user_id)
params = {'for_user': user.username, 'product_types': ['modis']}
response = api.fetch_production_products(params)
self.assertTrue('bilbo' in response[0]['orderid'])
@patch('api.external.lta.get_download_urls', lta.get_download_urls)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry',
mock_production_provider.set_product_retry)
def test_fetch_production_products_landsat(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
# need scenes with statuses of 'processing' and 'ordered'
self.mock_order.update_scenes(order_id, 'status', ['processing','oncache','ordered'])
user = User.find(self.user_id)
params = {'for_user': user.username, 'product_types': ['landsat']}
response = api.fetch_production_products(params)
self.assertTrue('bilbo' in response[0]['orderid'])
def test_fetch_production_products_plot(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
self.mock_order.update_scenes(order_id, 'status', ['complete'])
order = Order.find(order_id)
plot_scene = order.scenes()[0]
plot_scene.name = 'plot'
plot_scene.sensor_type = 'plot'
plot_scene.status = 'submitted'
plot_scene.save()
response = production_provider.handle_submitted_plot_products()
pscene = order.scenes({'status': 'oncache', 'sensor_type': 'plot'})
self.assertTrue(response is True)
self.assertEqual(len(pscene), 1)
def test_production_set_product_retry(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
order = Order.find(order_id)
scene = order.scenes()[3]
scene.update('retry_count', 4)
processing_loc = "get_products_to_process"
error = 'not available after EE call '
note = 'note this'
retry_after = datetime.datetime.now() + datetime.timedelta(hours=1)
retry_limit = 9
response = production_provider.set_product_retry(scene.name, order.orderid, processing_loc,
error, note, retry_after, retry_limit)
new = Scene.get('ordering_scene.status', scene.name, order.orderid)
self.assertTrue('retry' == new)
@patch('api.external.lta.update_order_status', mock_production_provider.respond_true)
def test_production_set_product_error_unavailable(self):
"""
Move a scene status from error to unavailable based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes({'name !=': 'plot'})[0]
production_provider.set_product_error(scene.name, order.orderid,
'get_products_to_process',
'include_dswe is an unavailable product option for OLITIRS')
self.assertTrue('unavailable' == Scene.get('ordering_scene.status', scene.name, order.orderid))
def test_production_set_product_error_submitted(self):
"""
Move a scene status from error to submitted based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes({'name !=': 'plot'})[0]
production_provider.set_product_error(scene.name, order.orderid,
'get_products_to_process',
'BLOCK, COMING FROM LST AS WELL: No such file or directory')
self.assertTrue('submitted' == Scene.get('ordering_scene.status', scene.name, order.orderid))
def test_production_set_product_error_retry(self):
"""
Move a scene status from error to retry based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[2]
production_provider.set_product_error(scene.name, order.orderid,
'somewhere',
'Verify the missing auxillary data products')
self.assertTrue('retry' == Scene.get('ordering_scene.status', scene.name, order.orderid))
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
def test_update_product_details_update_status(self):
"""
Set a scene status to Queued
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[0]
api.update_product_details('update_status',
{'name': scene.name,
'orderid': order.orderid,
'processing_loc': 'L8SRLEXAMPLE',
'status': 'Queued'})
self.assertTrue(Scene.get('ordering_scene.status', scene.name, order.orderid) == 'Queued')
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
@patch('api.external.onlinecache.capacity', onlinecache.capacity)
def test_update_product_details_set_product_error(self):
"""
Set a scene status to error
:return:
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[0]
production_provider.update_product('set_product_error',
name=scene.name, orderid=order.orderid,
processing_loc="L8SRLEXAMPLE",
error='problems yo')
self.assertTrue(Scene.find(scene.id).status == 'error')
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
def test_update_product_details_set_product_unavailable(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[0]
production_provider.update_product('set_product_unavailable',
name=scene.name, orderid=order.orderid,
processing_loc="L8SRLEXAMPLE",
error='include_dswe is an unavailable product option for OLITIRS')
self.assertTrue('unavailable' == Scene.get('ordering_scene.status', scene.name, order.orderid))
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
def test_update_product_details_mark_product_complete(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[0]
production_provider.update_product('mark_product_complete',
name=scene.name,
orderid=order.orderid,
processing_loc='L8SRLEXAMPLE',
completed_file_location='/some/loc',
cksum_file_location='some checksum',
log_file_contents='some log')
self.assertTrue('complete' == Scene.get('ordering_scene.status', scene.name, order.orderid))
@patch('api.external.lta.update_order_status', lta.update_order_status_fail)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
def test_update_product_details_fail_lta_mark_product_complete(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[1]
order.update('order_source', 'ee')
production_provider.update_product('mark_product_complete',
name=scene.name,
orderid=order.orderid,
processing_loc='L8SRLEXAMPLE',
completed_file_location='/some/loc',
cksum_file_location='some checksum',
log_file_contents='some log')
s = Scene.where({'name': scene.name, 'order_id': scene.order_id})[0]
self.assertTrue('C' == s.failed_lta_status_update)
@patch('api.providers.production.production_provider.ProductionProvider.send_initial_emails',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.handle_onorder_landsat_products',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.handle_retry_products',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.load_ee_orders',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.handle_submitted_products',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.finalize_orders',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.purge_orders',
mock_production_provider.respond_true)
def test_handle_orders_success(self):
self.assertTrue(api.handle_orders())
@patch('api.external.onlinecache.delete', mock_production_provider.respond_true)
@patch('api.notification.emails.send_purge_report', mock_production_provider.respond_true)
@patch('api.external.onlinecache.capacity', onlinecache.capacity)
def test_production_purge_orders(self):
new_completion_date = datetime.datetime.now() - datetime.timedelta(days=12)
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
order.update('status', 'complete')
order.update('completion_date', new_completion_date)
self.assertTrue(production_provider.purge_orders())
# need to figure a test for emails.send_email
@patch('api.notification.emails.Emails.send_email', mock_production_provider.respond_true)
def test_production_send_initial_emails(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
order.update('status', 'ordered')
self.assertTrue(emails.Emails().send_all_initial())
@patch('api.external.lta.get_order_status', lta.get_order_status)
@patch('api.external.lta.update_order_status', lta.update_order_status)
def test_production_handle_onorder_landsat_products(self):
tram_order_ids = lta.sample_tram_order_ids()[0:3]
scene_names = lta.sample_scene_names()[0:3]
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scenes = order.scenes()[0:3]
for idx, scene in enumerate(scenes):
scene.tram_order_id = tram_order_ids[idx]
scene.status = 'onorder'
# save() doesn't let you update name,
# b/c updating a scene name is not acceptable
# outside of testing
scene.update('name', scene_names[idx])
scene.save()
self.assertTrue(production_provider.handle_onorder_landsat_products())
def test_production_handle_retry_products(self):
prev = datetime.datetime.now() - datetime.timedelta(hours=1)
order_id = self.mock_order.generate_testing_order(self.user_id)
self.mock_order.update_scenes(order_id, 'status', ['retry'])
self.mock_order.update_scenes(order_id, 'retry_after', [prev])
production_provider.handle_retry_products()
for s in Scene.where({'order_id': order_id}):
self.assertTrue(s.status == 'submitted')
#@patch('api.external.lta.get_available_orders', lta.get_available_orders)
#@patch('api.external.lta.update_order_status', lta.update_order_status)
#@patch('api.external.lta.get_user_name', lta.get_user_name)
#def test_production_load_ee_orders(self):
# #production_provider.load_ee_orders()
# pass
@patch('api.external.lta.get_available_orders', lta.get_available_orders_partial)
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch('api.external.lta.get_user_name', lta.get_user_name)
def test_production_load_ee_orders_partial(self):
order = Order.find(self.mock_order.generate_ee_testing_order(self.user_id, partial=True))
self.assertTrue(order.product_opts == {'format': 'gtiff',
'etm7': {'inputs': ['LE70900652008327EDC00'],
'products': ['sr']}})
production_provider.load_ee_orders()
reorder = Order.find(order.id)
self.assertTrue(reorder.product_opts == {'format': 'gtiff',
'etm7': {'inputs': ['LE70900652008327EDC00'],
'products': ['sr']},
'tm5': {'inputs': ['LT50900652008327EDC00'],
'products': ['sr']}})
@patch('api.external.lta.update_order_status', lta.update_order_status)
def test_production_handle_failed_ee_updates(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.update('failed_lta_status_update', 'C')
production_provider.handle_failed_ee_updates()
scenes = Scene.where({'failed_lta_status_update IS NOT': None})
self.assertTrue(len(scenes) == 0)
@patch('api.providers.production.production_provider.ProductionProvider.handle_submitted_landsat_products',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.handle_submitted_modis_products',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.handle_submitted_plot_products',
mock_production_provider.respond_true)
def test_production_handle_submitted_products(self):
self.assertTrue(production_provider.handle_submitted_products())
@patch('api.providers.production.production_provider.ProductionProvider.mark_nlaps_unavailable',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.update_landsat_product_status',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.get_contactids_for_submitted_landsat_products',
mock_production_provider.contact_ids_list)
def test_production_handle_submitted_landsat_products(self):
self.assertTrue(production_provider.handle_submitted_landsat_products())
# !!! need to write test for nlaps.products_are_nlaps !!!
@patch('api.external.nlaps.products_are_nlaps', nlaps.products_are_nlaps)
@patch('api.providers.production.production_provider.ProductionProvider.set_products_unavailable',
mock_production_provider.respond_true)
def test_production_mark_nlaps_unavailable(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.status = 'submitted'
scene.sensor_type = 'landsat'
scene.save()
self.assertTrue(production_provider.mark_nlaps_unavailable())
@patch('api.external.lta.update_order_status', lta.update_order_status)
def test_production_set_products_unavailable(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
self.assertTrue(production_provider.set_products_unavailable(order.scenes(), "you want a reason?"))
@patch('api.external.lta.order_scenes', lta.order_scenes)
@patch('api.providers.production.production_provider.ProductionProvider.set_products_unavailable',
mock_production_provider.respond_true)
def test_production_update_landsat_product_status(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.status = 'submitted'
scene.sensor_type = 'landsat'
scene.save()
self.assertTrue(production_provider.update_landsat_product_status(User.find(self.user_id).contactid))
def test_production_get_contactids_for_submitted_landsat_products(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.status = 'submitted'
scene.sensor_type = 'landsat'
scene.save()
response = production_provider.get_contactids_for_submitted_landsat_products()
self.assertIsInstance(response, set)
self.assertTrue(len(response) > 0)
@patch('api.external.lpdaac.input_exists', lpdaac.input_exists_true)
def test_production_handle_submitted_modis_products_input_exists(self):
# handle oncache scenario
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.status = 'submitted'
scene.sensor_type = 'modis'
scene.save()
sid = scene.id
self.assertTrue(production_provider.handle_submitted_modis_products())
self.assertEquals(Scene.find(sid).status, "oncache")
@patch('api.external.lpdaac.input_exists', lpdaac.input_exists_false)
def test_production_handle_submitted_modis_products_input_missing(self):
# handle unavailable scenario
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.status = 'submitted'
scene.sensor_type = 'modis'
scene.save()
sid = scene.id
self.assertTrue(production_provider.handle_submitted_modis_products())
self.assertEquals(Scene.find(sid).status, "unavailable")
def test_production_handle_submitted_plot_products(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
order.status = 'ordered'
order.order_type = 'lpcs'
order.save()
plot_id = None
for idx, scene in enumerate(order.scenes()):
# at the moment, mock_order.generate_testing_order
# creates 21 products for the order. divvy those
# up between 'complete' and 'unavailable', setting
# one aside as the 'plot' product
if idx % 2 == 0:
if idx == 0:
# need to define a plot product
scene.update('status', 'submitted')
scene.update('sensor_type', 'plot')
plot_id = scene.id
else:
scene.update('status', 'complete')
else:
scene.update('status', 'unavailable')
self.assertTrue(production_provider.handle_submitted_plot_products())
self.assertEqual(Scene.find(plot_id).status, "oncache")
@patch('api.providers.production.production_provider.ProductionProvider.update_order_if_complete',
mock_production_provider.respond_true)
def test_production_finalize_orders(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
order.update('status', 'ordered')
self.assertTrue(production_provider.finalize_orders())
@patch('api.providers.production.production_provider.ProductionProvider.send_completion_email',
mock_production_provider.respond_true)
def test_production_update_order_if_complete(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
Scene.bulk_update([s.id for s in order.scenes()], {'status': 'retry'})
order.order_source = 'espa'
order.completion_email_sent = None
order.save()
self.assertTrue(production_provider.update_order_if_complete(order))
def test_production_queue_products_success(self):
names_tuple = self.mock_order.names_tuple(3, self.user_id)
processing_loc = "get_products_to_process"
job_name = 'jobname49'
params = (names_tuple, processing_loc, job_name)
response = api.queue_products(*params)
self.assertTrue(response)
def test_production_get_key(self):
key = 'system_message_title'
response = api.get_production_key(key)
val = response[key]
self.assertIsInstance(val, str)
def test_get_production_key_invalid(self):
bad_key = 'foobar'
response = api.get_production_key(bad_key)
self.assertEqual(response.keys(), ['msg'])
@patch('api.external.hadoop.HadoopHandler.job_names_ids',
hadoop.jobs_names_ids)
def test_catch_orphaned_scenes(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
# need scenes with statuses of 'queued'
self.mock_order.update_scenes(order_id, 'status', ['queued'])
response = production_provider.catch_orphaned_scenes()
self.assertTrue(response)
old_time = datetime.datetime.now() - datetime.timedelta(minutes=15)
for s in Scene.where({'order_id': order_id}):
self.assertTrue(s.reported_orphan is not None)
s.reported_orphan = old_time
s.save()
response = production_provider.catch_orphaned_scenes()
self.assertTrue(response)
for s in Scene.where({'order_id': order_id}):
self.assertTrue(s.orphaned)
class TestProductionAPI(unittest.TestCase):
def setUp(self):
os.environ['espa_api_testing'] = 'True'
# create a user
self.mock_user = MockUser()
self.mock_order = MockOrder()
self.user_id = self.mock_user.add_testing_user()
def tearDown(self):
# clean up orders
self.mock_order.tear_down_testing_orders()
# clean up users
self.mock_user.cleanup()
os.environ['espa_api_testing'] = ''
@patch('api.external.lpdaac.get_download_urls', lpdaac.get_download_urls)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry',
mock_production_provider.set_product_retry)
def test_fetch_production_products_modis(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
# need scenes with statuses of 'processing' and 'ordered'
self.mock_order.update_scenes(order_id, 'status', ['processing', 'ordered', 'oncache'])
user = User.find(self.user_id)
params = {'for_user': user.username, 'product_types': ['modis']}
response = api.fetch_production_products(params)
self.assertTrue('bilbo' in response[0]['orderid'])
@patch('api.external.lta.get_download_urls', lta.get_download_urls)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry',
mock_production_provider.set_product_retry)
def test_fetch_production_products_landsat(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
# need scenes with statuses of 'processing' and 'ordered'
self.mock_order.update_scenes(order_id, 'status', ['processing','oncache','ordered'])
user = User.find(self.user_id)
params = {'for_user': user.username, 'product_types': ['landsat']}
response = api.fetch_production_products(params)
self.assertTrue('bilbo' in response[0]['orderid'])
def test_fetch_production_products_plot(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
self.mock_order.update_scenes(order_id, 'status', ['complete'])
order = Order.find(order_id)
plot_scene = order.scenes()[0]
plot_scene.name = 'plot'
plot_scene.sensor_type = 'plot'
plot_scene.status = 'submitted'
plot_scene.save()
response = production_provider.handle_submitted_plot_products()
pscene = order.scenes({'status': 'oncache', 'sensor_type': 'plot'})
self.assertTrue(response is True)
self.assertEqual(len(pscene), 1)
def test_production_set_product_retry(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
order = Order.find(order_id)
scene = order.scenes()[3]
scene.update('retry_count', 4)
processing_loc = "get_products_to_process"
error = 'not available after EE call '
note = 'note this'
retry_after = datetime.datetime.now() + datetime.timedelta(hours=1)
retry_limit = 9
response = production_provider.set_product_retry(scene.name, order.orderid, processing_loc,
error, note, retry_after, retry_limit)
new = Scene.get('ordering_scene.status', scene.name, order.orderid)
self.assertTrue('retry' == new)
@patch('api.external.lta.update_order_status', mock_production_provider.respond_true)
def test_production_set_product_error_unavailable(self):
"""
Move a scene status from error to unavailable based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes({'name !=': 'plot'})[0]
production_provider.set_product_error(scene.name, order.orderid,
'get_products_to_process',
'include_dswe is an unavailable product option for OLITIRS')
self.assertTrue('unavailable' == Scene.get('ordering_scene.status', scene.name, order.orderid))
def test_production_set_product_error_submitted(self):
"""
Move a scene status from error to submitted based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes({'name !=': 'plot'})[0]
production_provider.set_product_error(scene.name, order.orderid,
'get_products_to_process',
'BLOCK, COMING FROM LST AS WELL: No such file or directory')
self.assertTrue('submitted' == Scene.get('ordering_scene.status', scene.name, order.orderid))
def test_production_set_product_error_retry(self):
"""
Move a scene status from error to retry based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[2]
production_provider.set_product_error(scene.name, order.orderid,
'somewhere',
'Verify the missing auxillary data products')
self.assertTrue('retry' == Scene.get('ordering_scene.status', scene.name, order.orderid))
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
def test_update_product_details_update_status(self):
"""
Set a scene status to Queued
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[0]
api.update_product_details('update_status',
{'name': scene.name,
'orderid': order.orderid,
'processing_loc': 'L8SRLEXAMPLE',
'status': 'Queued'})
self.assertTrue(Scene.get('ordering_scene.status', scene.name, order.orderid) == 'Queued')
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
@patch('api.external.onlinecache.capacity', onlinecache.capacity)
def test_update_product_details_set_product_error(self):
"""
Set a scene status to error
:return:
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[0]
production_provider.update_product('set_product_error',
name=scene.name, orderid=order.orderid,
processing_loc="L8SRLEXAMPLE",
error='problems yo')
self.assertTrue(Scene.find(scene.id).status == 'error')
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
def test_update_product_details_set_product_unavailable(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[0]
production_provider.update_product('set_product_unavailable',
name=scene.name, orderid=order.orderid,
processing_loc="L8SRLEXAMPLE",
error='include_dswe is an unavailable product option for OLITIRS')
self.assertTrue('unavailable' == Scene.get('ordering_scene.status', scene.name, order.orderid))
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
@patch('os.path.getsize', lambda y: 999)
def test_update_product_details_mark_product_complete(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[0]
production_provider.update_product('mark_product_complete',
name=scene.name,
orderid=order.orderid,
processing_loc='L8SRLEXAMPLE',
completed_file_location='/some/loc',
cksum_file_location='some checksum',
log_file_contents='some log')
self.assertTrue('complete' == Scene.get('ordering_scene.status', scene.name, order.orderid))
@patch('api.external.lta.update_order_status', lta.update_order_status_fail)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
@patch('os.path.getsize', lambda y: 999)
def test_update_product_details_fail_lta_mark_product_complete(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[1]
order.update('order_source', 'ee')
production_provider.update_product('mark_product_complete',
name=scene.name,
orderid=order.orderid,
processing_loc='L8SRLEXAMPLE',
completed_file_location='/some/loc',
cksum_file_location='some checksum',
log_file_contents='some log')
s = Scene.where({'name': scene.name, 'order_id': scene.order_id})[0]
self.assertTrue('C' == s.failed_lta_status_update)
@patch('api.providers.production.production_provider.ProductionProvider.send_initial_emails',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.handle_onorder_landsat_products',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.handle_retry_products',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.load_ee_orders',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.handle_submitted_products',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.finalize_orders',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.purge_orders',
mock_production_provider.respond_true)
def test_handle_orders_success(self):
self.assertTrue(api.handle_orders())
@patch('api.external.onlinecache.delete', mock_production_provider.respond_true)
@patch('api.notification.emails.send_purge_report', mock_production_provider.respond_true)
@patch('api.external.onlinecache.capacity', onlinecache.capacity)
def test_production_purge_orders(self):
new_completion_date = datetime.datetime.now() - datetime.timedelta(days=12)
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
order.update('status', 'complete')
order.update('completion_date', new_completion_date)
self.assertTrue(production_provider.purge_orders())
# need to figure a test for emails.send_email
@patch('api.notification.emails.Emails.send_email', mock_production_provider.respond_true)
def test_production_send_initial_emails(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
order.update('status', 'ordered')
self.assertTrue(emails.Emails().send_all_initial())
@patch('api.external.lta.get_order_status', lta.get_order_status)
@patch('api.external.lta.update_order_status', lta.update_order_status)
def test_production_handle_onorder_landsat_products(self):
tram_order_ids = lta.sample_tram_order_ids()[0:3]
scene_names = lta.sample_scene_names()[0:3]
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scenes = order.scenes()[0:3]
for idx, scene in enumerate(scenes):
scene.tram_order_id = tram_order_ids[idx]
scene.status = 'onorder'
# save() doesn't let you update name,
# b/c updating a scene name is not acceptable
# outside of testing
scene.update('name', scene_names[idx])
scene.save()
self.assertTrue(production_provider.handle_onorder_landsat_products())
def test_production_handle_retry_products(self):
prev = datetime.datetime.now() - datetime.timedelta(hours=1)
order_id = self.mock_order.generate_testing_order(self.user_id)
self.mock_order.update_scenes(order_id, 'status', ['retry'])
self.mock_order.update_scenes(order_id, 'retry_after', [prev])
production_provider.handle_retry_products()
for s in Scene.where({'order_id': order_id}):
self.assertTrue(s.status == 'submitted')
#@patch('api.external.lta.get_available_orders', lta.get_available_orders)
#@patch('api.external.lta.update_order_status', lta.update_order_status)
#@patch('api.external.lta.get_user_name', lta.get_user_name)
#def test_production_load_ee_orders(self):
# #production_provider.load_ee_orders()
# pass
@patch('api.external.lta.get_available_orders', lta.get_available_orders_partial)
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch('api.external.lta.get_user_name', lta.get_user_name)
def test_production_load_ee_orders_partial(self):
order = Order.find(self.mock_order.generate_ee_testing_order(self.user_id, partial=True))
self.assertTrue(order.product_opts == {'format': 'gtiff',
'etm7': {'inputs': ['LE70900652008327EDC00'],
'products': ['sr']}})
production_provider.load_ee_orders()
reorder = Order.find(order.id)
self.assertTrue(reorder.product_opts == {'format': 'gtiff',
'etm7': {'inputs': ['LE70900652008327EDC00'],
'products': ['sr']},
'tm5': {'inputs': ['LT50900652008327EDC00'],
'products': ['sr']}})
@patch('api.external.lta.update_order_status', lta.update_order_status)
def test_production_handle_failed_ee_updates(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.update('failed_lta_status_update', 'C')
production_provider.handle_failed_ee_updates()
scenes = Scene.where({'failed_lta_status_update IS NOT': None})
self.assertTrue(len(scenes) == 0)
@patch('api.providers.production.production_provider.ProductionProvider.handle_submitted_landsat_products',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.handle_submitted_modis_products',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.handle_submitted_plot_products',
mock_production_provider.respond_true)
def test_production_handle_submitted_products(self):
self.assertTrue(production_provider.handle_submitted_products())
@patch('api.providers.production.production_provider.ProductionProvider.mark_nlaps_unavailable',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.update_landsat_product_status',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.get_contactids_for_submitted_landsat_products',
mock_production_provider.contact_ids_list)
def test_production_handle_submitted_landsat_products(self):
self.assertTrue(production_provider.handle_submitted_landsat_products())
# !!! need to write test for nlaps.products_are_nlaps !!!
@patch('api.external.nlaps.products_are_nlaps', nlaps.products_are_nlaps)
@patch('api.providers.production.production_provider.ProductionProvider.set_products_unavailable',
mock_production_provider.respond_true)
def test_production_mark_nlaps_unavailable(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.status = 'submitted'
scene.sensor_type = 'landsat'
scene.save()
self.assertTrue(production_provider.mark_nlaps_unavailable())
@patch('api.external.lta.update_order_status', lta.update_order_status)
def test_production_set_products_unavailable(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
self.assertTrue(production_provider.set_products_unavailable(order.scenes(), "you want a reason?"))
@patch('api.external.lta.order_scenes', lta.order_scenes)
@patch('api.providers.production.production_provider.ProductionProvider.set_products_unavailable',
mock_production_provider.respond_true)
def test_production_update_landsat_product_status(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.status = 'submitted'
scene.sensor_type = 'landsat'
scene.save()
self.assertTrue(production_provider.update_landsat_product_status(User.find(self.user_id).contactid))
def test_production_get_contactids_for_submitted_landsat_products(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.status = 'submitted'
scene.sensor_type = 'landsat'
scene.save()
response = production_provider.get_contactids_for_submitted_landsat_products()
self.assertIsInstance(response, set)
self.assertTrue(len(response) > 0)
@patch('api.external.lpdaac.input_exists', lpdaac.input_exists_true)
def test_production_handle_submitted_modis_products_input_exists(self):
# handle oncache scenario
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.status = 'submitted'
scene.sensor_type = 'modis'
scene.save()
sid = scene.id
self.assertTrue(production_provider.handle_submitted_modis_products())
self.assertEquals(Scene.find(sid).status, "oncache")
@patch('api.external.lpdaac.input_exists', lpdaac.input_exists_false)
def test_production_handle_submitted_modis_products_input_missing(self):
# handle unavailable scenario
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.status = 'submitted'
scene.sensor_type = 'modis'
scene.save()
sid = scene.id
self.assertTrue(production_provider.handle_submitted_modis_products())
self.assertEquals(Scene.find(sid).status, "unavailable")
def test_production_handle_submitted_plot_products(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
order.status = 'ordered'
order.order_type = 'lpcs'
order.save()
plot_id = None
for idx, scene in enumerate(order.scenes()):
# at the moment, mock_order.generate_testing_order
# creates 21 products for the order. divvy those
# up between 'complete' and 'unavailable', setting
# one aside as the 'plot' product
if idx % 2 == 0:
if idx == 0:
# need to define a plot product
scene.update('status', 'submitted')
scene.update('sensor_type', 'plot')
plot_id = scene.id
else:
scene.update('status', 'complete')
else:
scene.update('status', 'unavailable')
self.assertTrue(production_provider.handle_submitted_plot_products())
self.assertEqual(Scene.find(plot_id).status, "oncache")
@patch('os.path.getsize', lambda y: 999)
def test_production_calc_scene_download_sizes(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scenes = order.scenes()
Scene.bulk_update([s.id for s in scenes], {'status': 'complete', 'download_size': 0})
self.assertTrue(production_provider.calc_scene_download_sizes())
upscenes = Scene.where({'status': 'complete', 'download_size': 999})
self.assertEqual(len(upscenes), len(scenes))
@patch('api.providers.production.production_provider.ProductionProvider.update_order_if_complete',
mock_production_provider.respond_true)
def test_production_finalize_orders(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
order.update('status', 'ordered')
self.assertTrue(production_provider.finalize_orders())
@patch('api.providers.production.production_provider.ProductionProvider.send_completion_email',
mock_production_provider.respond_true)
def test_production_update_order_if_complete(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
Scene.bulk_update([s.id for s in order.scenes()], {'status': 'retry'})
order.order_source = 'espa'
order.completion_email_sent = None
order.save()
self.assertTrue(production_provider.update_order_if_complete(order))
def test_production_queue_products_success(self):
names_tuple = self.mock_order.names_tuple(3, self.user_id)
processing_loc = "get_products_to_process"
job_name = 'jobname49'
params = (names_tuple, processing_loc, job_name)
response = api.queue_products(*params)
self.assertTrue(response)
def test_production_get_key(self):
key = 'system_message_title'
response = api.get_production_key(key)
val = response[key]
self.assertIsInstance(val, str)
def test_get_production_key_invalid(self):
bad_key = 'foobar'
response = api.get_production_key(bad_key)
self.assertEqual(response.keys(), ['msg'])
@patch('api.external.hadoop.HadoopHandler.job_names_ids',
hadoop.jobs_names_ids)
def test_catch_orphaned_scenes(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
# need scenes with statuses of 'queued'
self.mock_order.update_scenes(order_id, 'status', ['queued'])
response = production_provider.catch_orphaned_scenes()
self.assertTrue(response)
old_time = datetime.datetime.now() - datetime.timedelta(minutes=15)
for s in Scene.where({'order_id': order_id}):
self.assertTrue(s.reported_orphan is not None)
s.reported_orphan = old_time
s.save()
response = production_provider.catch_orphaned_scenes()
self.assertTrue(response)
for s in Scene.where({'order_id': order_id}):
self.assertTrue(s.orphaned)
def test_convert_product_options(self):
"""
Test the conversion procedure to make sure that the new format for orders converts
to the old format
"""
scenes = ['LE70480272012076EDC00', 'LC80500272013196LGN00',
'LT40480271983028PAC00', 'LT50490262009162PAC03']
includes = ['include_sr', 'include_sr_toa',
'include_cfmask', 'include_sr_thermal']
new_format = {u'etm7': {u'inputs': [u'LE70480272012076EDC00'],
u'products': [u'sr']},
u'olitirs8': {u'inputs': [u'LC80500272013196LGN00'],
u'products': [u'toa']},
u'tm4': {u'inputs': [u'LT40480271983028PAC00'],
u'products': [u'cloud']},
u'tm5': {u'inputs': [u'LT50490262009162PAC03'],
u'products': [u'bt']},
u'format': u'gtiff',
u'image_extents': {u'east': -2265585.0,
u'north': 3164805.0,
u'south': 3014805.0,
u'units': u'meters',
u'west': -2415585.0},
u'note': u'CONUS_h1v1',
u'projection': {u'aea': {u'central_meridian': -96,
u'datum': u'nad83',
u'false_easting': 0,
u'false_northing': 0,
u'latitude_of_origin': 23,
u'standard_parallel_1': 29.5,
u'standard_parallel_2': 45.5}},
u'resampling_method': u'cc'}
ruberic = {'central_meridian': -96,
'datum': u'nad83',
'false_easting': 0,
'false_northing': 0,
'image_extents': True,
'image_extents_units': u'meters',
'include_cfmask': False,
'include_customized_source_data': False,
'include_dswe': False,
'include_lst': False,
'include_solr_index': False,
'include_source_data': False,
'include_source_metadata': False,
'include_sr': False,
'include_sr_browse': False,
'include_sr_evi': False,
'include_sr_msavi': False,
'include_sr_nbr': False,
'include_sr_nbr2': False,
'include_sr_ndmi': False,
'include_sr_ndvi': False,
'include_sr_savi': False,
'include_sr_thermal': False,
'include_sr_toa': False,
'include_statistics': False,
'latitude_true_scale': None,
'longitude_pole': None,
'maxx': -2265585.0,
'maxy': 3164805.0,
'minx': -2415585.0,
'miny': 3014805.0,
'origin_lat': 23,
'output_format': u'gtiff',
'pixel_size': None,
'pixel_size_units': None,
'reproject': True,
'resample_method': 'cubic',
'resize': False,
'std_parallel_1': 29.5,
'std_parallel_2': 45.5,
'target_projection': u'aea',
'utm_north_south': None,
'utm_zone': None}
for scene, include in zip(scenes, includes):
ruberic[include] = True
old_format = OptionsConversion.convert(new=new_format, scenes=[scene])
self.assertDictEqual(ruberic, old_format)
ruberic[include] = False
