def test_order_exceeds_open_scene_limit(self):
"""
Build a large order, then attempt to place an order that would exceed the user's open scene limit.
"""
mock = MockOrder()
user = MockUser()
user_id = user.add_testing_user()
# base testing order will give a user 15 open scenes currently
order_id = mock.generate_testing_order(user_id)
# Add a second order that pushes scenes over the limit, make sure that this
# raises the appropriate exception
self.assertRaises(OpenSceneLimitException,
lambda: ordering_provider.check_open_scenes(order=mock.base_order,
user_id=user_id,
filters={'status': ('submitted',
'oncache',
'onorder',
'queued',
'processing')}))
class TestProductionAPI(unittest.TestCase):
def setUp(self):
os.environ['espa_api_testing'] = 'True'
# create a user
self.mock_user = MockUser()
self.mock_order = MockOrder()
self.user_id = self.mock_user.add_testing_user()
def tearDown(self):
# clean up orders
self.mock_order.tear_down_testing_orders()
# clean up users
self.mock_user.cleanup()
os.environ['espa_api_testing'] = ''
@patch('api.external.lpdaac.get_download_urls', lpdaac.get_download_urls)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry',
mock_production_provider.set_product_retry)
def test_fetch_production_products_modis(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
# need scenes with statuses of 'processing' and 'ordered'
self.mock_order.update_scenes(order_id, 'status', ['processing', 'ordered', 'oncache'])
user = User.find(self.user_id)
params = {'for_user': user.username, 'product_types': ['modis']}
response = api.fetch_production_products(params)
self.assertTrue('bilbo' in response[0]['orderid'])
@patch('api.external.lta.get_download_urls', lta.get_download_urls)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry',
mock_production_provider.set_product_retry)
def test_fetch_production_products_landsat(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
# need scenes with statuses of 'processing' and 'ordered'
self.mock_order.update_scenes(order_id, 'status', ['processing','oncache','ordered'])
user = User.find(self.user_id)
params = {'for_user': user.username, 'product_types': ['landsat']}
response = api.fetch_production_products(params)
self.assertTrue('bilbo' in response[0]['orderid'])
def test_fetch_production_products_plot(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
self.mock_order.update_scenes(order_id, 'status', ['complete'])
order = Order.find(order_id)
plot_scene = order.scenes()[0]
plot_scene.name = 'plot'
plot_scene.sensor_type = 'plot'
plot_scene.status = 'submitted'
plot_scene.save()
response = production_provider.handle_submitted_plot_products()
pscene = order.scenes({'status': 'oncache', 'sensor_type': 'plot'})
self.assertTrue(response is True)
self.assertEqual(len(pscene), 1)
def test_production_set_product_retry(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
order = Order.find(order_id)
scene = order.scenes()[3]
scene.update('retry_count', 4)
processing_loc = "get_products_to_process"
error = 'not available after EE call '
note = 'note this'
retry_after = datetime.datetime.now() + datetime.timedelta(hours=1)
retry_limit = 9
response = production_provider.set_product_retry(scene.name, order.orderid, processing_loc,
error, note, retry_after, retry_limit)
new = Scene.get('ordering_scene.status', scene.name, order.orderid)
self.assertTrue('retry' == new)
@patch('api.external.lta.update_order_status', mock_production_provider.respond_true)
def test_production_set_product_error_unavailable(self):
"""
Move a scene status from error to unavailable based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes({'name !=': 'plot'})[0]
production_provider.set_product_error(scene.name, order.orderid,
'get_products_to_process',
'include_dswe is an unavailable product option for OLITIRS')
self.assertTrue('unavailable' == Scene.get('ordering_scene.status', scene.name, order.orderid))
def test_production_set_product_error_submitted(self):
"""
Move a scene status from error to submitted based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes({'name !=': 'plot'})[0]
production_provider.set_product_error(scene.name, order.orderid,
'get_products_to_process',
'BLOCK, COMING FROM LST AS WELL: No such file or directory')
self.assertTrue('submitted' == Scene.get('ordering_scene.status', scene.name, order.orderid))
def test_production_set_product_error_retry(self):
"""
Move a scene status from error to retry based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[2]
production_provider.set_product_error(scene.name, order.orderid,
'somewhere',
'Verify the missing auxillary data products')
self.assertTrue('retry' == Scene.get('ordering_scene.status', scene.name, order.orderid))
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
def test_update_product_details_update_status(self):
"""
Set a scene status to Queued
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[0]
api.update_product_details('update_status',
{'name': scene.name,
'orderid': order.orderid,
'processing_loc': 'L8SRLEXAMPLE',
'status': 'Queued'})
self.assertTrue(Scene.get('ordering_scene.status', scene.name, order.orderid) == 'Queued')
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
@patch('api.external.onlinecache.capacity', onlinecache.capacity)
def test_update_product_details_set_product_error(self):
"""
Set a scene status to error
:return:
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[0]
production_provider.update_product('set_product_error',
name=scene.name, orderid=order.orderid,
processing_loc="L8SRLEXAMPLE",
error='problems yo')
self.assertTrue(Scene.find(scene.id).status == 'error')
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
def test_update_product_details_set_product_unavailable(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[0]
production_provider.update_product('set_product_unavailable',
name=scene.name, orderid=order.orderid,
processing_loc="L8SRLEXAMPLE",
error='include_dswe is an unavailable product option for OLITIRS')
self.assertTrue('unavailable' == Scene.get('ordering_scene.status', scene.name, order.orderid))
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
def test_update_product_details_mark_product_complete(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[0]
production_provider.update_product('mark_product_complete',
name=scene.name,
orderid=order.orderid,
processing_loc='L8SRLEXAMPLE',
completed_file_location='/some/loc',
cksum_file_location='some checksum',
log_file_contents='some log')
self.assertTrue('complete' == Scene.get('ordering_scene.status', scene.name, order.orderid))
@patch('api.external.lta.update_order_status', lta.update_order_status_fail)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
def test_update_product_details_fail_lta_mark_product_complete(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[1]
order.update('order_source', 'ee')
production_provider.update_product('mark_product_complete',
name=scene.name,
orderid=order.orderid,
processing_loc='L8SRLEXAMPLE',
completed_file_location='/some/loc',
cksum_file_location='some checksum',
log_file_contents='some log')
s = Scene.where({'name': scene.name, 'order_id': scene.order_id})[0]
self.assertTrue('C' == s.failed_lta_status_update)
@patch('api.providers.production.production_provider.ProductionProvider.send_initial_emails',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.handle_onorder_landsat_products',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.handle_retry_products',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.load_ee_orders',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.handle_submitted_products',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.finalize_orders',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.purge_orders',
mock_production_provider.respond_true)
def test_handle_orders_success(self):
self.assertTrue(api.handle_orders())
@patch('api.external.onlinecache.delete', mock_production_provider.respond_true)
@patch('api.notification.emails.send_purge_report', mock_production_provider.respond_true)
@patch('api.external.onlinecache.capacity', onlinecache.capacity)
def test_production_purge_orders(self):
new_completion_date = datetime.datetime.now() - datetime.timedelta(days=12)
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
order.update('status', 'complete')
order.update('completion_date', new_completion_date)
self.assertTrue(production_provider.purge_orders())
# need to figure a test for emails.send_email
@patch('api.notification.emails.Emails.send_email', mock_production_provider.respond_true)
def test_production_send_initial_emails(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
order.update('status', 'ordered')
self.assertTrue(emails.Emails().send_all_initial())
@patch('api.external.lta.get_order_status', lta.get_order_status)
@patch('api.external.lta.update_order_status', lta.update_order_status)
def test_production_handle_onorder_landsat_products(self):
tram_order_ids = lta.sample_tram_order_ids()[0:3]
scene_names = lta.sample_scene_names()[0:3]
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scenes = order.scenes()[0:3]
for idx, scene in enumerate(scenes):
scene.tram_order_id = tram_order_ids[idx]
scene.status = 'onorder'
# save() doesn't let you update name,
# b/c updating a scene name is not acceptable
# outside of testing
scene.update('name', scene_names[idx])
scene.save()
self.assertTrue(production_provider.handle_onorder_landsat_products())
def test_production_handle_retry_products(self):
prev = datetime.datetime.now() - datetime.timedelta(hours=1)
order_id = self.mock_order.generate_testing_order(self.user_id)
self.mock_order.update_scenes(order_id, 'status', ['retry'])
self.mock_order.update_scenes(order_id, 'retry_after', [prev])
production_provider.handle_retry_products()
for s in Scene.where({'order_id': order_id}):
self.assertTrue(s.status == 'submitted')
#@patch('api.external.lta.get_available_orders', lta.get_available_orders)
#@patch('api.external.lta.update_order_status', lta.update_order_status)
#@patch('api.external.lta.get_user_name', lta.get_user_name)
#def test_production_load_ee_orders(self):
# #production_provider.load_ee_orders()
# pass
@patch('api.external.lta.get_available_orders', lta.get_available_orders_partial)
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch('api.external.lta.get_user_name', lta.get_user_name)
def test_production_load_ee_orders_partial(self):
order = Order.find(self.mock_order.generate_ee_testing_order(self.user_id, partial=True))
self.assertTrue(order.product_opts == {'format': 'gtiff',
'etm7': {'inputs': ['LE70900652008327EDC00'],
'products': ['sr']}})
production_provider.load_ee_orders()
reorder = Order.find(order.id)
self.assertTrue(reorder.product_opts == {'format': 'gtiff',
'etm7': {'inputs': ['LE70900652008327EDC00'],
'products': ['sr']},
'tm5': {'inputs': ['LT50900652008327EDC00'],
'products': ['sr']}})
@patch('api.external.lta.update_order_status', lta.update_order_status)
def test_production_handle_failed_ee_updates(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.update('failed_lta_status_update', 'C')
production_provider.handle_failed_ee_updates()
scenes = Scene.where({'failed_lta_status_update IS NOT': None})
self.assertTrue(len(scenes) == 0)
@patch('api.providers.production.production_provider.ProductionProvider.handle_submitted_landsat_products',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.handle_submitted_modis_products',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.handle_submitted_plot_products',
mock_production_provider.respond_true)
def test_production_handle_submitted_products(self):
self.assertTrue(production_provider.handle_submitted_products())
@patch('api.providers.production.production_provider.ProductionProvider.mark_nlaps_unavailable',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.update_landsat_product_status',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.get_contactids_for_submitted_landsat_products',
mock_production_provider.contact_ids_list)
def test_production_handle_submitted_landsat_products(self):
self.assertTrue(production_provider.handle_submitted_landsat_products())
# !!! need to write test for nlaps.products_are_nlaps !!!
@patch('api.external.nlaps.products_are_nlaps', nlaps.products_are_nlaps)
@patch('api.providers.production.production_provider.ProductionProvider.set_products_unavailable',
mock_production_provider.respond_true)
def test_production_mark_nlaps_unavailable(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.status = 'submitted'
scene.sensor_type = 'landsat'
scene.save()
self.assertTrue(production_provider.mark_nlaps_unavailable())
@patch('api.external.lta.update_order_status', lta.update_order_status)
def test_production_set_products_unavailable(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
self.assertTrue(production_provider.set_products_unavailable(order.scenes(), "you want a reason?"))
@patch('api.external.lta.order_scenes', lta.order_scenes)
@patch('api.providers.production.production_provider.ProductionProvider.set_products_unavailable',
mock_production_provider.respond_true)
def test_production_update_landsat_product_status(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.status = 'submitted'
scene.sensor_type = 'landsat'
scene.save()
self.assertTrue(production_provider.update_landsat_product_status(User.find(self.user_id).contactid))
def test_production_get_contactids_for_submitted_landsat_products(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.status = 'submitted'
scene.sensor_type = 'landsat'
scene.save()
response = production_provider.get_contactids_for_submitted_landsat_products()
self.assertIsInstance(response, set)
self.assertTrue(len(response) > 0)
@patch('api.external.lpdaac.input_exists', lpdaac.input_exists_true)
def test_production_handle_submitted_modis_products_input_exists(self):
# handle oncache scenario
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.status = 'submitted'
scene.sensor_type = 'modis'
scene.save()
sid = scene.id
self.assertTrue(production_provider.handle_submitted_modis_products())
self.assertEquals(Scene.find(sid).status, "oncache")
@patch('api.external.lpdaac.input_exists', lpdaac.input_exists_false)
def test_production_handle_submitted_modis_products_input_missing(self):
# handle unavailable scenario
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.status = 'submitted'
scene.sensor_type = 'modis'
scene.save()
sid = scene.id
self.assertTrue(production_provider.handle_submitted_modis_products())
self.assertEquals(Scene.find(sid).status, "unavailable")
def test_production_handle_submitted_plot_products(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
order.status = 'ordered'
order.order_type = 'lpcs'
order.save()
plot_id = None
for idx, scene in enumerate(order.scenes()):
# at the moment, mock_order.generate_testing_order
# creates 21 products for the order. divvy those
# up between 'complete' and 'unavailable', setting
# one aside as the 'plot' product
if idx % 2 == 0:
if idx == 0:
# need to define a plot product
scene.update('status', 'submitted')
scene.update('sensor_type', 'plot')
plot_id = scene.id
else:
scene.update('status', 'complete')
else:
scene.update('status', 'unavailable')
self.assertTrue(production_provider.handle_submitted_plot_products())
self.assertEqual(Scene.find(plot_id).status, "oncache")
@patch('api.providers.production.production_provider.ProductionProvider.update_order_if_complete',
mock_production_provider.respond_true)
def test_production_finalize_orders(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
order.update('status', 'ordered')
self.assertTrue(production_provider.finalize_orders())
@patch('api.providers.production.production_provider.ProductionProvider.send_completion_email',
mock_production_provider.respond_true)
def test_production_update_order_if_complete(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
Scene.bulk_update([s.id for s in order.scenes()], {'status': 'retry'})
order.order_source = 'espa'
order.completion_email_sent = None
order.save()
self.assertTrue(production_provider.update_order_if_complete(order))
def test_production_queue_products_success(self):
names_tuple = self.mock_order.names_tuple(3, self.user_id)
processing_loc = "get_products_to_process"
job_name = 'jobname49'
params = (names_tuple, processing_loc, job_name)
response = api.queue_products(*params)
self.assertTrue(response)
def test_production_get_key(self):
key = 'system_message_title'
response = api.get_production_key(key)
val = response[key]
self.assertIsInstance(val, str)
def test_get_production_key_invalid(self):
bad_key = 'foobar'
response = api.get_production_key(bad_key)
self.assertEqual(response.keys(), ['msg'])
@patch('api.external.hadoop.HadoopHandler.job_names_ids',
hadoop.jobs_names_ids)
def test_catch_orphaned_scenes(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
# need scenes with statuses of 'queued'
self.mock_order.update_scenes(order_id, 'status', ['queued'])
response = production_provider.catch_orphaned_scenes()
self.assertTrue(response)
old_time = datetime.datetime.now() - datetime.timedelta(minutes=15)
for s in Scene.where({'order_id': order_id}):
self.assertTrue(s.reported_orphan is not None)
s.reported_orphan = old_time
s.save()
response = production_provider.catch_orphaned_scenes()
self.assertTrue(response)
for s in Scene.where({'order_id': order_id}):
self.assertTrue(s.orphaned)
class TransportTestCase(unittest.TestCase):
def setUp(self):
os.environ['espa_api_testing'] = 'True'
# create a user
self.mock_user = MockUser()
self.mock_order = MockOrder()
self.user = User.find(self.mock_user.add_testing_user())
self.order_id = self.mock_order.generate_testing_order(self.user.id)
self.app = http.app.test_client()
self.app.testing = True
self.sceneids = self.mock_order.scene_names_list(self.order_id)[0:2]
token = ''.format(self.user.username, 'foo')
auth_string = "Basic {}".format(base64.b64encode(token))
self.headers = {"Authorization": auth_string}
with db_instance() as db:
uidsql = "select user_id, orderid from ordering_order limit 1;"
db.select(uidsql)
self.userid = db[0]['user_id']
self.orderid = db[0]['orderid']
itemsql = "select name, order_id from ordering_scene limit 1;"
db.select(itemsql)
self.itemid = db[0][0]
itemorderid = db[0][1]
ordersql = "select orderid from ordering_order where id = {};".format(
itemorderid)
db.select(ordersql)
self.itemorderid = db[0][0]
self.base_order = lowercase_all(testorders.build_base_order())
def tearDown(self):
# clean up orders
self.mock_order.tear_down_testing_orders()
# clean up users
self.mock_user.cleanup()
os.environ['espa_api_testing'] = ''
def test_get_api_response_type(self):
response = self.app.get('/api',
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
assert response.content_type == 'application/json'
def test_get_api_response_content(self):
response = self.app.get('/api',
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertEqual(set(['1', '0']), set(resp_json.keys()))
@patch('api.domain.user.User.get', MockUser.get)
def test_get_api_info_response_content(self):
response = self.app.get('/api/v1',
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert "Version 1" in resp_json['description']
@patch('api.domain.user.User.get', MockUser.get)
@patch(
'api.providers.ordering.ordering_provider.OrderingProvider.available_products',
mock_api.available_products)
def test_get_available_prods(self):
url = '/api/v1/available-products/' + ",".join(self.sceneids)
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert "etm" in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
@patch(
'api.providers.ordering.ordering_provider.OrderingProvider.available_products',
mock_api.available_products)
def test_post_available_prods(self):
url = '/api/v1/available-products'
data_dict = {'inputs': self.sceneids}
response = self.app.post(url,
data=json.dumps(data_dict),
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert "etm" in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
@patch(
'api.providers.ordering.ordering_provider.OrderingProvider.fetch_user_orders',
mock_ordering_provider.fetch_user_orders)
def test_get_available_orders_user(self):
url = "/api/v1/list-orders"
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert resp_json.keys() == ['orders']
@patch('api.domain.user.User.get', MockUser.get)
def test_get_available_orders_email(self):
# email param comes in as unicode
url = "/api/v1/list-orders/" + str(self.user.email)
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert resp_json.keys() == ['orders']
@patch('api.domain.user.User.get', MockUser.get)
def test_get_order_by_ordernum(self):
url = "/api/v1/order/" + str(self.orderid)
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert 'orderid' in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
def test_get_order_status_by_ordernum(self):
url = "/api/v1/order-status/" + str(self.orderid)
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert 'orderid' in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
def test_get_item_status_by_ordernum(self):
url = "/api/v1/item-status/%s" % self.itemorderid
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert 'orderid' in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
def test_get_item_status_by_ordernum_itemnum(self):
url = "/api/v1/item-status/%s/%s" % (self.itemorderid, self.itemid)
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert 'orderid' in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
def test_get_current_user(self):
url = "/api/v1/user"
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert 'username' in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
def test_get_projections(self):
url = '/api/v1/projections'
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert 'aea' in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
def test_get_formats(self):
url = '/api/v1/formats'
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert 'formats' in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
def test_get_resampling(self):
url = '/api/v1/resampling-methods'
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert 'resampling_methods' in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
def test_get_order_schema(self):
url = '/api/v1/order-schema'
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert 'properties' in resp_json.keys()
# Waiting for DB mock-ups to be finished
def test_post_order(self):
pass
class TestProductionAPI(unittest.TestCase):
def setUp(self):
os.environ['espa_api_testing'] = 'True'
# create a user
self.mock_user = MockUser()
self.mock_order = MockOrder()
self.user_id = self.mock_user.add_testing_user()
def tearDown(self):
# clean up orders
self.mock_order.tear_down_testing_orders()
# clean up users
self.mock_user.cleanup()
os.environ['espa_api_testing'] = ''
@patch('api.external.inventory.available', lambda: True)
@patch(
'api.providers.production.production_provider.ProductionProvider.parse_urls_m2m',
lambda x, y: y)
def test_fetch_production_products_modis(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
# need scenes with statuses of 'processing'
self.mock_order.update_scenes(order_id, 'modis', 'status',
['processing', 'oncache'])
user = User.find(self.user_id)
params = {'product_types': ['modis']}
response = api.fetch_production_products(params)
self.assertTrue('bilbo' in response[0]['orderid'])
@patch('api.external.inventory.available', lambda: True)
@patch(
'api.providers.production.production_provider.ProductionProvider.parse_urls_m2m',
lambda x, y: y)
def test_fetch_production_products_landsat(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
# need scenes with statuses of 'processing'
self.mock_order.update_scenes(order_id, 'landsat', 'status',
['processing', 'oncache'])
user = User.find(self.user_id)
params = {'for_user': user.username, 'product_types': ['landsat']}
response = api.fetch_production_products(params)
self.assertTrue('bilbo' in response[0]['orderid'])
def test_fetch_production_products_plot(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
self.mock_order.update_scenes(order_id, ('landsat', 'modis'), 'status',
['complete'])
order = Order.find(order_id)
plot_scene = order.scenes({'name': 'plot'})[0]
plot_scene.name = 'plot'
plot_scene.sensor_type = 'plot'
plot_scene.status = 'submitted'
plot_scene.save()
scenes = Order.find(order_id).scenes({
'name': 'plot',
'status': 'submitted'
})
response = production_provider.handle_submitted_plot_products(scenes)
pscene = order.scenes({'status': 'oncache', 'sensor_type': 'plot'})
self.assertTrue(response is True)
self.assertEqual(len(pscene), 1)
def test_production_set_product_retry(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
order = Order.find(order_id)
scene = order.scenes()[3]
scene.update('retry_count', 4)
processing_loc = "get_products_to_process"
error = 'not available after EE call '
note = 'note this'
retry_after = datetime.datetime.now() + datetime.timedelta(hours=1)
retry_limit = 9
response = production_provider.set_product_retry(
scene.name, order.orderid, processing_loc, error, note,
retry_after, retry_limit)
new = Scene.get('ordering_scene.status', scene.name, order.orderid)
self.assertTrue('retry' == new)
@patch('api.external.lta.update_order_status',
mock_production_provider.respond_true)
def test_production_set_product_error_unavailable(self):
"""
Move a scene status from error to unavailable based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes({'name !=': 'plot'})[0]
production_provider.set_product_error(
scene.name, order.orderid, 'get_products_to_process',
'include_dswe is an unavailable product option for OLITIRS')
self.assertTrue('unavailable' == Scene.get('ordering_scene.status',
scene.name, order.orderid))
def test_production_set_product_error_submitted(self):
"""
Move a scene status from error to submitted based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes({'name !=': 'plot'})[0]
production_provider.set_product_error(
scene.name, order.orderid, 'get_products_to_process',
'BLOCK, COMING FROM LST AS WELL: No such file or directory')
self.assertTrue('submitted' == Scene.get('ordering_scene.status',
scene.name, order.orderid))
def test_production_set_product_error_retry(self):
"""
Move a scene status from error to retry based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes()[2]
production_provider.set_product_error(
scene.name, order.orderid, 'somewhere',
'Verify the missing auxillary data products')
self.assertTrue('retry' == Scene.get('ordering_scene.status',
scene.name, order.orderid))
def test_production_set_product_error_retry_lasrc_segfault(self):
"""
Move a scene status from error to retry based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes({'sensor_type': 'landsat'})[-1]
production_provider.set_product_error(
scene.name, order.orderid, 'somewhere',
'runSr sh: line 1: 1010 Segmentation fault lasrc --xml=')
self.assertTrue('retry' == Scene.get('ordering_scene.status',
scene.name, order.orderid))
def test_production_set_product_error_unavail_reproject(self):
"""
Move a scene status from error to retry based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes({'sensor_type': 'landsat'})[-1]
log_file_contents = ('BLAH BLAH BLAH WarpVerificationError: Failed to '
'compute statistics, no valid pixels found in '
'sampling BLAH BLAH BLAH')
production_provider.set_product_error(scene.name, order.orderid,
'somewhere', log_file_contents)
self.assertEqual(
'unavailable',
Scene.get('ordering_scene.status', scene.name, order.orderid))
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch(
'api.providers.production.production_provider.ProductionProvider.set_product_retry',
mock_production_provider.set_product_retry)
def test_update_product_details_update_status(self):
"""
Set a scene status to Queued
"""
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes()[0]
api.update_product_details(
'update_status', {
'name': scene.name,
'orderid': order.orderid,
'processing_loc': 'L8SRLEXAMPLE',
'status': 'Queued'
})
self.assertTrue(
Scene.get('ordering_scene.status', scene.name, order.orderid) ==
'Queued')
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch(
'api.providers.production.production_provider.ProductionProvider.set_product_retry',
mock_production_provider.set_product_retry)
@patch('api.external.onlinecache.capacity', onlinecache.capacity)
def test_update_product_details_set_product_error(self):
"""
Set a scene status to error
:return:
"""
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes()[0]
production_provider.update_product('set_product_error',
name=scene.name,
orderid=order.orderid,
processing_loc="L8SRLEXAMPLE",
error='problems yo')
self.assertTrue(Scene.find(scene.id).status == 'error')
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch(
'api.providers.production.production_provider.ProductionProvider.set_product_retry',
mock_production_provider.set_product_retry)
def test_update_product_details_set_product_unavailable(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes()[0]
production_provider.update_product(
'set_product_unavailable',
name=scene.name,
orderid=order.orderid,
processing_loc="L8SRLEXAMPLE",
error='include_dswe is an unavailable product option for OLITIRS')
self.assertTrue('unavailable' == Scene.get('ordering_scene.status',
scene.name, order.orderid))
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch(
'api.providers.production.production_provider.ProductionProvider.set_product_retry',
mock_production_provider.set_product_retry)
@patch('os.path.getsize', lambda y: 999)
def test_update_product_details_mark_product_complete(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes()[0]
production_provider.update_product('mark_product_complete',
name=scene.name,
orderid=order.orderid,
processing_loc='L8SRLEXAMPLE',
completed_file_location='/some/loc',
cksum_file_location='some checksum',
log_file_contents='some log')
self.assertTrue('complete' == Scene.get('ordering_scene.status',
scene.name, order.orderid))
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch(
'api.providers.production.production_provider.ProductionProvider.set_product_retry',
mock_production_provider.set_product_retry)
@patch('os.path.getsize', lambda y: 999)
def test_update_product_details_mark_product_processing(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes()[0]
res = production_provider.update_product('update_status',
name=scene.name,
orderid=order.orderid,
processing_loc='L8SRLEXAMPLE',
status='processing')
order = Order.find(order.id)
scene = order.scenes({'id': scene.id})[0]
self.assertEqual('processing', scene.status)
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch(
'api.providers.production.production_provider.ProductionProvider.set_product_retry',
mock_production_provider.set_product_retry)
@patch('os.path.getsize', lambda y: 999)
def test_update_product_details_mark_cancelled_product_processing(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
order.status = 'cancelled'
order.save()
scene = order.scenes()[0]
res = production_provider.update_product('update_status',
name=scene.name,
orderid=order.orderid,
processing_loc='L8SRLEXAMPLE',
status='processing')
self.assertFalse(res)
def test_production_set_product_error_unavailable_night(self):
"""
Move a scene status from error to unavailable based on the solar zenith (TOA)
error message
"""
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes({'name !=': 'plot'})[0]
production_provider.set_product_error(
name=scene.name,
orderid=order.orderid,
processing_loc='L8SRLEXAMPLE',
error='solar zenith angle out of range')
scene = Scene.by_name_orderid(name=scene.name, order_id=order.id)
self.assertTrue('unavailable' == scene.status)
self.assertTrue(
'Solar zenith angle out of range, cannot process night scene' in
scene.note)
def test_production_set_product_error_unavailable_almost_night(self):
"""
Move a scene status from error to unavailable based on the solar zenith (SR)
error message
"""
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes({'name !=': 'plot'})[0]
production_provider.set_product_error(
name=scene.name,
orderid=order.orderid,
processing_loc='L8SRLEXAMPLE',
error='solar zenith angle is too large')
scene = Scene.by_name_orderid(name=scene.name, order_id=order.id)
self.assertTrue('unavailable' == scene.status)
self.assertTrue(
'Solar zenith angle is too large, cannot process scene to SR' in
scene.note)
@patch('api.external.lta.update_order_status',
lta.update_order_status_fail)
@patch(
'api.providers.production.production_provider.ProductionProvider.set_product_retry',
mock_production_provider.set_product_retry)
@patch('os.path.getsize', lambda y: 999)
def test_update_product_details_fail_lta_mark_product_complete(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes()[1]
order.update('order_source', 'ee')
production_provider.update_product('mark_product_complete',
name=scene.name,
orderid=order.orderid,
processing_loc='L8SRLEXAMPLE',
completed_file_location='/some/loc',
cksum_file_location='some checksum',
log_file_contents='some log')
s = Scene.where({'name': scene.name, 'order_id': scene.order_id})[0]
self.assertTrue('C' == s.failed_lta_status_update)
@patch(
'api.providers.production.production_provider.ProductionProvider.send_initial_emails',
mock_production_provider.respond_true)
@patch(
'api.providers.production.production_provider.ProductionProvider.handle_onorder_landsat_products',
mock_production_provider.respond_true)
@patch(
'api.providers.production.production_provider.ProductionProvider.handle_retry_products',
mock_production_provider.respond_true)
@patch(
'api.providers.production.production_provider.ProductionProvider.load_ee_orders',
mock_production_provider.respond_true)
@patch(
'api.providers.production.production_provider.ProductionProvider.finalize_orders',
mock_production_provider.respond_true)
@patch(
'api.providers.production.production_provider.ProductionProvider.purge_orders',
mock_production_provider.respond_true)
def test_handle_orders_success(self):
_ = self.mock_order.generate_testing_order(self.user_id)
self.assertTrue(
api.handle_orders({'username': User.find(self.user_id)}))
@patch('api.external.onlinecache.delete',
mock_production_provider.respond_true)
@patch('api.notification.emails.send_purge_report',
mock_production_provider.respond_true)
@patch('api.external.onlinecache.capacity', onlinecache.mock_capacity)
@patch('api.external.onlinecache.exists', onlinecache.mock_exists)
@patch('api.external.onlinecache.delete', onlinecache.mock_delete)
def test_production_purge_orders(self):
new_completion_date = datetime.datetime.now() - datetime.timedelta(
days=12)
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
order.update('status', 'complete')
order.update('completion_date', new_completion_date)
self.assertTrue(production_provider.purge_orders())
# need to figure a test for emails.send_email
@patch('api.notification.emails.Emails.send_email',
mock_production_provider.respond_true)
def test_production_send_initial_emails(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
order.update('status', 'ordered')
self.assertTrue(emails.Emails().send_all_initial([order]))
@patch('api.external.lta.get_order_status', lta.get_order_status)
@patch('api.external.lta.update_order_status', lta.update_order_status)
def test_production_handle_onorder_landsat_products(self):
tram_order_ids = lta.sample_tram_order_ids()[0:3]
scene_names = lta.sample_scene_names()[0:3]
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scenes = order.scenes()[0:3]
for idx, scene in enumerate(scenes):
scene.tram_order_id = tram_order_ids[idx]
scene.status = 'onorder'
# save() doesn't let you update name,
# b/c updating a scene name is not acceptable
# outside of testing
scene.update('name', scene_names[idx])
scene.save()
self.assertTrue(
production_provider.handle_onorder_landsat_products(scenes))
def test_production_handle_retry_products(self):
prev = datetime.datetime.now() - datetime.timedelta(hours=1)
order_id = self.mock_order.generate_testing_order(self.user_id)
self.mock_order.update_scenes(order_id, 'landsat', 'status', ['retry'])
self.mock_order.update_scenes(order_id, 'landsat', 'retry_after',
[prev])
scenes = Order.find(order_id).scenes({'status': 'retry'})
production_provider.handle_retry_products(scenes)
for s in Scene.where({'order_id': order_id, 'sensor_type': 'landsat'}):
self.assertTrue(s.status == 'submitted')
@patch('api.external.lta.get_available_orders',
lta.get_available_orders_partial)
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch('api.external.inventory.get_user_name', inventory.get_user_name)
@patch('api.external.inventory.get_session', lambda: True)
def test_production_load_ee_orders_partial(self):
order = Order.find(
self.mock_order.generate_ee_testing_order(self.user_id,
partial=True))
self.assertEqual(
order.product_opts, {
'format': 'gtiff',
'etm7_collection': {
'inputs': ['LE07_L1TP_026027_20170912_20171008_01_T1'],
'products': ['sr']
}
})
key = 'system.load_ee_orders_enabled'
self.assertEqual(api.get_production_key(key)[key], 'True')
production_provider.load_ee_orders()
reorder = Order.find(order.id)
self.assertEqual(
reorder.product_opts, {
'format': 'gtiff',
'etm7_collection': {
'inputs': ['LE07_L1TP_026027_20170912_20171008_01_T1'],
'products': ['sr']
},
'tm5_collection': {
'inputs': ['LT05_L1TP_025027_20110913_20160830_01_T1'],
'products': ['sr']
}
})
@patch('api.external.lta.update_order_status', lta.update_order_status)
def test_production_handle_failed_ee_updates(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
for scene in order.scenes():
scene.update('failed_lta_status_update', 'C')
scenes = Order.find(order.id).scenes(
{'failed_lta_status_update IS NOT': None})
production_provider.handle_failed_ee_updates(scenes)
scenes = Scene.where({'failed_lta_status_update IS NOT': None})
self.assertTrue(len(scenes) == 0)
@patch(
'api.providers.production.production_provider.ProductionProvider.update_landsat_product_status',
mock_production_provider.respond_true)
@patch(
'api.providers.production.production_provider.ProductionProvider.get_contactids_for_submitted_landsat_products',
mock_production_provider.contact_ids_list)
@patch('api.external.inventory.available', lambda: True)
def test_production_handle_submitted_landsat_products(self):
orders = Order.find(
self.mock_order.generate_testing_order(self.user_id))
scenes = orders.scenes({'sensor_type': 'landsat'})
self.assertTrue(
production_provider.handle_submitted_landsat_products(scenes))
@patch('api.external.lta.update_order_status', lta.update_order_status)
def test_production_set_products_unavailable(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
self.assertTrue(
production_provider.set_products_unavailable(
order.scenes(), "you want a reason?"))
@patch(
'api.providers.production.production_provider.ProductionProvider.check_dependencies_for_products',
mock_production_provider.check_dependencies_for_products)
@patch('api.external.inventory.get_cached_session',
inventory.get_cached_session)
@patch('api.external.inventory.check_valid', inventory.check_valid_landsat)
def test_production_update_landsat_product_status(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
for scene in order.scenes({'name !=': 'plot'}):
scene.status = 'submitted'
scene.sensor_type = 'landsat'
user = User.find(self.user_id)
scene.save()
self.assertTrue(
production_provider.update_landsat_product_status(user.contactid))
def test_production_get_contactids_for_submitted_landsat_products(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
for scene in order.scenes({'name !=': 'plot'}):
scene.status = 'submitted'
scene.sensor_type = 'landsat'
scene.save()
scenes = Order.find(order.id).scenes({'sensor_type': 'landsat'})
response = production_provider.get_contactids_for_submitted_landsat_products(
scenes)
self.assertIsInstance(response, set)
self.assertTrue(len(response) > 0)
@patch('api.external.inventory.available', lambda: True)
@patch('api.external.inventory.get_cached_session',
inventory.get_cached_session)
@patch('api.external.inventory.check_valid', inventory.check_valid_modis)
def test_production_handle_submitted_modis_products_input_exists(self):
# handle oncache scenario
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
for scene in order.scenes({'name !=': 'plot'}):
scene.status = 'submitted'
scene.sensor_type = 'modis'
scene.save()
sid = scene.id
scenes = order.scenes({'sensor_type': 'modis'})
self.assertTrue(
production_provider.handle_submitted_modis_products(scenes))
#self.assertEquals(Scene.find(sid).status, "oncache")
@patch('api.external.inventory.available', lambda: True)
@patch('api.external.inventory.get_cached_session',
inventory.get_cached_session)
@patch('api.external.inventory.check_valid', inventory.check_valid_modis)
def test_production_handle_submitted_modis_products_input_missing(self):
# handle unavailable scenario
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
for scene in order.scenes({'name !=': 'plot'}):
scene.status = 'submitted'
scene.sensor_type = 'modis'
scene.save()
sid = scene.id
scenes = order.scenes({'sensor_type': 'modis'})
self.assertTrue(
production_provider.handle_submitted_modis_products(scenes))
self.assertEquals(Scene.find(sid).status, "unavailable")
def test_production_handle_submitted_plot_products(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
order.status = 'ordered'
order.order_type = 'lpcs'
order.save()
plot_id = None
for idx, scene in enumerate(order.scenes()):
# at the moment, mock_order.generate_testing_order
# creates 21 products for the order. divvy those
# up between 'complete' and 'unavailable', setting
# one aside as the 'plot' product
if scene.sensor_type == 'plot':
# need to define a plot product
scene.update('status', 'submitted')
plot_id = scene.id
else:
if idx % 2 == 0:
scene.update('status', 'complete')
else:
scene.update('status', 'unavailable')
scenes = order.scenes()
self.assertTrue(
production_provider.handle_submitted_plot_products(scenes))
self.assertEqual(Scene.find(plot_id).status, "oncache")
@patch('os.path.exists', lambda y: True)
@patch('os.path.getsize', lambda y: 999)
def test_production_calc_scene_download_sizes(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scenes = order.scenes()
Scene.bulk_update([s.id for s in scenes], {
'status': 'complete',
'download_size': 0
})
scenes = Order.find(order.id).scenes({
'status': 'complete',
'download_size': 0
})
self.assertTrue(production_provider.calc_scene_download_sizes(scenes))
upscenes = Scene.where({'status': 'complete', 'download_size': 999})
self.assertEqual(len(upscenes), len(scenes))
@patch(
'api.providers.production.production_provider.ProductionProvider.update_order_if_complete',
mock_production_provider.respond_true)
def test_production_finalize_orders(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
order.update('status', 'ordered')
self.assertTrue(production_provider.finalize_orders([order]))
@patch(
'api.providers.production.production_provider.ProductionProvider.send_completion_email',
mock_production_provider.respond_true)
def test_production_update_order_if_complete(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
Scene.bulk_update([s.id for s in order.scenes()], {'status': 'retry'})
order.order_source = 'espa'
order.completion_email_sent = None
order.save()
self.assertTrue(production_provider.update_order_if_complete(order))
def test_production_queue_products_success(self):
names_tuple = self.mock_order.names_tuple(3, self.user_id)
processing_loc = "get_products_to_process"
job_name = 'jobname49'
params = (names_tuple, processing_loc, job_name)
response = api.queue_products(*params)
self.assertTrue(response)
def test_production_get_key(self):
key = 'system_message_title'
response = api.get_production_key(key)
val = response[key]
self.assertIsInstance(val, str)
def test_get_production_key_invalid(self):
bad_key = 'foobar'
response = api.get_production_key(bad_key)
self.assertEqual(response.keys(), ['msg'])
@patch('api.external.hadoop.HadoopHandler.job_names_ids',
hadoop.jobs_names_ids)
def test_catch_orphaned_scenes(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
# need scenes with statuses of 'queued'
self.mock_order.update_scenes(order_id, ('landsat', 'modis', 'plot'),
'status', ['queued'])
response = production_provider.catch_orphaned_scenes()
self.assertTrue(response)
old_time = datetime.datetime.now() - datetime.timedelta(minutes=15)
for s in Scene.where({'order_id': order_id}):
self.assertTrue(s.reported_orphan is not None)
s.reported_orphan = old_time
s.save()
response = production_provider.catch_orphaned_scenes()
self.assertTrue(response)
for s in Scene.where({'order_id': order_id}):
self.assertTrue(s.orphaned)
@patch('api.external.hadoop.HadoopHandler.job_names_ids',
hadoop.jobs_names_ids)
def test_handle_stuck_jobs(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
# Make some really old jobs
self.mock_order.update_scenes(order_id, ('landsat', 'modis', 'plot'),
'status', ['processing'])
self.mock_order.update_scenes(order_id, ('landsat', 'modis', 'plot'),
'status_modified',
[datetime.datetime(1900, 1, 1)])
scenes = Scene.where({'status': 'processing', 'order_id': order_id})
n_scenes = len(scenes)
response = production_provider.handle_stuck_jobs(scenes)
self.assertTrue(response)
scenes = Scene.where({
'status': 'processing',
'order_id': order_id,
'reported_orphan is not': None
})
self.assertEqual(n_scenes, len(scenes))
self.mock_order.update_scenes(order_id, ('landsat', 'modis', 'plot'),
'reported_orphan',
[datetime.datetime(1900, 1, 1)])
response = production_provider.handle_stuck_jobs(scenes)
self.assertTrue(response)
scenes = Scene.where({'status': 'processing', 'order_id': order_id})
self.assertEqual(0, len(scenes))
def test_convert_product_options(self):
"""
Test the conversion procedure to make sure that the new format for orders converts
to the old format
"""
scenes = [
'LE07_L1TP_026027_20170912_20171008_01_T1',
'LC08_L1TP_025027_20160521_20170223_01_T1',
'LT05_L1TP_025027_20110913_20160830_01_T1'
]
includes = ['include_sr', 'include_sr_toa', 'include_sr_thermal']
new_format = {
u'etm7_collection': {
u'inputs': [u'LE07_L1TP_026027_20170912_20171008_01_T1'],
u'products': [u'sr']
},
u'olitirs8_collection': {
u'inputs': [u'LC08_L1TP_025027_20160521_20170223_01_T1'],
u'products': [u'toa']
},
u'tm5_collection': {
u'inputs': [u'LT05_L1TP_025027_20110913_20160830_01_T1'],
u'products': [u'bt']
},
u'format': u'gtiff',
u'image_extents': {
u'east': -2265585.0,
u'north': 3164805.0,
u'south': 3014805.0,
u'units': u'meters',
u'west': -2415585.0
},
u'note': u'CONUS_h1v1',
u'projection': {
u'aea': {
u'central_meridian': -96,
u'datum': u'nad83',
u'false_easting': 0,
u'false_northing': 0,
u'latitude_of_origin': 23,
u'standard_parallel_1': 29.5,
u'standard_parallel_2': 45.5
}
},
u'resampling_method': u'cc'
}
ruberic = {
'central_meridian': -96,
'datum': u'nad83',
'false_easting': 0,
'false_northing': 0,
'image_extents': True,
'image_extents_units': u'meters',
'include_customized_source_data': False,
'include_dswe': False,
'include_st': False,
'include_solr_index': False,
'include_source_data': False,
'include_source_metadata': False,
'include_sr': False,
'include_sr_browse': False,
'include_sr_evi': False,
'include_sr_msavi': False,
'include_sr_nbr': False,
'include_sr_nbr2': False,
'include_sr_ndmi': False,
'include_sr_ndvi': False,
'include_sr_savi': False,
'include_sr_thermal': False,
'include_sr_toa': False,
'include_statistics': False,
'latitude_true_scale': None,
'longitude_pole': None,
'maxx': -2265585.0,
'maxy': 3164805.0,
'minx': -2415585.0,
'miny': 3014805.0,
'origin_lat': 23,
'output_format': u'gtiff',
'pixel_size': None,
'pixel_size_units': None,
'reproject': True,
'resample_method': 'cubic',
'resize': False,
'std_parallel_1': 29.5,
'std_parallel_2': 45.5,
'target_projection': u'aea',
'utm_north_south': None,
'utm_zone': None
}
for scene, include in zip(scenes, includes):
ruberic[include] = True
old_format = OptionsConversion.convert(new=new_format,
scenes=[scene])
self.assertDictEqual(ruberic, old_format)
ruberic[include] = False
def test_status_modified(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
scene = Scene.where({'order_id': order_id}).pop()
scene.status = 'oncache'
scene.save()
old_time = scene.status_modified
scene.status = 'queued'
scene.save()
new_time = scene.status_modified
self.assertGreater(new_time, old_time)
@patch('api.external.hadoop.HadoopHandler.list_jobs',
hadoop.jobs_names_ids)
@patch('api.external.hadoop.HadoopHandler.kill_job', lambda x, y: True)
def test_hadoop_reset_status(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
scenes = Scene.where({'order_id': order_id})
Scene.bulk_update([s.id for s in scenes], {'status': 'processing'})
self.assertTrue(production_provider.reset_processing_status())
scenes = Scene.where({'order_id': order_id})
self.assertEqual({'submitted'}, set([s.status for s in scenes]))
class TestAPI(unittest.TestCase):
def setUp(self):
logger.warning('Testing API started...')
os.environ['espa_api_testing'] = 'True'
# create a user
self.mock_user = MockUser()
self.mock_order = MockOrder()
user_id = self.mock_user.add_testing_user()
order_id = self.mock_order.generate_testing_order(user_id)
self.order = Order.find(order_id)
self.user = User.find(user_id)
self.product_id = 'LT05_L1TP_032028_20120425_20160830_01_T1'
self.sensor_id = 'tm5_collection'
self.staff_product_id = 'LE07_L1TP_010028_20050420_20160925_01_T1'
self.staff_sensor = 'etm7_collection'
self.global_product_id = 'LE70450302003206EDC01'
staff_user_id = self.mock_user.add_testing_user()
self.staff_user = User.find(staff_user_id)
self.staff_user.update('is_staff', True)
staff_order_id = self.mock_order.generate_testing_order(staff_user_id)
staff_order = Order.find(staff_order_id)
staff_scene = staff_order.scenes()[0]
staff_scene.update('name', self.staff_product_id)
user_scene = self.order.scenes()[0]
user_scene.update('name', self.staff_product_id)
with open(os.path.join(__location__, 'domain/restricted.yaml')) as f:
self.restricted = yaml.load(f.read())
self.restricted['all']['role'].remove('restricted_prod')
def tearDown(self):
logger.warning('Testing API done.')
# clean up orders
self.mock_order.tear_down_testing_orders()
# clean up users
self.mock_user.cleanup()
os.environ['espa_api_testing'] = ''
def test_api_versions_key_val(self):
self.assertEqual(set(api.api_versions().keys()), set(['v0', 'v1']))
def test_get_available_products_key_val(self):
self.assertEqual(
api.available_products(self.product_id,
self.user.username).keys()[0],
self.sensor_id)
def test_get_available_products_by_staff(self):
# staff should see all available products
self.user.update('is_staff', True)
return_dict = api.available_products(self.staff_product_id,
self.staff_user.username)
for item in self.restricted['all']['role']:
self.assertTrue(item in return_dict[self.staff_sensor]['products'])
def test_get_available_products_by_public(self):
# public should not see products listed in api/domain.restricted.yaml
self.user.update('is_staff', False)
return_dict = api.available_products(self.staff_product_id,
self.user.username)
for item in self.restricted['all']['role']:
self.assertFalse(
item in return_dict[self.staff_sensor]['products'])
def test_get_available_products_global_restriction(self):
# testing of landsat pre-collection restricted
self.user.update('is_staff', False)
return_dict = api.available_products(self.global_product_id,
self.user.username)
self.assertIn('ordering_restricted', return_dict)
self.assertEqual(return_dict['ordering_restricted']['etm7'],
[self.global_product_id])
def test_fetch_user_orders_by_email_val(self):
orders = api.fetch_user_orders(email=self.user.email)
self.assertTrue(len(orders) > 1)
self.assertIn(self.order.orderid, [o.orderid for o in orders])
def test_fetch_user_orders_by_username_val(self):
orders = api.fetch_user_orders(username=self.user.username)
self.assertTrue(len(orders) > 1)
self.assertIn(self.order.orderid, [o.orderid for o in orders])
def test_fetch_order_by_orderid_val(self):
order = api.fetch_order(self.order.orderid)
self.assertEqual(1, len(order))
self.assertEqual(order[0].orderid, self.order.orderid)
def test_fetch_order_by_orderid_invalid(self):
invalid_orderid = 'invalidorderid'
response = api.fetch_order(invalid_orderid)
self.assertEqual(response, list())
def test_fetch_item_status_valid(self):
response = api.item_status(self.order.orderid)
self.assertEqual(set(response), {self.order.orderid})
self.assertIsInstance(response[self.order.orderid], list)
self.assertEqual(set([s.name for s in self.order.scenes()]),
set([s.name for s in response[self.order.orderid]]))
class ProductionTransportTestCase(unittest.TestCase):
def setUp(self):
os.environ['espa_api_testing'] = 'True'
# create a user
self.mock_user = MockUser()
self.mock_order = MockOrder()
self.user = User.find(self.mock_user.add_testing_user())
self.order_id = self.mock_order.generate_testing_order(self.user.id)
self.app = http.app.test_client()
self.app.testing = True
self.sceneids = self.mock_order.scene_names_list(self.order_id)[0:2]
with db_instance() as db:
uidsql = "select user_id, orderid from ordering_order limit 1;"
db.select(uidsql)
self.userid = db[0]['user_id']
self.orderid = db[0]['orderid']
itemsql = "select name, order_id from ordering_scene limit 1;"
db.select(itemsql)
self.itemid = db[0][0]
itemorderid = db[0][1]
ordersql = "select orderid from ordering_order where id = {};".format(
itemorderid)
db.select(ordersql)
self.itemorderid = db[0][0]
self.base_order = lowercase_all(testorders.build_base_order())
def tearDown(self):
# clean up orders
self.mock_order.tear_down_testing_orders()
# clean up users
self.mock_user.cleanup()
os.environ['espa_api_testing'] = ''
@patch('api.interfaces.production.version1.API.get_production_whitelist',
api.get_production_whitelist)
def test_get_production_api(self):
response = self.app.get('/production-api',
environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
assert response.content_type == 'application/json'
assert set(response_data.keys()) == set(['0', '1'])
@patch('api.interfaces.production.version1.API.get_production_whitelist',
api.get_production_whitelist)
def test_get_production_api_v1(self):
response = self.app.get('/production-api/v1',
environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
assert set(response_data.keys()) == set(["operations", "description"])
assert "ESPA Production" in response_data['description']
@patch(
'api.providers.production.production_provider.ProductionProvider.get_products_to_process',
production_provider.get_products_to_process_inputs)
@patch('api.interfaces.production.version1.API.get_production_whitelist',
api.get_production_whitelist)
def test_get_production_api_products_modis(self):
url = "/production-api/v1/products?for_user=bilbo&product_types=modis"
response = self.app.get(url, environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
correct_resp = {
'encode_urls': False,
'for_user': '******',
'record_limit': 500,
'product_types': 'modis',
'priority': None
}
assert response_data == correct_resp
@patch(
'api.providers.production.production_provider.ProductionProvider.get_products_to_process',
production_provider.get_products_to_process_inputs)
@patch('api.interfaces.production.version1.API.get_production_whitelist',
api.get_production_whitelist)
def test_get_production_api_products_landsat(self):
url = "/production-api/v1/products?for_user=bilbo&product_types=landsat"
response = self.app.get(url, environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
correct_resp = {
'encode_urls': False,
'for_user': '******',
'record_limit': 500,
'product_types': 'landsat',
'priority': None
}
assert response_data == correct_resp
@patch('api.interfaces.production.version1.API.get_production_whitelist',
api.get_production_whitelist)
@patch(
'api.providers.production.production_provider.ProductionProvider.update_status',
production_provider.update_status_inputs)
def test_post_production_api_update_status(self):
url = "/production-api/v1/update_status"
data_dict = {
'name': 't10000xyz401',
'orderid': '[email protected]',
'processing_loc': 'update_status',
'status': 'updated'
}
response = self.app.post(url,
data=json.dumps(data_dict),
environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
assert response_data == data_dict
@patch(
'api.providers.production.production_provider.ProductionProvider.set_product_error',
production_provider.set_product_error_inputs)
@patch('api.interfaces.production.version1.API.get_production_whitelist',
api.get_production_whitelist)
def test_post_production_api_set_product_error(self):
url = "/production-api/v1/set_product_error"
data_dict = {
'name': 't10000xyz401',
'orderid': '[email protected]',
'processing_loc': 'xyz',
'error': 'oopsie'
}
response = self.app.post(url,
data=json.dumps(data_dict),
environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
assert response_data == data_dict
@patch(
'api.providers.production.production_provider.ProductionProvider.set_product_unavailable',
production_provider.set_product_unavailable_inputs)
@patch('api.interfaces.production.version1.API.get_production_whitelist',
api.get_production_whitelist)
def test_post_production_api_set_product_unavailable(self):
url = "/production-api/v1/set_product_unavailable"
data_dict = {
'name': 't10000xyz401',
'orderid': '[email protected]',
'processing_loc': 'xyz',
'error': 'oopsie',
'note': 'them notes'
}
response = self.app.post(url,
data=json.dumps(data_dict),
environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
assert response_data == data_dict
@patch(
'api.providers.production.production_provider.ProductionProvider.mark_product_complete',
production_provider.set_mark_product_complete_inputs)
@patch('api.interfaces.production.version1.API.get_production_whitelist',
api.get_production_whitelist)
def test_post_production_api_mark_product_complete(self):
url = "/production-api/v1/mark_product_complete"
data_dict = {
'name': 't10000xyz401',
'orderid': '[email protected]',
'processing_loc': 'xyz',
'completed_file_location': '/tmp',
'cksum_file_location': '/tmp/txt.txt',
'log_file_contents': 'details'
}
response = self.app.post(url,
data=json.dumps(data_dict),
environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
assert response_data == data_dict
@patch(
'api.providers.production.production_provider.ProductionProvider.handle_orders',
production_provider.respond_true)
@patch('api.interfaces.production.version1.API.get_production_whitelist',
api.get_production_whitelist)
def test_post_production_api_handle_orders(self):
url = "/production-api/v1/handle-orders"
response = self.app.get(url,
data=json.dumps({}),
environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
assert response_data is True
@patch(
'api.providers.production.production_provider.ProductionProvider.queue_products',
production_provider.queue_products_inputs)
@patch('api.interfaces.production.version1.API.get_production_whitelist',
api.get_production_whitelist)
def test_post_production_api_queue_products(self):
url = "/production-api/v1/queue-products"
data_dict = {
'order_name_tuple_list': 'order_name_tuple_list',
'processing_location': 'processing_location',
'job_name': 'job_name'
}
response = self.app.post(url,
data=json.dumps(data_dict),
environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
assert response_data == data_dict
@patch('api.interfaces.production.version1.API.get_production_whitelist',
api.get_production_whitelist)
def test_get_production_api_configurations(self):
url = "/production-api/v1/configuration/system_message_title"
response = self.app.get(url, environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
assert response_data.keys() == ['system_message_title']
@patch('api.interfaces.admin.version1.API.get_stat_whitelist',
api.get_stat_whitelist)
def test_get_production_api_stat_products_complete_24_hrs(self):
url = "/production-api/v1/statistics/stat_products_complete_24_hrs"
response = self.app.get(url, environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
assert response_data == 0
class TestProductionAPI(unittest.TestCase):
def setUp(self):
os.environ['espa_api_testing'] = 'True'
# create a user
self.mock_user = MockUser()
self.mock_order = MockOrder()
self.user_id = self.mock_user.add_testing_user()
def tearDown(self):
# clean up orders
self.mock_order.tear_down_testing_orders()
# clean up users
self.mock_user.cleanup()
os.environ['espa_api_testing'] = ''
@patch('api.external.lpdaac.get_download_urls', lpdaac.get_download_urls)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry',
mock_production_provider.set_product_retry)
def test_fetch_production_products_modis(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
# need scenes with statuses of 'processing' and 'ordered'
self.mock_order.update_scenes(order_id, 'status', ['processing', 'ordered', 'oncache'])
user = User.find(self.user_id)
params = {'for_user': user.username, 'product_types': ['modis']}
response = api.fetch_production_products(params)
self.assertTrue('bilbo' in response[0]['orderid'])
@patch('api.external.lta.get_download_urls', lta.get_download_urls)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry',
mock_production_provider.set_product_retry)
def test_fetch_production_products_landsat(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
# need scenes with statuses of 'processing' and 'ordered'
self.mock_order.update_scenes(order_id, 'status', ['processing','oncache','ordered'])
user = User.find(self.user_id)
params = {'for_user': user.username, 'product_types': ['landsat']}
response = api.fetch_production_products(params)
self.assertTrue('bilbo' in response[0]['orderid'])
def test_fetch_production_products_plot(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
self.mock_order.update_scenes(order_id, 'status', ['complete'])
order = Order.find(order_id)
plot_scene = order.scenes()[0]
plot_scene.name = 'plot'
plot_scene.sensor_type = 'plot'
plot_scene.status = 'submitted'
plot_scene.save()
response = production_provider.handle_submitted_plot_products()
pscene = order.scenes({'status': 'oncache', 'sensor_type': 'plot'})
self.assertTrue(response is True)
self.assertEqual(len(pscene), 1)
def test_production_set_product_retry(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
order = Order.find(order_id)
scene = order.scenes()[3]
scene.update('retry_count', 4)
processing_loc = "get_products_to_process"
error = 'not available after EE call '
note = 'note this'
retry_after = datetime.datetime.now() + datetime.timedelta(hours=1)
retry_limit = 9
response = production_provider.set_product_retry(scene.name, order.orderid, processing_loc,
error, note, retry_after, retry_limit)
new = Scene.get('ordering_scene.status', scene.name, order.orderid)
self.assertTrue('retry' == new)
@patch('api.external.lta.update_order_status', mock_production_provider.respond_true)
def test_production_set_product_error_unavailable(self):
"""
Move a scene status from error to unavailable based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes({'name !=': 'plot'})[0]
production_provider.set_product_error(scene.name, order.orderid,
'get_products_to_process',
'include_dswe is an unavailable product option for OLITIRS')
self.assertTrue('unavailable' == Scene.get('ordering_scene.status', scene.name, order.orderid))
def test_production_set_product_error_submitted(self):
"""
Move a scene status from error to submitted based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes({'name !=': 'plot'})[0]
production_provider.set_product_error(scene.name, order.orderid,
'get_products_to_process',
'BLOCK, COMING FROM LST AS WELL: No such file or directory')
self.assertTrue('submitted' == Scene.get('ordering_scene.status', scene.name, order.orderid))
def test_production_set_product_error_retry(self):
"""
Move a scene status from error to retry based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[2]
production_provider.set_product_error(scene.name, order.orderid,
'somewhere',
'Verify the missing auxillary data products')
self.assertTrue('retry' == Scene.get('ordering_scene.status', scene.name, order.orderid))
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
def test_update_product_details_update_status(self):
"""
Set a scene status to Queued
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[0]
api.update_product_details('update_status',
{'name': scene.name,
'orderid': order.orderid,
'processing_loc': 'L8SRLEXAMPLE',
'status': 'Queued'})
self.assertTrue(Scene.get('ordering_scene.status', scene.name, order.orderid) == 'Queued')
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
@patch('api.external.onlinecache.capacity', onlinecache.capacity)
def test_update_product_details_set_product_error(self):
"""
Set a scene status to error
:return:
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[0]
production_provider.update_product('set_product_error',
name=scene.name, orderid=order.orderid,
processing_loc="L8SRLEXAMPLE",
error='problems yo')
self.assertTrue(Scene.find(scene.id).status == 'error')
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
def test_update_product_details_set_product_unavailable(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[0]
production_provider.update_product('set_product_unavailable',
name=scene.name, orderid=order.orderid,
processing_loc="L8SRLEXAMPLE",
error='include_dswe is an unavailable product option for OLITIRS')
self.assertTrue('unavailable' == Scene.get('ordering_scene.status', scene.name, order.orderid))
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
@patch('os.path.getsize', lambda y: 999)
def test_update_product_details_mark_product_complete(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[0]
production_provider.update_product('mark_product_complete',
name=scene.name,
orderid=order.orderid,
processing_loc='L8SRLEXAMPLE',
completed_file_location='/some/loc',
cksum_file_location='some checksum',
log_file_contents='some log')
self.assertTrue('complete' == Scene.get('ordering_scene.status', scene.name, order.orderid))
@patch('api.external.lta.update_order_status', lta.update_order_status_fail)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
@patch('os.path.getsize', lambda y: 999)
def test_update_product_details_fail_lta_mark_product_complete(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[1]
order.update('order_source', 'ee')
production_provider.update_product('mark_product_complete',
name=scene.name,
orderid=order.orderid,
processing_loc='L8SRLEXAMPLE',
completed_file_location='/some/loc',
cksum_file_location='some checksum',
log_file_contents='some log')
s = Scene.where({'name': scene.name, 'order_id': scene.order_id})[0]
self.assertTrue('C' == s.failed_lta_status_update)
@patch('api.providers.production.production_provider.ProductionProvider.send_initial_emails',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.handle_onorder_landsat_products',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.handle_retry_products',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.load_ee_orders',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.handle_submitted_products',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.finalize_orders',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.purge_orders',
mock_production_provider.respond_true)
def test_handle_orders_success(self):
self.assertTrue(api.handle_orders())
@patch('api.external.onlinecache.delete', mock_production_provider.respond_true)
@patch('api.notification.emails.send_purge_report', mock_production_provider.respond_true)
@patch('api.external.onlinecache.capacity', onlinecache.capacity)
def test_production_purge_orders(self):
new_completion_date = datetime.datetime.now() - datetime.timedelta(days=12)
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
order.update('status', 'complete')
order.update('completion_date', new_completion_date)
self.assertTrue(production_provider.purge_orders())
# need to figure a test for emails.send_email
@patch('api.notification.emails.Emails.send_email', mock_production_provider.respond_true)
def test_production_send_initial_emails(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
order.update('status', 'ordered')
self.assertTrue(emails.Emails().send_all_initial())
@patch('api.external.lta.get_order_status', lta.get_order_status)
@patch('api.external.lta.update_order_status', lta.update_order_status)
def test_production_handle_onorder_landsat_products(self):
tram_order_ids = lta.sample_tram_order_ids()[0:3]
scene_names = lta.sample_scene_names()[0:3]
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scenes = order.scenes()[0:3]
for idx, scene in enumerate(scenes):
scene.tram_order_id = tram_order_ids[idx]
scene.status = 'onorder'
# save() doesn't let you update name,
# b/c updating a scene name is not acceptable
# outside of testing
scene.update('name', scene_names[idx])
scene.save()
self.assertTrue(production_provider.handle_onorder_landsat_products())
def test_production_handle_retry_products(self):
prev = datetime.datetime.now() - datetime.timedelta(hours=1)
order_id = self.mock_order.generate_testing_order(self.user_id)
self.mock_order.update_scenes(order_id, 'status', ['retry'])
self.mock_order.update_scenes(order_id, 'retry_after', [prev])
production_provider.handle_retry_products()
for s in Scene.where({'order_id': order_id}):
self.assertTrue(s.status == 'submitted')
#@patch('api.external.lta.get_available_orders', lta.get_available_orders)
#@patch('api.external.lta.update_order_status', lta.update_order_status)
#@patch('api.external.lta.get_user_name', lta.get_user_name)
#def test_production_load_ee_orders(self):
# #production_provider.load_ee_orders()
# pass
@patch('api.external.lta.get_available_orders', lta.get_available_orders_partial)
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch('api.external.lta.get_user_name', lta.get_user_name)
def test_production_load_ee_orders_partial(self):
order = Order.find(self.mock_order.generate_ee_testing_order(self.user_id, partial=True))
self.assertTrue(order.product_opts == {'format': 'gtiff',
'etm7': {'inputs': ['LE70900652008327EDC00'],
'products': ['sr']}})
production_provider.load_ee_orders()
reorder = Order.find(order.id)
self.assertTrue(reorder.product_opts == {'format': 'gtiff',
'etm7': {'inputs': ['LE70900652008327EDC00'],
'products': ['sr']},
'tm5': {'inputs': ['LT50900652008327EDC00'],
'products': ['sr']}})
@patch('api.external.lta.update_order_status', lta.update_order_status)
def test_production_handle_failed_ee_updates(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.update('failed_lta_status_update', 'C')
production_provider.handle_failed_ee_updates()
scenes = Scene.where({'failed_lta_status_update IS NOT': None})
self.assertTrue(len(scenes) == 0)
@patch('api.providers.production.production_provider.ProductionProvider.handle_submitted_landsat_products',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.handle_submitted_modis_products',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.handle_submitted_plot_products',
mock_production_provider.respond_true)
def test_production_handle_submitted_products(self):
self.assertTrue(production_provider.handle_submitted_products())
@patch('api.providers.production.production_provider.ProductionProvider.mark_nlaps_unavailable',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.update_landsat_product_status',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.get_contactids_for_submitted_landsat_products',
mock_production_provider.contact_ids_list)
def test_production_handle_submitted_landsat_products(self):
self.assertTrue(production_provider.handle_submitted_landsat_products())
# !!! need to write test for nlaps.products_are_nlaps !!!
@patch('api.external.nlaps.products_are_nlaps', nlaps.products_are_nlaps)
@patch('api.providers.production.production_provider.ProductionProvider.set_products_unavailable',
mock_production_provider.respond_true)
def test_production_mark_nlaps_unavailable(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.status = 'submitted'
scene.sensor_type = 'landsat'
scene.save()
self.assertTrue(production_provider.mark_nlaps_unavailable())
@patch('api.external.lta.update_order_status', lta.update_order_status)
def test_production_set_products_unavailable(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
self.assertTrue(production_provider.set_products_unavailable(order.scenes(), "you want a reason?"))
@patch('api.external.lta.order_scenes', lta.order_scenes)
@patch('api.providers.production.production_provider.ProductionProvider.set_products_unavailable',
mock_production_provider.respond_true)
def test_production_update_landsat_product_status(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.status = 'submitted'
scene.sensor_type = 'landsat'
scene.save()
self.assertTrue(production_provider.update_landsat_product_status(User.find(self.user_id).contactid))
def test_production_get_contactids_for_submitted_landsat_products(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.status = 'submitted'
scene.sensor_type = 'landsat'
scene.save()
response = production_provider.get_contactids_for_submitted_landsat_products()
self.assertIsInstance(response, set)
self.assertTrue(len(response) > 0)
@patch('api.external.lpdaac.input_exists', lpdaac.input_exists_true)
def test_production_handle_submitted_modis_products_input_exists(self):
# handle oncache scenario
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.status = 'submitted'
scene.sensor_type = 'modis'
scene.save()
sid = scene.id
self.assertTrue(production_provider.handle_submitted_modis_products())
self.assertEquals(Scene.find(sid).status, "oncache")
@patch('api.external.lpdaac.input_exists', lpdaac.input_exists_false)
def test_production_handle_submitted_modis_products_input_missing(self):
# handle unavailable scenario
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.status = 'submitted'
scene.sensor_type = 'modis'
scene.save()
sid = scene.id
self.assertTrue(production_provider.handle_submitted_modis_products())
self.assertEquals(Scene.find(sid).status, "unavailable")
def test_production_handle_submitted_plot_products(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
order.status = 'ordered'
order.order_type = 'lpcs'
order.save()
plot_id = None
for idx, scene in enumerate(order.scenes()):
# at the moment, mock_order.generate_testing_order
# creates 21 products for the order. divvy those
# up between 'complete' and 'unavailable', setting
# one aside as the 'plot' product
if idx % 2 == 0:
if idx == 0:
# need to define a plot product
scene.update('status', 'submitted')
scene.update('sensor_type', 'plot')
plot_id = scene.id
else:
scene.update('status', 'complete')
else:
scene.update('status', 'unavailable')
self.assertTrue(production_provider.handle_submitted_plot_products())
self.assertEqual(Scene.find(plot_id).status, "oncache")
@patch('os.path.getsize', lambda y: 999)
def test_production_calc_scene_download_sizes(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scenes = order.scenes()
Scene.bulk_update([s.id for s in scenes], {'status': 'complete', 'download_size': 0})
self.assertTrue(production_provider.calc_scene_download_sizes())
upscenes = Scene.where({'status': 'complete', 'download_size': 999})
self.assertEqual(len(upscenes), len(scenes))
@patch('api.providers.production.production_provider.ProductionProvider.update_order_if_complete',
mock_production_provider.respond_true)
def test_production_finalize_orders(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
order.update('status', 'ordered')
self.assertTrue(production_provider.finalize_orders())
@patch('api.providers.production.production_provider.ProductionProvider.send_completion_email',
mock_production_provider.respond_true)
def test_production_update_order_if_complete(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
Scene.bulk_update([s.id for s in order.scenes()], {'status': 'retry'})
order.order_source = 'espa'
order.completion_email_sent = None
order.save()
self.assertTrue(production_provider.update_order_if_complete(order))
def test_production_queue_products_success(self):
names_tuple = self.mock_order.names_tuple(3, self.user_id)
processing_loc = "get_products_to_process"
job_name = 'jobname49'
params = (names_tuple, processing_loc, job_name)
response = api.queue_products(*params)
self.assertTrue(response)
def test_production_get_key(self):
key = 'system_message_title'
response = api.get_production_key(key)
val = response[key]
self.assertIsInstance(val, str)
def test_get_production_key_invalid(self):
bad_key = 'foobar'
response = api.get_production_key(bad_key)
self.assertEqual(response.keys(), ['msg'])
@patch('api.external.hadoop.HadoopHandler.job_names_ids',
hadoop.jobs_names_ids)
def test_catch_orphaned_scenes(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
# need scenes with statuses of 'queued'
self.mock_order.update_scenes(order_id, 'status', ['queued'])
response = production_provider.catch_orphaned_scenes()
self.assertTrue(response)
old_time = datetime.datetime.now() - datetime.timedelta(minutes=15)
for s in Scene.where({'order_id': order_id}):
self.assertTrue(s.reported_orphan is not None)
s.reported_orphan = old_time
s.save()
response = production_provider.catch_orphaned_scenes()
self.assertTrue(response)
for s in Scene.where({'order_id': order_id}):
self.assertTrue(s.orphaned)
def test_convert_product_options(self):
"""
Test the conversion procedure to make sure that the new format for orders converts
to the old format
"""
scenes = ['LE70480272012076EDC00', 'LC80500272013196LGN00',
'LT40480271983028PAC00', 'LT50490262009162PAC03']
includes = ['include_sr', 'include_sr_toa',
'include_cfmask', 'include_sr_thermal']
new_format = {u'etm7': {u'inputs': [u'LE70480272012076EDC00'],
u'products': [u'sr']},
u'olitirs8': {u'inputs': [u'LC80500272013196LGN00'],
u'products': [u'toa']},
u'tm4': {u'inputs': [u'LT40480271983028PAC00'],
u'products': [u'cloud']},
u'tm5': {u'inputs': [u'LT50490262009162PAC03'],
u'products': [u'bt']},
u'format': u'gtiff',
u'image_extents': {u'east': -2265585.0,
u'north': 3164805.0,
u'south': 3014805.0,
u'units': u'meters',
u'west': -2415585.0},
u'note': u'CONUS_h1v1',
u'projection': {u'aea': {u'central_meridian': -96,
u'datum': u'nad83',
u'false_easting': 0,
u'false_northing': 0,
u'latitude_of_origin': 23,
u'standard_parallel_1': 29.5,
u'standard_parallel_2': 45.5}},
u'resampling_method': u'cc'}
ruberic = {'central_meridian': -96,
'datum': u'nad83',
'false_easting': 0,
'false_northing': 0,
'image_extents': True,
'image_extents_units': u'meters',
'include_cfmask': False,
'include_customized_source_data': False,
'include_dswe': False,
'include_lst': False,
'include_solr_index': False,
'include_source_data': False,
'include_source_metadata': False,
'include_sr': False,
'include_sr_browse': False,
'include_sr_evi': False,
'include_sr_msavi': False,
'include_sr_nbr': False,
'include_sr_nbr2': False,
'include_sr_ndmi': False,
'include_sr_ndvi': False,
'include_sr_savi': False,
'include_sr_thermal': False,
'include_sr_toa': False,
'include_statistics': False,
'latitude_true_scale': None,
'longitude_pole': None,
'maxx': -2265585.0,
'maxy': 3164805.0,
'minx': -2415585.0,
'miny': 3014805.0,
'origin_lat': 23,
'output_format': u'gtiff',
'pixel_size': None,
'pixel_size_units': None,
'reproject': True,
'resample_method': 'cubic',
'resize': False,
'std_parallel_1': 29.5,
'std_parallel_2': 45.5,
'target_projection': u'aea',
'utm_north_south': None,
'utm_zone': None}
for scene, include in zip(scenes, includes):
ruberic[include] = True
old_format = OptionsConversion.convert(new=new_format, scenes=[scene])
self.assertDictEqual(ruberic, old_format)
ruberic[include] = False
class TestAPI(unittest.TestCase):
def setUp(self):
os.environ['espa_api_testing'] = 'True'
# create a user
self.mock_user = MockUser()
self.mock_order = MockOrder()
user_id = self.mock_user.add_testing_user()
order_id = self.mock_order.generate_testing_order(user_id)
self.order = Order.find(order_id)
self.user = User.find(user_id)
self.product_id = 'LT50150401987120XXX02'
self.staff_product_id = 'LE70450302003206EDC01'
staff_user_id = self.mock_user.add_testing_user()
self.staff_user = User.find(staff_user_id)
self.staff_user.update('is_staff', True)
staff_order_id = self.mock_order.generate_testing_order(staff_user_id)
staff_order = Order.find(staff_order_id)
staff_scene = staff_order.scenes()[0]
staff_scene.update('name', self.staff_product_id)
user_scene = self.order.scenes()[0]
user_scene.update('name', self.staff_product_id)
with open('api/domain/restricted.yaml') as f:
self.restricted = yaml.load(f.read())
self.restricted['all']['role'].remove('restricted_prod')
def tearDown(self):
# clean up orders
self.mock_order.tear_down_testing_orders()
# clean up users
self.mock_user.cleanup()
os.environ['espa_api_testing'] = ''
def test_api_versions_key_val(self):
self.assertEqual(set(api.api_versions().keys()), set(['0', '1']))
def test_get_available_products_key_val(self):
self.assertEqual(
api.available_products(self.product_id,
self.user.username).keys()[0], "tm5")
def test_get_available_products_by_staff(self):
# staff should see all available products
self.user.update('is_staff', True)
return_dict = api.available_products(self.staff_product_id,
self.staff_user.username)
for item in self.restricted['all']['role']:
self.assertTrue(item in return_dict['etm7']['products'])
def test_get_available_products_by_public(self):
# public should not see products listed in api/domain.restricted.yaml
self.user.update('is_staff', False)
return_dict = api.available_products(self.staff_product_id,
self.user.username)
for item in self.restricted['all']['role']:
self.assertFalse(item in return_dict['etm7']['products'])
def test_fetch_user_orders_by_email_val(self):
orders = api.fetch_user_orders(self.user.email)
self.assertEqual(orders.keys()[0], "orders")
def test_fetch_user_orders_by_username_val(self):
orders = api.fetch_user_orders(self.user.username)
self.assertEqual(orders.keys()[0], "orders")
def test_fetch_order_by_orderid_val(self):
order = api.fetch_order(self.order.orderid)
self.assertEqual(order['orderid'], self.order.orderid)
def test_fetch_order_status_valid(self):
response = api.order_status(self.order.orderid)
self.assertEqual(response.keys(), ['orderid', 'status'])
def test_fetch_order_status_invalid(self):
invalid_orderid = 'invalidorderid'
response = api.order_status(invalid_orderid)
self.assertEqual(response.keys(), ['msg'])
def test_fetch_item_status_valid(self):
response = api.item_status(self.order.orderid)
self.assertEqual(response.keys(), ['orderid'])
self.assertIsInstance(response['orderid'], dict)
def test_fetch_item_status_invalid(self):
invalid_orderid = 'invalidorderid'
response = api.order_status(invalid_orderid)
self.assertEqual(response.keys(), ['msg'])
class TransportTestCase(unittest.TestCase):
def setUp(self):
os.environ['espa_api_testing'] = 'True'
# create a user
self.mock_user = MockUser()
self.mock_order = MockOrder()
self.user = User.find(self.mock_user.add_testing_user())
self.order_id = self.mock_order.generate_testing_order(self.user.id)
self.app = http.app.test_client()
self.app.testing = True
self.sceneids = self.mock_order.scene_names_list(self.order_id)[0:2]
token = ''.format(self.user.username, 'foo')
auth_string = "Basic {}".format(base64.b64encode(token))
self.headers = {"Authorization": auth_string}
with db_instance() as db:
uidsql = "select user_id, orderid from ordering_order limit 1;"
db.select(uidsql)
self.userid = db[0]['user_id']
self.orderid = db[0]['orderid']
itemsql = "select name, order_id from ordering_scene limit 1;"
db.select(itemsql)
self.itemid = db[0][0]
itemorderid = db[0][1]
ordersql = "select orderid from ordering_order where id = {};".format(itemorderid)
db.select(ordersql)
self.itemorderid = db[0][0]
self.base_order = lowercase_all(testorders.build_base_order())
def tearDown(self):
# clean up orders
self.mock_order.tear_down_testing_orders()
# clean up users
self.mock_user.cleanup()
os.environ['espa_api_testing'] = ''
def test_get_api_response_type(self):
response = self.app.get('/api', headers=self.headers, environ_base={'REMOTE_ADDR': '127.0.0.1'})
assert response.content_type == 'application/json'
def test_get_api_response_content(self):
response = self.app.get('/api', headers=self.headers, environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertEqual(set(['1', '0']), set(resp_json.keys()))
@patch('api.domain.user.User.get', MockUser.get)
def test_get_api_info_response_content(self):
response = self.app.get('/api/v1', headers=self.headers, environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert "Version 1" in resp_json['description']
@patch('api.domain.user.User.get', MockUser.get)
@patch('api.providers.ordering.ordering_provider.OrderingProvider.available_products', mock_api.available_products)
def test_get_available_prods(self):
url = '/api/v1/available-products/' + ",".join(self.sceneids)
response = self.app.get(url, headers=self.headers, environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert "etm" in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
@patch('api.providers.ordering.ordering_provider.OrderingProvider.available_products', mock_api.available_products)
def test_post_available_prods(self):
url = '/api/v1/available-products'
data_dict = {'inputs': self.sceneids}
response = self.app.post(url, data=json.dumps(data_dict), headers=self.headers, environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert "etm" in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
@patch('api.providers.ordering.ordering_provider.OrderingProvider.fetch_user_orders', mock_ordering_provider.fetch_user_orders)
def test_get_available_orders_user(self):
url = "/api/v1/list-orders"
response = self.app.get(url, headers=self.headers, environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert resp_json.keys() == ['orders']
@patch('api.domain.user.User.get', MockUser.get)
def test_get_available_orders_email(self):
# email param comes in as unicode
url = "/api/v1/list-orders/" + str(self.user.email)
response = self.app.get(url, headers=self.headers, environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert resp_json.keys() == ['orders']
@patch('api.domain.user.User.get', MockUser.get)
def test_get_order_by_ordernum(self):
url = "/api/v1/order/" + str(self.orderid)
response = self.app.get(url, headers=self.headers, environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert 'orderid' in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
def test_get_order_status_by_ordernum(self):
url = "/api/v1/order-status/" + str(self.orderid)
response = self.app.get(url, headers=self.headers, environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert 'orderid' in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
def test_get_item_status_by_ordernum(self):
url = "/api/v1/item-status/%s" % self.itemorderid
response = self.app.get(url, headers=self.headers, environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert 'orderid' in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
def test_get_item_status_by_ordernum_itemnum(self):
url = "/api/v1/item-status/%s/%s" % (self.itemorderid, self.itemid)
response = self.app.get(url, headers=self.headers, environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert 'orderid' in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
def test_get_current_user(self):
url = "/api/v1/user"
response = self.app.get(url, headers=self.headers, environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert 'username' in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
def test_get_projections(self):
url = '/api/v1/projections'
response = self.app.get(url, headers=self.headers, environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert 'aea' in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
def test_get_formats(self):
url = '/api/v1/formats'
response = self.app.get(url, headers=self.headers, environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert 'formats' in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
def test_get_resampling(self):
url = '/api/v1/resampling-methods'
response = self.app.get(url, headers=self.headers, environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert 'resampling_methods' in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
def test_get_order_schema(self):
url = '/api/v1/order-schema'
response = self.app.get(url, headers=self.headers, environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert 'properties' in resp_json.keys()
# Waiting for DB mock-ups to be finished
def test_post_order(self):
pass
class TransportTestCase(unittest.TestCase):
def setUp(self):
os.environ['espa_api_testing'] = 'True'
# create a user
self.mock_user = MockUser()
self.mock_order = MockOrder()
self.user = User.find(self.mock_user.add_testing_user())
self.order_id = self.mock_order.generate_testing_order(self.user.id)
self.app = http.app.test_client()
self.app.testing = True
self.sceneids = self.mock_order.scene_names_list(self.order_id)[0:2]
token = ':'.join((self.user.username, 'foo'))
auth_string = "Basic {}".format(base64.b64encode(token))
self.headers = {"Authorization": auth_string}
with db_instance() as db:
uidsql = "select user_id, orderid from ordering_order limit 1;"
db.select(uidsql)
self.userid = db[0]['user_id']
self.orderid = db[0]['orderid']
itemsql = "select name, order_id from ordering_scene limit 1;"
db.select(itemsql)
self.itemid = db[0][0]
itemorderid = db[0][1]
ordersql = "select orderid from ordering_order where id = {};".format(
itemorderid)
db.select(ordersql)
self.itemorderid = db[0][0]
self.base_order = lowercase_all(testorders.build_base_order())
self.sensors = [
k for k in self.base_order.keys()
if isinstance(self.base_order[k], dict)
and 'inputs' in self.base_order[k]
]
self.inputs = {s: self.base_order[s]['inputs'] for s in self.sensors}
self.input_names_all = set([
i for sublist in [s for k, s in self.inputs.items()]
for i in sublist
])
def tearDown(self):
# clean up orders
self.mock_order.tear_down_testing_orders()
# clean up users
self.mock_user.cleanup()
os.environ['espa_api_testing'] = ''
def test_get_api_response_type(self):
response = self.app.get('/api',
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
self.assertEqual(response.content_type, 'application/json')
@patch('api.domain.user.User.get', MockUser.get)
def test_get_api_response_content(self):
response = self.app.get('/api',
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertEqual(set(['v1', 'v0']), set(resp_json.keys()))
self.assertEqual(200, response.status_code)
@patch('api.domain.user.User.get', MockUser.get)
def test_get_api_info_response_content(self):
response = self.app.get('/api/v1',
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertIn("Version 1", resp_json['description'])
self.assertEqual(200, response.status_code)
@patch('api.domain.user.User.get', MockUser.get)
@patch(
'api.providers.ordering.ordering_provider.OrderingProvider.available_products',
mock_api.available_products)
def test_get_available_prods(self):
url = '/api/v1/available-products/' + ",".join(self.sceneids)
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertIn("etm", resp_json.keys())
self.assertEqual(200, response.status_code)
@patch('api.domain.user.User.get', MockUser.get)
@patch(
'api.providers.ordering.ordering_provider.OrderingProvider.available_products',
mock_api.available_products)
def test_get_available_prods_json(self):
url = '/api/v1/available-products'
data_dict = {'inputs': self.sceneids}
response = self.app.get(url,
data=json.dumps(data_dict),
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertIn('etm', resp_json.keys())
self.assertEqual(200, response.status_code)
@patch('api.domain.user.User.get', MockUser.get)
def test_get_available_orders_user(self):
url = "/api/v1/list-orders"
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertIsInstance(resp_json, list)
self.assertListEqual(resp_json, [self.orderid])
self.assertEqual(200, response.status_code)
@patch('api.domain.user.User.get', MockUser.get)
def test_get_available_orders_email(self):
# email param comes in as unicode
url = "/api/v1/list-orders/" + str(self.user.email)
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertIsInstance(resp_json, list)
self.assertListEqual(resp_json, [self.orderid])
self.assertEqual(200, response.status_code)
@patch('api.domain.user.User.get', MockUser.get)
def test_get_order_by_ordernum(self):
url = "/api/v1/order/" + str(self.orderid)
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
items = {
'orderid', 'note', 'order_source', 'order_type', 'product_opts',
'priority', 'completion_date', 'status', 'order_date',
'product_options'
}
self.assertEqual(items, set(resp_json))
self.assertEqual(200, response.status_code)
@patch('api.domain.user.User.get', MockUser.get)
def test_get_order_status_by_ordernum(self):
url = "/api/v1/order-status/" + str(self.orderid)
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertEqual({'orderid', 'status'}, set(resp_json))
self.assertEqual(self.orderid, resp_json.get('orderid'))
self.assertEqual('ordered', resp_json.get('status'))
self.assertEqual(200, response.status_code)
@patch('api.domain.user.User.get', MockUser.get)
def test_get_item_status_by_ordernum(self):
url = "/api/v1/item-status/%s" % self.itemorderid
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
all_names = set([s['name'].lower() for s in resp_json[self.orderid]])
all_names -= {'plot'}
self.assertEqual(self.input_names_all, all_names)
self.assertEqual(200, response.status_code)
@patch('api.domain.user.User.get', MockUser.get)
def test_get_item_status_by_ordernum_itemnum(self):
url = "/api/v1/item-status/%s/%s" % (self.itemorderid, self.itemid)
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
all_names = set([s['name'].lower() for s in resp_json[self.orderid]])
self.assertEqual({self.itemid.lower()}, all_names)
self.assertEqual(200, response.status_code)
@patch('api.domain.user.User.get', MockUser.get)
def test_get_current_user(self):
url = "/api/v1/user"
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertIn('username', resp_json.keys())
self.assertEqual(200, response.status_code)
@patch('api.domain.user.User.get', MockUser.get)
def test_get_projections(self):
url = '/api/v1/projections'
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertIn('aea', resp_json.keys())
self.assertEqual(200, response.status_code)
@patch('api.domain.user.User.get', MockUser.get)
def test_get_formats(self):
url = '/api/v1/formats'
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertIn('formats', resp_json.keys())
self.assertEqual(200, response.status_code)
@patch('api.domain.user.User.get', MockUser.get)
def test_get_resampling(self):
url = '/api/v1/resampling-methods'
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertIn('resampling_methods', resp_json.keys())
self.assertEqual(200, response.status_code)
@patch('api.domain.user.User.get', MockUser.get)
def test_get_order_schema(self):
url = '/api/v1/order-schema'
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertIn('properties', resp_json.keys())
self.assertEqual(200, response.status_code)
@patch('api.domain.user.User.get', MockUser.get)
def test_bad_method(self):
url = '/api/v1/available-products/'
response = self.app.post(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertEqual(405, response.status_code)
self.assertIn('messages', resp_json)
self.assertIn('errors', resp_json['messages'])
@patch('api.domain.user.User.get', MockUser.get)
def test_bad_data(self):
url = '/api/v1/order'
data = '{"inputs": [}'
response = self.app.post(url,
data=data,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertEqual(400, response.status_code)
self.assertIn('messages', resp_json)
self.assertIn('errors', resp_json['messages'])
@patch('api.domain.user.User.get', MockUser.get)
def test_bad_validation_inputs(self):
url = '/api/v1/order'
data = '{"inputs": []}'
response = self.app.post(url,
data=data,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertEqual(400, response.status_code)
self.assertIn('messages', resp_json)
self.assertIn('errors', resp_json['messages'])
self.assertIn('Schema errors', resp_json['messages']['errors'][0])
@patch('api.domain.user.User.get', MockUser.get)
def test_bad_validation_sensor_inputs(self):
url = '/api/v1/order'
data = '{"etm7_collection": {"inputs": ["LE07_L1TP_010028_20050420_20160925_01_T1"]}}'
response = self.app.post(url,
data=data,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertEqual(400, response.status_code)
self.assertIn('messages', resp_json)
self.assertIn('errors', resp_json['messages'])
self.assertIn('2 validation errors',
resp_json['messages']['errors'][0])
@patch('api.domain.user.User.get', MockUser.get)
def test_bad_data_avail_inputs(self):
url = '/api/v1/available-products/'
data = '{"bad": []}'
response = self.app.get(url,
data=data,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertEqual(400, response.status_code)
self.assertIn('messages', resp_json)
self.assertIn('errors', resp_json['messages'])
self.assertIn('No input products supplied',
resp_json['messages']['errors'][0])
@patch('api.external.ers.ERSApi._api_post',
lambda x, y, z: {'errors': True})
def test_messages_field_acc_denied(self):
url = '/api/v1/available-products/'
response = self.app.get(url, environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertEqual(401, response.status_code)
self.assertIn('messages', resp_json)
self.assertIn('errors', resp_json['messages'])
@patch('api.domain.user.User.get', MockUser.get)
def test_not_found(self):
url = '/api/v1/not-valid'
response = self.app.get(url, environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertEqual(404, response.status_code)
self.assertIn('messages', resp_json)
self.assertIn("errors", resp_json['messages'])
@patch('api.domain.user.User.get', MockUser.get)
@patch('api.interfaces.ordering.version1.API.place_order',
MockOrder.place_order)
def test_post_order(self):
url = '/api/v1/order'
data = {'etm7_collection': {'inputs': [''], 'products': ['']}}
response = self.app.post(url,
headers=self.headers,
data=json.dumps(data),
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertEqual(201, response.status_code)
self.assertEqual({'orderid', 'status'}, set(resp_json.keys()))
@patch('api.domain.user.User.get', MockUser.get)
@patch('api.interfaces.ordering.version1.API.cancel_order',
MockOrder.cancel_order)
def test_cancel_order(self):
url = '/api/v1/order'
data = {'orderid': self.orderid, 'status': 'cancelled'}
response = self.app.put(url,
headers=self.headers,
data=json.dumps(data),
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertEqual(data, resp_json)
self.assertEqual(202, response.status_code)
class ProductionTransportTestCase(unittest.TestCase):
def setUp(self):
os.environ['espa_api_testing'] = 'True'
# create a user
self.mock_user = MockUser()
self.mock_order = MockOrder()
self.user = User.find(self.mock_user.add_testing_user())
self.order_id = self.mock_order.generate_testing_order(self.user.id)
self.app = http.app.test_client()
self.app.testing = True
self.sceneids = self.mock_order.scene_names_list(self.order_id)[0:2]
with db_instance() as db:
uidsql = "select user_id, orderid from ordering_order limit 1;"
db.select(uidsql)
self.userid = db[0]['user_id']
self.orderid = db[0]['orderid']
itemsql = "select name, order_id from ordering_scene limit 1;"
db.select(itemsql)
self.itemid = db[0][0]
itemorderid = db[0][1]
ordersql = "select orderid from ordering_order where id = {};".format(itemorderid)
db.select(ordersql)
self.itemorderid = db[0][0]
self.base_order = lowercase_all(testorders.build_base_order())
def tearDown(self):
# clean up orders
self.mock_order.tear_down_testing_orders()
# clean up users
self.mock_user.cleanup()
os.environ['espa_api_testing'] = ''
@patch('api.interfaces.production.version1.API.get_production_whitelist', api.get_production_whitelist)
def test_get_production_api(self):
response = self.app.get('/production-api', environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
assert response.content_type == 'application/json'
assert set(response_data.keys()) == set(['0', '1'])
@patch('api.interfaces.production.version1.API.get_production_whitelist', api.get_production_whitelist)
def test_get_production_api_v1(self):
response = self.app.get('/production-api/v1', environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
assert set(response_data.keys()) == set(["operations", "description"])
assert "ESPA Production" in response_data['description']
@patch('api.providers.production.production_provider.ProductionProvider.get_products_to_process',
production_provider.get_products_to_process_inputs)
@patch('api.interfaces.production.version1.API.get_production_whitelist', api.get_production_whitelist)
def test_get_production_api_products_modis(self):
url = "/production-api/v1/products?for_user=bilbo&product_types=modis"
response = self.app.get(url, environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
correct_resp = {'encode_urls': False, 'for_user': '******',
'record_limit': 500, 'product_types': 'modis',
'priority': None}
assert response_data == correct_resp
@patch('api.providers.production.production_provider.ProductionProvider.get_products_to_process',
production_provider.get_products_to_process_inputs)
@patch('api.interfaces.production.version1.API.get_production_whitelist', api.get_production_whitelist)
def test_get_production_api_products_landsat(self):
url = "/production-api/v1/products?for_user=bilbo&product_types=landsat"
response = self.app.get(url, environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
correct_resp = {'encode_urls': False, 'for_user': '******',
'record_limit': 500, 'product_types': 'landsat',
'priority': None}
assert response_data == correct_resp
@patch('api.interfaces.production.version1.API.get_production_whitelist', api.get_production_whitelist)
@patch('api.providers.production.production_provider.ProductionProvider.update_status',
production_provider.update_status_inputs)
def test_post_production_api_update_status(self):
url = "/production-api/v1/update_status"
data_dict = {'name': 't10000xyz401', 'orderid': '[email protected]',
'processing_loc': 'update_status', 'status': 'updated'}
response = self.app.post(url, data=json.dumps(data_dict), environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
assert response_data == data_dict
@patch('api.providers.production.production_provider.ProductionProvider.set_product_error',
production_provider.set_product_error_inputs)
@patch('api.interfaces.production.version1.API.get_production_whitelist', api.get_production_whitelist)
def test_post_production_api_set_product_error(self):
url = "/production-api/v1/set_product_error"
data_dict = {'name': 't10000xyz401', 'orderid': '[email protected]',
'processing_loc': 'xyz', 'error': 'oopsie'}
response = self.app.post(url, data=json.dumps(data_dict), environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
assert response_data == data_dict
@patch('api.providers.production.production_provider.ProductionProvider.set_product_unavailable',
production_provider.set_product_unavailable_inputs)
@patch('api.interfaces.production.version1.API.get_production_whitelist', api.get_production_whitelist)
def test_post_production_api_set_product_unavailable(self):
url = "/production-api/v1/set_product_unavailable"
data_dict = {'name': 't10000xyz401', 'orderid': '[email protected]',
'processing_loc': 'xyz', 'error': 'oopsie', 'note': 'them notes'}
response = self.app.post(url, data=json.dumps(data_dict), environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
assert response_data == data_dict
@patch('api.providers.production.production_provider.ProductionProvider.mark_product_complete',
production_provider.set_mark_product_complete_inputs)
@patch('api.interfaces.production.version1.API.get_production_whitelist', api.get_production_whitelist)
def test_post_production_api_mark_product_complete(self):
url = "/production-api/v1/mark_product_complete"
data_dict = {'name': 't10000xyz401', 'orderid': '[email protected]',
'processing_loc': 'xyz',
'completed_file_location': '/tmp',
'cksum_file_location': '/tmp/txt.txt',
'log_file_contents': 'details'}
response = self.app.post(url, data=json.dumps(data_dict), environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
assert response_data == data_dict
@patch('api.providers.production.production_provider.ProductionProvider.handle_orders',
production_provider.respond_true)
@patch('api.interfaces.production.version1.API.get_production_whitelist', api.get_production_whitelist)
def test_post_production_api_handle_orders(self):
url = "/production-api/v1/handle-orders"
response = self.app.get(url, data=json.dumps({}), environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
assert response_data is True
@patch('api.providers.production.production_provider.ProductionProvider.queue_products',
production_provider.queue_products_inputs)
@patch('api.interfaces.production.version1.API.get_production_whitelist', api.get_production_whitelist)
def test_post_production_api_queue_products(self):
url = "/production-api/v1/queue-products"
data_dict = {'order_name_tuple_list': 'order_name_tuple_list',
'processing_location': 'processing_location',
'job_name': 'job_name'}
response = self.app.post(url, data=json.dumps(data_dict), environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
assert response_data == data_dict
@patch('api.interfaces.production.version1.API.get_production_whitelist', api.get_production_whitelist)
def test_get_production_api_configurations(self):
url = "/production-api/v1/configuration/system_message_title"
response = self.app.get(url, environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
assert response_data.keys() == ['system_message_title']
class TestAPI(unittest.TestCase):
def setUp(self):
os.environ['espa_api_testing'] = 'True'
# create a user
self.mock_user = MockUser()
self.mock_order = MockOrder()
user_id = self.mock_user.add_testing_user()
order_id = self.mock_order.generate_testing_order(user_id)
self.order = Order.find(order_id)
self.user = User.find(user_id)
self.product_id = 'LT50150401987120XXX02'
self.staff_product_id = 'LE70450302003206EDC01'
staff_user_id = self.mock_user.add_testing_user()
self.staff_user = User.find(staff_user_id)
self.staff_user.update('is_staff', True)
staff_order_id = self.mock_order.generate_testing_order(staff_user_id)
staff_order = Order.find(staff_order_id)
staff_scene = staff_order.scenes()[0]
staff_scene.update('name', self.staff_product_id)
user_scene = self.order.scenes()[0]
user_scene.update('name', self.staff_product_id)
with open('api/domain/restricted.yaml') as f:
self.restricted = yaml.load(f.read())
self.restricted['all']['role'].remove('restricted_prod')
def tearDown(self):
# clean up orders
self.mock_order.tear_down_testing_orders()
# clean up users
self.mock_user.cleanup()
os.environ['espa_api_testing'] = ''
def test_api_versions_key_val(self):
self.assertEqual(set(api.api_versions().keys()), set(['0', '1']))
def test_get_available_products_key_val(self):
self.assertEqual(api.available_products(self.product_id, self.user.username).keys()[0], "tm5")
def test_get_available_products_by_staff(self):
# staff should see all available products
self.user.update('is_staff', True)
return_dict = api.available_products(self.staff_product_id, self.staff_user.username)
for item in self.restricted['all']['role']:
self.assertTrue(item in return_dict['etm7']['products'])
def test_get_available_products_by_public(self):
# public should not see products listed in api/domain.restricted.yaml
self.user.update('is_staff', False)
return_dict = api.available_products(self.staff_product_id, self.user.username)
for item in self.restricted['all']['role']:
self.assertFalse(item in return_dict['etm7']['products'])
def test_fetch_user_orders_by_email_val(self):
orders = api.fetch_user_orders(self.user.email)
self.assertEqual(orders.keys()[0], "orders")
def test_fetch_user_orders_by_username_val(self):
orders = api.fetch_user_orders(self.user.username)
self.assertEqual(orders.keys()[0], "orders")
def test_fetch_order_by_orderid_val(self):
order = api.fetch_order(self.order.orderid)
self.assertEqual(order['orderid'], self.order.orderid)
def test_fetch_order_status_valid(self):
response = api.order_status(self.order.orderid)
self.assertEqual(response.keys(), ['orderid', 'status'])
def test_fetch_order_status_invalid(self):
invalid_orderid = 'invalidorderid'
response = api.order_status(invalid_orderid)
self.assertEqual(response.keys(), ['msg'])
def test_fetch_item_status_valid(self):
response = api.item_status(self.order.orderid)
self.assertEqual(response.keys(), ['orderid'])
self.assertIsInstance(response['orderid'], dict)
def test_fetch_item_status_invalid(self):
invalid_orderid = 'invalidorderid'
response = api.order_status(invalid_orderid)
self.assertEqual(response.keys(), ['msg'])