def setUp(self):
logger.warning('Testing API started...')
os.environ['espa_api_testing'] = 'True'
# create a user
self.mock_user = MockUser()
self.mock_order = MockOrder()
user_id = self.mock_user.add_testing_user()
order_id = self.mock_order.generate_testing_order(user_id)
self.order = Order.find(order_id)
self.user = User.find(user_id)
self.product_id = 'LT05_L1TP_032028_20120425_20160830_01_T1'
self.sensor_id = 'tm5_collection'
self.staff_product_id = 'LE07_L1TP_010028_20050420_20160925_01_T1'
self.staff_sensor = 'etm7_collection'
self.global_product_id = 'LE70450302003206EDC01'
staff_user_id = self.mock_user.add_testing_user()
self.staff_user = User.find(staff_user_id)
self.staff_user.update('is_staff', True)
staff_order_id = self.mock_order.generate_testing_order(staff_user_id)
staff_order = Order.find(staff_order_id)
staff_scene = staff_order.scenes()[0]
staff_scene.update('name', self.staff_product_id)
user_scene = self.order.scenes()[0]
user_scene.update('name', self.staff_product_id)
with open(os.path.join(__location__, 'domain/restricted.yaml')) as f:
self.restricted = yaml.load(f.read())
self.restricted['all']['role'].remove('restricted_prod')
def setUp(self):
os.environ['espa_api_testing'] = 'True'
# create a user
self.mock_user = MockUser()
self.mock_order = MockOrder()
user_id = self.mock_user.add_testing_user()
order_id = self.mock_order.generate_testing_order(user_id)
self.order = Order.find(order_id)
self.user = User.find(user_id)
self.product_id = 'LT50150401987120XXX02'
self.staff_product_id = 'LE70450302003206EDC01'
staff_user_id = self.mock_user.add_testing_user()
self.staff_user = User.find(staff_user_id)
self.staff_user.update('is_staff', True)
staff_order_id = self.mock_order.generate_testing_order(staff_user_id)
staff_order = Order.find(staff_order_id)
staff_scene = staff_order.scenes()[0]
staff_scene.update('name', self.staff_product_id)
user_scene = self.order.scenes()[0]
user_scene.update('name', self.staff_product_id)
with open('api/domain/restricted.yaml') as f:
self.restricted = yaml.load(f.read())
self.restricted['all']['role'].remove('restricted_prod')
def setUp(self):
os.environ['espa_api_testing'] = 'True'
# create a user
self.mock_user = MockUser()
self.mock_order = MockOrder()
self.user = User.find(self.mock_user.add_testing_user())
self.order_id = self.mock_order.generate_testing_order(self.user.id)
self.app = http.app.test_client()
self.app.testing = True
self.sceneids = self.mock_order.scene_names_list(self.order_id)[0:2]
token = ''.format(self.user.username, 'foo')
auth_string = "Basic {}".format(base64.b64encode(token))
self.headers = {"Authorization": auth_string}
with db_instance() as db:
uidsql = "select user_id, orderid from ordering_order limit 1;"
db.select(uidsql)
self.userid = db[0]['user_id']
self.orderid = db[0]['orderid']
itemsql = "select name, order_id from ordering_scene limit 1;"
db.select(itemsql)
self.itemid = db[0][0]
itemorderid = db[0][1]
ordersql = "select orderid from ordering_order where id = {};".format(
itemorderid)
db.select(ordersql)
self.itemorderid = db[0][0]
self.base_order = lowercase_all(testorders.build_base_order())
def test_get_scenes_for_new_user(self):
"""
Make sure that checking the number of open scenes for a new user
with no orders will not raise an exception
"""
mock = MockOrder()
user = MockUser()
user_id = user.add_testing_user()
user_id = user_id + random.randint(1, 200)
user_orders = Order.where({'user_id': user_id})
self.assertTrue(len(user_orders) == 0)
try:
ordering_provider.check_open_scenes(order=mock.base_order,
user_id=user_id,
filters={
'status':
('submitted', 'oncache',
'onorder', 'queued',
'processing')
})
except:
self.fail(
'ordering_provider.check_open_scenes() raised Exception for new user'
)
def setUp(self):
os.environ['espa_api_testing'] = 'True'
self.mock_user = MockUser()
self.staffuser = User.find(self.mock_user.add_testing_user())
self.staffuser.update('is_staff', True)
self.base_order = lowercase_all(testorders.build_base_order())
self.base_schema = BaseValidationSchema.request_schema
def setUp(self):
os.environ['espa_api_testing'] = 'True'
# create a user
self.mock_user = MockUser()
self.mock_order = MockOrder()
self.user = User.find(self.mock_user.add_testing_user())
self.order_id = self.mock_order.generate_testing_order(self.user.id)
self.app = http.app.test_client()
self.app.testing = True
self.sceneids = self.mock_order.scene_names_list(self.order_id)[0:2]
token = ''.format(self.user.username, 'foo')
auth_string = "Basic {}".format(base64.b64encode(token))
self.headers = {"Authorization": auth_string}
with db_instance() as db:
uidsql = "select user_id, orderid from ordering_order limit 1;"
db.select(uidsql)
self.userid = db[0]['user_id']
self.orderid = db[0]['orderid']
itemsql = "select name, order_id from ordering_scene limit 1;"
db.select(itemsql)
self.itemid = db[0][0]
itemorderid = db[0][1]
ordersql = "select orderid from ordering_order where id = {};".format(itemorderid)
db.select(ordersql)
self.itemorderid = db[0][0]
self.base_order = lowercase_all(testorders.build_base_order())
def setUp(self):
os.environ['espa_api_testing'] = 'True'
# create a user
self.mock_user = MockUser()
self.mock_order = MockOrder()
user_id = self.mock_user.add_testing_user()
order_id = self.mock_order.generate_testing_order(user_id)
self.order = Order.find(order_id)
self.user = User.find(user_id)
self.product_id = 'LT50150401987120XXX02'
self.staff_product_id = 'LE70450302003206EDC01'
staff_user_id = self.mock_user.add_testing_user()
self.staff_user = User.find(staff_user_id)
self.staff_user.update('is_staff', True)
staff_order_id = self.mock_order.generate_testing_order(staff_user_id)
staff_order = Order.find(staff_order_id)
staff_scene = staff_order.scenes()[0]
staff_scene.update('name', self.staff_product_id)
user_scene = self.order.scenes()[0]
user_scene.update('name', self.staff_product_id)
with open('api/domain/restricted.yaml') as f:
self.restricted = yaml.load(f.read())
self.restricted['all']['role'].remove('restricted_prod')
def setUp(self):
os.environ['espa_api_testing'] = 'True'
self.mock_user = MockUser()
self.staffuser = User.find(self.mock_user.add_testing_user())
self.staffuser.update('is_staff', True)
self.base_order = lowercase_all(testorders.build_base_order())
self.base_schema = BaseValidationSchema.request_schema
def setUp(self):
os.environ['espa_api_testing'] = 'True'
# create a user
self.mock_user = MockUser()
self.mock_order = MockOrder()
self.user = User.find(self.mock_user.add_testing_user())
self.order_id = self.mock_order.generate_testing_order(self.user.id)
self.app = http.app.test_client()
self.app.testing = True
self.sceneids = self.mock_order.scene_names_list(self.order_id)[0:2]
token = ':'.join((self.user.username, 'foo'))
auth_string = "Basic {}".format(base64.b64encode(token))
self.headers = {"Authorization": auth_string}
with db_instance() as db:
uidsql = "select user_id, orderid from ordering_order limit 1;"
db.select(uidsql)
self.userid = db[0]['user_id']
self.orderid = db[0]['orderid']
itemsql = "select name, order_id from ordering_scene limit 1;"
db.select(itemsql)
self.itemid = db[0][0]
itemorderid = db[0][1]
ordersql = "select orderid from ordering_order where id = {};".format(
itemorderid)
db.select(ordersql)
self.itemorderid = db[0][0]
self.base_order = lowercase_all(testorders.build_base_order())
self.sensors = [
k for k in self.base_order.keys()
if isinstance(self.base_order[k], dict)
and 'inputs' in self.base_order[k]
]
self.inputs = {s: self.base_order[s]['inputs'] for s in self.sensors}
self.input_names_all = set([
i for sublist in [s for k, s in self.inputs.items()]
for i in sublist
])
def test_order_exceeds_open_scene_limit(self):
"""
Build a large order, then attempt to place an order that would exceed the user's open scene limit.
"""
mock = MockOrder()
user = MockUser()
user_id = user.add_testing_user()
# base testing order will give a user 15 open scenes currently
order_id = mock.generate_testing_order(user_id)
# Add a second order that pushes scenes over the limit, make sure that this
# raises the appropriate exception
self.assertRaises(OpenSceneLimitException,
lambda: ordering_provider.check_open_scenes(order=mock.base_order,
user_id=user_id,
filters={'status': ('submitted',
'oncache',
'onorder',
'queued',
'processing')}))
class TestProductionAPI(unittest.TestCase):
def setUp(self):
os.environ['espa_api_testing'] = 'True'
# create a user
self.mock_user = MockUser()
self.mock_order = MockOrder()
self.user_id = self.mock_user.add_testing_user()
def tearDown(self):
# clean up orders
self.mock_order.tear_down_testing_orders()
# clean up users
self.mock_user.cleanup()
os.environ['espa_api_testing'] = ''
@patch('api.external.inventory.available', lambda: True)
@patch(
'api.providers.production.production_provider.ProductionProvider.parse_urls_m2m',
lambda x, y: y)
def test_fetch_production_products_modis(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
# need scenes with statuses of 'processing'
self.mock_order.update_scenes(order_id, 'modis', 'status',
['processing', 'oncache'])
user = User.find(self.user_id)
params = {'product_types': ['modis']}
response = api.fetch_production_products(params)
self.assertTrue('bilbo' in response[0]['orderid'])
@patch('api.external.inventory.available', lambda: True)
@patch(
'api.providers.production.production_provider.ProductionProvider.parse_urls_m2m',
lambda x, y: y)
def test_fetch_production_products_landsat(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
# need scenes with statuses of 'processing'
self.mock_order.update_scenes(order_id, 'landsat', 'status',
['processing', 'oncache'])
user = User.find(self.user_id)
params = {'for_user': user.username, 'product_types': ['landsat']}
response = api.fetch_production_products(params)
self.assertTrue('bilbo' in response[0]['orderid'])
def test_fetch_production_products_plot(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
self.mock_order.update_scenes(order_id, ('landsat', 'modis'), 'status',
['complete'])
order = Order.find(order_id)
plot_scene = order.scenes({'name': 'plot'})[0]
plot_scene.name = 'plot'
plot_scene.sensor_type = 'plot'
plot_scene.status = 'submitted'
plot_scene.save()
scenes = Order.find(order_id).scenes({
'name': 'plot',
'status': 'submitted'
})
response = production_provider.handle_submitted_plot_products(scenes)
pscene = order.scenes({'status': 'oncache', 'sensor_type': 'plot'})
self.assertTrue(response is True)
self.assertEqual(len(pscene), 1)
def test_production_set_product_retry(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
order = Order.find(order_id)
scene = order.scenes()[3]
scene.update('retry_count', 4)
processing_loc = "get_products_to_process"
error = 'not available after EE call '
note = 'note this'
retry_after = datetime.datetime.now() + datetime.timedelta(hours=1)
retry_limit = 9
response = production_provider.set_product_retry(
scene.name, order.orderid, processing_loc, error, note,
retry_after, retry_limit)
new = Scene.get('ordering_scene.status', scene.name, order.orderid)
self.assertTrue('retry' == new)
@patch('api.external.lta.update_order_status',
mock_production_provider.respond_true)
def test_production_set_product_error_unavailable(self):
"""
Move a scene status from error to unavailable based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes({'name !=': 'plot'})[0]
production_provider.set_product_error(
scene.name, order.orderid, 'get_products_to_process',
'include_dswe is an unavailable product option for OLITIRS')
self.assertTrue('unavailable' == Scene.get('ordering_scene.status',
scene.name, order.orderid))
def test_production_set_product_error_submitted(self):
"""
Move a scene status from error to submitted based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes({'name !=': 'plot'})[0]
production_provider.set_product_error(
scene.name, order.orderid, 'get_products_to_process',
'BLOCK, COMING FROM LST AS WELL: No such file or directory')
self.assertTrue('submitted' == Scene.get('ordering_scene.status',
scene.name, order.orderid))
def test_production_set_product_error_retry(self):
"""
Move a scene status from error to retry based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes()[2]
production_provider.set_product_error(
scene.name, order.orderid, 'somewhere',
'Verify the missing auxillary data products')
self.assertTrue('retry' == Scene.get('ordering_scene.status',
scene.name, order.orderid))
def test_production_set_product_error_retry_lasrc_segfault(self):
"""
Move a scene status from error to retry based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes({'sensor_type': 'landsat'})[-1]
production_provider.set_product_error(
scene.name, order.orderid, 'somewhere',
'runSr sh: line 1: 1010 Segmentation fault lasrc --xml=')
self.assertTrue('retry' == Scene.get('ordering_scene.status',
scene.name, order.orderid))
def test_production_set_product_error_unavail_reproject(self):
"""
Move a scene status from error to retry based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes({'sensor_type': 'landsat'})[-1]
log_file_contents = ('BLAH BLAH BLAH WarpVerificationError: Failed to '
'compute statistics, no valid pixels found in '
'sampling BLAH BLAH BLAH')
production_provider.set_product_error(scene.name, order.orderid,
'somewhere', log_file_contents)
self.assertEqual(
'unavailable',
Scene.get('ordering_scene.status', scene.name, order.orderid))
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch(
'api.providers.production.production_provider.ProductionProvider.set_product_retry',
mock_production_provider.set_product_retry)
def test_update_product_details_update_status(self):
"""
Set a scene status to Queued
"""
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes()[0]
api.update_product_details(
'update_status', {
'name': scene.name,
'orderid': order.orderid,
'processing_loc': 'L8SRLEXAMPLE',
'status': 'Queued'
})
self.assertTrue(
Scene.get('ordering_scene.status', scene.name, order.orderid) ==
'Queued')
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch(
'api.providers.production.production_provider.ProductionProvider.set_product_retry',
mock_production_provider.set_product_retry)
@patch('api.external.onlinecache.capacity', onlinecache.capacity)
def test_update_product_details_set_product_error(self):
"""
Set a scene status to error
:return:
"""
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes()[0]
production_provider.update_product('set_product_error',
name=scene.name,
orderid=order.orderid,
processing_loc="L8SRLEXAMPLE",
error='problems yo')
self.assertTrue(Scene.find(scene.id).status == 'error')
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch(
'api.providers.production.production_provider.ProductionProvider.set_product_retry',
mock_production_provider.set_product_retry)
def test_update_product_details_set_product_unavailable(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes()[0]
production_provider.update_product(
'set_product_unavailable',
name=scene.name,
orderid=order.orderid,
processing_loc="L8SRLEXAMPLE",
error='include_dswe is an unavailable product option for OLITIRS')
self.assertTrue('unavailable' == Scene.get('ordering_scene.status',
scene.name, order.orderid))
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch(
'api.providers.production.production_provider.ProductionProvider.set_product_retry',
mock_production_provider.set_product_retry)
@patch('os.path.getsize', lambda y: 999)
def test_update_product_details_mark_product_complete(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes()[0]
production_provider.update_product('mark_product_complete',
name=scene.name,
orderid=order.orderid,
processing_loc='L8SRLEXAMPLE',
completed_file_location='/some/loc',
cksum_file_location='some checksum',
log_file_contents='some log')
self.assertTrue('complete' == Scene.get('ordering_scene.status',
scene.name, order.orderid))
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch(
'api.providers.production.production_provider.ProductionProvider.set_product_retry',
mock_production_provider.set_product_retry)
@patch('os.path.getsize', lambda y: 999)
def test_update_product_details_mark_product_processing(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes()[0]
res = production_provider.update_product('update_status',
name=scene.name,
orderid=order.orderid,
processing_loc='L8SRLEXAMPLE',
status='processing')
order = Order.find(order.id)
scene = order.scenes({'id': scene.id})[0]
self.assertEqual('processing', scene.status)
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch(
'api.providers.production.production_provider.ProductionProvider.set_product_retry',
mock_production_provider.set_product_retry)
@patch('os.path.getsize', lambda y: 999)
def test_update_product_details_mark_cancelled_product_processing(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
order.status = 'cancelled'
order.save()
scene = order.scenes()[0]
res = production_provider.update_product('update_status',
name=scene.name,
orderid=order.orderid,
processing_loc='L8SRLEXAMPLE',
status='processing')
self.assertFalse(res)
def test_production_set_product_error_unavailable_night(self):
"""
Move a scene status from error to unavailable based on the solar zenith (TOA)
error message
"""
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes({'name !=': 'plot'})[0]
production_provider.set_product_error(
name=scene.name,
orderid=order.orderid,
processing_loc='L8SRLEXAMPLE',
error='solar zenith angle out of range')
scene = Scene.by_name_orderid(name=scene.name, order_id=order.id)
self.assertTrue('unavailable' == scene.status)
self.assertTrue(
'Solar zenith angle out of range, cannot process night scene' in
scene.note)
def test_production_set_product_error_unavailable_almost_night(self):
"""
Move a scene status from error to unavailable based on the solar zenith (SR)
error message
"""
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes({'name !=': 'plot'})[0]
production_provider.set_product_error(
name=scene.name,
orderid=order.orderid,
processing_loc='L8SRLEXAMPLE',
error='solar zenith angle is too large')
scene = Scene.by_name_orderid(name=scene.name, order_id=order.id)
self.assertTrue('unavailable' == scene.status)
self.assertTrue(
'Solar zenith angle is too large, cannot process scene to SR' in
scene.note)
@patch('api.external.lta.update_order_status',
lta.update_order_status_fail)
@patch(
'api.providers.production.production_provider.ProductionProvider.set_product_retry',
mock_production_provider.set_product_retry)
@patch('os.path.getsize', lambda y: 999)
def test_update_product_details_fail_lta_mark_product_complete(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scene = order.scenes()[1]
order.update('order_source', 'ee')
production_provider.update_product('mark_product_complete',
name=scene.name,
orderid=order.orderid,
processing_loc='L8SRLEXAMPLE',
completed_file_location='/some/loc',
cksum_file_location='some checksum',
log_file_contents='some log')
s = Scene.where({'name': scene.name, 'order_id': scene.order_id})[0]
self.assertTrue('C' == s.failed_lta_status_update)
@patch(
'api.providers.production.production_provider.ProductionProvider.send_initial_emails',
mock_production_provider.respond_true)
@patch(
'api.providers.production.production_provider.ProductionProvider.handle_onorder_landsat_products',
mock_production_provider.respond_true)
@patch(
'api.providers.production.production_provider.ProductionProvider.handle_retry_products',
mock_production_provider.respond_true)
@patch(
'api.providers.production.production_provider.ProductionProvider.load_ee_orders',
mock_production_provider.respond_true)
@patch(
'api.providers.production.production_provider.ProductionProvider.finalize_orders',
mock_production_provider.respond_true)
@patch(
'api.providers.production.production_provider.ProductionProvider.purge_orders',
mock_production_provider.respond_true)
def test_handle_orders_success(self):
_ = self.mock_order.generate_testing_order(self.user_id)
self.assertTrue(
api.handle_orders({'username': User.find(self.user_id)}))
@patch('api.external.onlinecache.delete',
mock_production_provider.respond_true)
@patch('api.notification.emails.send_purge_report',
mock_production_provider.respond_true)
@patch('api.external.onlinecache.capacity', onlinecache.mock_capacity)
@patch('api.external.onlinecache.exists', onlinecache.mock_exists)
@patch('api.external.onlinecache.delete', onlinecache.mock_delete)
def test_production_purge_orders(self):
new_completion_date = datetime.datetime.now() - datetime.timedelta(
days=12)
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
order.update('status', 'complete')
order.update('completion_date', new_completion_date)
self.assertTrue(production_provider.purge_orders())
# need to figure a test for emails.send_email
@patch('api.notification.emails.Emails.send_email',
mock_production_provider.respond_true)
def test_production_send_initial_emails(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
order.update('status', 'ordered')
self.assertTrue(emails.Emails().send_all_initial([order]))
@patch('api.external.lta.get_order_status', lta.get_order_status)
@patch('api.external.lta.update_order_status', lta.update_order_status)
def test_production_handle_onorder_landsat_products(self):
tram_order_ids = lta.sample_tram_order_ids()[0:3]
scene_names = lta.sample_scene_names()[0:3]
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scenes = order.scenes()[0:3]
for idx, scene in enumerate(scenes):
scene.tram_order_id = tram_order_ids[idx]
scene.status = 'onorder'
# save() doesn't let you update name,
# b/c updating a scene name is not acceptable
# outside of testing
scene.update('name', scene_names[idx])
scene.save()
self.assertTrue(
production_provider.handle_onorder_landsat_products(scenes))
def test_production_handle_retry_products(self):
prev = datetime.datetime.now() - datetime.timedelta(hours=1)
order_id = self.mock_order.generate_testing_order(self.user_id)
self.mock_order.update_scenes(order_id, 'landsat', 'status', ['retry'])
self.mock_order.update_scenes(order_id, 'landsat', 'retry_after',
[prev])
scenes = Order.find(order_id).scenes({'status': 'retry'})
production_provider.handle_retry_products(scenes)
for s in Scene.where({'order_id': order_id, 'sensor_type': 'landsat'}):
self.assertTrue(s.status == 'submitted')
@patch('api.external.lta.get_available_orders',
lta.get_available_orders_partial)
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch('api.external.inventory.get_user_name', inventory.get_user_name)
@patch('api.external.inventory.get_session', lambda: True)
def test_production_load_ee_orders_partial(self):
order = Order.find(
self.mock_order.generate_ee_testing_order(self.user_id,
partial=True))
self.assertEqual(
order.product_opts, {
'format': 'gtiff',
'etm7_collection': {
'inputs': ['LE07_L1TP_026027_20170912_20171008_01_T1'],
'products': ['sr']
}
})
key = 'system.load_ee_orders_enabled'
self.assertEqual(api.get_production_key(key)[key], 'True')
production_provider.load_ee_orders()
reorder = Order.find(order.id)
self.assertEqual(
reorder.product_opts, {
'format': 'gtiff',
'etm7_collection': {
'inputs': ['LE07_L1TP_026027_20170912_20171008_01_T1'],
'products': ['sr']
},
'tm5_collection': {
'inputs': ['LT05_L1TP_025027_20110913_20160830_01_T1'],
'products': ['sr']
}
})
@patch('api.external.lta.update_order_status', lta.update_order_status)
def test_production_handle_failed_ee_updates(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
for scene in order.scenes():
scene.update('failed_lta_status_update', 'C')
scenes = Order.find(order.id).scenes(
{'failed_lta_status_update IS NOT': None})
production_provider.handle_failed_ee_updates(scenes)
scenes = Scene.where({'failed_lta_status_update IS NOT': None})
self.assertTrue(len(scenes) == 0)
@patch(
'api.providers.production.production_provider.ProductionProvider.update_landsat_product_status',
mock_production_provider.respond_true)
@patch(
'api.providers.production.production_provider.ProductionProvider.get_contactids_for_submitted_landsat_products',
mock_production_provider.contact_ids_list)
@patch('api.external.inventory.available', lambda: True)
def test_production_handle_submitted_landsat_products(self):
orders = Order.find(
self.mock_order.generate_testing_order(self.user_id))
scenes = orders.scenes({'sensor_type': 'landsat'})
self.assertTrue(
production_provider.handle_submitted_landsat_products(scenes))
@patch('api.external.lta.update_order_status', lta.update_order_status)
def test_production_set_products_unavailable(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
self.assertTrue(
production_provider.set_products_unavailable(
order.scenes(), "you want a reason?"))
@patch(
'api.providers.production.production_provider.ProductionProvider.check_dependencies_for_products',
mock_production_provider.check_dependencies_for_products)
@patch('api.external.inventory.get_cached_session',
inventory.get_cached_session)
@patch('api.external.inventory.check_valid', inventory.check_valid_landsat)
def test_production_update_landsat_product_status(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
for scene in order.scenes({'name !=': 'plot'}):
scene.status = 'submitted'
scene.sensor_type = 'landsat'
user = User.find(self.user_id)
scene.save()
self.assertTrue(
production_provider.update_landsat_product_status(user.contactid))
def test_production_get_contactids_for_submitted_landsat_products(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
for scene in order.scenes({'name !=': 'plot'}):
scene.status = 'submitted'
scene.sensor_type = 'landsat'
scene.save()
scenes = Order.find(order.id).scenes({'sensor_type': 'landsat'})
response = production_provider.get_contactids_for_submitted_landsat_products(
scenes)
self.assertIsInstance(response, set)
self.assertTrue(len(response) > 0)
@patch('api.external.inventory.available', lambda: True)
@patch('api.external.inventory.get_cached_session',
inventory.get_cached_session)
@patch('api.external.inventory.check_valid', inventory.check_valid_modis)
def test_production_handle_submitted_modis_products_input_exists(self):
# handle oncache scenario
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
for scene in order.scenes({'name !=': 'plot'}):
scene.status = 'submitted'
scene.sensor_type = 'modis'
scene.save()
sid = scene.id
scenes = order.scenes({'sensor_type': 'modis'})
self.assertTrue(
production_provider.handle_submitted_modis_products(scenes))
#self.assertEquals(Scene.find(sid).status, "oncache")
@patch('api.external.inventory.available', lambda: True)
@patch('api.external.inventory.get_cached_session',
inventory.get_cached_session)
@patch('api.external.inventory.check_valid', inventory.check_valid_modis)
def test_production_handle_submitted_modis_products_input_missing(self):
# handle unavailable scenario
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
for scene in order.scenes({'name !=': 'plot'}):
scene.status = 'submitted'
scene.sensor_type = 'modis'
scene.save()
sid = scene.id
scenes = order.scenes({'sensor_type': 'modis'})
self.assertTrue(
production_provider.handle_submitted_modis_products(scenes))
self.assertEquals(Scene.find(sid).status, "unavailable")
def test_production_handle_submitted_plot_products(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
order.status = 'ordered'
order.order_type = 'lpcs'
order.save()
plot_id = None
for idx, scene in enumerate(order.scenes()):
# at the moment, mock_order.generate_testing_order
# creates 21 products for the order. divvy those
# up between 'complete' and 'unavailable', setting
# one aside as the 'plot' product
if scene.sensor_type == 'plot':
# need to define a plot product
scene.update('status', 'submitted')
plot_id = scene.id
else:
if idx % 2 == 0:
scene.update('status', 'complete')
else:
scene.update('status', 'unavailable')
scenes = order.scenes()
self.assertTrue(
production_provider.handle_submitted_plot_products(scenes))
self.assertEqual(Scene.find(plot_id).status, "oncache")
@patch('os.path.exists', lambda y: True)
@patch('os.path.getsize', lambda y: 999)
def test_production_calc_scene_download_sizes(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
scenes = order.scenes()
Scene.bulk_update([s.id for s in scenes], {
'status': 'complete',
'download_size': 0
})
scenes = Order.find(order.id).scenes({
'status': 'complete',
'download_size': 0
})
self.assertTrue(production_provider.calc_scene_download_sizes(scenes))
upscenes = Scene.where({'status': 'complete', 'download_size': 999})
self.assertEqual(len(upscenes), len(scenes))
@patch(
'api.providers.production.production_provider.ProductionProvider.update_order_if_complete',
mock_production_provider.respond_true)
def test_production_finalize_orders(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
order.update('status', 'ordered')
self.assertTrue(production_provider.finalize_orders([order]))
@patch(
'api.providers.production.production_provider.ProductionProvider.send_completion_email',
mock_production_provider.respond_true)
def test_production_update_order_if_complete(self):
order = Order.find(self.mock_order.generate_testing_order(
self.user_id))
Scene.bulk_update([s.id for s in order.scenes()], {'status': 'retry'})
order.order_source = 'espa'
order.completion_email_sent = None
order.save()
self.assertTrue(production_provider.update_order_if_complete(order))
def test_production_queue_products_success(self):
names_tuple = self.mock_order.names_tuple(3, self.user_id)
processing_loc = "get_products_to_process"
job_name = 'jobname49'
params = (names_tuple, processing_loc, job_name)
response = api.queue_products(*params)
self.assertTrue(response)
def test_production_get_key(self):
key = 'system_message_title'
response = api.get_production_key(key)
val = response[key]
self.assertIsInstance(val, str)
def test_get_production_key_invalid(self):
bad_key = 'foobar'
response = api.get_production_key(bad_key)
self.assertEqual(response.keys(), ['msg'])
@patch('api.external.hadoop.HadoopHandler.job_names_ids',
hadoop.jobs_names_ids)
def test_catch_orphaned_scenes(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
# need scenes with statuses of 'queued'
self.mock_order.update_scenes(order_id, ('landsat', 'modis', 'plot'),
'status', ['queued'])
response = production_provider.catch_orphaned_scenes()
self.assertTrue(response)
old_time = datetime.datetime.now() - datetime.timedelta(minutes=15)
for s in Scene.where({'order_id': order_id}):
self.assertTrue(s.reported_orphan is not None)
s.reported_orphan = old_time
s.save()
response = production_provider.catch_orphaned_scenes()
self.assertTrue(response)
for s in Scene.where({'order_id': order_id}):
self.assertTrue(s.orphaned)
@patch('api.external.hadoop.HadoopHandler.job_names_ids',
hadoop.jobs_names_ids)
def test_handle_stuck_jobs(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
# Make some really old jobs
self.mock_order.update_scenes(order_id, ('landsat', 'modis', 'plot'),
'status', ['processing'])
self.mock_order.update_scenes(order_id, ('landsat', 'modis', 'plot'),
'status_modified',
[datetime.datetime(1900, 1, 1)])
scenes = Scene.where({'status': 'processing', 'order_id': order_id})
n_scenes = len(scenes)
response = production_provider.handle_stuck_jobs(scenes)
self.assertTrue(response)
scenes = Scene.where({
'status': 'processing',
'order_id': order_id,
'reported_orphan is not': None
})
self.assertEqual(n_scenes, len(scenes))
self.mock_order.update_scenes(order_id, ('landsat', 'modis', 'plot'),
'reported_orphan',
[datetime.datetime(1900, 1, 1)])
response = production_provider.handle_stuck_jobs(scenes)
self.assertTrue(response)
scenes = Scene.where({'status': 'processing', 'order_id': order_id})
self.assertEqual(0, len(scenes))
def test_convert_product_options(self):
"""
Test the conversion procedure to make sure that the new format for orders converts
to the old format
"""
scenes = [
'LE07_L1TP_026027_20170912_20171008_01_T1',
'LC08_L1TP_025027_20160521_20170223_01_T1',
'LT05_L1TP_025027_20110913_20160830_01_T1'
]
includes = ['include_sr', 'include_sr_toa', 'include_sr_thermal']
new_format = {
u'etm7_collection': {
u'inputs': [u'LE07_L1TP_026027_20170912_20171008_01_T1'],
u'products': [u'sr']
},
u'olitirs8_collection': {
u'inputs': [u'LC08_L1TP_025027_20160521_20170223_01_T1'],
u'products': [u'toa']
},
u'tm5_collection': {
u'inputs': [u'LT05_L1TP_025027_20110913_20160830_01_T1'],
u'products': [u'bt']
},
u'format': u'gtiff',
u'image_extents': {
u'east': -2265585.0,
u'north': 3164805.0,
u'south': 3014805.0,
u'units': u'meters',
u'west': -2415585.0
},
u'note': u'CONUS_h1v1',
u'projection': {
u'aea': {
u'central_meridian': -96,
u'datum': u'nad83',
u'false_easting': 0,
u'false_northing': 0,
u'latitude_of_origin': 23,
u'standard_parallel_1': 29.5,
u'standard_parallel_2': 45.5
}
},
u'resampling_method': u'cc'
}
ruberic = {
'central_meridian': -96,
'datum': u'nad83',
'false_easting': 0,
'false_northing': 0,
'image_extents': True,
'image_extents_units': u'meters',
'include_customized_source_data': False,
'include_dswe': False,
'include_st': False,
'include_solr_index': False,
'include_source_data': False,
'include_source_metadata': False,
'include_sr': False,
'include_sr_browse': False,
'include_sr_evi': False,
'include_sr_msavi': False,
'include_sr_nbr': False,
'include_sr_nbr2': False,
'include_sr_ndmi': False,
'include_sr_ndvi': False,
'include_sr_savi': False,
'include_sr_thermal': False,
'include_sr_toa': False,
'include_statistics': False,
'latitude_true_scale': None,
'longitude_pole': None,
'maxx': -2265585.0,
'maxy': 3164805.0,
'minx': -2415585.0,
'miny': 3014805.0,
'origin_lat': 23,
'output_format': u'gtiff',
'pixel_size': None,
'pixel_size_units': None,
'reproject': True,
'resample_method': 'cubic',
'resize': False,
'std_parallel_1': 29.5,
'std_parallel_2': 45.5,
'target_projection': u'aea',
'utm_north_south': None,
'utm_zone': None
}
for scene, include in zip(scenes, includes):
ruberic[include] = True
old_format = OptionsConversion.convert(new=new_format,
scenes=[scene])
self.assertDictEqual(ruberic, old_format)
ruberic[include] = False
def test_status_modified(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
scene = Scene.where({'order_id': order_id}).pop()
scene.status = 'oncache'
scene.save()
old_time = scene.status_modified
scene.status = 'queued'
scene.save()
new_time = scene.status_modified
self.assertGreater(new_time, old_time)
@patch('api.external.hadoop.HadoopHandler.list_jobs',
hadoop.jobs_names_ids)
@patch('api.external.hadoop.HadoopHandler.kill_job', lambda x, y: True)
def test_hadoop_reset_status(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
scenes = Scene.where({'order_id': order_id})
Scene.bulk_update([s.id for s in scenes], {'status': 'processing'})
self.assertTrue(production_provider.reset_processing_status())
scenes = Scene.where({'order_id': order_id})
self.assertEqual({'submitted'}, set([s.status for s in scenes]))
class TestAPI(unittest.TestCase):
def setUp(self):
os.environ['espa_api_testing'] = 'True'
# create a user
self.mock_user = MockUser()
self.mock_order = MockOrder()
user_id = self.mock_user.add_testing_user()
order_id = self.mock_order.generate_testing_order(user_id)
self.order = Order.find(order_id)
self.user = User.find(user_id)
self.product_id = 'LT50150401987120XXX02'
self.staff_product_id = 'LE70450302003206EDC01'
staff_user_id = self.mock_user.add_testing_user()
self.staff_user = User.find(staff_user_id)
self.staff_user.update('is_staff', True)
staff_order_id = self.mock_order.generate_testing_order(staff_user_id)
staff_order = Order.find(staff_order_id)
staff_scene = staff_order.scenes()[0]
staff_scene.update('name', self.staff_product_id)
user_scene = self.order.scenes()[0]
user_scene.update('name', self.staff_product_id)
with open('api/domain/restricted.yaml') as f:
self.restricted = yaml.load(f.read())
self.restricted['all']['role'].remove('restricted_prod')
def tearDown(self):
# clean up orders
self.mock_order.tear_down_testing_orders()
# clean up users
self.mock_user.cleanup()
os.environ['espa_api_testing'] = ''
def test_api_versions_key_val(self):
self.assertEqual(set(api.api_versions().keys()), set(['0', '1']))
def test_get_available_products_key_val(self):
self.assertEqual(api.available_products(self.product_id, self.user.username).keys()[0], "tm5")
def test_get_available_products_by_staff(self):
# staff should see all available products
self.user.update('is_staff', True)
return_dict = api.available_products(self.staff_product_id, self.staff_user.username)
for item in self.restricted['all']['role']:
self.assertTrue(item in return_dict['etm7']['products'])
def test_get_available_products_by_public(self):
# public should not see products listed in api/domain.restricted.yaml
self.user.update('is_staff', False)
return_dict = api.available_products(self.staff_product_id, self.user.username)
for item in self.restricted['all']['role']:
self.assertFalse(item in return_dict['etm7']['products'])
def test_fetch_user_orders_by_email_val(self):
orders = api.fetch_user_orders(self.user.email)
self.assertEqual(orders.keys()[0], "orders")
def test_fetch_user_orders_by_username_val(self):
orders = api.fetch_user_orders(self.user.username)
self.assertEqual(orders.keys()[0], "orders")
def test_fetch_order_by_orderid_val(self):
order = api.fetch_order(self.order.orderid)
self.assertEqual(order['orderid'], self.order.orderid)
def test_fetch_order_status_valid(self):
response = api.order_status(self.order.orderid)
self.assertEqual(response.keys(), ['orderid', 'status'])
def test_fetch_order_status_invalid(self):
invalid_orderid = 'invalidorderid'
response = api.order_status(invalid_orderid)
self.assertEqual(response.keys(), ['msg'])
def test_fetch_item_status_valid(self):
response = api.item_status(self.order.orderid)
self.assertEqual(response.keys(), ['orderid'])
self.assertIsInstance(response['orderid'], dict)
def test_fetch_item_status_invalid(self):
invalid_orderid = 'invalidorderid'
response = api.order_status(invalid_orderid)
self.assertEqual(response.keys(), ['msg'])
class TestAPI(unittest.TestCase):
def setUp(self):
logger.warning('Testing API started...')
os.environ['espa_api_testing'] = 'True'
# create a user
self.mock_user = MockUser()
self.mock_order = MockOrder()
user_id = self.mock_user.add_testing_user()
order_id = self.mock_order.generate_testing_order(user_id)
self.order = Order.find(order_id)
self.user = User.find(user_id)
self.product_id = 'LT05_L1TP_032028_20120425_20160830_01_T1'
self.sensor_id = 'tm5_collection'
self.staff_product_id = 'LE07_L1TP_010028_20050420_20160925_01_T1'
self.staff_sensor = 'etm7_collection'
self.global_product_id = 'LE70450302003206EDC01'
staff_user_id = self.mock_user.add_testing_user()
self.staff_user = User.find(staff_user_id)
self.staff_user.update('is_staff', True)
staff_order_id = self.mock_order.generate_testing_order(staff_user_id)
staff_order = Order.find(staff_order_id)
staff_scene = staff_order.scenes()[0]
staff_scene.update('name', self.staff_product_id)
user_scene = self.order.scenes()[0]
user_scene.update('name', self.staff_product_id)
with open(os.path.join(__location__, 'domain/restricted.yaml')) as f:
self.restricted = yaml.load(f.read())
self.restricted['all']['role'].remove('restricted_prod')
def tearDown(self):
logger.warning('Testing API done.')
# clean up orders
self.mock_order.tear_down_testing_orders()
# clean up users
self.mock_user.cleanup()
os.environ['espa_api_testing'] = ''
def test_api_versions_key_val(self):
self.assertEqual(set(api.api_versions().keys()), set(['v0', 'v1']))
def test_get_available_products_key_val(self):
self.assertEqual(
api.available_products(self.product_id,
self.user.username).keys()[0],
self.sensor_id)
def test_get_available_products_by_staff(self):
# staff should see all available products
self.user.update('is_staff', True)
return_dict = api.available_products(self.staff_product_id,
self.staff_user.username)
for item in self.restricted['all']['role']:
self.assertTrue(item in return_dict[self.staff_sensor]['products'])
def test_get_available_products_by_public(self):
# public should not see products listed in api/domain.restricted.yaml
self.user.update('is_staff', False)
return_dict = api.available_products(self.staff_product_id,
self.user.username)
for item in self.restricted['all']['role']:
self.assertFalse(
item in return_dict[self.staff_sensor]['products'])
def test_get_available_products_global_restriction(self):
# testing of landsat pre-collection restricted
self.user.update('is_staff', False)
return_dict = api.available_products(self.global_product_id,
self.user.username)
self.assertIn('ordering_restricted', return_dict)
self.assertEqual(return_dict['ordering_restricted']['etm7'],
[self.global_product_id])
def test_fetch_user_orders_by_email_val(self):
orders = api.fetch_user_orders(email=self.user.email)
self.assertTrue(len(orders) > 1)
self.assertIn(self.order.orderid, [o.orderid for o in orders])
def test_fetch_user_orders_by_username_val(self):
orders = api.fetch_user_orders(username=self.user.username)
self.assertTrue(len(orders) > 1)
self.assertIn(self.order.orderid, [o.orderid for o in orders])
def test_fetch_order_by_orderid_val(self):
order = api.fetch_order(self.order.orderid)
self.assertEqual(1, len(order))
self.assertEqual(order[0].orderid, self.order.orderid)
def test_fetch_order_by_orderid_invalid(self):
invalid_orderid = 'invalidorderid'
response = api.fetch_order(invalid_orderid)
self.assertEqual(response, list())
def test_fetch_item_status_valid(self):
response = api.item_status(self.order.orderid)
self.assertEqual(set(response), {self.order.orderid})
self.assertIsInstance(response[self.order.orderid], list)
self.assertEqual(set([s.name for s in self.order.scenes()]),
set([s.name for s in response[self.order.orderid]]))
class TestValidation(unittest.TestCase):
def setUp(self):
logger.warning('Testing Validation started...')
os.environ['espa_api_testing'] = 'True'
self.mock_user = MockUser()
self.staffuser = User.find(self.mock_user.add_testing_user())
self.staffuser.update('is_staff', True)
self.base_order = lowercase_all(testorders.build_base_order())
self.base_schema = BaseValidationSchema.request_schema
def test_validation_get_order_schema(self):
"""
Make sure the ordering schema is retrievable as a dict
"""
self.assertIsInstance(api.validation.fetch_order_schema(), dict)
def test_validation_get_valid_formats(self):
"""
Make sure the file format options are retrievable as a dict
"""
self.assertIsInstance(api.validation.fetch_formats(), dict)
def test_validation_get_valid_resampling(self):
"""
Make sure the resampling options are retrievable as a dict
"""
self.assertIsInstance(api.validation.fetch_resampling(), dict)
def test_validation_get_valid_projections(self):
"""
Make sure the projection options are retrievable as a dict
"""
self.assertIsInstance(api.validation.fetch_projections(), dict)
def test_validate_good_order(self):
"""
Test a series of known good orders
"""
for proj in testorders.good_test_projections:
valid_order = copy.deepcopy(self.base_order)
valid_order['projection'] = {
proj: testorders.good_test_projections[proj]
}
if 'lonlat' not in valid_order['projection']:
valid_order['resize'] = {
"pixel_size": 30,
"pixel_size_units": "meters"
}
try:
good = api.validation(valid_order, self.staffuser.username)
except ValidationException as e:
self.fail('Raised ValidationException: {}'.format(e.message))
def test_modis_resize(self):
"""
Most common issue of orders resizing MODIS to 30m pixels, without setting the extents
"""
modis_order = {
'mod09a1': {
'inputs': ['mod09a1.a2016305.h11v04.006.2016314200836'],
'products': ['l1']
},
'resampling_method': 'cc',
'resize': {
'pixel_size': 30,
'pixel_size_units': 'meters'
},
'format': 'gtiff'
}
exc = 'pixel count is greater than maximum size of'
exc_key = '1 validation errors'
try:
api.validation(modis_order, self.staffuser.username)
except Exception as e:
self.assertIn(exc_key, e.response)
self.assertIsInstance(e.response[exc_key], list)
self.assertIn(exc, str(e.response[exc_key]))
else:
self.fail('Failed MODIS pixel resize test')
def test_validate_bad_orders(self):
"""
Build a series of invalid orders to try and catch any potential errors in a
submitted order
Check to make sure the invalid order raises ValidationException, then check
the exception message for the expected error message
The abbreviated flag for the InvalidOrders class changes the number of invalid
orders that will get tested.
abbreviated=True - test each constraint type once
abbreviated=False - test each constraint on each value location in the nested structure
"""
exc_type = ValidationException
invalid_order = copy.deepcopy(self.base_order)
for proj in testorders.good_test_projections:
invalid_order['projection'] = {
proj: testorders.good_test_projections[proj]
}
invalid_list = testorders.InvalidOrders(invalid_order,
self.base_schema,
abbreviated=False)
for order, test, exc in invalid_list:
# empty lists cause assertRaisesRegExp to fail
exc = str(exc).replace('[', '\[')
with self.assertRaisesRegexp(exc_type, exc):
api.validation(order, self.staffuser.username)
def test_validate_sr_restricted_human_readable(self):
"""
Assert that a human readable response is returned for unavailable or date restricted products
"""
exc_type = ValidationException
invalid_list = {
'olitirs8_collection': {
'inputs': ['lc08_l1tp_031043_20160225_20170224_01_t1'],
'products': ['sr'],
'err_msg': 'Requested {} products are restricted by date'
},
'oli8_collection': {
'inputs': ['lo08_l1tp_021049_20150304_20170227_01_t1'],
'products': ['sr'],
'err_msg': 'Requested {} products are not available'
}
}
for stype in invalid_list:
invalid_order = copy.deepcopy(self.base_order)
invalid_order[stype]['inputs'] = invalid_list[stype]['inputs']
invalid_order[stype]['products'] = invalid_list[stype]['products']
for p in invalid_order[stype]['products']:
err_message = invalid_list[stype]['err_msg'].format(p)
with self.assertRaisesRegexp(exc_type, err_message):
api.validation.validate(invalid_order,
self.staffuser.username)
def test_projection_units_geographic(self):
"""
Make sure Geographic (latlon) projection only accepts "dd" units
"""
part_order = {
"olitirs8_collection": {
"inputs": ['lc08_l1tp_015035_20140713_20170304_01_t1'],
"products": ["l1"]
},
"projection": {
"lonlat": None
},
"format": "gtiff",
"resampling_method": "cc"
}
bad_parts = {
"resize": {
"pixel_size": 30,
"pixel_size_units": "meters"
},
"image_extents": {
"north": 80,
"south": -80,
"east": 170,
"west": -170,
"units": "meters"
},
}
err_msg = '{} units must be in "dd" for projection "lonlat"'
exc_type = ValidationException
for bname in bad_parts:
invalid_order = copy.deepcopy(part_order)
invalid_order.update({bname: bad_parts.get(bname)})
with self.assertRaisesRegexp(exc_type, err_msg.format(bname)):
api.validation.validate(invalid_order, self.staffuser.username)
def test_l1_only_restricted(self):
""" Landsat Level-1 data needs to go through other channels """
invalid_order = {
"olitirs8_collection": {
"inputs": ["lc08_l1tp_015035_20140713_20170304_01_t1"],
"products": ["l1"]
},
"format": "gtiff"
}
with self.assertRaisesRegexp(ValidationException,
'Landsat Level-1 data products'):
api.validation.validate(invalid_order, self.staffuser.username)
def test_l1_only_restricted_override(self):
""" Customizations or other sensors should override Level-1 restrictions """
valid_orders = [{
"olitirs8_collection": {
"inputs": ["lc08_l1tp_015035_20140713_20170304_01_t1"],
"products": ["l1"]
},
"format": "envi"
}, {
"olitirs8_collection": {
"inputs": ["lc08_l1tp_015035_20140713_20170304_01_t1"],
"products": ["l1"]
},
"myd13a2": {
"inputs": ["myd13a2.a2017249.h19v06.006.2017265235022"],
"products": ["l1"]
},
"format": "gtiff"
}]
for vorder in valid_orders:
api.validation.validate(vorder, self.staffuser.username)
class ProductionTransportTestCase(unittest.TestCase):
def setUp(self):
os.environ['espa_api_testing'] = 'True'
# create a user
self.mock_user = MockUser()
self.mock_order = MockOrder()
self.user = User.find(self.mock_user.add_testing_user())
self.order_id = self.mock_order.generate_testing_order(self.user.id)
self.app = http.app.test_client()
self.app.testing = True
self.sceneids = self.mock_order.scene_names_list(self.order_id)[0:2]
with db_instance() as db:
uidsql = "select user_id, orderid from ordering_order limit 1;"
db.select(uidsql)
self.userid = db[0]['user_id']
self.orderid = db[0]['orderid']
itemsql = "select name, order_id from ordering_scene limit 1;"
db.select(itemsql)
self.itemid = db[0][0]
itemorderid = db[0][1]
ordersql = "select orderid from ordering_order where id = {};".format(
itemorderid)
db.select(ordersql)
self.itemorderid = db[0][0]
self.base_order = lowercase_all(testorders.build_base_order())
def tearDown(self):
# clean up orders
self.mock_order.tear_down_testing_orders()
# clean up users
self.mock_user.cleanup()
os.environ['espa_api_testing'] = ''
@patch('api.interfaces.production.version1.API.get_production_whitelist',
api.get_production_whitelist)
def test_get_production_api(self):
response = self.app.get('/production-api',
environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
assert response.content_type == 'application/json'
assert set(response_data.keys()) == set(['0', '1'])
@patch('api.interfaces.production.version1.API.get_production_whitelist',
api.get_production_whitelist)
def test_get_production_api_v1(self):
response = self.app.get('/production-api/v1',
environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
assert set(response_data.keys()) == set(["operations", "description"])
assert "ESPA Production" in response_data['description']
@patch(
'api.providers.production.production_provider.ProductionProvider.get_products_to_process',
production_provider.get_products_to_process_inputs)
@patch('api.interfaces.production.version1.API.get_production_whitelist',
api.get_production_whitelist)
def test_get_production_api_products_modis(self):
url = "/production-api/v1/products?for_user=bilbo&product_types=modis"
response = self.app.get(url, environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
correct_resp = {
'encode_urls': False,
'for_user': '******',
'record_limit': 500,
'product_types': 'modis',
'priority': None
}
assert response_data == correct_resp
@patch(
'api.providers.production.production_provider.ProductionProvider.get_products_to_process',
production_provider.get_products_to_process_inputs)
@patch('api.interfaces.production.version1.API.get_production_whitelist',
api.get_production_whitelist)
def test_get_production_api_products_landsat(self):
url = "/production-api/v1/products?for_user=bilbo&product_types=landsat"
response = self.app.get(url, environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
correct_resp = {
'encode_urls': False,
'for_user': '******',
'record_limit': 500,
'product_types': 'landsat',
'priority': None
}
assert response_data == correct_resp
@patch('api.interfaces.production.version1.API.get_production_whitelist',
api.get_production_whitelist)
@patch(
'api.providers.production.production_provider.ProductionProvider.update_status',
production_provider.update_status_inputs)
def test_post_production_api_update_status(self):
url = "/production-api/v1/update_status"
data_dict = {
'name': 't10000xyz401',
'orderid': '[email protected]',
'processing_loc': 'update_status',
'status': 'updated'
}
response = self.app.post(url,
data=json.dumps(data_dict),
environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
assert response_data == data_dict
@patch(
'api.providers.production.production_provider.ProductionProvider.set_product_error',
production_provider.set_product_error_inputs)
@patch('api.interfaces.production.version1.API.get_production_whitelist',
api.get_production_whitelist)
def test_post_production_api_set_product_error(self):
url = "/production-api/v1/set_product_error"
data_dict = {
'name': 't10000xyz401',
'orderid': '[email protected]',
'processing_loc': 'xyz',
'error': 'oopsie'
}
response = self.app.post(url,
data=json.dumps(data_dict),
environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
assert response_data == data_dict
@patch(
'api.providers.production.production_provider.ProductionProvider.set_product_unavailable',
production_provider.set_product_unavailable_inputs)
@patch('api.interfaces.production.version1.API.get_production_whitelist',
api.get_production_whitelist)
def test_post_production_api_set_product_unavailable(self):
url = "/production-api/v1/set_product_unavailable"
data_dict = {
'name': 't10000xyz401',
'orderid': '[email protected]',
'processing_loc': 'xyz',
'error': 'oopsie',
'note': 'them notes'
}
response = self.app.post(url,
data=json.dumps(data_dict),
environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
assert response_data == data_dict
@patch(
'api.providers.production.production_provider.ProductionProvider.mark_product_complete',
production_provider.set_mark_product_complete_inputs)
@patch('api.interfaces.production.version1.API.get_production_whitelist',
api.get_production_whitelist)
def test_post_production_api_mark_product_complete(self):
url = "/production-api/v1/mark_product_complete"
data_dict = {
'name': 't10000xyz401',
'orderid': '[email protected]',
'processing_loc': 'xyz',
'completed_file_location': '/tmp',
'cksum_file_location': '/tmp/txt.txt',
'log_file_contents': 'details'
}
response = self.app.post(url,
data=json.dumps(data_dict),
environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
assert response_data == data_dict
@patch(
'api.providers.production.production_provider.ProductionProvider.handle_orders',
production_provider.respond_true)
@patch('api.interfaces.production.version1.API.get_production_whitelist',
api.get_production_whitelist)
def test_post_production_api_handle_orders(self):
url = "/production-api/v1/handle-orders"
response = self.app.get(url,
data=json.dumps({}),
environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
assert response_data is True
@patch(
'api.providers.production.production_provider.ProductionProvider.queue_products',
production_provider.queue_products_inputs)
@patch('api.interfaces.production.version1.API.get_production_whitelist',
api.get_production_whitelist)
def test_post_production_api_queue_products(self):
url = "/production-api/v1/queue-products"
data_dict = {
'order_name_tuple_list': 'order_name_tuple_list',
'processing_location': 'processing_location',
'job_name': 'job_name'
}
response = self.app.post(url,
data=json.dumps(data_dict),
environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
assert response_data == data_dict
@patch('api.interfaces.production.version1.API.get_production_whitelist',
api.get_production_whitelist)
def test_get_production_api_configurations(self):
url = "/production-api/v1/configuration/system_message_title"
response = self.app.get(url, environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
assert response_data.keys() == ['system_message_title']
@patch('api.interfaces.admin.version1.API.get_stat_whitelist',
api.get_stat_whitelist)
def test_get_production_api_stat_products_complete_24_hrs(self):
url = "/production-api/v1/statistics/stat_products_complete_24_hrs"
response = self.app.get(url, environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
assert response_data == 0
def setUp(self):
os.environ['espa_api_testing'] = 'True'
# create a user
self.mock_user = MockUser()
self.mock_order = MockOrder()
self.user_id = self.mock_user.add_testing_user()
class TestProductionAPI(unittest.TestCase):
def setUp(self):
os.environ['espa_api_testing'] = 'True'
# create a user
self.mock_user = MockUser()
self.mock_order = MockOrder()
self.user_id = self.mock_user.add_testing_user()
def tearDown(self):
# clean up orders
self.mock_order.tear_down_testing_orders()
# clean up users
self.mock_user.cleanup()
os.environ['espa_api_testing'] = ''
@patch('api.external.lpdaac.get_download_urls', lpdaac.get_download_urls)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry',
mock_production_provider.set_product_retry)
def test_fetch_production_products_modis(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
# need scenes with statuses of 'processing' and 'ordered'
self.mock_order.update_scenes(order_id, 'status', ['processing', 'ordered', 'oncache'])
user = User.find(self.user_id)
params = {'for_user': user.username, 'product_types': ['modis']}
response = api.fetch_production_products(params)
self.assertTrue('bilbo' in response[0]['orderid'])
@patch('api.external.lta.get_download_urls', lta.get_download_urls)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry',
mock_production_provider.set_product_retry)
def test_fetch_production_products_landsat(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
# need scenes with statuses of 'processing' and 'ordered'
self.mock_order.update_scenes(order_id, 'status', ['processing','oncache','ordered'])
user = User.find(self.user_id)
params = {'for_user': user.username, 'product_types': ['landsat']}
response = api.fetch_production_products(params)
self.assertTrue('bilbo' in response[0]['orderid'])
def test_fetch_production_products_plot(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
self.mock_order.update_scenes(order_id, 'status', ['complete'])
order = Order.find(order_id)
plot_scene = order.scenes()[0]
plot_scene.name = 'plot'
plot_scene.sensor_type = 'plot'
plot_scene.status = 'submitted'
plot_scene.save()
response = production_provider.handle_submitted_plot_products()
pscene = order.scenes({'status': 'oncache', 'sensor_type': 'plot'})
self.assertTrue(response is True)
self.assertEqual(len(pscene), 1)
def test_production_set_product_retry(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
order = Order.find(order_id)
scene = order.scenes()[3]
scene.update('retry_count', 4)
processing_loc = "get_products_to_process"
error = 'not available after EE call '
note = 'note this'
retry_after = datetime.datetime.now() + datetime.timedelta(hours=1)
retry_limit = 9
response = production_provider.set_product_retry(scene.name, order.orderid, processing_loc,
error, note, retry_after, retry_limit)
new = Scene.get('ordering_scene.status', scene.name, order.orderid)
self.assertTrue('retry' == new)
@patch('api.external.lta.update_order_status', mock_production_provider.respond_true)
def test_production_set_product_error_unavailable(self):
"""
Move a scene status from error to unavailable based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes({'name !=': 'plot'})[0]
production_provider.set_product_error(scene.name, order.orderid,
'get_products_to_process',
'include_dswe is an unavailable product option for OLITIRS')
self.assertTrue('unavailable' == Scene.get('ordering_scene.status', scene.name, order.orderid))
def test_production_set_product_error_submitted(self):
"""
Move a scene status from error to submitted based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes({'name !=': 'plot'})[0]
production_provider.set_product_error(scene.name, order.orderid,
'get_products_to_process',
'BLOCK, COMING FROM LST AS WELL: No such file or directory')
self.assertTrue('submitted' == Scene.get('ordering_scene.status', scene.name, order.orderid))
def test_production_set_product_error_retry(self):
"""
Move a scene status from error to retry based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[2]
production_provider.set_product_error(scene.name, order.orderid,
'somewhere',
'Verify the missing auxillary data products')
self.assertTrue('retry' == Scene.get('ordering_scene.status', scene.name, order.orderid))
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
def test_update_product_details_update_status(self):
"""
Set a scene status to Queued
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[0]
api.update_product_details('update_status',
{'name': scene.name,
'orderid': order.orderid,
'processing_loc': 'L8SRLEXAMPLE',
'status': 'Queued'})
self.assertTrue(Scene.get('ordering_scene.status', scene.name, order.orderid) == 'Queued')
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
@patch('api.external.onlinecache.capacity', onlinecache.capacity)
def test_update_product_details_set_product_error(self):
"""
Set a scene status to error
:return:
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[0]
production_provider.update_product('set_product_error',
name=scene.name, orderid=order.orderid,
processing_loc="L8SRLEXAMPLE",
error='problems yo')
self.assertTrue(Scene.find(scene.id).status == 'error')
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
def test_update_product_details_set_product_unavailable(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[0]
production_provider.update_product('set_product_unavailable',
name=scene.name, orderid=order.orderid,
processing_loc="L8SRLEXAMPLE",
error='include_dswe is an unavailable product option for OLITIRS')
self.assertTrue('unavailable' == Scene.get('ordering_scene.status', scene.name, order.orderid))
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
def test_update_product_details_mark_product_complete(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[0]
production_provider.update_product('mark_product_complete',
name=scene.name,
orderid=order.orderid,
processing_loc='L8SRLEXAMPLE',
completed_file_location='/some/loc',
cksum_file_location='some checksum',
log_file_contents='some log')
self.assertTrue('complete' == Scene.get('ordering_scene.status', scene.name, order.orderid))
@patch('api.external.lta.update_order_status', lta.update_order_status_fail)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
def test_update_product_details_fail_lta_mark_product_complete(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[1]
order.update('order_source', 'ee')
production_provider.update_product('mark_product_complete',
name=scene.name,
orderid=order.orderid,
processing_loc='L8SRLEXAMPLE',
completed_file_location='/some/loc',
cksum_file_location='some checksum',
log_file_contents='some log')
s = Scene.where({'name': scene.name, 'order_id': scene.order_id})[0]
self.assertTrue('C' == s.failed_lta_status_update)
@patch('api.providers.production.production_provider.ProductionProvider.send_initial_emails',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.handle_onorder_landsat_products',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.handle_retry_products',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.load_ee_orders',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.handle_submitted_products',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.finalize_orders',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.purge_orders',
mock_production_provider.respond_true)
def test_handle_orders_success(self):
self.assertTrue(api.handle_orders())
@patch('api.external.onlinecache.delete', mock_production_provider.respond_true)
@patch('api.notification.emails.send_purge_report', mock_production_provider.respond_true)
@patch('api.external.onlinecache.capacity', onlinecache.capacity)
def test_production_purge_orders(self):
new_completion_date = datetime.datetime.now() - datetime.timedelta(days=12)
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
order.update('status', 'complete')
order.update('completion_date', new_completion_date)
self.assertTrue(production_provider.purge_orders())
# need to figure a test for emails.send_email
@patch('api.notification.emails.Emails.send_email', mock_production_provider.respond_true)
def test_production_send_initial_emails(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
order.update('status', 'ordered')
self.assertTrue(emails.Emails().send_all_initial())
@patch('api.external.lta.get_order_status', lta.get_order_status)
@patch('api.external.lta.update_order_status', lta.update_order_status)
def test_production_handle_onorder_landsat_products(self):
tram_order_ids = lta.sample_tram_order_ids()[0:3]
scene_names = lta.sample_scene_names()[0:3]
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scenes = order.scenes()[0:3]
for idx, scene in enumerate(scenes):
scene.tram_order_id = tram_order_ids[idx]
scene.status = 'onorder'
# save() doesn't let you update name,
# b/c updating a scene name is not acceptable
# outside of testing
scene.update('name', scene_names[idx])
scene.save()
self.assertTrue(production_provider.handle_onorder_landsat_products())
def test_production_handle_retry_products(self):
prev = datetime.datetime.now() - datetime.timedelta(hours=1)
order_id = self.mock_order.generate_testing_order(self.user_id)
self.mock_order.update_scenes(order_id, 'status', ['retry'])
self.mock_order.update_scenes(order_id, 'retry_after', [prev])
production_provider.handle_retry_products()
for s in Scene.where({'order_id': order_id}):
self.assertTrue(s.status == 'submitted')
#@patch('api.external.lta.get_available_orders', lta.get_available_orders)
#@patch('api.external.lta.update_order_status', lta.update_order_status)
#@patch('api.external.lta.get_user_name', lta.get_user_name)
#def test_production_load_ee_orders(self):
# #production_provider.load_ee_orders()
# pass
@patch('api.external.lta.get_available_orders', lta.get_available_orders_partial)
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch('api.external.lta.get_user_name', lta.get_user_name)
def test_production_load_ee_orders_partial(self):
order = Order.find(self.mock_order.generate_ee_testing_order(self.user_id, partial=True))
self.assertTrue(order.product_opts == {'format': 'gtiff',
'etm7': {'inputs': ['LE70900652008327EDC00'],
'products': ['sr']}})
production_provider.load_ee_orders()
reorder = Order.find(order.id)
self.assertTrue(reorder.product_opts == {'format': 'gtiff',
'etm7': {'inputs': ['LE70900652008327EDC00'],
'products': ['sr']},
'tm5': {'inputs': ['LT50900652008327EDC00'],
'products': ['sr']}})
@patch('api.external.lta.update_order_status', lta.update_order_status)
def test_production_handle_failed_ee_updates(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.update('failed_lta_status_update', 'C')
production_provider.handle_failed_ee_updates()
scenes = Scene.where({'failed_lta_status_update IS NOT': None})
self.assertTrue(len(scenes) == 0)
@patch('api.providers.production.production_provider.ProductionProvider.handle_submitted_landsat_products',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.handle_submitted_modis_products',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.handle_submitted_plot_products',
mock_production_provider.respond_true)
def test_production_handle_submitted_products(self):
self.assertTrue(production_provider.handle_submitted_products())
@patch('api.providers.production.production_provider.ProductionProvider.mark_nlaps_unavailable',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.update_landsat_product_status',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.get_contactids_for_submitted_landsat_products',
mock_production_provider.contact_ids_list)
def test_production_handle_submitted_landsat_products(self):
self.assertTrue(production_provider.handle_submitted_landsat_products())
# !!! need to write test for nlaps.products_are_nlaps !!!
@patch('api.external.nlaps.products_are_nlaps', nlaps.products_are_nlaps)
@patch('api.providers.production.production_provider.ProductionProvider.set_products_unavailable',
mock_production_provider.respond_true)
def test_production_mark_nlaps_unavailable(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.status = 'submitted'
scene.sensor_type = 'landsat'
scene.save()
self.assertTrue(production_provider.mark_nlaps_unavailable())
@patch('api.external.lta.update_order_status', lta.update_order_status)
def test_production_set_products_unavailable(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
self.assertTrue(production_provider.set_products_unavailable(order.scenes(), "you want a reason?"))
@patch('api.external.lta.order_scenes', lta.order_scenes)
@patch('api.providers.production.production_provider.ProductionProvider.set_products_unavailable',
mock_production_provider.respond_true)
def test_production_update_landsat_product_status(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.status = 'submitted'
scene.sensor_type = 'landsat'
scene.save()
self.assertTrue(production_provider.update_landsat_product_status(User.find(self.user_id).contactid))
def test_production_get_contactids_for_submitted_landsat_products(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.status = 'submitted'
scene.sensor_type = 'landsat'
scene.save()
response = production_provider.get_contactids_for_submitted_landsat_products()
self.assertIsInstance(response, set)
self.assertTrue(len(response) > 0)
@patch('api.external.lpdaac.input_exists', lpdaac.input_exists_true)
def test_production_handle_submitted_modis_products_input_exists(self):
# handle oncache scenario
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.status = 'submitted'
scene.sensor_type = 'modis'
scene.save()
sid = scene.id
self.assertTrue(production_provider.handle_submitted_modis_products())
self.assertEquals(Scene.find(sid).status, "oncache")
@patch('api.external.lpdaac.input_exists', lpdaac.input_exists_false)
def test_production_handle_submitted_modis_products_input_missing(self):
# handle unavailable scenario
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.status = 'submitted'
scene.sensor_type = 'modis'
scene.save()
sid = scene.id
self.assertTrue(production_provider.handle_submitted_modis_products())
self.assertEquals(Scene.find(sid).status, "unavailable")
def test_production_handle_submitted_plot_products(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
order.status = 'ordered'
order.order_type = 'lpcs'
order.save()
plot_id = None
for idx, scene in enumerate(order.scenes()):
# at the moment, mock_order.generate_testing_order
# creates 21 products for the order. divvy those
# up between 'complete' and 'unavailable', setting
# one aside as the 'plot' product
if idx % 2 == 0:
if idx == 0:
# need to define a plot product
scene.update('status', 'submitted')
scene.update('sensor_type', 'plot')
plot_id = scene.id
else:
scene.update('status', 'complete')
else:
scene.update('status', 'unavailable')
self.assertTrue(production_provider.handle_submitted_plot_products())
self.assertEqual(Scene.find(plot_id).status, "oncache")
@patch('api.providers.production.production_provider.ProductionProvider.update_order_if_complete',
mock_production_provider.respond_true)
def test_production_finalize_orders(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
order.update('status', 'ordered')
self.assertTrue(production_provider.finalize_orders())
@patch('api.providers.production.production_provider.ProductionProvider.send_completion_email',
mock_production_provider.respond_true)
def test_production_update_order_if_complete(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
Scene.bulk_update([s.id for s in order.scenes()], {'status': 'retry'})
order.order_source = 'espa'
order.completion_email_sent = None
order.save()
self.assertTrue(production_provider.update_order_if_complete(order))
def test_production_queue_products_success(self):
names_tuple = self.mock_order.names_tuple(3, self.user_id)
processing_loc = "get_products_to_process"
job_name = 'jobname49'
params = (names_tuple, processing_loc, job_name)
response = api.queue_products(*params)
self.assertTrue(response)
def test_production_get_key(self):
key = 'system_message_title'
response = api.get_production_key(key)
val = response[key]
self.assertIsInstance(val, str)
def test_get_production_key_invalid(self):
bad_key = 'foobar'
response = api.get_production_key(bad_key)
self.assertEqual(response.keys(), ['msg'])
@patch('api.external.hadoop.HadoopHandler.job_names_ids',
hadoop.jobs_names_ids)
def test_catch_orphaned_scenes(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
# need scenes with statuses of 'queued'
self.mock_order.update_scenes(order_id, 'status', ['queued'])
response = production_provider.catch_orphaned_scenes()
self.assertTrue(response)
old_time = datetime.datetime.now() - datetime.timedelta(minutes=15)
for s in Scene.where({'order_id': order_id}):
self.assertTrue(s.reported_orphan is not None)
s.reported_orphan = old_time
s.save()
response = production_provider.catch_orphaned_scenes()
self.assertTrue(response)
for s in Scene.where({'order_id': order_id}):
self.assertTrue(s.orphaned)
class TestAPI(unittest.TestCase):
def setUp(self):
os.environ['espa_api_testing'] = 'True'
# create a user
self.mock_user = MockUser()
self.mock_order = MockOrder()
user_id = self.mock_user.add_testing_user()
order_id = self.mock_order.generate_testing_order(user_id)
self.order = Order.find(order_id)
self.user = User.find(user_id)
self.product_id = 'LT50150401987120XXX02'
self.staff_product_id = 'LE70450302003206EDC01'
staff_user_id = self.mock_user.add_testing_user()
self.staff_user = User.find(staff_user_id)
self.staff_user.update('is_staff', True)
staff_order_id = self.mock_order.generate_testing_order(staff_user_id)
staff_order = Order.find(staff_order_id)
staff_scene = staff_order.scenes()[0]
staff_scene.update('name', self.staff_product_id)
user_scene = self.order.scenes()[0]
user_scene.update('name', self.staff_product_id)
with open('api/domain/restricted.yaml') as f:
self.restricted = yaml.load(f.read())
self.restricted['all']['role'].remove('restricted_prod')
def tearDown(self):
# clean up orders
self.mock_order.tear_down_testing_orders()
# clean up users
self.mock_user.cleanup()
os.environ['espa_api_testing'] = ''
def test_api_versions_key_val(self):
self.assertEqual(set(api.api_versions().keys()), set(['0', '1']))
def test_get_available_products_key_val(self):
self.assertEqual(
api.available_products(self.product_id,
self.user.username).keys()[0], "tm5")
def test_get_available_products_by_staff(self):
# staff should see all available products
self.user.update('is_staff', True)
return_dict = api.available_products(self.staff_product_id,
self.staff_user.username)
for item in self.restricted['all']['role']:
self.assertTrue(item in return_dict['etm7']['products'])
def test_get_available_products_by_public(self):
# public should not see products listed in api/domain.restricted.yaml
self.user.update('is_staff', False)
return_dict = api.available_products(self.staff_product_id,
self.user.username)
for item in self.restricted['all']['role']:
self.assertFalse(item in return_dict['etm7']['products'])
def test_fetch_user_orders_by_email_val(self):
orders = api.fetch_user_orders(self.user.email)
self.assertEqual(orders.keys()[0], "orders")
def test_fetch_user_orders_by_username_val(self):
orders = api.fetch_user_orders(self.user.username)
self.assertEqual(orders.keys()[0], "orders")
def test_fetch_order_by_orderid_val(self):
order = api.fetch_order(self.order.orderid)
self.assertEqual(order['orderid'], self.order.orderid)
def test_fetch_order_status_valid(self):
response = api.order_status(self.order.orderid)
self.assertEqual(response.keys(), ['orderid', 'status'])
def test_fetch_order_status_invalid(self):
invalid_orderid = 'invalidorderid'
response = api.order_status(invalid_orderid)
self.assertEqual(response.keys(), ['msg'])
def test_fetch_item_status_valid(self):
response = api.item_status(self.order.orderid)
self.assertEqual(response.keys(), ['orderid'])
self.assertIsInstance(response['orderid'], dict)
def test_fetch_item_status_invalid(self):
invalid_orderid = 'invalidorderid'
response = api.order_status(invalid_orderid)
self.assertEqual(response.keys(), ['msg'])
class TestValidation(unittest.TestCase):
def setUp(self):
os.environ['espa_api_testing'] = 'True'
self.mock_user = MockUser()
self.staffuser = User.find(self.mock_user.add_testing_user())
self.staffuser.update('is_staff', True)
self.base_order = lowercase_all(testorders.build_base_order())
self.base_schema = BaseValidationSchema.request_schema
def test_validation_get_order_schema(self):
"""
Make sure the ordering schema is retrievable as a dict
"""
self.assertIsInstance(api.validation.fetch_order_schema(), dict)
def test_validation_get_valid_formats(self):
"""
Make sure the file format options are retrievable as a dict
"""
self.assertIsInstance(api.validation.fetch_formats(), dict)
def test_validation_get_valid_resampling(self):
"""
Make sure the resampling options are retrievable as a dict
"""
self.assertIsInstance(api.validation.fetch_resampling(), dict)
def test_validation_get_valid_projections(self):
"""
Make sure the projection options are retrievable as a dict
"""
self.assertIsInstance(api.validation.fetch_projections(), dict)
def test_validate_good_order(self):
"""
Test a series of known good orders
"""
for proj in testorders.good_test_projections:
valid_order = copy.deepcopy(self.base_order)
valid_order['projection'] = {proj: testorders.good_test_projections[proj]}
try:
good = api.validation(valid_order, self.staffuser.username)
except ValidationException as e:
self.fail('Raised ValidationException: {}'.format(e.message))
def test_modis_resize(self):
"""
Most common issue of orders resizing MODIS to 30m pixels, without setting the extents
"""
modis_order = {'mod09a1': {'inputs': 'mod09a1.a2000072.h02v09.005.2008237032813',
'products': ['l1']},
'resampling_method': 'cc',
'resize': {'pixel_size': 30,
'pixel_size_units': 'meters'},
'format': 'gtiff'}
exc = 'pixel count value is greater than maximum size of'
try:
api.validation(modis_order, self.staffuser.username)
except Exception as e:
assert(exc in str(e))
else:
self.fail('Failed MODIS pixel resize test')
def test_validate_bad_orders(self):
"""
Build a series of invalid orders to try and catch any potential errors in a
submitted order
Check to make sure the invalid order raises ValidationException, then check
the exception message for the expected error message
The abbreviated flag for the InvalidOrders class changes the number of invalid
orders that will get tested.
abbreviated=True - test each constraint type once
abbreviated=False - test each constraint on each value location in the nested structure
"""
exc_type = ValidationException
invalid_order = copy.deepcopy(self.base_order)
c = 0 # For initial debugging
for proj in testorders.good_test_projections:
invalid_order['projection'] = {proj: testorders.good_test_projections[proj]}
invalid_list = testorders.InvalidOrders(invalid_order, self.base_schema, abbreviated=True)
for order, test, exc in invalid_list:
# issues getting assertRaisesRegExp to work correctly
with self.assertRaises(exc_type):
try:
c += 1
api.validation(order, self.staffuser.username)
except exc_type as e:
if str(exc) in str(e):
raise
else:
self.fail('\n\nExpected in exception message:\n{}'
'\n\nException message raised:\n{}'
'\n\nUsing test {}'.format(str(exc), str(e), test))
else:
self.fail('\n{} Exception was not raised\n'
'\nExpected exception message:\n{}\n'
'\nUsing test: {}'.format(exc_type, str(exc), test))
class TestValidation(unittest.TestCase):
def setUp(self):
os.environ['espa_api_testing'] = 'True'
self.mock_user = MockUser()
self.staffuser = User.find(self.mock_user.add_testing_user())
self.staffuser.update('is_staff', True)
self.base_order = lowercase_all(testorders.build_base_order())
self.base_schema = BaseValidationSchema.request_schema
def test_validation_get_order_schema(self):
"""
Make sure the ordering schema is retrievable as a dict
"""
self.assertIsInstance(api.validation.fetch_order_schema(), dict)
def test_validation_get_valid_formats(self):
"""
Make sure the file format options are retrievable as a dict
"""
self.assertIsInstance(api.validation.fetch_formats(), dict)
def test_validation_get_valid_resampling(self):
"""
Make sure the resampling options are retrievable as a dict
"""
self.assertIsInstance(api.validation.fetch_resampling(), dict)
def test_validation_get_valid_projections(self):
"""
Make sure the projection options are retrievable as a dict
"""
self.assertIsInstance(api.validation.fetch_projections(), dict)
def test_validate_good_order(self):
"""
Test a series of known good orders
"""
for proj in testorders.good_test_projections:
valid_order = copy.deepcopy(self.base_order)
valid_order['projection'] = {
proj: testorders.good_test_projections[proj]
}
try:
good = api.validation(valid_order, self.staffuser.username)
except ValidationException as e:
self.fail('Raised ValidationException: {}'.format(e.message))
def test_modis_resize(self):
"""
Most common issue of orders resizing MODIS to 30m pixels, without setting the extents
"""
modis_order = {
'mod09a1': {
'inputs': 'mod09a1.a2000072.h02v09.005.2008237032813',
'products': ['l1']
},
'resampling_method': 'cc',
'resize': {
'pixel_size': 30,
'pixel_size_units': 'meters'
},
'format': 'gtiff'
}
exc = 'pixel count value is greater than maximum size of'
try:
api.validation(modis_order, self.staffuser.username)
except Exception as e:
assert (exc in str(e))
else:
self.fail('Failed MODIS pixel resize test')
def test_validate_bad_orders(self):
"""
Build a series of invalid orders to try and catch any potential errors in a
submitted order
Check to make sure the invalid order raises ValidationException, then check
the exception message for the expected error message
The abbreviated flag for the InvalidOrders class changes the number of invalid
orders that will get tested.
abbreviated=True - test each constraint type once
abbreviated=False - test each constraint on each value location in the nested structure
"""
exc_type = ValidationException
invalid_order = copy.deepcopy(self.base_order)
c = 0 # For initial debugging
for proj in testorders.good_test_projections:
invalid_order['projection'] = {
proj: testorders.good_test_projections[proj]
}
invalid_list = testorders.InvalidOrders(invalid_order,
self.base_schema,
abbreviated=True)
for order, test, exc in invalid_list:
# issues getting assertRaisesRegExp to work correctly
with self.assertRaises(exc_type):
try:
c += 1
api.validation(order, self.staffuser.username)
except exc_type as e:
if str(exc) in str(e):
raise
else:
self.fail('\n\nExpected in exception message:\n{}'
'\n\nException message raised:\n{}'
'\n\nUsing test {}'.format(
str(exc), str(e), test))
else:
self.fail('\n{} Exception was not raised\n'
'\nExpected exception message:\n{}\n'
'\nUsing test: {}'.format(
exc_type, str(exc), test))
class ProductionTransportTestCase(unittest.TestCase):
def setUp(self):
os.environ['espa_api_testing'] = 'True'
# create a user
self.mock_user = MockUser()
self.mock_order = MockOrder()
self.user = User.find(self.mock_user.add_testing_user())
self.order_id = self.mock_order.generate_testing_order(self.user.id)
self.app = http.app.test_client()
self.app.testing = True
self.sceneids = self.mock_order.scene_names_list(self.order_id)[0:2]
with db_instance() as db:
uidsql = "select user_id, orderid from ordering_order limit 1;"
db.select(uidsql)
self.userid = db[0]['user_id']
self.orderid = db[0]['orderid']
itemsql = "select name, order_id from ordering_scene limit 1;"
db.select(itemsql)
self.itemid = db[0][0]
itemorderid = db[0][1]
ordersql = "select orderid from ordering_order where id = {};".format(itemorderid)
db.select(ordersql)
self.itemorderid = db[0][0]
self.base_order = lowercase_all(testorders.build_base_order())
def tearDown(self):
# clean up orders
self.mock_order.tear_down_testing_orders()
# clean up users
self.mock_user.cleanup()
os.environ['espa_api_testing'] = ''
@patch('api.interfaces.production.version1.API.get_production_whitelist', api.get_production_whitelist)
def test_get_production_api(self):
response = self.app.get('/production-api', environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
assert response.content_type == 'application/json'
assert set(response_data.keys()) == set(['0', '1'])
@patch('api.interfaces.production.version1.API.get_production_whitelist', api.get_production_whitelist)
def test_get_production_api_v1(self):
response = self.app.get('/production-api/v1', environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
assert set(response_data.keys()) == set(["operations", "description"])
assert "ESPA Production" in response_data['description']
@patch('api.providers.production.production_provider.ProductionProvider.get_products_to_process',
production_provider.get_products_to_process_inputs)
@patch('api.interfaces.production.version1.API.get_production_whitelist', api.get_production_whitelist)
def test_get_production_api_products_modis(self):
url = "/production-api/v1/products?for_user=bilbo&product_types=modis"
response = self.app.get(url, environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
correct_resp = {'encode_urls': False, 'for_user': '******',
'record_limit': 500, 'product_types': 'modis',
'priority': None}
assert response_data == correct_resp
@patch('api.providers.production.production_provider.ProductionProvider.get_products_to_process',
production_provider.get_products_to_process_inputs)
@patch('api.interfaces.production.version1.API.get_production_whitelist', api.get_production_whitelist)
def test_get_production_api_products_landsat(self):
url = "/production-api/v1/products?for_user=bilbo&product_types=landsat"
response = self.app.get(url, environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
correct_resp = {'encode_urls': False, 'for_user': '******',
'record_limit': 500, 'product_types': 'landsat',
'priority': None}
assert response_data == correct_resp
@patch('api.interfaces.production.version1.API.get_production_whitelist', api.get_production_whitelist)
@patch('api.providers.production.production_provider.ProductionProvider.update_status',
production_provider.update_status_inputs)
def test_post_production_api_update_status(self):
url = "/production-api/v1/update_status"
data_dict = {'name': 't10000xyz401', 'orderid': '[email protected]',
'processing_loc': 'update_status', 'status': 'updated'}
response = self.app.post(url, data=json.dumps(data_dict), environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
assert response_data == data_dict
@patch('api.providers.production.production_provider.ProductionProvider.set_product_error',
production_provider.set_product_error_inputs)
@patch('api.interfaces.production.version1.API.get_production_whitelist', api.get_production_whitelist)
def test_post_production_api_set_product_error(self):
url = "/production-api/v1/set_product_error"
data_dict = {'name': 't10000xyz401', 'orderid': '[email protected]',
'processing_loc': 'xyz', 'error': 'oopsie'}
response = self.app.post(url, data=json.dumps(data_dict), environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
assert response_data == data_dict
@patch('api.providers.production.production_provider.ProductionProvider.set_product_unavailable',
production_provider.set_product_unavailable_inputs)
@patch('api.interfaces.production.version1.API.get_production_whitelist', api.get_production_whitelist)
def test_post_production_api_set_product_unavailable(self):
url = "/production-api/v1/set_product_unavailable"
data_dict = {'name': 't10000xyz401', 'orderid': '[email protected]',
'processing_loc': 'xyz', 'error': 'oopsie', 'note': 'them notes'}
response = self.app.post(url, data=json.dumps(data_dict), environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
assert response_data == data_dict
@patch('api.providers.production.production_provider.ProductionProvider.mark_product_complete',
production_provider.set_mark_product_complete_inputs)
@patch('api.interfaces.production.version1.API.get_production_whitelist', api.get_production_whitelist)
def test_post_production_api_mark_product_complete(self):
url = "/production-api/v1/mark_product_complete"
data_dict = {'name': 't10000xyz401', 'orderid': '[email protected]',
'processing_loc': 'xyz',
'completed_file_location': '/tmp',
'cksum_file_location': '/tmp/txt.txt',
'log_file_contents': 'details'}
response = self.app.post(url, data=json.dumps(data_dict), environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
assert response_data == data_dict
@patch('api.providers.production.production_provider.ProductionProvider.handle_orders',
production_provider.respond_true)
@patch('api.interfaces.production.version1.API.get_production_whitelist', api.get_production_whitelist)
def test_post_production_api_handle_orders(self):
url = "/production-api/v1/handle-orders"
response = self.app.get(url, data=json.dumps({}), environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
assert response_data is True
@patch('api.providers.production.production_provider.ProductionProvider.queue_products',
production_provider.queue_products_inputs)
@patch('api.interfaces.production.version1.API.get_production_whitelist', api.get_production_whitelist)
def test_post_production_api_queue_products(self):
url = "/production-api/v1/queue-products"
data_dict = {'order_name_tuple_list': 'order_name_tuple_list',
'processing_location': 'processing_location',
'job_name': 'job_name'}
response = self.app.post(url, data=json.dumps(data_dict), environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
assert response_data == data_dict
@patch('api.interfaces.production.version1.API.get_production_whitelist', api.get_production_whitelist)
def test_get_production_api_configurations(self):
url = "/production-api/v1/configuration/system_message_title"
response = self.app.get(url, environ_base={'REMOTE_ADDR': '127.0.0.1'})
response_data = json.loads(response.get_data())
assert response_data.keys() == ['system_message_title']
class TransportTestCase(unittest.TestCase):
def setUp(self):
os.environ['espa_api_testing'] = 'True'
# create a user
self.mock_user = MockUser()
self.mock_order = MockOrder()
self.user = User.find(self.mock_user.add_testing_user())
self.order_id = self.mock_order.generate_testing_order(self.user.id)
self.app = http.app.test_client()
self.app.testing = True
self.sceneids = self.mock_order.scene_names_list(self.order_id)[0:2]
token = ''.format(self.user.username, 'foo')
auth_string = "Basic {}".format(base64.b64encode(token))
self.headers = {"Authorization": auth_string}
with db_instance() as db:
uidsql = "select user_id, orderid from ordering_order limit 1;"
db.select(uidsql)
self.userid = db[0]['user_id']
self.orderid = db[0]['orderid']
itemsql = "select name, order_id from ordering_scene limit 1;"
db.select(itemsql)
self.itemid = db[0][0]
itemorderid = db[0][1]
ordersql = "select orderid from ordering_order where id = {};".format(itemorderid)
db.select(ordersql)
self.itemorderid = db[0][0]
self.base_order = lowercase_all(testorders.build_base_order())
def tearDown(self):
# clean up orders
self.mock_order.tear_down_testing_orders()
# clean up users
self.mock_user.cleanup()
os.environ['espa_api_testing'] = ''
def test_get_api_response_type(self):
response = self.app.get('/api', headers=self.headers, environ_base={'REMOTE_ADDR': '127.0.0.1'})
assert response.content_type == 'application/json'
def test_get_api_response_content(self):
response = self.app.get('/api', headers=self.headers, environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertEqual(set(['1', '0']), set(resp_json.keys()))
@patch('api.domain.user.User.get', MockUser.get)
def test_get_api_info_response_content(self):
response = self.app.get('/api/v1', headers=self.headers, environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert "Version 1" in resp_json['description']
@patch('api.domain.user.User.get', MockUser.get)
@patch('api.providers.ordering.ordering_provider.OrderingProvider.available_products', mock_api.available_products)
def test_get_available_prods(self):
url = '/api/v1/available-products/' + ",".join(self.sceneids)
response = self.app.get(url, headers=self.headers, environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert "etm" in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
@patch('api.providers.ordering.ordering_provider.OrderingProvider.available_products', mock_api.available_products)
def test_post_available_prods(self):
url = '/api/v1/available-products'
data_dict = {'inputs': self.sceneids}
response = self.app.post(url, data=json.dumps(data_dict), headers=self.headers, environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert "etm" in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
@patch('api.providers.ordering.ordering_provider.OrderingProvider.fetch_user_orders', mock_ordering_provider.fetch_user_orders)
def test_get_available_orders_user(self):
url = "/api/v1/list-orders"
response = self.app.get(url, headers=self.headers, environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert resp_json.keys() == ['orders']
@patch('api.domain.user.User.get', MockUser.get)
def test_get_available_orders_email(self):
# email param comes in as unicode
url = "/api/v1/list-orders/" + str(self.user.email)
response = self.app.get(url, headers=self.headers, environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert resp_json.keys() == ['orders']
@patch('api.domain.user.User.get', MockUser.get)
def test_get_order_by_ordernum(self):
url = "/api/v1/order/" + str(self.orderid)
response = self.app.get(url, headers=self.headers, environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert 'orderid' in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
def test_get_order_status_by_ordernum(self):
url = "/api/v1/order-status/" + str(self.orderid)
response = self.app.get(url, headers=self.headers, environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert 'orderid' in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
def test_get_item_status_by_ordernum(self):
url = "/api/v1/item-status/%s" % self.itemorderid
response = self.app.get(url, headers=self.headers, environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert 'orderid' in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
def test_get_item_status_by_ordernum_itemnum(self):
url = "/api/v1/item-status/%s/%s" % (self.itemorderid, self.itemid)
response = self.app.get(url, headers=self.headers, environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert 'orderid' in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
def test_get_current_user(self):
url = "/api/v1/user"
response = self.app.get(url, headers=self.headers, environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert 'username' in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
def test_get_projections(self):
url = '/api/v1/projections'
response = self.app.get(url, headers=self.headers, environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert 'aea' in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
def test_get_formats(self):
url = '/api/v1/formats'
response = self.app.get(url, headers=self.headers, environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert 'formats' in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
def test_get_resampling(self):
url = '/api/v1/resampling-methods'
response = self.app.get(url, headers=self.headers, environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert 'resampling_methods' in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
def test_get_order_schema(self):
url = '/api/v1/order-schema'
response = self.app.get(url, headers=self.headers, environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert 'properties' in resp_json.keys()
# Waiting for DB mock-ups to be finished
def test_post_order(self):
pass
class TransportTestCase(unittest.TestCase):
def setUp(self):
os.environ['espa_api_testing'] = 'True'
# create a user
self.mock_user = MockUser()
self.mock_order = MockOrder()
self.user = User.find(self.mock_user.add_testing_user())
self.order_id = self.mock_order.generate_testing_order(self.user.id)
self.app = http.app.test_client()
self.app.testing = True
self.sceneids = self.mock_order.scene_names_list(self.order_id)[0:2]
token = ''.format(self.user.username, 'foo')
auth_string = "Basic {}".format(base64.b64encode(token))
self.headers = {"Authorization": auth_string}
with db_instance() as db:
uidsql = "select user_id, orderid from ordering_order limit 1;"
db.select(uidsql)
self.userid = db[0]['user_id']
self.orderid = db[0]['orderid']
itemsql = "select name, order_id from ordering_scene limit 1;"
db.select(itemsql)
self.itemid = db[0][0]
itemorderid = db[0][1]
ordersql = "select orderid from ordering_order where id = {};".format(
itemorderid)
db.select(ordersql)
self.itemorderid = db[0][0]
self.base_order = lowercase_all(testorders.build_base_order())
def tearDown(self):
# clean up orders
self.mock_order.tear_down_testing_orders()
# clean up users
self.mock_user.cleanup()
os.environ['espa_api_testing'] = ''
def test_get_api_response_type(self):
response = self.app.get('/api',
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
assert response.content_type == 'application/json'
def test_get_api_response_content(self):
response = self.app.get('/api',
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertEqual(set(['1', '0']), set(resp_json.keys()))
@patch('api.domain.user.User.get', MockUser.get)
def test_get_api_info_response_content(self):
response = self.app.get('/api/v1',
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert "Version 1" in resp_json['description']
@patch('api.domain.user.User.get', MockUser.get)
@patch(
'api.providers.ordering.ordering_provider.OrderingProvider.available_products',
mock_api.available_products)
def test_get_available_prods(self):
url = '/api/v1/available-products/' + ",".join(self.sceneids)
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert "etm" in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
@patch(
'api.providers.ordering.ordering_provider.OrderingProvider.available_products',
mock_api.available_products)
def test_post_available_prods(self):
url = '/api/v1/available-products'
data_dict = {'inputs': self.sceneids}
response = self.app.post(url,
data=json.dumps(data_dict),
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert "etm" in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
@patch(
'api.providers.ordering.ordering_provider.OrderingProvider.fetch_user_orders',
mock_ordering_provider.fetch_user_orders)
def test_get_available_orders_user(self):
url = "/api/v1/list-orders"
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert resp_json.keys() == ['orders']
@patch('api.domain.user.User.get', MockUser.get)
def test_get_available_orders_email(self):
# email param comes in as unicode
url = "/api/v1/list-orders/" + str(self.user.email)
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert resp_json.keys() == ['orders']
@patch('api.domain.user.User.get', MockUser.get)
def test_get_order_by_ordernum(self):
url = "/api/v1/order/" + str(self.orderid)
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert 'orderid' in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
def test_get_order_status_by_ordernum(self):
url = "/api/v1/order-status/" + str(self.orderid)
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert 'orderid' in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
def test_get_item_status_by_ordernum(self):
url = "/api/v1/item-status/%s" % self.itemorderid
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert 'orderid' in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
def test_get_item_status_by_ordernum_itemnum(self):
url = "/api/v1/item-status/%s/%s" % (self.itemorderid, self.itemid)
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert 'orderid' in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
def test_get_current_user(self):
url = "/api/v1/user"
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert 'username' in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
def test_get_projections(self):
url = '/api/v1/projections'
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert 'aea' in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
def test_get_formats(self):
url = '/api/v1/formats'
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert 'formats' in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
def test_get_resampling(self):
url = '/api/v1/resampling-methods'
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert 'resampling_methods' in resp_json.keys()
@patch('api.domain.user.User.get', MockUser.get)
def test_get_order_schema(self):
url = '/api/v1/order-schema'
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
assert 'properties' in resp_json.keys()
# Waiting for DB mock-ups to be finished
def test_post_order(self):
pass
class TestProductionAPI(unittest.TestCase):
def setUp(self):
os.environ['espa_api_testing'] = 'True'
# create a user
self.mock_user = MockUser()
self.mock_order = MockOrder()
self.user_id = self.mock_user.add_testing_user()
def tearDown(self):
# clean up orders
self.mock_order.tear_down_testing_orders()
# clean up users
self.mock_user.cleanup()
os.environ['espa_api_testing'] = ''
@patch('api.external.lpdaac.get_download_urls', lpdaac.get_download_urls)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry',
mock_production_provider.set_product_retry)
def test_fetch_production_products_modis(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
# need scenes with statuses of 'processing' and 'ordered'
self.mock_order.update_scenes(order_id, 'status', ['processing', 'ordered', 'oncache'])
user = User.find(self.user_id)
params = {'for_user': user.username, 'product_types': ['modis']}
response = api.fetch_production_products(params)
self.assertTrue('bilbo' in response[0]['orderid'])
@patch('api.external.lta.get_download_urls', lta.get_download_urls)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry',
mock_production_provider.set_product_retry)
def test_fetch_production_products_landsat(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
# need scenes with statuses of 'processing' and 'ordered'
self.mock_order.update_scenes(order_id, 'status', ['processing','oncache','ordered'])
user = User.find(self.user_id)
params = {'for_user': user.username, 'product_types': ['landsat']}
response = api.fetch_production_products(params)
self.assertTrue('bilbo' in response[0]['orderid'])
def test_fetch_production_products_plot(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
self.mock_order.update_scenes(order_id, 'status', ['complete'])
order = Order.find(order_id)
plot_scene = order.scenes()[0]
plot_scene.name = 'plot'
plot_scene.sensor_type = 'plot'
plot_scene.status = 'submitted'
plot_scene.save()
response = production_provider.handle_submitted_plot_products()
pscene = order.scenes({'status': 'oncache', 'sensor_type': 'plot'})
self.assertTrue(response is True)
self.assertEqual(len(pscene), 1)
def test_production_set_product_retry(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
order = Order.find(order_id)
scene = order.scenes()[3]
scene.update('retry_count', 4)
processing_loc = "get_products_to_process"
error = 'not available after EE call '
note = 'note this'
retry_after = datetime.datetime.now() + datetime.timedelta(hours=1)
retry_limit = 9
response = production_provider.set_product_retry(scene.name, order.orderid, processing_loc,
error, note, retry_after, retry_limit)
new = Scene.get('ordering_scene.status', scene.name, order.orderid)
self.assertTrue('retry' == new)
@patch('api.external.lta.update_order_status', mock_production_provider.respond_true)
def test_production_set_product_error_unavailable(self):
"""
Move a scene status from error to unavailable based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes({'name !=': 'plot'})[0]
production_provider.set_product_error(scene.name, order.orderid,
'get_products_to_process',
'include_dswe is an unavailable product option for OLITIRS')
self.assertTrue('unavailable' == Scene.get('ordering_scene.status', scene.name, order.orderid))
def test_production_set_product_error_submitted(self):
"""
Move a scene status from error to submitted based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes({'name !=': 'plot'})[0]
production_provider.set_product_error(scene.name, order.orderid,
'get_products_to_process',
'BLOCK, COMING FROM LST AS WELL: No such file or directory')
self.assertTrue('submitted' == Scene.get('ordering_scene.status', scene.name, order.orderid))
def test_production_set_product_error_retry(self):
"""
Move a scene status from error to retry based on the error
message
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[2]
production_provider.set_product_error(scene.name, order.orderid,
'somewhere',
'Verify the missing auxillary data products')
self.assertTrue('retry' == Scene.get('ordering_scene.status', scene.name, order.orderid))
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
def test_update_product_details_update_status(self):
"""
Set a scene status to Queued
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[0]
api.update_product_details('update_status',
{'name': scene.name,
'orderid': order.orderid,
'processing_loc': 'L8SRLEXAMPLE',
'status': 'Queued'})
self.assertTrue(Scene.get('ordering_scene.status', scene.name, order.orderid) == 'Queued')
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
@patch('api.external.onlinecache.capacity', onlinecache.capacity)
def test_update_product_details_set_product_error(self):
"""
Set a scene status to error
:return:
"""
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[0]
production_provider.update_product('set_product_error',
name=scene.name, orderid=order.orderid,
processing_loc="L8SRLEXAMPLE",
error='problems yo')
self.assertTrue(Scene.find(scene.id).status == 'error')
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
def test_update_product_details_set_product_unavailable(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[0]
production_provider.update_product('set_product_unavailable',
name=scene.name, orderid=order.orderid,
processing_loc="L8SRLEXAMPLE",
error='include_dswe is an unavailable product option for OLITIRS')
self.assertTrue('unavailable' == Scene.get('ordering_scene.status', scene.name, order.orderid))
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
@patch('os.path.getsize', lambda y: 999)
def test_update_product_details_mark_product_complete(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[0]
production_provider.update_product('mark_product_complete',
name=scene.name,
orderid=order.orderid,
processing_loc='L8SRLEXAMPLE',
completed_file_location='/some/loc',
cksum_file_location='some checksum',
log_file_contents='some log')
self.assertTrue('complete' == Scene.get('ordering_scene.status', scene.name, order.orderid))
@patch('api.external.lta.update_order_status', lta.update_order_status_fail)
@patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
@patch('os.path.getsize', lambda y: 999)
def test_update_product_details_fail_lta_mark_product_complete(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scene = order.scenes()[1]
order.update('order_source', 'ee')
production_provider.update_product('mark_product_complete',
name=scene.name,
orderid=order.orderid,
processing_loc='L8SRLEXAMPLE',
completed_file_location='/some/loc',
cksum_file_location='some checksum',
log_file_contents='some log')
s = Scene.where({'name': scene.name, 'order_id': scene.order_id})[0]
self.assertTrue('C' == s.failed_lta_status_update)
@patch('api.providers.production.production_provider.ProductionProvider.send_initial_emails',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.handle_onorder_landsat_products',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.handle_retry_products',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.load_ee_orders',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.handle_submitted_products',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.finalize_orders',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.purge_orders',
mock_production_provider.respond_true)
def test_handle_orders_success(self):
self.assertTrue(api.handle_orders())
@patch('api.external.onlinecache.delete', mock_production_provider.respond_true)
@patch('api.notification.emails.send_purge_report', mock_production_provider.respond_true)
@patch('api.external.onlinecache.capacity', onlinecache.capacity)
def test_production_purge_orders(self):
new_completion_date = datetime.datetime.now() - datetime.timedelta(days=12)
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
order.update('status', 'complete')
order.update('completion_date', new_completion_date)
self.assertTrue(production_provider.purge_orders())
# need to figure a test for emails.send_email
@patch('api.notification.emails.Emails.send_email', mock_production_provider.respond_true)
def test_production_send_initial_emails(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
order.update('status', 'ordered')
self.assertTrue(emails.Emails().send_all_initial())
@patch('api.external.lta.get_order_status', lta.get_order_status)
@patch('api.external.lta.update_order_status', lta.update_order_status)
def test_production_handle_onorder_landsat_products(self):
tram_order_ids = lta.sample_tram_order_ids()[0:3]
scene_names = lta.sample_scene_names()[0:3]
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scenes = order.scenes()[0:3]
for idx, scene in enumerate(scenes):
scene.tram_order_id = tram_order_ids[idx]
scene.status = 'onorder'
# save() doesn't let you update name,
# b/c updating a scene name is not acceptable
# outside of testing
scene.update('name', scene_names[idx])
scene.save()
self.assertTrue(production_provider.handle_onorder_landsat_products())
def test_production_handle_retry_products(self):
prev = datetime.datetime.now() - datetime.timedelta(hours=1)
order_id = self.mock_order.generate_testing_order(self.user_id)
self.mock_order.update_scenes(order_id, 'status', ['retry'])
self.mock_order.update_scenes(order_id, 'retry_after', [prev])
production_provider.handle_retry_products()
for s in Scene.where({'order_id': order_id}):
self.assertTrue(s.status == 'submitted')
#@patch('api.external.lta.get_available_orders', lta.get_available_orders)
#@patch('api.external.lta.update_order_status', lta.update_order_status)
#@patch('api.external.lta.get_user_name', lta.get_user_name)
#def test_production_load_ee_orders(self):
# #production_provider.load_ee_orders()
# pass
@patch('api.external.lta.get_available_orders', lta.get_available_orders_partial)
@patch('api.external.lta.update_order_status', lta.update_order_status)
@patch('api.external.lta.get_user_name', lta.get_user_name)
def test_production_load_ee_orders_partial(self):
order = Order.find(self.mock_order.generate_ee_testing_order(self.user_id, partial=True))
self.assertTrue(order.product_opts == {'format': 'gtiff',
'etm7': {'inputs': ['LE70900652008327EDC00'],
'products': ['sr']}})
production_provider.load_ee_orders()
reorder = Order.find(order.id)
self.assertTrue(reorder.product_opts == {'format': 'gtiff',
'etm7': {'inputs': ['LE70900652008327EDC00'],
'products': ['sr']},
'tm5': {'inputs': ['LT50900652008327EDC00'],
'products': ['sr']}})
@patch('api.external.lta.update_order_status', lta.update_order_status)
def test_production_handle_failed_ee_updates(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.update('failed_lta_status_update', 'C')
production_provider.handle_failed_ee_updates()
scenes = Scene.where({'failed_lta_status_update IS NOT': None})
self.assertTrue(len(scenes) == 0)
@patch('api.providers.production.production_provider.ProductionProvider.handle_submitted_landsat_products',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.handle_submitted_modis_products',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.handle_submitted_plot_products',
mock_production_provider.respond_true)
def test_production_handle_submitted_products(self):
self.assertTrue(production_provider.handle_submitted_products())
@patch('api.providers.production.production_provider.ProductionProvider.mark_nlaps_unavailable',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.update_landsat_product_status',
mock_production_provider.respond_true)
@patch('api.providers.production.production_provider.ProductionProvider.get_contactids_for_submitted_landsat_products',
mock_production_provider.contact_ids_list)
def test_production_handle_submitted_landsat_products(self):
self.assertTrue(production_provider.handle_submitted_landsat_products())
# !!! need to write test for nlaps.products_are_nlaps !!!
@patch('api.external.nlaps.products_are_nlaps', nlaps.products_are_nlaps)
@patch('api.providers.production.production_provider.ProductionProvider.set_products_unavailable',
mock_production_provider.respond_true)
def test_production_mark_nlaps_unavailable(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.status = 'submitted'
scene.sensor_type = 'landsat'
scene.save()
self.assertTrue(production_provider.mark_nlaps_unavailable())
@patch('api.external.lta.update_order_status', lta.update_order_status)
def test_production_set_products_unavailable(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
self.assertTrue(production_provider.set_products_unavailable(order.scenes(), "you want a reason?"))
@patch('api.external.lta.order_scenes', lta.order_scenes)
@patch('api.providers.production.production_provider.ProductionProvider.set_products_unavailable',
mock_production_provider.respond_true)
def test_production_update_landsat_product_status(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.status = 'submitted'
scene.sensor_type = 'landsat'
scene.save()
self.assertTrue(production_provider.update_landsat_product_status(User.find(self.user_id).contactid))
def test_production_get_contactids_for_submitted_landsat_products(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.status = 'submitted'
scene.sensor_type = 'landsat'
scene.save()
response = production_provider.get_contactids_for_submitted_landsat_products()
self.assertIsInstance(response, set)
self.assertTrue(len(response) > 0)
@patch('api.external.lpdaac.input_exists', lpdaac.input_exists_true)
def test_production_handle_submitted_modis_products_input_exists(self):
# handle oncache scenario
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.status = 'submitted'
scene.sensor_type = 'modis'
scene.save()
sid = scene.id
self.assertTrue(production_provider.handle_submitted_modis_products())
self.assertEquals(Scene.find(sid).status, "oncache")
@patch('api.external.lpdaac.input_exists', lpdaac.input_exists_false)
def test_production_handle_submitted_modis_products_input_missing(self):
# handle unavailable scenario
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
for scene in order.scenes():
scene.status = 'submitted'
scene.sensor_type = 'modis'
scene.save()
sid = scene.id
self.assertTrue(production_provider.handle_submitted_modis_products())
self.assertEquals(Scene.find(sid).status, "unavailable")
def test_production_handle_submitted_plot_products(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
order.status = 'ordered'
order.order_type = 'lpcs'
order.save()
plot_id = None
for idx, scene in enumerate(order.scenes()):
# at the moment, mock_order.generate_testing_order
# creates 21 products for the order. divvy those
# up between 'complete' and 'unavailable', setting
# one aside as the 'plot' product
if idx % 2 == 0:
if idx == 0:
# need to define a plot product
scene.update('status', 'submitted')
scene.update('sensor_type', 'plot')
plot_id = scene.id
else:
scene.update('status', 'complete')
else:
scene.update('status', 'unavailable')
self.assertTrue(production_provider.handle_submitted_plot_products())
self.assertEqual(Scene.find(plot_id).status, "oncache")
@patch('os.path.getsize', lambda y: 999)
def test_production_calc_scene_download_sizes(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
scenes = order.scenes()
Scene.bulk_update([s.id for s in scenes], {'status': 'complete', 'download_size': 0})
self.assertTrue(production_provider.calc_scene_download_sizes())
upscenes = Scene.where({'status': 'complete', 'download_size': 999})
self.assertEqual(len(upscenes), len(scenes))
@patch('api.providers.production.production_provider.ProductionProvider.update_order_if_complete',
mock_production_provider.respond_true)
def test_production_finalize_orders(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
order.update('status', 'ordered')
self.assertTrue(production_provider.finalize_orders())
@patch('api.providers.production.production_provider.ProductionProvider.send_completion_email',
mock_production_provider.respond_true)
def test_production_update_order_if_complete(self):
order = Order.find(self.mock_order.generate_testing_order(self.user_id))
Scene.bulk_update([s.id for s in order.scenes()], {'status': 'retry'})
order.order_source = 'espa'
order.completion_email_sent = None
order.save()
self.assertTrue(production_provider.update_order_if_complete(order))
def test_production_queue_products_success(self):
names_tuple = self.mock_order.names_tuple(3, self.user_id)
processing_loc = "get_products_to_process"
job_name = 'jobname49'
params = (names_tuple, processing_loc, job_name)
response = api.queue_products(*params)
self.assertTrue(response)
def test_production_get_key(self):
key = 'system_message_title'
response = api.get_production_key(key)
val = response[key]
self.assertIsInstance(val, str)
def test_get_production_key_invalid(self):
bad_key = 'foobar'
response = api.get_production_key(bad_key)
self.assertEqual(response.keys(), ['msg'])
@patch('api.external.hadoop.HadoopHandler.job_names_ids',
hadoop.jobs_names_ids)
def test_catch_orphaned_scenes(self):
order_id = self.mock_order.generate_testing_order(self.user_id)
# need scenes with statuses of 'queued'
self.mock_order.update_scenes(order_id, 'status', ['queued'])
response = production_provider.catch_orphaned_scenes()
self.assertTrue(response)
old_time = datetime.datetime.now() - datetime.timedelta(minutes=15)
for s in Scene.where({'order_id': order_id}):
self.assertTrue(s.reported_orphan is not None)
s.reported_orphan = old_time
s.save()
response = production_provider.catch_orphaned_scenes()
self.assertTrue(response)
for s in Scene.where({'order_id': order_id}):
self.assertTrue(s.orphaned)
def test_convert_product_options(self):
"""
Test the conversion procedure to make sure that the new format for orders converts
to the old format
"""
scenes = ['LE70480272012076EDC00', 'LC80500272013196LGN00',
'LT40480271983028PAC00', 'LT50490262009162PAC03']
includes = ['include_sr', 'include_sr_toa',
'include_cfmask', 'include_sr_thermal']
new_format = {u'etm7': {u'inputs': [u'LE70480272012076EDC00'],
u'products': [u'sr']},
u'olitirs8': {u'inputs': [u'LC80500272013196LGN00'],
u'products': [u'toa']},
u'tm4': {u'inputs': [u'LT40480271983028PAC00'],
u'products': [u'cloud']},
u'tm5': {u'inputs': [u'LT50490262009162PAC03'],
u'products': [u'bt']},
u'format': u'gtiff',
u'image_extents': {u'east': -2265585.0,
u'north': 3164805.0,
u'south': 3014805.0,
u'units': u'meters',
u'west': -2415585.0},
u'note': u'CONUS_h1v1',
u'projection': {u'aea': {u'central_meridian': -96,
u'datum': u'nad83',
u'false_easting': 0,
u'false_northing': 0,
u'latitude_of_origin': 23,
u'standard_parallel_1': 29.5,
u'standard_parallel_2': 45.5}},
u'resampling_method': u'cc'}
ruberic = {'central_meridian': -96,
'datum': u'nad83',
'false_easting': 0,
'false_northing': 0,
'image_extents': True,
'image_extents_units': u'meters',
'include_cfmask': False,
'include_customized_source_data': False,
'include_dswe': False,
'include_lst': False,
'include_solr_index': False,
'include_source_data': False,
'include_source_metadata': False,
'include_sr': False,
'include_sr_browse': False,
'include_sr_evi': False,
'include_sr_msavi': False,
'include_sr_nbr': False,
'include_sr_nbr2': False,
'include_sr_ndmi': False,
'include_sr_ndvi': False,
'include_sr_savi': False,
'include_sr_thermal': False,
'include_sr_toa': False,
'include_statistics': False,
'latitude_true_scale': None,
'longitude_pole': None,
'maxx': -2265585.0,
'maxy': 3164805.0,
'minx': -2415585.0,
'miny': 3014805.0,
'origin_lat': 23,
'output_format': u'gtiff',
'pixel_size': None,
'pixel_size_units': None,
'reproject': True,
'resample_method': 'cubic',
'resize': False,
'std_parallel_1': 29.5,
'std_parallel_2': 45.5,
'target_projection': u'aea',
'utm_north_south': None,
'utm_zone': None}
for scene, include in zip(scenes, includes):
ruberic[include] = True
old_format = OptionsConversion.convert(new=new_format, scenes=[scene])
self.assertDictEqual(ruberic, old_format)
ruberic[include] = False
class TransportTestCase(unittest.TestCase):
def setUp(self):
os.environ['espa_api_testing'] = 'True'
# create a user
self.mock_user = MockUser()
self.mock_order = MockOrder()
self.user = User.find(self.mock_user.add_testing_user())
self.order_id = self.mock_order.generate_testing_order(self.user.id)
self.app = http.app.test_client()
self.app.testing = True
self.sceneids = self.mock_order.scene_names_list(self.order_id)[0:2]
token = ':'.join((self.user.username, 'foo'))
auth_string = "Basic {}".format(base64.b64encode(token))
self.headers = {"Authorization": auth_string}
with db_instance() as db:
uidsql = "select user_id, orderid from ordering_order limit 1;"
db.select(uidsql)
self.userid = db[0]['user_id']
self.orderid = db[0]['orderid']
itemsql = "select name, order_id from ordering_scene limit 1;"
db.select(itemsql)
self.itemid = db[0][0]
itemorderid = db[0][1]
ordersql = "select orderid from ordering_order where id = {};".format(
itemorderid)
db.select(ordersql)
self.itemorderid = db[0][0]
self.base_order = lowercase_all(testorders.build_base_order())
self.sensors = [
k for k in self.base_order.keys()
if isinstance(self.base_order[k], dict)
and 'inputs' in self.base_order[k]
]
self.inputs = {s: self.base_order[s]['inputs'] for s in self.sensors}
self.input_names_all = set([
i for sublist in [s for k, s in self.inputs.items()]
for i in sublist
])
def tearDown(self):
# clean up orders
self.mock_order.tear_down_testing_orders()
# clean up users
self.mock_user.cleanup()
os.environ['espa_api_testing'] = ''
def test_get_api_response_type(self):
response = self.app.get('/api',
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
self.assertEqual(response.content_type, 'application/json')
@patch('api.domain.user.User.get', MockUser.get)
def test_get_api_response_content(self):
response = self.app.get('/api',
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertEqual(set(['v1', 'v0']), set(resp_json.keys()))
self.assertEqual(200, response.status_code)
@patch('api.domain.user.User.get', MockUser.get)
def test_get_api_info_response_content(self):
response = self.app.get('/api/v1',
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertIn("Version 1", resp_json['description'])
self.assertEqual(200, response.status_code)
@patch('api.domain.user.User.get', MockUser.get)
@patch(
'api.providers.ordering.ordering_provider.OrderingProvider.available_products',
mock_api.available_products)
def test_get_available_prods(self):
url = '/api/v1/available-products/' + ",".join(self.sceneids)
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertIn("etm", resp_json.keys())
self.assertEqual(200, response.status_code)
@patch('api.domain.user.User.get', MockUser.get)
@patch(
'api.providers.ordering.ordering_provider.OrderingProvider.available_products',
mock_api.available_products)
def test_get_available_prods_json(self):
url = '/api/v1/available-products'
data_dict = {'inputs': self.sceneids}
response = self.app.get(url,
data=json.dumps(data_dict),
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertIn('etm', resp_json.keys())
self.assertEqual(200, response.status_code)
@patch('api.domain.user.User.get', MockUser.get)
def test_get_available_orders_user(self):
url = "/api/v1/list-orders"
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertIsInstance(resp_json, list)
self.assertListEqual(resp_json, [self.orderid])
self.assertEqual(200, response.status_code)
@patch('api.domain.user.User.get', MockUser.get)
def test_get_available_orders_email(self):
# email param comes in as unicode
url = "/api/v1/list-orders/" + str(self.user.email)
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertIsInstance(resp_json, list)
self.assertListEqual(resp_json, [self.orderid])
self.assertEqual(200, response.status_code)
@patch('api.domain.user.User.get', MockUser.get)
def test_get_order_by_ordernum(self):
url = "/api/v1/order/" + str(self.orderid)
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
items = {
'orderid', 'note', 'order_source', 'order_type', 'product_opts',
'priority', 'completion_date', 'status', 'order_date',
'product_options'
}
self.assertEqual(items, set(resp_json))
self.assertEqual(200, response.status_code)
@patch('api.domain.user.User.get', MockUser.get)
def test_get_order_status_by_ordernum(self):
url = "/api/v1/order-status/" + str(self.orderid)
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertEqual({'orderid', 'status'}, set(resp_json))
self.assertEqual(self.orderid, resp_json.get('orderid'))
self.assertEqual('ordered', resp_json.get('status'))
self.assertEqual(200, response.status_code)
@patch('api.domain.user.User.get', MockUser.get)
def test_get_item_status_by_ordernum(self):
url = "/api/v1/item-status/%s" % self.itemorderid
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
all_names = set([s['name'].lower() for s in resp_json[self.orderid]])
all_names -= {'plot'}
self.assertEqual(self.input_names_all, all_names)
self.assertEqual(200, response.status_code)
@patch('api.domain.user.User.get', MockUser.get)
def test_get_item_status_by_ordernum_itemnum(self):
url = "/api/v1/item-status/%s/%s" % (self.itemorderid, self.itemid)
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
all_names = set([s['name'].lower() for s in resp_json[self.orderid]])
self.assertEqual({self.itemid.lower()}, all_names)
self.assertEqual(200, response.status_code)
@patch('api.domain.user.User.get', MockUser.get)
def test_get_current_user(self):
url = "/api/v1/user"
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertIn('username', resp_json.keys())
self.assertEqual(200, response.status_code)
@patch('api.domain.user.User.get', MockUser.get)
def test_get_projections(self):
url = '/api/v1/projections'
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertIn('aea', resp_json.keys())
self.assertEqual(200, response.status_code)
@patch('api.domain.user.User.get', MockUser.get)
def test_get_formats(self):
url = '/api/v1/formats'
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertIn('formats', resp_json.keys())
self.assertEqual(200, response.status_code)
@patch('api.domain.user.User.get', MockUser.get)
def test_get_resampling(self):
url = '/api/v1/resampling-methods'
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertIn('resampling_methods', resp_json.keys())
self.assertEqual(200, response.status_code)
@patch('api.domain.user.User.get', MockUser.get)
def test_get_order_schema(self):
url = '/api/v1/order-schema'
response = self.app.get(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertIn('properties', resp_json.keys())
self.assertEqual(200, response.status_code)
@patch('api.domain.user.User.get', MockUser.get)
def test_bad_method(self):
url = '/api/v1/available-products/'
response = self.app.post(url,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertEqual(405, response.status_code)
self.assertIn('messages', resp_json)
self.assertIn('errors', resp_json['messages'])
@patch('api.domain.user.User.get', MockUser.get)
def test_bad_data(self):
url = '/api/v1/order'
data = '{"inputs": [}'
response = self.app.post(url,
data=data,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertEqual(400, response.status_code)
self.assertIn('messages', resp_json)
self.assertIn('errors', resp_json['messages'])
@patch('api.domain.user.User.get', MockUser.get)
def test_bad_validation_inputs(self):
url = '/api/v1/order'
data = '{"inputs": []}'
response = self.app.post(url,
data=data,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertEqual(400, response.status_code)
self.assertIn('messages', resp_json)
self.assertIn('errors', resp_json['messages'])
self.assertIn('Schema errors', resp_json['messages']['errors'][0])
@patch('api.domain.user.User.get', MockUser.get)
def test_bad_validation_sensor_inputs(self):
url = '/api/v1/order'
data = '{"etm7_collection": {"inputs": ["LE07_L1TP_010028_20050420_20160925_01_T1"]}}'
response = self.app.post(url,
data=data,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertEqual(400, response.status_code)
self.assertIn('messages', resp_json)
self.assertIn('errors', resp_json['messages'])
self.assertIn('2 validation errors',
resp_json['messages']['errors'][0])
@patch('api.domain.user.User.get', MockUser.get)
def test_bad_data_avail_inputs(self):
url = '/api/v1/available-products/'
data = '{"bad": []}'
response = self.app.get(url,
data=data,
headers=self.headers,
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertEqual(400, response.status_code)
self.assertIn('messages', resp_json)
self.assertIn('errors', resp_json['messages'])
self.assertIn('No input products supplied',
resp_json['messages']['errors'][0])
@patch('api.external.ers.ERSApi._api_post',
lambda x, y, z: {'errors': True})
def test_messages_field_acc_denied(self):
url = '/api/v1/available-products/'
response = self.app.get(url, environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertEqual(401, response.status_code)
self.assertIn('messages', resp_json)
self.assertIn('errors', resp_json['messages'])
@patch('api.domain.user.User.get', MockUser.get)
def test_not_found(self):
url = '/api/v1/not-valid'
response = self.app.get(url, environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertEqual(404, response.status_code)
self.assertIn('messages', resp_json)
self.assertIn("errors", resp_json['messages'])
@patch('api.domain.user.User.get', MockUser.get)
@patch('api.interfaces.ordering.version1.API.place_order',
MockOrder.place_order)
def test_post_order(self):
url = '/api/v1/order'
data = {'etm7_collection': {'inputs': [''], 'products': ['']}}
response = self.app.post(url,
headers=self.headers,
data=json.dumps(data),
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertEqual(201, response.status_code)
self.assertEqual({'orderid', 'status'}, set(resp_json.keys()))
@patch('api.domain.user.User.get', MockUser.get)
@patch('api.interfaces.ordering.version1.API.cancel_order',
MockOrder.cancel_order)
def test_cancel_order(self):
url = '/api/v1/order'
data = {'orderid': self.orderid, 'status': 'cancelled'}
response = self.app.put(url,
headers=self.headers,
data=json.dumps(data),
environ_base={'REMOTE_ADDR': '127.0.0.1'})
resp_json = json.loads(response.get_data())
self.assertEqual(data, resp_json)
self.assertEqual(202, response.status_code)
def setUp(self):
os.environ['espa_api_testing'] = 'True'
# create a user
self.mock_user = MockUser()
self.mock_order = MockOrder()
self.user_id = self.mock_user.add_testing_user()
class TestValidation(unittest.TestCase):
def setUp(self):
logger.warning('Testing Validation started...')
os.environ['espa_api_testing'] = 'True'
self.mock_user = MockUser()
self.staffuser = User.find(self.mock_user.add_testing_user())
self.staffuser.update('is_staff', True)
self.base_order = lowercase_all(testorders.build_base_order())
self.base_schema = BaseValidationSchema.request_schema
def test_validation_get_order_schema(self):
"""
Make sure the ordering schema is retrievable as a dict
"""
self.assertIsInstance(api.validation.fetch_order_schema(), dict)
def test_validation_get_valid_formats(self):
"""
Make sure the file format options are retrievable as a dict
"""
self.assertIsInstance(api.validation.fetch_formats(), dict)
def test_validation_get_valid_resampling(self):
"""
Make sure the resampling options are retrievable as a dict
"""
self.assertIsInstance(api.validation.fetch_resampling(), dict)
def test_validation_get_valid_projections(self):
"""
Make sure the projection options are retrievable as a dict
"""
self.assertIsInstance(api.validation.fetch_projections(), dict)
def test_validate_good_order(self):
"""
Test a series of known good orders
"""
for proj in testorders.good_test_projections:
valid_order = copy.deepcopy(self.base_order)
valid_order['projection'] = {proj: testorders.good_test_projections[proj]}
if 'lonlat' not in valid_order['projection']:
valid_order['resize'] = {"pixel_size": 30, "pixel_size_units": "meters"}
try:
good = api.validation(valid_order, self.staffuser.username)
except ValidationException as e:
self.fail('Raised ValidationException: {}'.format(e.message))
def test_modis_resize(self):
"""
Most common issue of orders resizing MODIS to 30m pixels, without setting the extents
"""
modis_order = {'mod09a1': {'inputs': ['mod09a1.a2016305.h11v04.006.2016314200836'],
'products': ['l1']},
'resampling_method': 'cc',
'resize': {'pixel_size': 30,
'pixel_size_units': 'meters'},
'format': 'gtiff'}
exc = 'pixel count is greater than maximum size of'
exc_key = '1 validation errors'
try:
api.validation(modis_order, self.staffuser.username)
except Exception as e:
self.assertIn(exc_key, e.response)
self.assertIsInstance(e.response[exc_key], list)
self.assertIn(exc, str(e.response[exc_key]))
else:
self.fail('Failed MODIS pixel resize test')
def test_validate_bad_orders(self):
"""
Build a series of invalid orders to try and catch any potential errors in a
submitted order
Check to make sure the invalid order raises ValidationException, then check
the exception message for the expected error message
The abbreviated flag for the InvalidOrders class changes the number of invalid
orders that will get tested.
abbreviated=True - test each constraint type once
abbreviated=False - test each constraint on each value location in the nested structure
"""
exc_type = ValidationException
invalid_order = copy.deepcopy(self.base_order)
for proj in testorders.good_test_projections:
invalid_order['projection'] = {proj: testorders.good_test_projections[proj]}
invalid_list = testorders.InvalidOrders(invalid_order, self.base_schema, abbreviated=False)
for order, test, exc in invalid_list:
# empty lists cause assertRaisesRegExp to fail
exc = str(exc).replace('[', '\[')
with self.assertRaisesRegexp(exc_type, exc):
api.validation(order, self.staffuser.username)
@patch('api.external.inventory.available', lambda: True)
@patch('api.providers.production.production_provider.ProductionProvider.parse_urls_m2m',
lambda x, y: y)
def test_order_exceeds_open_scene_limit(self):
"""
Build a large order, then attempt to place an order that would exceed the user's open scene limit.
"""
mock = MockOrder()
user = MockUser()
user_id = user.add_testing_user()
# base testing order will give a user 15 open scenes currently
order_id = mock.generate_testing_order(user_id)
# Add a second order that pushes scenes over the limit, make sure that this
# raises the appropriate exception
self.assertRaises(OpenSceneLimitException,
lambda: ordering_provider.check_open_scenes(order=mock.base_order,
user_id=user_id,
filters={'status': ('submitted',
'oncache',
'onorder',
'queued',
'processing')}))
def test_get_scenes_for_new_user(self):
"""
Make sure that checking the number of open scenes for a new user
with no orders will not raise an exception
"""
mock = MockOrder()
user = MockUser()
user_id = user.add_testing_user()
user_id = user_id + random.randint(1, 200)
user_orders = Order.where({'user_id': user_id})
self.assertTrue(len(user_orders) == 0)
try:
ordering_provider.check_open_scenes(order=mock.base_order,
user_id=user_id,
filters={'status': ('submitted',
'oncache',
'onorder',
'queued',
'processing')})
except:
self.fail('ordering_provider.check_open_scenes() raised Exception for new user')
def test_validate_sr_restricted_human_readable(self):
"""
Assert that a human readable response is returned for unavailable or date restricted products
"""
exc_type = ValidationException
invalid_list = {'olitirs8_collection': {'inputs': ['lc08_l1tp_031043_20160225_20170224_01_t1'],
'products': ['sr'],
'err_msg': 'Requested {} products are restricted by date'},
'oli8_collection': {'inputs': ['lo08_l1tp_021049_20150304_20170227_01_t1'],
'products': ['sr'],
'err_msg': 'Requested {} products are not available'}}
for stype in invalid_list:
invalid_order = copy.deepcopy(self.base_order)
invalid_order[stype]['inputs'] = invalid_list[stype]['inputs']
invalid_order[stype]['products'] = invalid_list[stype]['products']
for p in invalid_order[stype]['products']:
err_message = invalid_list[stype]['err_msg'].format(p)
with self.assertRaisesRegexp(exc_type, err_message):
api.validation.validate(invalid_order, self.staffuser.username)
def test_projection_units_geographic(self):
"""
Make sure Geographic (latlon) projection only accepts "dd" units
"""
part_order = {
"olitirs8_collection": {
"inputs": ['lc08_l1tp_015035_20140713_20170304_01_t1'],
"products": ["l1"]
},
"projection": {"lonlat": None},
"format": "gtiff",
"resampling_method": "cc"
}
bad_parts = {
"resize": {"pixel_size": 30, "pixel_size_units": "meters"},
"image_extents": {"north": 80, "south": -80,
"east": 170, "west": -170, "units": "meters"},
}
err_msg = '{} units must be in "dd" for projection "lonlat"'
exc_type = ValidationException
for bname in bad_parts:
invalid_order = copy.deepcopy(part_order)
invalid_order.update({bname: bad_parts.get(bname)})
with self.assertRaisesRegexp(exc_type, err_msg.format(bname)):
api.validation.validate(invalid_order, self.staffuser.username)
def test_l1_only_restricted(self):
""" Landsat Level-1 data needs to go through other channels """
invalid_orders = [
{
"olitirs8_collection": {
"inputs": ["lc08_l1tp_015035_20140713_20170304_01_t1"],
"products": ["l1"]
},
"format": "gtiff"
},
{
"mod09ga": {
"inputs": ["mod09ga.a2017249.h29v10.006.2016256236022"],
"products": ["modis_ndvi"]
},
"olitirs8_collection": {
"inputs": ["lc08_l1tp_015035_20140713_20170304_01_t1"],
"products": ["l1"]
},
"format": "gtiff"
},
{
"mod09ga": {
"inputs": ["mod09ga.a2017249.h29v10.006.2016256236022"],
"products": ["l1"]
},
"olitirs8_collection": {
"inputs": ["lc08_l1tp_015035_20140713_20170304_01_t1"],
"products": ["sr"]
},
"format": "hdf-eos2"
},
{
"vnp09ga": {
"inputs": ["vnp09ga.a2017249.h19v06.001.2016265235022"],
"products": ["l1"],
},
"format": "hdf-eos2"
}
]
for iorder in invalid_orders:
with self.assertRaisesRegexp(ValidationException, "non-customized products"):
api.validation.validate(iorder, self.staffuser.username)
def test_l1_only_restricted_override(self):
""" Customizations should override Level-1 restrictions """
valid_orders = [
{
"olitirs8_collection": {
"inputs": ["lc08_l1tp_015035_20140713_20170304_01_t1"],
"products": ["l1"]
},
"format": "envi"
},
{
"projection": {
"aea": {
"standard_parallel_1": 29.5,
"central_meridian": -96,
"datum": "wgs84",
"latitude_of_origin": 23,
"standard_parallel_2": 45.5,
"false_northing": 0,
"false_easting": 0
}
},
"olitirs8_collection": {
"inputs": ["lc08_l1tp_015035_20140713_20170304_01_t1"],
"products": ["l1"]
},
"myd13a2": {
"inputs": ["myd13a2.a2017249.h19v06.006.2017265235022"],
"products": ["l1"]
},
"format": "gtiff"
},
{
"projection": {
"aea": {
"standard_parallel_1": 29.5,
"central_meridian": -96,
"datum": "wgs84",
"latitude_of_origin": 23,
"standard_parallel_2": 45.5,
"false_northing": 0,
"false_easting": 0
}
},
"olitirs8_collection": {
"inputs": ["lc08_l1tp_015035_20140713_20170304_01_t1"],
"products": ["l1"]
},
"myd13a2": {
"inputs": ["myd13a2.a2017249.h19v06.006.2017265235022"],
"products": ["l1"]
},
"format": "hdf-eos2"
},
{
"vnp09ga": {
"inputs": ["vnp09ga.a2017249.h19v06.001.2016265235022"],
"products": ["viirs_ndvi"],
},
"format": "hdf-eos2"
}
]
for vorder in valid_orders:
api.validation.validate(vorder, self.staffuser.username)
# def test_validate_utm_zone(self):
# invalid_order = copy.deepcopy(self.base_order)
# invalid_order['projection'] = {'utm': {'zone': 50, 'zone_ns': 'north'}}
# invalid_order['image_extents'] = {'east': 32.5, 'north': 114.9, 'south': 113.5, 'units': u'dd', 'west': 31.5}
# with self.assertRaisesRegexp(ValidationException, 'are not near the requested UTM zone'):
# api.validation.validate(invalid_order, self.staffuser.username)
def test_modis_viirs_ndvi_restricted(self):
""" Users should only be able to order NDVI for
Daily Surface Reflectance 500-m MODIS products """
invalid_orders = [
{
"projection": {
"aea": {
"standard_parallel_1": 29.5,
"central_meridian": -96,
"datum": "wgs84",
"latitude_of_origin": 23,
"standard_parallel_2": 45.5,
"false_northing": 0,
"false_easting": 0
}
},
"myd13a2": {
"inputs": ["myd13a2.a2017249.h19v06.006.2017265235022"],
"products": ["l1"]
},
"mod09ga": {
"inputs": ["mod09ga.a2017249.h29v10.006.2016256236022"],
"products": ["modis_ndvi"]
},
"format": "gtiff"
},
{
"projection": {
"aea": {
"standard_parallel_1": 29.5,
"central_meridian": -96,
"datum": "wgs84",
"latitude_of_origin": 23,
"standard_parallel_2": 45.5,
"false_northing": 0,
"false_easting": 0
}
},
"myd13a2": {
"inputs": ["myd13a2.a2017249.h19v06.006.2017265235022"],
"products": ["l1"]
},
"myd09ga": {
"inputs": ["myd09ga.a2017249.h19v06.006.2017265235022"],
"products": ["modis_ndvi"]
},
"vnp09ga": {
"inputs": ["vnp09ga.a2017249.h19v06.001.2016265235022"],
"products": ["viirs_ndvi"]
},
"format": "gtiff"
}
]
for iorder in invalid_orders:
with self.assertRaisesRegexp(ValidationException, "NDVI not available for requested products"):
api.validation.validate(iorder, self.staffuser.username)
valid_orders = [
{
"myd09ga": {
"inputs": ["myd09ga.a2017249.h19v06.006.2017265235022"],
"products": ["modis_ndvi"]
},
"vnp09ga": {
"inputs": ["vnp09ga.a2017249.h19v06.001.2016265235022"],
"products": ["viirs_ndvi"],
},
"format": "gtiff"
},
{
"myd13a2": {
"inputs": ["myd13a2.a2017249.h19v06.006.2017265235022"],
"products": ["l1"]
},
"vnp09ga": {
"inputs": ["vnp09ga.a2017249.h19v06.001.2016265235022"],
"products": ["viirs_ndvi"],
},
"format": "gtiff"
},
{
"myd09ga": {
"inputs": ["myd09ga.a2017249.h19v06.006.2017265235022"],
"products": ["modis_ndvi"]
},
"vnp09ga": {
"inputs": ["vnp09ga.a2017249.h19v06.001.2016265235022"],
"products": ["viirs_ndvi"],
},
"olitirs8_collection": {
"inputs": ["lc08_l1tp_015035_20140713_20170304_01_t1"],
"products": ["sr"]
},
"format": "gtiff"
}
]
for vorder in valid_orders:
api.validation.validate(vorder, self.staffuser.username)
def test_orca_sensor_restricted(self):
""" Users should only be able to order LaORCA for
Landsat 8 OLI and/or OLITIRS"""
valid_orders = [
{
"olitirs8_collection": {
"inputs": ["lc08_l1tp_015035_20140713_20170304_01_t1"],
"products": ["orca"]
},
"oli8_collection": {
"inputs": ["lo08_l1tp_021049_20150304_20170227_01_t1"],
"products": ["orca"]
},
"format": "gtiff"
}]
invalid_orders = [
{
"olitirs8_collection": {
"inputs": ["lc08_l1tp_015035_20140713_20170304_01_t1"],
"products": ["orca"]
},
"etm7_collection": {
"inputs": ["le07_l1tp_151041_20190218_20190316_01_t1"],
"products": ["orca"]
},
"format": "gtiff"
}]
for iorder in invalid_orders:
with self.assertRaisesRegexp(ValidationException, "LaORCA currently only available for Landsat 8 OLI or OLI/TIRS"):
api.validation.validate(iorder, self.staffuser.username)
for vorder in valid_orders:
api.validation.validate(vorder, self.staffuser.username)
