def cancel_order(self, orderid, request_ip_address): """ Cancels an order, and all scenes contained within it :return: """ order = Order.where({'id': orderid}) if len(order) != 1: raise OrderingProviderException('Order not found') else: order = order.pop() logger.info('Received request to cancel {} from {}'.format( orderid, request_ip_address)) killable_scene_states = ('submitted', 'oncache', 'onorder', 'queued', 'retry', 'error', 'unavailable', 'complete') scenes = order.scenes(sql_dict={'status': killable_scene_states}) if len(scenes) > 0: Scene.bulk_update([s.id for s in scenes], Scene.cancel_opts()) else: logger.info('No scenes to cancel for order {}'.format( orderid, request_ip_address)) order.status = 'cancelled' order.save() logger.info('Request to cancel {} from {} successful.'.format( orderid, request_ip_address)) return order
def test_handle_stuck_jobs(self): time_jobs_stuck = datetime.datetime.now() - datetime.timedelta(hours=6) order_id = self.mock_order.generate_testing_order(self.user_id) # Make some really old jobs self.mock_order.update_scenes( order_id, ('landsat', 'modis', 'viirs', 'sentinel', 'plot'), 'status', ['scheduled']) self.mock_order.update_scenes( order_id, ('landsat', 'modis', 'viirs', 'sentinel', 'plot'), 'status_modified', [datetime.datetime(1900, 1, 1)]) scenes = Scene.where({ 'status': ('tasked', 'scheduled', 'processing'), 'order_id': order_id }) n_scenes = len(scenes) response = production_provider.handle_stuck_jobs(scenes) self.assertTrue(response) scenes = Scene.where({ 'status': ('tasked', 'scheduled', 'processing'), 'order_id': order_id }) self.assertEqual(0, len(scenes)) scenes = Scene.where({'status': 'oncache', 'order_id': order_id}) self.assertEqual(n_scenes, len(scenes))
def error_to(self, orderid, state): order = Order.find(orderid) err_scenes = order.scenes({'status': 'error'}) try: if state == 'submitted': Scene.bulk_update( [s.id for s in err_scenes], { 'status': state, 'orphaned': None, 'reported_orphan': None, 'log_file_contents': '', 'note': '', 'retry_count': 0 }) order.status = 'ordered' order.completion_email_sent = None order.save() else: Scene.bulk_update([s.id for s in err_scenes], {'status': state}) return True except SceneException as e: logger.critical('ERR admin provider error_to\ntrace: {}'.format( e.message)) raise AdministrationProviderException('ERR updating with error_to')
def test_hadoop_reset_status(self): order_id = self.mock_order.generate_testing_order(self.user_id) scenes = Scene.where({'order_id': order_id}) Scene.bulk_update([s.id for s in scenes], {'status': 'processing'}) self.assertTrue(production_provider.reset_processing_status()) scenes = Scene.where({'order_id': order_id}) self.assertEqual({'submitted'}, set([s.status for s in scenes]))
def test_production_calc_scene_download_sizes(self): order = Order.find(self.mock_order.generate_testing_order(self.user_id)) scenes = order.scenes() Scene.bulk_update([s.id for s in scenes], {'status': 'complete', 'download_size': 0}) self.assertTrue(production_provider.calc_scene_download_sizes()) upscenes = Scene.where({'status': 'complete', 'download_size': 999}) self.assertEqual(len(upscenes), len(scenes))
def test_handle_stuck_jobs(self): order_id = self.mock_order.generate_testing_order(self.user_id) # Make some really old jobs self.mock_order.update_scenes(order_id, ('landsat', 'modis', 'plot'), 'status', ['processing']) self.mock_order.update_scenes(order_id, ('landsat', 'modis', 'plot'), 'status_modified', [datetime.datetime(1900, 1, 1)]) scenes = Scene.where({'status': 'processing', 'order_id': order_id}) n_scenes = len(scenes) response = production_provider.handle_stuck_jobs(scenes) self.assertTrue(response) scenes = Scene.where({ 'status': 'processing', 'order_id': order_id, 'reported_orphan is not': None }) self.assertEqual(n_scenes, len(scenes)) self.mock_order.update_scenes(order_id, ('landsat', 'modis', 'plot'), 'reported_orphan', [datetime.datetime(1900, 1, 1)]) response = production_provider.handle_stuck_jobs(scenes) self.assertTrue(response) scenes = Scene.where({'status': 'processing', 'order_id': order_id}) self.assertEqual(0, len(scenes))
def test_production_update_order_if_complete(self): order = Order.find(self.mock_order.generate_testing_order(self.user_id)) Scene.bulk_update([s.id for s in order.scenes()], {'status': 'retry'}) order.order_source = 'espa' order.completion_email_sent = None order.save() self.assertTrue(production_provider.update_order_if_complete(order))
def test_production_calc_scene_download_sizes(self): order = Order.find(self.mock_order.generate_testing_order(self.user_id)) scenes = order.scenes() Scene.bulk_update([s.id for s in scenes], {'status': 'complete', 'download_size': 0}) scenes = Order.find(order.id).scenes({'status': 'complete', 'download_size': 0}) self.assertTrue(production_provider.calc_scene_download_sizes(scenes)) upscenes = Scene.where({'status': 'complete', 'download_size': 999}) self.assertEqual(len(upscenes), len(scenes))
def test_production_order_completion_email_error(self): """ Make sure that order status is not set to complete if the completion email failed to send """ order = Order.find(self.mock_order.generate_testing_order( self.user_id)) Scene.bulk_update([s.id for s in order.scenes()], {'status': 'complete'}) order.order_source = 'espa' order.completion_email_sent = None order.save() production_provider.update_order_if_complete(order) self.assertEquals(order.status, 'ordered')
def test_production_handle_submitted_plot_products(self): order = Order.find(self.mock_order.generate_testing_order( self.user_id)) order.status = 'ordered' order.order_type = 'lpcs' order.save() plot_id = None for idx, scene in enumerate(order.scenes()): # at the moment, mock_order.generate_testing_order # creates 21 products for the order. divvy those # up between 'complete' and 'unavailable', setting # one aside as the 'plot' product if scene.sensor_type == 'plot': # need to define a plot product scene.update('status', 'submitted') plot_id = scene.id else: if idx % 2 == 0: scene.update('status', 'complete') else: scene.update('status', 'unavailable') scenes = order.scenes() self.assertTrue( production_provider.handle_submitted_plot_products(scenes)) self.assertEqual(Scene.find(plot_id).status, "oncache")
def test_update_product_details_set_product_unavailable(self): order = Order.find(self.mock_order.generate_testing_order(self.user_id)) scene = order.scenes()[0] production_provider.update_product('set_product_unavailable', name=scene.name, orderid=order.orderid, processing_loc="L8SRLEXAMPLE", error='include_dswe is an unavailable product option for OLITIRS') self.assertTrue('unavailable' == Scene.get('ordering_scene.status', scene.name, order.orderid))
def update_scenes(self, order_id, attribute, values): scenes = Scene.where({'order_id': order_id}) xscenes = chunkify(scenes, len(values)) for idx, value in enumerate(values): for scene in xscenes[idx]: scene.update(attribute, value) return True
def update_scenes(self, order_id, stype, attribute, values): scenes = Scene.where({'order_id': order_id, 'sensor_type': stype}) xscenes = chunkify(scenes, len(values)) for idx, value in enumerate(values): for scene in xscenes[idx]: scene.update(attribute, value) return True
def test_production_handle_retry_products(self): prev = datetime.datetime.now() - datetime.timedelta(hours=1) order_id = self.mock_order.generate_testing_order(self.user_id) self.mock_order.update_scenes(order_id, 'status', ['retry']) self.mock_order.update_scenes(order_id, 'retry_after', [prev]) production_provider.handle_retry_products() for s in Scene.where({'order_id': order_id}): self.assertTrue(s.status == 'submitted')
def test_catch_orphaned_scenes(self): order_id = self.mock_order.generate_testing_order(self.user_id) # need scenes with statuses of 'queued' self.mock_order.update_scenes(order_id, 'status', ['queued']) response = production_provider.catch_orphaned_scenes() self.assertTrue(response) old_time = datetime.datetime.now() - datetime.timedelta(minutes=15) for s in Scene.where({'order_id': order_id}): self.assertTrue(s.reported_orphan is not None) s.reported_orphan = old_time s.save() response = production_provider.catch_orphaned_scenes() self.assertTrue(response) for s in Scene.where({'order_id': order_id}): self.assertTrue(s.orphaned)
def test_production_handle_failed_ee_updates(self): order = Order.find(self.mock_order.generate_testing_order(self.user_id)) for scene in order.scenes(): scene.update('failed_lta_status_update', 'C') production_provider.handle_failed_ee_updates() scenes = Scene.where({'failed_lta_status_update IS NOT': None}) self.assertTrue(len(scenes) == 0)
def test_status_modified(self): order_id = self.mock_order.generate_testing_order(self.user_id) scene = Scene.where({'order_id': order_id}).pop() scene.status = 'oncache' scene.save() old_time = scene.status_modified scene.status = 'queued' scene.save() new_time = scene.status_modified self.assertGreater(new_time, old_time)
def test_fetch_production_products_sentinel(self): order_id = self.mock_order.generate_testing_order(self.user_id) # need scenes with statuses of 'processing' scenes = Scene.where({'order_id': order_id, 'sensor_type': 'sentinel'}) self.mock_order.update_scenes(order_id, 'sentinel', 'status', ['processing', 'oncache']) user = User.find(self.user_id) params = {'for_user': user.username, 'product_types': ['sentinel']} response = api.fetch_production_products(params) self.assertTrue('bilbo' in response[0]['orderid'])
def test_production_handle_submitted_modis_products_input_missing(self): # handle unavailable scenario order = Order.find(self.mock_order.generate_testing_order(self.user_id)) for scene in order.scenes(): scene.status = 'submitted' scene.sensor_type = 'modis' scene.save() sid = scene.id self.assertTrue(production_provider.handle_submitted_modis_products()) self.assertEquals(Scene.find(sid).status, "unavailable")
def test_production_set_product_error_retry_lasrc_segfault(self): """ Move a scene status from error to retry based on the error message """ order = Order.find(self.mock_order.generate_testing_order(self.user_id)) scene = order.scenes({'sensor_type': 'landsat'})[-1] production_provider.set_product_error(scene.name, order.orderid, 'somewhere', 'runSr sh: line 1: 1010 Segmentation fault lasrc --xml=') self.assertTrue('retry' == Scene.get('ordering_scene.status', scene.name, order.orderid))
def test_production_handle_submitted_modis_products_input_exists(self): # handle oncache scenario order = Order.find(self.mock_order.generate_testing_order(self.user_id)) for scene in order.scenes({'name !=': 'plot'}): scene.status = 'submitted' scene.sensor_type = 'modis' scene.save() sid = scene.id scenes = order.scenes({'sensor_type': 'modis'}) self.assertTrue(production_provider.handle_submitted_modis_products(scenes)) self.assertEquals(Scene.find(sid).status, "oncache")
def test_production_set_product_error_submitted(self): """ Move a scene status from error to submitted based on the error message """ order = Order.find(self.mock_order.generate_testing_order(self.user_id)) scene = order.scenes({'name !=': 'plot'})[0] production_provider.set_product_error(scene.name, order.orderid, 'get_products_to_process', 'BLOCK, COMING FROM LST AS WELL: No such file or directory') self.assertTrue('submitted' == Scene.get('ordering_scene.status', scene.name, order.orderid))
def test_production_set_product_error_retry(self): """ Move a scene status from error to retry based on the error message """ order = Order.find(self.mock_order.generate_testing_order(self.user_id)) scene = order.scenes()[2] production_provider.set_product_error(scene.name, order.orderid, 'somewhere', 'Verify the missing auxillary data products') self.assertTrue('retry' == Scene.get('ordering_scene.status', scene.name, order.orderid))
def test_production_set_product_error_unavailable(self): """ Move a scene status from error to unavailable based on the error message """ order = Order.find(self.mock_order.generate_testing_order(self.user_id)) scene = order.scenes({'name !=': 'plot'})[0] production_provider.set_product_error(scene.name, order.orderid, 'get_products_to_process', 'include_dswe is an unavailable product option for OLITIRS') self.assertTrue('unavailable' == Scene.get('ordering_scene.status', scene.name, order.orderid))
def set_product_retry(self, name, orderid, processing_loc, error, note, retry_after, retry_limit=None): try: order_id = Scene.get('order_id', name, orderid) assert(type(retry_after) is int) assert(type(processing_loc) is str) assert(type(error) is str) assert(type(note) is str) except Exception: raise return True
def test_update_product_details_update_status(self): """ Set a scene status to Queued """ order = Order.find(self.mock_order.generate_testing_order(self.user_id)) scene = order.scenes()[0] api.update_product_details('update_status', {'name': scene.name, 'orderid': order.orderid, 'processing_loc': 'L8SRLEXAMPLE', 'status': 'Queued'}) self.assertTrue(Scene.get('ordering_scene.status', scene.name, order.orderid) == 'Queued')
def test_update_product_details_set_product_error(self): """ Set a scene status to error :return: """ order = Order.find(self.mock_order.generate_testing_order(self.user_id)) scene = order.scenes()[0] production_provider.update_product('set_product_error', name=scene.name, orderid=order.orderid, processing_loc="L8SRLEXAMPLE", error='problems yo') self.assertTrue(Scene.find(scene.id).status == 'error')
def test_update_product_details_mark_product_complete(self): order = Order.find(self.mock_order.generate_testing_order(self.user_id)) scene = order.scenes()[0] production_provider.update_product('mark_product_complete', name=scene.name, orderid=order.orderid, processing_loc='L8SRLEXAMPLE', completed_file_location='/some/loc', cksum_file_location='some checksum', log_file_contents='some log') self.assertTrue('complete' == Scene.get('ordering_scene.status', scene.name, order.orderid))
def active_hadoop_job_names(self): order_ids = tuple([ o.id for o in Order.where({ 'status': 'ordered', 'user_id': self.id }) ]) return [ s.job_name for s in Scene.where({ 'status': ('processing', 'queued'), 'order_id': order_ids }) ]
def test_production_set_product_error_unavail_reproject(self): """ Move a scene status from error to retry based on the error message """ order = Order.find(self.mock_order.generate_testing_order(self.user_id)) scene = order.scenes({'sensor_type': 'landsat'})[-1] log_file_contents = ('BLAH BLAH BLAH WarpVerificationError: Failed to ' 'compute statistics, no valid pixels found in ' 'sampling BLAH BLAH BLAH') production_provider.set_product_error(scene.name, order.orderid, 'somewhere', log_file_contents) self.assertEqual('unavailable', Scene.get('ordering_scene.status', scene.name, order.orderid))
def test_update_product_details_fail_lta_mark_product_complete(self): order = Order.find(self.mock_order.generate_testing_order(self.user_id)) scene = order.scenes()[1] order.update('order_source', 'ee') production_provider.update_product('mark_product_complete', name=scene.name, orderid=order.orderid, processing_loc='L8SRLEXAMPLE', completed_file_location='/some/loc', cksum_file_location='some checksum', log_file_contents='some log') s = Scene.where({'name': scene.name, 'order_id': scene.order_id})[0] self.assertTrue('C' == s.failed_lta_status_update)
def scenes(self, sql_dict=None): """ Retrieve a list of Scene objects related to this initialized Order object :param sql_dict: dictionary object for sql parameters :return: list of Scene objects """ if sql_dict: sql_dict['order_id'] = self.id else: sql_dict = {'order_id': self.id} return Scene.where(sql_dict)
def test_production_set_product_retry(self): order_id = self.mock_order.generate_testing_order(self.user_id) order = Order.find(order_id) scene = order.scenes()[3] scene.update('retry_count', 4) processing_loc = "get_products_to_process" error = 'not available after EE call ' note = 'note this' retry_after = datetime.datetime.now() + datetime.timedelta(hours=1) retry_limit = 9 response = production_provider.set_product_retry(scene.name, order.orderid, processing_loc, error, note, retry_after, retry_limit) new = Scene.get('ordering_scene.status', scene.name, order.orderid) self.assertTrue('retry' == new)
def test_production_handle_submitted_plot_products(self): order = Order.find(self.mock_order.generate_testing_order(self.user_id)) order.status = 'ordered' order.order_type = 'lpcs' order.save() plot_id = None for idx, scene in enumerate(order.scenes()): # at the moment, mock_order.generate_testing_order # creates 21 products for the order. divvy those # up between 'complete' and 'unavailable', setting # one aside as the 'plot' product if idx % 2 == 0: if idx == 0: # need to define a plot product scene.update('status', 'submitted') scene.update('sensor_type', 'plot') plot_id = scene.id else: scene.update('status', 'complete') else: scene.update('status', 'unavailable') self.assertTrue(production_provider.handle_submitted_plot_products()) self.assertEqual(Scene.find(plot_id).status, "oncache")
def create(cls, params): """ Place a new order into the system :param params: dict of required parameters to be used params = {'product_opts': {dictionary object of the order received} 'orderid': id generated from generate_order_id 'user_id': EE user id 'order_type': typically 'level2_ondemand' 'status': 'ordered' 'note': user notes 'ee_order_id': earth explorer order id, or '' 'order_source': 'espa' or 'ee' 'order_date': date time string 'priority': legacy item, should be 'normal' 'email': user's contact email 'product_options': legacy column} :return: order object """ opts = params['product_opts'] params['product_opts'] = json.dumps(params['product_opts']) sql = ('INSERT INTO ordering_order ' '(orderid, user_id, order_type, status, note, ' 'product_opts, ee_order_id, order_source, order_date, ' 'priority, email, product_options) ' 'VALUES (%(orderid)s, %(user_id)s, %(order_type)s, ' '%(status)s, %(note)s, %(product_opts)s, ' '%(ee_order_id)s, %(order_source)s, %(order_date)s, ' '%(priority)s, %(email)s, %(product_options)s)') logger.info('Order creation parameters: {}'.format(params)) log_sql = '' try: with db_instance() as db: log_sql = db.cursor.mogrify(sql, params) logger.info('New order complete SQL: {}' .format(log_sql)) db.execute(sql, params) db.commit() except DBConnectException as e: logger.debug('Error creating new order: {}\n' 'sql: {}'.format(e.message, log_sql)) raise OrderException(e) order = Order.find(params['orderid']) # Let the load_ee_order method handle the scene injection # as there is special logic for interacting with LTA if params['ee_order_id']: return order sensor_keys = sensor.SensorCONST.instances.keys() bulk_ls = [] for key in opts: if key in sensor_keys: sensor_type = '' item1 = opts[key]['inputs'][0] if isinstance(sensor.instance(item1), sensor.Landsat): sensor_type = 'landsat' elif isinstance(sensor.instance(item1), sensor.Modis): sensor_type = 'modis' for s in opts[key]['inputs']: scene_dict = {'name': s, 'sensor_type': sensor_type, 'order_id': order.id, 'status': 'submitted', 'ee_unit_id': None} bulk_ls.append(scene_dict) if 'plot_statistics' in opts and opts['plot_statistics']: scene_dict = {'name': 'plot', 'sensor_type': 'plot', 'order_id': order.id, 'status': 'submitted', 'ee_unit_id': None} bulk_ls.append(scene_dict) try: Scene.create(bulk_ls) except SceneException as e: logger.debug('Order creation failed on scene injection, ' 'order: {}\nexception: {}' .format(order.orderid, e.message)) with db_instance() as db: db.execute('delete ordering_order where id = %s', order.id) db.commit() raise OrderException(e) return order
def scene_names_list(self, order_id): scenes = Scene.where({'order_id': order_id}) names_list = [s.name for s in scenes] return names_list