class TestProductionAPI(unittest.TestCase):
    def setUp(self):
        os.environ['espa_api_testing'] = 'True'
        # create a user
        self.mock_user = MockUser()
        self.mock_order = MockOrder()
        self.user_id = self.mock_user.add_testing_user()

    def tearDown(self):
        # clean up orders
        self.mock_order.tear_down_testing_orders()
        # clean up users
        self.mock_user.cleanup()
        os.environ['espa_api_testing'] = ''

    @patch('api.external.inventory.available', lambda: True)
    @patch(
        'api.providers.production.production_provider.ProductionProvider.parse_urls_m2m',
        lambda x, y: y)
    def test_fetch_production_products_modis(self):
        order_id = self.mock_order.generate_testing_order(self.user_id)
        # need scenes with statuses of 'processing'
        self.mock_order.update_scenes(order_id, 'modis', 'status',
                                      ['processing', 'oncache'])
        user = User.find(self.user_id)
        params = {'product_types': ['modis']}
        response = api.fetch_production_products(params)
        self.assertTrue('bilbo' in response[0]['orderid'])

    @patch('api.external.inventory.available', lambda: True)
    @patch(
        'api.providers.production.production_provider.ProductionProvider.parse_urls_m2m',
        lambda x, y: y)
    def test_fetch_production_products_landsat(self):
        order_id = self.mock_order.generate_testing_order(self.user_id)
        # need scenes with statuses of 'processing'
        self.mock_order.update_scenes(order_id, 'landsat', 'status',
                                      ['processing', 'oncache'])
        user = User.find(self.user_id)
        params = {'for_user': user.username, 'product_types': ['landsat']}
        response = api.fetch_production_products(params)
        self.assertTrue('bilbo' in response[0]['orderid'])

    def test_fetch_production_products_plot(self):
        order_id = self.mock_order.generate_testing_order(self.user_id)
        self.mock_order.update_scenes(order_id, ('landsat', 'modis'), 'status',
                                      ['complete'])
        order = Order.find(order_id)
        plot_scene = order.scenes({'name': 'plot'})[0]
        plot_scene.name = 'plot'
        plot_scene.sensor_type = 'plot'
        plot_scene.status = 'submitted'
        plot_scene.save()
        scenes = Order.find(order_id).scenes({
            'name': 'plot',
            'status': 'submitted'
        })
        response = production_provider.handle_submitted_plot_products(scenes)
        pscene = order.scenes({'status': 'oncache', 'sensor_type': 'plot'})
        self.assertTrue(response is True)
        self.assertEqual(len(pscene), 1)

    def test_production_set_product_retry(self):
        order_id = self.mock_order.generate_testing_order(self.user_id)
        order = Order.find(order_id)
        scene = order.scenes()[3]
        scene.update('retry_count', 4)
        processing_loc = "get_products_to_process"
        error = 'not available after EE call '
        note = 'note this'
        retry_after = datetime.datetime.now() + datetime.timedelta(hours=1)
        retry_limit = 9
        response = production_provider.set_product_retry(
            scene.name, order.orderid, processing_loc, error, note,
            retry_after, retry_limit)

        new = Scene.get('ordering_scene.status', scene.name, order.orderid)
        self.assertTrue('retry' == new)

    @patch('api.external.lta.update_order_status',
           mock_production_provider.respond_true)
    def test_production_set_product_error_unavailable(self):
        """
        Move a scene status from error to unavailable based on the error
        message
        """
        order = Order.find(self.mock_order.generate_testing_order(
            self.user_id))
        scene = order.scenes({'name !=': 'plot'})[0]
        production_provider.set_product_error(
            scene.name, order.orderid, 'get_products_to_process',
            'include_dswe is an unavailable product option for OLITIRS')
        self.assertTrue('unavailable' == Scene.get('ordering_scene.status',
                                                   scene.name, order.orderid))

    def test_production_set_product_error_submitted(self):
        """
        Move a scene status from error to submitted based on the error
        message
        """
        order = Order.find(self.mock_order.generate_testing_order(
            self.user_id))
        scene = order.scenes({'name !=': 'plot'})[0]
        production_provider.set_product_error(
            scene.name, order.orderid, 'get_products_to_process',
            'BLOCK, COMING FROM LST AS WELL: No such file or directory')
        self.assertTrue('submitted' == Scene.get('ordering_scene.status',
                                                 scene.name, order.orderid))

    def test_production_set_product_error_retry(self):
        """
        Move a scene status from error to retry based on the error
        message
        """
        order = Order.find(self.mock_order.generate_testing_order(
            self.user_id))
        scene = order.scenes()[2]
        production_provider.set_product_error(
            scene.name, order.orderid, 'somewhere',
            'Verify the missing auxillary data products')
        self.assertTrue('retry' == Scene.get('ordering_scene.status',
                                             scene.name, order.orderid))

    def test_production_set_product_error_retry_lasrc_segfault(self):
        """
        Move a scene status from error to retry based on the error
        message
        """
        order = Order.find(self.mock_order.generate_testing_order(
            self.user_id))
        scene = order.scenes({'sensor_type': 'landsat'})[-1]
        production_provider.set_product_error(
            scene.name, order.orderid, 'somewhere',
            'runSr  sh: line 1: 1010 Segmentation fault lasrc --xml=')
        self.assertTrue('retry' == Scene.get('ordering_scene.status',
                                             scene.name, order.orderid))

    def test_production_set_product_error_unavail_reproject(self):
        """
        Move a scene status from error to retry based on the error
        message
        """
        order = Order.find(self.mock_order.generate_testing_order(
            self.user_id))
        scene = order.scenes({'sensor_type': 'landsat'})[-1]
        log_file_contents = ('BLAH BLAH BLAH WarpVerificationError: Failed to '
                             'compute statistics, no valid pixels found in '
                             'sampling BLAH BLAH BLAH')
        production_provider.set_product_error(scene.name, order.orderid,
                                              'somewhere', log_file_contents)
        self.assertEqual(
            'unavailable',
            Scene.get('ordering_scene.status', scene.name, order.orderid))

    @patch('api.external.lta.update_order_status', lta.update_order_status)
    @patch(
        'api.providers.production.production_provider.ProductionProvider.set_product_retry',
        mock_production_provider.set_product_retry)
    def test_update_product_details_update_status(self):
        """
        Set a scene status to Queued
        """
        order = Order.find(self.mock_order.generate_testing_order(
            self.user_id))
        scene = order.scenes()[0]
        api.update_product_details(
            'update_status', {
                'name': scene.name,
                'orderid': order.orderid,
                'processing_loc': 'L8SRLEXAMPLE',
                'status': 'Queued'
            })
        self.assertTrue(
            Scene.get('ordering_scene.status', scene.name, order.orderid) ==
            'Queued')

    @patch('api.external.lta.update_order_status', lta.update_order_status)
    @patch(
        'api.providers.production.production_provider.ProductionProvider.set_product_retry',
        mock_production_provider.set_product_retry)
    @patch('api.external.onlinecache.capacity', onlinecache.capacity)
    def test_update_product_details_set_product_error(self):
        """
        Set a scene status to error
        :return:
        """
        order = Order.find(self.mock_order.generate_testing_order(
            self.user_id))
        scene = order.scenes()[0]
        production_provider.update_product('set_product_error',
                                           name=scene.name,
                                           orderid=order.orderid,
                                           processing_loc="L8SRLEXAMPLE",
                                           error='problems yo')
        self.assertTrue(Scene.find(scene.id).status == 'error')

    @patch('api.external.lta.update_order_status', lta.update_order_status)
    @patch(
        'api.providers.production.production_provider.ProductionProvider.set_product_retry',
        mock_production_provider.set_product_retry)
    def test_update_product_details_set_product_unavailable(self):
        order = Order.find(self.mock_order.generate_testing_order(
            self.user_id))
        scene = order.scenes()[0]
        production_provider.update_product(
            'set_product_unavailable',
            name=scene.name,
            orderid=order.orderid,
            processing_loc="L8SRLEXAMPLE",
            error='include_dswe is an unavailable product option for OLITIRS')
        self.assertTrue('unavailable' == Scene.get('ordering_scene.status',
                                                   scene.name, order.orderid))

    @patch('api.external.lta.update_order_status', lta.update_order_status)
    @patch(
        'api.providers.production.production_provider.ProductionProvider.set_product_retry',
        mock_production_provider.set_product_retry)
    @patch('os.path.getsize', lambda y: 999)
    def test_update_product_details_mark_product_complete(self):
        order = Order.find(self.mock_order.generate_testing_order(
            self.user_id))
        scene = order.scenes()[0]
        production_provider.update_product('mark_product_complete',
                                           name=scene.name,
                                           orderid=order.orderid,
                                           processing_loc='L8SRLEXAMPLE',
                                           completed_file_location='/some/loc',
                                           cksum_file_location='some checksum',
                                           log_file_contents='some log')

        self.assertTrue('complete' == Scene.get('ordering_scene.status',
                                                scene.name, order.orderid))

    @patch('api.external.lta.update_order_status', lta.update_order_status)
    @patch(
        'api.providers.production.production_provider.ProductionProvider.set_product_retry',
        mock_production_provider.set_product_retry)
    @patch('os.path.getsize', lambda y: 999)
    def test_update_product_details_mark_product_processing(self):
        order = Order.find(self.mock_order.generate_testing_order(
            self.user_id))
        scene = order.scenes()[0]
        res = production_provider.update_product('update_status',
                                                 name=scene.name,
                                                 orderid=order.orderid,
                                                 processing_loc='L8SRLEXAMPLE',
                                                 status='processing')
        order = Order.find(order.id)
        scene = order.scenes({'id': scene.id})[0]
        self.assertEqual('processing', scene.status)

    @patch('api.external.lta.update_order_status', lta.update_order_status)
    @patch(
        'api.providers.production.production_provider.ProductionProvider.set_product_retry',
        mock_production_provider.set_product_retry)
    @patch('os.path.getsize', lambda y: 999)
    def test_update_product_details_mark_cancelled_product_processing(self):
        order = Order.find(self.mock_order.generate_testing_order(
            self.user_id))
        order.status = 'cancelled'
        order.save()
        scene = order.scenes()[0]
        res = production_provider.update_product('update_status',
                                                 name=scene.name,
                                                 orderid=order.orderid,
                                                 processing_loc='L8SRLEXAMPLE',
                                                 status='processing')
        self.assertFalse(res)

    def test_production_set_product_error_unavailable_night(self):
        """
        Move a scene status from error to unavailable based on the solar zenith (TOA)
        error message
        """
        order = Order.find(self.mock_order.generate_testing_order(
            self.user_id))
        scene = order.scenes({'name !=': 'plot'})[0]
        production_provider.set_product_error(
            name=scene.name,
            orderid=order.orderid,
            processing_loc='L8SRLEXAMPLE',
            error='solar zenith angle out of range')
        scene = Scene.by_name_orderid(name=scene.name, order_id=order.id)
        self.assertTrue('unavailable' == scene.status)
        self.assertTrue(
            'Solar zenith angle out of range, cannot process night scene' in
            scene.note)

    def test_production_set_product_error_unavailable_almost_night(self):
        """
        Move a scene status from error to unavailable based on the solar zenith (SR)
        error message
        """
        order = Order.find(self.mock_order.generate_testing_order(
            self.user_id))
        scene = order.scenes({'name !=': 'plot'})[0]
        production_provider.set_product_error(
            name=scene.name,
            orderid=order.orderid,
            processing_loc='L8SRLEXAMPLE',
            error='solar zenith angle is too large')
        scene = Scene.by_name_orderid(name=scene.name, order_id=order.id)
        self.assertTrue('unavailable' == scene.status)
        self.assertTrue(
            'Solar zenith angle is too large, cannot process scene to SR' in
            scene.note)

    @patch('api.external.lta.update_order_status',
           lta.update_order_status_fail)
    @patch(
        'api.providers.production.production_provider.ProductionProvider.set_product_retry',
        mock_production_provider.set_product_retry)
    @patch('os.path.getsize', lambda y: 999)
    def test_update_product_details_fail_lta_mark_product_complete(self):
        order = Order.find(self.mock_order.generate_testing_order(
            self.user_id))
        scene = order.scenes()[1]
        order.update('order_source', 'ee')
        production_provider.update_product('mark_product_complete',
                                           name=scene.name,
                                           orderid=order.orderid,
                                           processing_loc='L8SRLEXAMPLE',
                                           completed_file_location='/some/loc',
                                           cksum_file_location='some checksum',
                                           log_file_contents='some log')

        s = Scene.where({'name': scene.name, 'order_id': scene.order_id})[0]
        self.assertTrue('C' == s.failed_lta_status_update)

    @patch(
        'api.providers.production.production_provider.ProductionProvider.send_initial_emails',
        mock_production_provider.respond_true)
    @patch(
        'api.providers.production.production_provider.ProductionProvider.handle_onorder_landsat_products',
        mock_production_provider.respond_true)
    @patch(
        'api.providers.production.production_provider.ProductionProvider.handle_retry_products',
        mock_production_provider.respond_true)
    @patch(
        'api.providers.production.production_provider.ProductionProvider.load_ee_orders',
        mock_production_provider.respond_true)
    @patch(
        'api.providers.production.production_provider.ProductionProvider.finalize_orders',
        mock_production_provider.respond_true)
    @patch(
        'api.providers.production.production_provider.ProductionProvider.purge_orders',
        mock_production_provider.respond_true)
    def test_handle_orders_success(self):
        _ = self.mock_order.generate_testing_order(self.user_id)
        self.assertTrue(
            api.handle_orders({'username': User.find(self.user_id)}))

    @patch('api.external.onlinecache.delete',
           mock_production_provider.respond_true)
    @patch('api.notification.emails.send_purge_report',
           mock_production_provider.respond_true)
    @patch('api.external.onlinecache.capacity', onlinecache.mock_capacity)
    @patch('api.external.onlinecache.exists', onlinecache.mock_exists)
    @patch('api.external.onlinecache.delete', onlinecache.mock_delete)
    def test_production_purge_orders(self):
        new_completion_date = datetime.datetime.now() - datetime.timedelta(
            days=12)
        order = Order.find(self.mock_order.generate_testing_order(
            self.user_id))
        order.update('status', 'complete')
        order.update('completion_date', new_completion_date)
        self.assertTrue(production_provider.purge_orders())

    # need to figure a test for emails.send_email
    @patch('api.notification.emails.Emails.send_email',
           mock_production_provider.respond_true)
    def test_production_send_initial_emails(self):
        order = Order.find(self.mock_order.generate_testing_order(
            self.user_id))
        order.update('status', 'ordered')
        self.assertTrue(emails.Emails().send_all_initial([order]))

    @patch('api.external.lta.get_order_status', lta.get_order_status)
    @patch('api.external.lta.update_order_status', lta.update_order_status)
    def test_production_handle_onorder_landsat_products(self):
        tram_order_ids = lta.sample_tram_order_ids()[0:3]
        scene_names = lta.sample_scene_names()[0:3]
        order = Order.find(self.mock_order.generate_testing_order(
            self.user_id))
        scenes = order.scenes()[0:3]
        for idx, scene in enumerate(scenes):
            scene.tram_order_id = tram_order_ids[idx]
            scene.status = 'onorder'
            # save() doesn't let you update name,
            # b/c updating a scene name is not acceptable
            # outside of testing
            scene.update('name', scene_names[idx])
            scene.save()
        self.assertTrue(
            production_provider.handle_onorder_landsat_products(scenes))

    def test_production_handle_retry_products(self):
        prev = datetime.datetime.now() - datetime.timedelta(hours=1)
        order_id = self.mock_order.generate_testing_order(self.user_id)
        self.mock_order.update_scenes(order_id, 'landsat', 'status', ['retry'])
        self.mock_order.update_scenes(order_id, 'landsat', 'retry_after',
                                      [prev])
        scenes = Order.find(order_id).scenes({'status': 'retry'})
        production_provider.handle_retry_products(scenes)
        for s in Scene.where({'order_id': order_id, 'sensor_type': 'landsat'}):
            self.assertTrue(s.status == 'submitted')

    @patch('api.external.lta.get_available_orders',
           lta.get_available_orders_partial)
    @patch('api.external.lta.update_order_status', lta.update_order_status)
    @patch('api.external.inventory.get_user_name', inventory.get_user_name)
    @patch('api.external.inventory.get_session', lambda: True)
    def test_production_load_ee_orders_partial(self):
        order = Order.find(
            self.mock_order.generate_ee_testing_order(self.user_id,
                                                      partial=True))
        self.assertEqual(
            order.product_opts, {
                'format': 'gtiff',
                'etm7_collection': {
                    'inputs': ['LE07_L1TP_026027_20170912_20171008_01_T1'],
                    'products': ['sr']
                }
            })
        key = 'system.load_ee_orders_enabled'
        self.assertEqual(api.get_production_key(key)[key], 'True')
        production_provider.load_ee_orders()
        reorder = Order.find(order.id)
        self.assertEqual(
            reorder.product_opts, {
                'format': 'gtiff',
                'etm7_collection': {
                    'inputs': ['LE07_L1TP_026027_20170912_20171008_01_T1'],
                    'products': ['sr']
                },
                'tm5_collection': {
                    'inputs': ['LT05_L1TP_025027_20110913_20160830_01_T1'],
                    'products': ['sr']
                }
            })

    @patch('api.external.lta.update_order_status', lta.update_order_status)
    def test_production_handle_failed_ee_updates(self):
        order = Order.find(self.mock_order.generate_testing_order(
            self.user_id))
        for scene in order.scenes():
            scene.update('failed_lta_status_update', 'C')

        scenes = Order.find(order.id).scenes(
            {'failed_lta_status_update IS NOT': None})
        production_provider.handle_failed_ee_updates(scenes)

        scenes = Scene.where({'failed_lta_status_update IS NOT': None})
        self.assertTrue(len(scenes) == 0)

    @patch(
        'api.providers.production.production_provider.ProductionProvider.update_landsat_product_status',
        mock_production_provider.respond_true)
    @patch(
        'api.providers.production.production_provider.ProductionProvider.get_contactids_for_submitted_landsat_products',
        mock_production_provider.contact_ids_list)
    @patch('api.external.inventory.available', lambda: True)
    def test_production_handle_submitted_landsat_products(self):
        orders = Order.find(
            self.mock_order.generate_testing_order(self.user_id))
        scenes = orders.scenes({'sensor_type': 'landsat'})
        self.assertTrue(
            production_provider.handle_submitted_landsat_products(scenes))

    @patch('api.external.lta.update_order_status', lta.update_order_status)
    def test_production_set_products_unavailable(self):
        order = Order.find(self.mock_order.generate_testing_order(
            self.user_id))
        self.assertTrue(
            production_provider.set_products_unavailable(
                order.scenes(), "you want a reason?"))

    @patch(
        'api.providers.production.production_provider.ProductionProvider.check_dependencies_for_products',
        mock_production_provider.check_dependencies_for_products)
    @patch('api.external.inventory.get_cached_session',
           inventory.get_cached_session)
    @patch('api.external.inventory.check_valid', inventory.check_valid_landsat)
    def test_production_update_landsat_product_status(self):
        order = Order.find(self.mock_order.generate_testing_order(
            self.user_id))
        for scene in order.scenes({'name !=': 'plot'}):
            scene.status = 'submitted'
            scene.sensor_type = 'landsat'
            user = User.find(self.user_id)
            scene.save()
        self.assertTrue(
            production_provider.update_landsat_product_status(user.contactid))

    def test_production_get_contactids_for_submitted_landsat_products(self):
        order = Order.find(self.mock_order.generate_testing_order(
            self.user_id))
        for scene in order.scenes({'name !=': 'plot'}):
            scene.status = 'submitted'
            scene.sensor_type = 'landsat'
            scene.save()
        scenes = Order.find(order.id).scenes({'sensor_type': 'landsat'})
        response = production_provider.get_contactids_for_submitted_landsat_products(
            scenes)
        self.assertIsInstance(response, set)
        self.assertTrue(len(response) > 0)

    @patch('api.external.inventory.available', lambda: True)
    @patch('api.external.inventory.get_cached_session',
           inventory.get_cached_session)
    @patch('api.external.inventory.check_valid', inventory.check_valid_modis)
    def test_production_handle_submitted_modis_products_input_exists(self):
        # handle oncache scenario
        order = Order.find(self.mock_order.generate_testing_order(
            self.user_id))
        for scene in order.scenes({'name !=': 'plot'}):
            scene.status = 'submitted'
            scene.sensor_type = 'modis'
            scene.save()
            sid = scene.id
        scenes = order.scenes({'sensor_type': 'modis'})
        self.assertTrue(
            production_provider.handle_submitted_modis_products(scenes))
        #self.assertEquals(Scene.find(sid).status, "oncache")

    @patch('api.external.inventory.available', lambda: True)
    @patch('api.external.inventory.get_cached_session',
           inventory.get_cached_session)
    @patch('api.external.inventory.check_valid', inventory.check_valid_modis)
    def test_production_handle_submitted_modis_products_input_missing(self):
        # handle unavailable scenario
        order = Order.find(self.mock_order.generate_testing_order(
            self.user_id))
        for scene in order.scenes({'name !=': 'plot'}):
            scene.status = 'submitted'
            scene.sensor_type = 'modis'
            scene.save()
            sid = scene.id
        scenes = order.scenes({'sensor_type': 'modis'})
        self.assertTrue(
            production_provider.handle_submitted_modis_products(scenes))
        self.assertEquals(Scene.find(sid).status, "unavailable")

    def test_production_handle_submitted_plot_products(self):
        order = Order.find(self.mock_order.generate_testing_order(
            self.user_id))
        order.status = 'ordered'
        order.order_type = 'lpcs'
        order.save()
        plot_id = None
        for idx, scene in enumerate(order.scenes()):
            # at the moment, mock_order.generate_testing_order
            # creates 21 products for the order. divvy those
            # up between 'complete' and 'unavailable', setting
            # one aside as the 'plot' product
            if scene.sensor_type == 'plot':
                # need to define a plot product
                scene.update('status', 'submitted')
                plot_id = scene.id
            else:
                if idx % 2 == 0:
                    scene.update('status', 'complete')
                else:
                    scene.update('status', 'unavailable')

        scenes = order.scenes()
        self.assertTrue(
            production_provider.handle_submitted_plot_products(scenes))
        self.assertEqual(Scene.find(plot_id).status, "oncache")

    @patch('os.path.exists', lambda y: True)
    @patch('os.path.getsize', lambda y: 999)
    def test_production_calc_scene_download_sizes(self):
        order = Order.find(self.mock_order.generate_testing_order(
            self.user_id))
        scenes = order.scenes()
        Scene.bulk_update([s.id for s in scenes], {
            'status': 'complete',
            'download_size': 0
        })
        scenes = Order.find(order.id).scenes({
            'status': 'complete',
            'download_size': 0
        })
        self.assertTrue(production_provider.calc_scene_download_sizes(scenes))
        upscenes = Scene.where({'status': 'complete', 'download_size': 999})
        self.assertEqual(len(upscenes), len(scenes))

    @patch(
        'api.providers.production.production_provider.ProductionProvider.update_order_if_complete',
        mock_production_provider.respond_true)
    def test_production_finalize_orders(self):
        order = Order.find(self.mock_order.generate_testing_order(
            self.user_id))
        order.update('status', 'ordered')
        self.assertTrue(production_provider.finalize_orders([order]))

    @patch(
        'api.providers.production.production_provider.ProductionProvider.send_completion_email',
        mock_production_provider.respond_true)
    def test_production_update_order_if_complete(self):
        order = Order.find(self.mock_order.generate_testing_order(
            self.user_id))
        Scene.bulk_update([s.id for s in order.scenes()], {'status': 'retry'})
        order.order_source = 'espa'
        order.completion_email_sent = None
        order.save()
        self.assertTrue(production_provider.update_order_if_complete(order))

    def test_production_queue_products_success(self):
        names_tuple = self.mock_order.names_tuple(3, self.user_id)
        processing_loc = "get_products_to_process"
        job_name = 'jobname49'
        params = (names_tuple, processing_loc, job_name)
        response = api.queue_products(*params)
        self.assertTrue(response)

    def test_production_get_key(self):
        key = 'system_message_title'
        response = api.get_production_key(key)
        val = response[key]
        self.assertIsInstance(val, str)

    def test_get_production_key_invalid(self):
        bad_key = 'foobar'
        response = api.get_production_key(bad_key)
        self.assertEqual(response.keys(), ['msg'])

    @patch('api.external.hadoop.HadoopHandler.job_names_ids',
           hadoop.jobs_names_ids)
    def test_catch_orphaned_scenes(self):
        order_id = self.mock_order.generate_testing_order(self.user_id)
        # need scenes with statuses of 'queued'
        self.mock_order.update_scenes(order_id, ('landsat', 'modis', 'plot'),
                                      'status', ['queued'])
        response = production_provider.catch_orphaned_scenes()
        self.assertTrue(response)

        old_time = datetime.datetime.now() - datetime.timedelta(minutes=15)

        for s in Scene.where({'order_id': order_id}):
            self.assertTrue(s.reported_orphan is not None)
            s.reported_orphan = old_time
            s.save()

        response = production_provider.catch_orphaned_scenes()
        self.assertTrue(response)
        for s in Scene.where({'order_id': order_id}):
            self.assertTrue(s.orphaned)

    @patch('api.external.hadoop.HadoopHandler.job_names_ids',
           hadoop.jobs_names_ids)
    def test_handle_stuck_jobs(self):
        order_id = self.mock_order.generate_testing_order(self.user_id)
        # Make some really old jobs
        self.mock_order.update_scenes(order_id, ('landsat', 'modis', 'plot'),
                                      'status', ['processing'])
        self.mock_order.update_scenes(order_id, ('landsat', 'modis', 'plot'),
                                      'status_modified',
                                      [datetime.datetime(1900, 1, 1)])
        scenes = Scene.where({'status': 'processing', 'order_id': order_id})
        n_scenes = len(scenes)
        response = production_provider.handle_stuck_jobs(scenes)
        self.assertTrue(response)

        scenes = Scene.where({
            'status': 'processing',
            'order_id': order_id,
            'reported_orphan is not': None
        })
        self.assertEqual(n_scenes, len(scenes))

        self.mock_order.update_scenes(order_id, ('landsat', 'modis', 'plot'),
                                      'reported_orphan',
                                      [datetime.datetime(1900, 1, 1)])
        response = production_provider.handle_stuck_jobs(scenes)
        self.assertTrue(response)

        scenes = Scene.where({'status': 'processing', 'order_id': order_id})
        self.assertEqual(0, len(scenes))

    def test_convert_product_options(self):
        """
        Test the conversion procedure to make sure that the new format for orders converts
        to the old format
        """
        scenes = [
            'LE07_L1TP_026027_20170912_20171008_01_T1',
            'LC08_L1TP_025027_20160521_20170223_01_T1',
            'LT05_L1TP_025027_20110913_20160830_01_T1'
        ]

        includes = ['include_sr', 'include_sr_toa', 'include_sr_thermal']

        new_format = {
            u'etm7_collection': {
                u'inputs': [u'LE07_L1TP_026027_20170912_20171008_01_T1'],
                u'products': [u'sr']
            },
            u'olitirs8_collection': {
                u'inputs': [u'LC08_L1TP_025027_20160521_20170223_01_T1'],
                u'products': [u'toa']
            },
            u'tm5_collection': {
                u'inputs': [u'LT05_L1TP_025027_20110913_20160830_01_T1'],
                u'products': [u'bt']
            },
            u'format': u'gtiff',
            u'image_extents': {
                u'east': -2265585.0,
                u'north': 3164805.0,
                u'south': 3014805.0,
                u'units': u'meters',
                u'west': -2415585.0
            },
            u'note': u'CONUS_h1v1',
            u'projection': {
                u'aea': {
                    u'central_meridian': -96,
                    u'datum': u'nad83',
                    u'false_easting': 0,
                    u'false_northing': 0,
                    u'latitude_of_origin': 23,
                    u'standard_parallel_1': 29.5,
                    u'standard_parallel_2': 45.5
                }
            },
            u'resampling_method': u'cc'
        }

        ruberic = {
            'central_meridian': -96,
            'datum': u'nad83',
            'false_easting': 0,
            'false_northing': 0,
            'image_extents': True,
            'image_extents_units': u'meters',
            'include_customized_source_data': False,
            'include_dswe': False,
            'include_st': False,
            'include_solr_index': False,
            'include_source_data': False,
            'include_source_metadata': False,
            'include_sr': False,
            'include_sr_browse': False,
            'include_sr_evi': False,
            'include_sr_msavi': False,
            'include_sr_nbr': False,
            'include_sr_nbr2': False,
            'include_sr_ndmi': False,
            'include_sr_ndvi': False,
            'include_sr_savi': False,
            'include_sr_thermal': False,
            'include_sr_toa': False,
            'include_statistics': False,
            'latitude_true_scale': None,
            'longitude_pole': None,
            'maxx': -2265585.0,
            'maxy': 3164805.0,
            'minx': -2415585.0,
            'miny': 3014805.0,
            'origin_lat': 23,
            'output_format': u'gtiff',
            'pixel_size': None,
            'pixel_size_units': None,
            'reproject': True,
            'resample_method': 'cubic',
            'resize': False,
            'std_parallel_1': 29.5,
            'std_parallel_2': 45.5,
            'target_projection': u'aea',
            'utm_north_south': None,
            'utm_zone': None
        }

        for scene, include in zip(scenes, includes):
            ruberic[include] = True
            old_format = OptionsConversion.convert(new=new_format,
                                                   scenes=[scene])

            self.assertDictEqual(ruberic, old_format)

            ruberic[include] = False

    def test_status_modified(self):
        order_id = self.mock_order.generate_testing_order(self.user_id)
        scene = Scene.where({'order_id': order_id}).pop()
        scene.status = 'oncache'
        scene.save()
        old_time = scene.status_modified
        scene.status = 'queued'
        scene.save()
        new_time = scene.status_modified
        self.assertGreater(new_time, old_time)

    @patch('api.external.hadoop.HadoopHandler.list_jobs',
           hadoop.jobs_names_ids)
    @patch('api.external.hadoop.HadoopHandler.kill_job', lambda x, y: True)
    def test_hadoop_reset_status(self):
        order_id = self.mock_order.generate_testing_order(self.user_id)
        scenes = Scene.where({'order_id': order_id})
        Scene.bulk_update([s.id for s in scenes], {'status': 'processing'})
        self.assertTrue(production_provider.reset_processing_status())
        scenes = Scene.where({'order_id': order_id})
        self.assertEqual({'submitted'}, set([s.status for s in scenes]))
Beispiel #2
0
class TestProductionAPI(unittest.TestCase):
    def setUp(self):
        os.environ['espa_api_testing'] = 'True'
        # create a user
        self.mock_user = MockUser()
        self.mock_order = MockOrder()
        self.user_id = self.mock_user.add_testing_user()

    def tearDown(self):
        # clean up orders
        self.mock_order.tear_down_testing_orders()
        # clean up users
        self.mock_user.cleanup()
        os.environ['espa_api_testing'] = ''

    @patch('api.external.lpdaac.get_download_urls', lpdaac.get_download_urls)
    @patch('api.providers.production.production_provider.ProductionProvider.set_product_retry',
           mock_production_provider.set_product_retry)
    def test_fetch_production_products_modis(self):
        order_id = self.mock_order.generate_testing_order(self.user_id)
        # need scenes with statuses of 'processing' and 'ordered'
        self.mock_order.update_scenes(order_id, 'status', ['processing', 'ordered', 'oncache'])
        user = User.find(self.user_id)
        params = {'for_user': user.username, 'product_types': ['modis']}
        response = api.fetch_production_products(params)
        self.assertTrue('bilbo' in response[0]['orderid'])

    @patch('api.external.lta.get_download_urls', lta.get_download_urls)
    @patch('api.providers.production.production_provider.ProductionProvider.set_product_retry',
           mock_production_provider.set_product_retry)
    def test_fetch_production_products_landsat(self):
        order_id = self.mock_order.generate_testing_order(self.user_id)
        # need scenes with statuses of 'processing' and 'ordered'
        self.mock_order.update_scenes(order_id, 'status', ['processing','oncache','ordered'])
        user = User.find(self.user_id)
        params = {'for_user': user.username, 'product_types': ['landsat']}
        response = api.fetch_production_products(params)
        self.assertTrue('bilbo' in response[0]['orderid'])

    def test_fetch_production_products_plot(self):
        order_id = self.mock_order.generate_testing_order(self.user_id)
        self.mock_order.update_scenes(order_id, 'status', ['complete'])
        order = Order.find(order_id)
        plot_scene = order.scenes()[0]
        plot_scene.name = 'plot'
        plot_scene.sensor_type = 'plot'
        plot_scene.status = 'submitted'
        plot_scene.save()
        response = production_provider.handle_submitted_plot_products()
        pscene = order.scenes({'status': 'oncache', 'sensor_type': 'plot'})
        self.assertTrue(response is True)
        self.assertEqual(len(pscene), 1)

    def test_production_set_product_retry(self):
        order_id = self.mock_order.generate_testing_order(self.user_id)
        order = Order.find(order_id)
        scene = order.scenes()[3]
        scene.update('retry_count', 4)
        processing_loc = "get_products_to_process"
        error = 'not available after EE call '
        note = 'note this'
        retry_after = datetime.datetime.now() + datetime.timedelta(hours=1)
        retry_limit = 9
        response = production_provider.set_product_retry(scene.name, order.orderid, processing_loc,
                                                         error, note, retry_after, retry_limit)

        new = Scene.get('ordering_scene.status', scene.name, order.orderid)
        self.assertTrue('retry' == new)

    @patch('api.external.lta.update_order_status', mock_production_provider.respond_true)
    def test_production_set_product_error_unavailable(self):
        """
        Move a scene status from error to unavailable based on the error
        message
        """
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        scene = order.scenes({'name !=': 'plot'})[0]
        production_provider.set_product_error(scene.name, order.orderid,
                                              'get_products_to_process',
                                              'include_dswe is an unavailable product option for OLITIRS')
        self.assertTrue('unavailable' == Scene.get('ordering_scene.status', scene.name, order.orderid))

    def test_production_set_product_error_submitted(self):
        """
        Move a scene status from error to submitted based on the error
        message
        """
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        scene = order.scenes({'name !=': 'plot'})[0]
        production_provider.set_product_error(scene.name, order.orderid,
                                              'get_products_to_process',
                                              'BLOCK, COMING FROM LST AS WELL: No such file or directory')
        self.assertTrue('submitted' == Scene.get('ordering_scene.status', scene.name, order.orderid))

    def test_production_set_product_error_retry(self):
        """
        Move a scene status from error to retry based on the error
        message
        """
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        scene = order.scenes()[2]
        production_provider.set_product_error(scene.name, order.orderid,
                                              'somewhere',
                                              'Verify the missing auxillary data products')
        self.assertTrue('retry' == Scene.get('ordering_scene.status', scene.name, order.orderid))

    @patch('api.external.lta.update_order_status', lta.update_order_status)
    @patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
    def test_update_product_details_update_status(self):
        """
        Set a scene status to Queued
        """
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        scene = order.scenes()[0]
        api.update_product_details('update_status',
                                   {'name': scene.name,
                                    'orderid': order.orderid,
                                    'processing_loc': 'L8SRLEXAMPLE',
                                    'status': 'Queued'})
        self.assertTrue(Scene.get('ordering_scene.status', scene.name, order.orderid) == 'Queued')

    @patch('api.external.lta.update_order_status', lta.update_order_status)
    @patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
    @patch('api.external.onlinecache.capacity', onlinecache.capacity)
    def test_update_product_details_set_product_error(self):
        """
        Set a scene status to error
        :return:
        """
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        scene = order.scenes()[0]
        production_provider.update_product('set_product_error',
                                           name=scene.name, orderid=order.orderid,
                                           processing_loc="L8SRLEXAMPLE",
                                           error='problems yo')
        self.assertTrue(Scene.find(scene.id).status == 'error')

    @patch('api.external.lta.update_order_status', lta.update_order_status)
    @patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
    def test_update_product_details_set_product_unavailable(self):
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        scene = order.scenes()[0]
        production_provider.update_product('set_product_unavailable',
                                           name=scene.name, orderid=order.orderid,
                                           processing_loc="L8SRLEXAMPLE",
                                           error='include_dswe is an unavailable product option for OLITIRS')
        self.assertTrue('unavailable' == Scene.get('ordering_scene.status', scene.name, order.orderid))

    @patch('api.external.lta.update_order_status', lta.update_order_status)
    @patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
    def test_update_product_details_mark_product_complete(self):
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        scene = order.scenes()[0]
        production_provider.update_product('mark_product_complete',
                                           name=scene.name,
                                           orderid=order.orderid,
                                           processing_loc='L8SRLEXAMPLE',
                                           completed_file_location='/some/loc',
                                           cksum_file_location='some checksum',
                                           log_file_contents='some log')

        self.assertTrue('complete' == Scene.get('ordering_scene.status', scene.name, order.orderid))

    @patch('api.external.lta.update_order_status', lta.update_order_status_fail)
    @patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
    def test_update_product_details_fail_lta_mark_product_complete(self):
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        scene = order.scenes()[1]
        order.update('order_source', 'ee')
        production_provider.update_product('mark_product_complete',
                                           name=scene.name,
                                           orderid=order.orderid,
                                           processing_loc='L8SRLEXAMPLE',
                                           completed_file_location='/some/loc',
                                           cksum_file_location='some checksum',
                                           log_file_contents='some log')

        s = Scene.where({'name': scene.name, 'order_id': scene.order_id})[0]
        self.assertTrue('C' == s.failed_lta_status_update)

    @patch('api.providers.production.production_provider.ProductionProvider.send_initial_emails',
           mock_production_provider.respond_true)
    @patch('api.providers.production.production_provider.ProductionProvider.handle_onorder_landsat_products',
           mock_production_provider.respond_true)
    @patch('api.providers.production.production_provider.ProductionProvider.handle_retry_products',
           mock_production_provider.respond_true)
    @patch('api.providers.production.production_provider.ProductionProvider.load_ee_orders',
           mock_production_provider.respond_true)
    @patch('api.providers.production.production_provider.ProductionProvider.handle_submitted_products',
           mock_production_provider.respond_true)
    @patch('api.providers.production.production_provider.ProductionProvider.finalize_orders',
           mock_production_provider.respond_true)
    @patch('api.providers.production.production_provider.ProductionProvider.purge_orders',
           mock_production_provider.respond_true)
    def test_handle_orders_success(self):
        self.assertTrue(api.handle_orders())

    @patch('api.external.onlinecache.delete', mock_production_provider.respond_true)
    @patch('api.notification.emails.send_purge_report', mock_production_provider.respond_true)
    @patch('api.external.onlinecache.capacity', onlinecache.capacity)
    def test_production_purge_orders(self):
        new_completion_date = datetime.datetime.now() - datetime.timedelta(days=12)
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        order.update('status', 'complete')
        order.update('completion_date', new_completion_date)
        self.assertTrue(production_provider.purge_orders())

    # need to figure a test for emails.send_email
    @patch('api.notification.emails.Emails.send_email', mock_production_provider.respond_true)
    def test_production_send_initial_emails(self):
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        order.update('status', 'ordered')
        self.assertTrue(emails.Emails().send_all_initial())

    @patch('api.external.lta.get_order_status', lta.get_order_status)
    @patch('api.external.lta.update_order_status', lta.update_order_status)
    def test_production_handle_onorder_landsat_products(self):
        tram_order_ids = lta.sample_tram_order_ids()[0:3]
        scene_names = lta.sample_scene_names()[0:3]
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        scenes = order.scenes()[0:3]
        for idx, scene in enumerate(scenes):
            scene.tram_order_id = tram_order_ids[idx]
            scene.status = 'onorder'
            # save() doesn't let you update name,
            # b/c updating a scene name is not acceptable
            # outside of testing
            scene.update('name', scene_names[idx])
            scene.save()
        self.assertTrue(production_provider.handle_onorder_landsat_products())

    def test_production_handle_retry_products(self):
        prev = datetime.datetime.now() - datetime.timedelta(hours=1)
        order_id = self.mock_order.generate_testing_order(self.user_id)
        self.mock_order.update_scenes(order_id, 'status', ['retry'])
        self.mock_order.update_scenes(order_id, 'retry_after', [prev])
        production_provider.handle_retry_products()
        for s in Scene.where({'order_id': order_id}):
            self.assertTrue(s.status == 'submitted')

    #@patch('api.external.lta.get_available_orders', lta.get_available_orders)
    #@patch('api.external.lta.update_order_status', lta.update_order_status)
    #@patch('api.external.lta.get_user_name', lta.get_user_name)
    #def test_production_load_ee_orders(self):
    #    #production_provider.load_ee_orders()
    #    pass

    @patch('api.external.lta.get_available_orders', lta.get_available_orders_partial)
    @patch('api.external.lta.update_order_status', lta.update_order_status)
    @patch('api.external.lta.get_user_name', lta.get_user_name)
    def test_production_load_ee_orders_partial(self):
        order = Order.find(self.mock_order.generate_ee_testing_order(self.user_id, partial=True))
        self.assertTrue(order.product_opts == {'format': 'gtiff',
                                               'etm7': {'inputs': ['LE70900652008327EDC00'],
                                                        'products': ['sr']}})
        production_provider.load_ee_orders()
        reorder = Order.find(order.id)
        self.assertTrue(reorder.product_opts == {'format': 'gtiff',
                                               'etm7': {'inputs': ['LE70900652008327EDC00'],
                                                        'products': ['sr']},
                                               'tm5': {'inputs': ['LT50900652008327EDC00'],
                                                        'products': ['sr']}})

    @patch('api.external.lta.update_order_status', lta.update_order_status)
    def test_production_handle_failed_ee_updates(self):
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        for scene in order.scenes():
            scene.update('failed_lta_status_update', 'C')

        production_provider.handle_failed_ee_updates()

        scenes = Scene.where({'failed_lta_status_update IS NOT': None})
        self.assertTrue(len(scenes) == 0)

    @patch('api.providers.production.production_provider.ProductionProvider.handle_submitted_landsat_products',
           mock_production_provider.respond_true)
    @patch('api.providers.production.production_provider.ProductionProvider.handle_submitted_modis_products',
           mock_production_provider.respond_true)
    @patch('api.providers.production.production_provider.ProductionProvider.handle_submitted_plot_products',
           mock_production_provider.respond_true)
    def test_production_handle_submitted_products(self):
        self.assertTrue(production_provider.handle_submitted_products())

    @patch('api.providers.production.production_provider.ProductionProvider.mark_nlaps_unavailable',
           mock_production_provider.respond_true)
    @patch('api.providers.production.production_provider.ProductionProvider.update_landsat_product_status',
           mock_production_provider.respond_true)
    @patch('api.providers.production.production_provider.ProductionProvider.get_contactids_for_submitted_landsat_products',
           mock_production_provider.contact_ids_list)
    def test_production_handle_submitted_landsat_products(self):
        self.assertTrue(production_provider.handle_submitted_landsat_products())

    # !!! need to write test for nlaps.products_are_nlaps !!!
    @patch('api.external.nlaps.products_are_nlaps', nlaps.products_are_nlaps)
    @patch('api.providers.production.production_provider.ProductionProvider.set_products_unavailable',
           mock_production_provider.respond_true)
    def test_production_mark_nlaps_unavailable(self):
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        for scene in order.scenes():
            scene.status = 'submitted'
            scene.sensor_type = 'landsat'
            scene.save()
        self.assertTrue(production_provider.mark_nlaps_unavailable())

    @patch('api.external.lta.update_order_status', lta.update_order_status)
    def test_production_set_products_unavailable(self):
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        self.assertTrue(production_provider.set_products_unavailable(order.scenes(), "you want a reason?"))

    @patch('api.external.lta.order_scenes', lta.order_scenes)
    @patch('api.providers.production.production_provider.ProductionProvider.set_products_unavailable',
           mock_production_provider.respond_true)
    def test_production_update_landsat_product_status(self):
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        for scene in order.scenes():
            scene.status = 'submitted'
            scene.sensor_type = 'landsat'
            scene.save()
        self.assertTrue(production_provider.update_landsat_product_status(User.find(self.user_id).contactid))

    def test_production_get_contactids_for_submitted_landsat_products(self):
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        for scene in order.scenes():
            scene.status = 'submitted'
            scene.sensor_type = 'landsat'
            scene.save()
        response = production_provider.get_contactids_for_submitted_landsat_products()
        self.assertIsInstance(response, set)
        self.assertTrue(len(response) > 0)

    @patch('api.external.lpdaac.input_exists', lpdaac.input_exists_true)
    def test_production_handle_submitted_modis_products_input_exists(self):
        # handle oncache scenario
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        for scene in order.scenes():
            scene.status = 'submitted'
            scene.sensor_type = 'modis'
            scene.save()
            sid = scene.id
        self.assertTrue(production_provider.handle_submitted_modis_products())
        self.assertEquals(Scene.find(sid).status, "oncache")

    @patch('api.external.lpdaac.input_exists', lpdaac.input_exists_false)
    def test_production_handle_submitted_modis_products_input_missing(self):
        # handle unavailable scenario
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        for scene in order.scenes():
            scene.status = 'submitted'
            scene.sensor_type = 'modis'
            scene.save()
            sid = scene.id
        self.assertTrue(production_provider.handle_submitted_modis_products())
        self.assertEquals(Scene.find(sid).status, "unavailable")

    def test_production_handle_submitted_plot_products(self):
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        order.status = 'ordered'
        order.order_type = 'lpcs'
        order.save()
        plot_id = None
        for idx, scene in enumerate(order.scenes()):
            # at the moment, mock_order.generate_testing_order
            # creates 21 products for the order. divvy those
            # up between 'complete' and 'unavailable', setting
            # one aside as the 'plot' product
            if idx % 2 == 0:
                if idx == 0:
                    # need to define a plot product
                    scene.update('status', 'submitted')
                    scene.update('sensor_type', 'plot')
                    plot_id = scene.id
                else:
                    scene.update('status', 'complete')
            else:
                scene.update('status', 'unavailable')

        self.assertTrue(production_provider.handle_submitted_plot_products())
        self.assertEqual(Scene.find(plot_id).status, "oncache")


    @patch('api.providers.production.production_provider.ProductionProvider.update_order_if_complete',
           mock_production_provider.respond_true)
    def test_production_finalize_orders(self):
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        order.update('status', 'ordered')
        self.assertTrue(production_provider.finalize_orders())

    @patch('api.providers.production.production_provider.ProductionProvider.send_completion_email',
           mock_production_provider.respond_true)
    def test_production_update_order_if_complete(self):
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        Scene.bulk_update([s.id for s in order.scenes()], {'status': 'retry'})
        order.order_source = 'espa'
        order.completion_email_sent = None
        order.save()
        self.assertTrue(production_provider.update_order_if_complete(order))

    def test_production_queue_products_success(self):
        names_tuple = self.mock_order.names_tuple(3, self.user_id)
        processing_loc = "get_products_to_process"
        job_name = 'jobname49'
        params = (names_tuple, processing_loc, job_name)
        response = api.queue_products(*params)
        self.assertTrue(response)

    def test_production_get_key(self):
        key = 'system_message_title'
        response = api.get_production_key(key)
        val = response[key]
        self.assertIsInstance(val, str)

    def test_get_production_key_invalid(self):
        bad_key = 'foobar'
        response = api.get_production_key(bad_key)
        self.assertEqual(response.keys(), ['msg'])

    @patch('api.external.hadoop.HadoopHandler.job_names_ids',
           hadoop.jobs_names_ids)
    def test_catch_orphaned_scenes(self):
        order_id = self.mock_order.generate_testing_order(self.user_id)
        # need scenes with statuses of 'queued'
        self.mock_order.update_scenes(order_id, 'status', ['queued'])
        response = production_provider.catch_orphaned_scenes()
        self.assertTrue(response)

        old_time = datetime.datetime.now() - datetime.timedelta(minutes=15)

        for s in Scene.where({'order_id': order_id}):
            self.assertTrue(s.reported_orphan is not None)
            s.reported_orphan = old_time
            s.save()

        response = production_provider.catch_orphaned_scenes()
        self.assertTrue(response)
        for s in Scene.where({'order_id': order_id}):
            self.assertTrue(s.orphaned)
class TestProductionAPI(unittest.TestCase):
    def setUp(self):
        os.environ['espa_api_testing'] = 'True'
        # create a user
        self.mock_user = MockUser()
        self.mock_order = MockOrder()
        self.user_id = self.mock_user.add_testing_user()

    def tearDown(self):
        # clean up orders
        self.mock_order.tear_down_testing_orders()
        # clean up users
        self.mock_user.cleanup()
        os.environ['espa_api_testing'] = ''

    @patch('api.external.lpdaac.get_download_urls', lpdaac.get_download_urls)
    @patch('api.providers.production.production_provider.ProductionProvider.set_product_retry',
           mock_production_provider.set_product_retry)
    def test_fetch_production_products_modis(self):
        order_id = self.mock_order.generate_testing_order(self.user_id)
        # need scenes with statuses of 'processing' and 'ordered'
        self.mock_order.update_scenes(order_id, 'status', ['processing', 'ordered', 'oncache'])
        user = User.find(self.user_id)
        params = {'for_user': user.username, 'product_types': ['modis']}
        response = api.fetch_production_products(params)
        self.assertTrue('bilbo' in response[0]['orderid'])

    @patch('api.external.lta.get_download_urls', lta.get_download_urls)
    @patch('api.providers.production.production_provider.ProductionProvider.set_product_retry',
           mock_production_provider.set_product_retry)
    def test_fetch_production_products_landsat(self):
        order_id = self.mock_order.generate_testing_order(self.user_id)
        # need scenes with statuses of 'processing' and 'ordered'
        self.mock_order.update_scenes(order_id, 'status', ['processing','oncache','ordered'])
        user = User.find(self.user_id)
        params = {'for_user': user.username, 'product_types': ['landsat']}
        response = api.fetch_production_products(params)
        self.assertTrue('bilbo' in response[0]['orderid'])

    def test_fetch_production_products_plot(self):
        order_id = self.mock_order.generate_testing_order(self.user_id)
        self.mock_order.update_scenes(order_id, 'status', ['complete'])
        order = Order.find(order_id)
        plot_scene = order.scenes()[0]
        plot_scene.name = 'plot'
        plot_scene.sensor_type = 'plot'
        plot_scene.status = 'submitted'
        plot_scene.save()
        response = production_provider.handle_submitted_plot_products()
        pscene = order.scenes({'status': 'oncache', 'sensor_type': 'plot'})
        self.assertTrue(response is True)
        self.assertEqual(len(pscene), 1)

    def test_production_set_product_retry(self):
        order_id = self.mock_order.generate_testing_order(self.user_id)
        order = Order.find(order_id)
        scene = order.scenes()[3]
        scene.update('retry_count', 4)
        processing_loc = "get_products_to_process"
        error = 'not available after EE call '
        note = 'note this'
        retry_after = datetime.datetime.now() + datetime.timedelta(hours=1)
        retry_limit = 9
        response = production_provider.set_product_retry(scene.name, order.orderid, processing_loc,
                                                         error, note, retry_after, retry_limit)

        new = Scene.get('ordering_scene.status', scene.name, order.orderid)
        self.assertTrue('retry' == new)

    @patch('api.external.lta.update_order_status', mock_production_provider.respond_true)
    def test_production_set_product_error_unavailable(self):
        """
        Move a scene status from error to unavailable based on the error
        message
        """
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        scene = order.scenes({'name !=': 'plot'})[0]
        production_provider.set_product_error(scene.name, order.orderid,
                                              'get_products_to_process',
                                              'include_dswe is an unavailable product option for OLITIRS')
        self.assertTrue('unavailable' == Scene.get('ordering_scene.status', scene.name, order.orderid))

    def test_production_set_product_error_submitted(self):
        """
        Move a scene status from error to submitted based on the error
        message
        """
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        scene = order.scenes({'name !=': 'plot'})[0]
        production_provider.set_product_error(scene.name, order.orderid,
                                              'get_products_to_process',
                                              'BLOCK, COMING FROM LST AS WELL: No such file or directory')
        self.assertTrue('submitted' == Scene.get('ordering_scene.status', scene.name, order.orderid))

    def test_production_set_product_error_retry(self):
        """
        Move a scene status from error to retry based on the error
        message
        """
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        scene = order.scenes()[2]
        production_provider.set_product_error(scene.name, order.orderid,
                                              'somewhere',
                                              'Verify the missing auxillary data products')
        self.assertTrue('retry' == Scene.get('ordering_scene.status', scene.name, order.orderid))

    @patch('api.external.lta.update_order_status', lta.update_order_status)
    @patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
    def test_update_product_details_update_status(self):
        """
        Set a scene status to Queued
        """
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        scene = order.scenes()[0]
        api.update_product_details('update_status',
                                   {'name': scene.name,
                                    'orderid': order.orderid,
                                    'processing_loc': 'L8SRLEXAMPLE',
                                    'status': 'Queued'})
        self.assertTrue(Scene.get('ordering_scene.status', scene.name, order.orderid) == 'Queued')

    @patch('api.external.lta.update_order_status', lta.update_order_status)
    @patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
    @patch('api.external.onlinecache.capacity', onlinecache.capacity)
    def test_update_product_details_set_product_error(self):
        """
        Set a scene status to error
        :return:
        """
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        scene = order.scenes()[0]
        production_provider.update_product('set_product_error',
                                           name=scene.name, orderid=order.orderid,
                                           processing_loc="L8SRLEXAMPLE",
                                           error='problems yo')
        self.assertTrue(Scene.find(scene.id).status == 'error')

    @patch('api.external.lta.update_order_status', lta.update_order_status)
    @patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
    def test_update_product_details_set_product_unavailable(self):
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        scene = order.scenes()[0]
        production_provider.update_product('set_product_unavailable',
                                           name=scene.name, orderid=order.orderid,
                                           processing_loc="L8SRLEXAMPLE",
                                           error='include_dswe is an unavailable product option for OLITIRS')
        self.assertTrue('unavailable' == Scene.get('ordering_scene.status', scene.name, order.orderid))

    @patch('api.external.lta.update_order_status', lta.update_order_status)
    @patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
    @patch('os.path.getsize', lambda y: 999)
    def test_update_product_details_mark_product_complete(self):
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        scene = order.scenes()[0]
        production_provider.update_product('mark_product_complete',
                                           name=scene.name,
                                           orderid=order.orderid,
                                           processing_loc='L8SRLEXAMPLE',
                                           completed_file_location='/some/loc',
                                           cksum_file_location='some checksum',
                                           log_file_contents='some log')

        self.assertTrue('complete' == Scene.get('ordering_scene.status', scene.name, order.orderid))

    @patch('api.external.lta.update_order_status', lta.update_order_status_fail)
    @patch('api.providers.production.production_provider.ProductionProvider.set_product_retry', mock_production_provider.set_product_retry)
    @patch('os.path.getsize', lambda y: 999)
    def test_update_product_details_fail_lta_mark_product_complete(self):
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        scene = order.scenes()[1]
        order.update('order_source', 'ee')
        production_provider.update_product('mark_product_complete',
                                           name=scene.name,
                                           orderid=order.orderid,
                                           processing_loc='L8SRLEXAMPLE',
                                           completed_file_location='/some/loc',
                                           cksum_file_location='some checksum',
                                           log_file_contents='some log')

        s = Scene.where({'name': scene.name, 'order_id': scene.order_id})[0]
        self.assertTrue('C' == s.failed_lta_status_update)

    @patch('api.providers.production.production_provider.ProductionProvider.send_initial_emails',
           mock_production_provider.respond_true)
    @patch('api.providers.production.production_provider.ProductionProvider.handle_onorder_landsat_products',
           mock_production_provider.respond_true)
    @patch('api.providers.production.production_provider.ProductionProvider.handle_retry_products',
           mock_production_provider.respond_true)
    @patch('api.providers.production.production_provider.ProductionProvider.load_ee_orders',
           mock_production_provider.respond_true)
    @patch('api.providers.production.production_provider.ProductionProvider.handle_submitted_products',
           mock_production_provider.respond_true)
    @patch('api.providers.production.production_provider.ProductionProvider.finalize_orders',
           mock_production_provider.respond_true)
    @patch('api.providers.production.production_provider.ProductionProvider.purge_orders',
           mock_production_provider.respond_true)
    def test_handle_orders_success(self):
        self.assertTrue(api.handle_orders())

    @patch('api.external.onlinecache.delete', mock_production_provider.respond_true)
    @patch('api.notification.emails.send_purge_report', mock_production_provider.respond_true)
    @patch('api.external.onlinecache.capacity', onlinecache.capacity)
    def test_production_purge_orders(self):
        new_completion_date = datetime.datetime.now() - datetime.timedelta(days=12)
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        order.update('status', 'complete')
        order.update('completion_date', new_completion_date)
        self.assertTrue(production_provider.purge_orders())

    # need to figure a test for emails.send_email
    @patch('api.notification.emails.Emails.send_email', mock_production_provider.respond_true)
    def test_production_send_initial_emails(self):
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        order.update('status', 'ordered')
        self.assertTrue(emails.Emails().send_all_initial())

    @patch('api.external.lta.get_order_status', lta.get_order_status)
    @patch('api.external.lta.update_order_status', lta.update_order_status)
    def test_production_handle_onorder_landsat_products(self):
        tram_order_ids = lta.sample_tram_order_ids()[0:3]
        scene_names = lta.sample_scene_names()[0:3]
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        scenes = order.scenes()[0:3]
        for idx, scene in enumerate(scenes):
            scene.tram_order_id = tram_order_ids[idx]
            scene.status = 'onorder'
            # save() doesn't let you update name,
            # b/c updating a scene name is not acceptable
            # outside of testing
            scene.update('name', scene_names[idx])
            scene.save()
        self.assertTrue(production_provider.handle_onorder_landsat_products())

    def test_production_handle_retry_products(self):
        prev = datetime.datetime.now() - datetime.timedelta(hours=1)
        order_id = self.mock_order.generate_testing_order(self.user_id)
        self.mock_order.update_scenes(order_id, 'status', ['retry'])
        self.mock_order.update_scenes(order_id, 'retry_after', [prev])
        production_provider.handle_retry_products()
        for s in Scene.where({'order_id': order_id}):
            self.assertTrue(s.status == 'submitted')

    #@patch('api.external.lta.get_available_orders', lta.get_available_orders)
    #@patch('api.external.lta.update_order_status', lta.update_order_status)
    #@patch('api.external.lta.get_user_name', lta.get_user_name)
    #def test_production_load_ee_orders(self):
    #    #production_provider.load_ee_orders()
    #    pass

    @patch('api.external.lta.get_available_orders', lta.get_available_orders_partial)
    @patch('api.external.lta.update_order_status', lta.update_order_status)
    @patch('api.external.lta.get_user_name', lta.get_user_name)
    def test_production_load_ee_orders_partial(self):
        order = Order.find(self.mock_order.generate_ee_testing_order(self.user_id, partial=True))
        self.assertTrue(order.product_opts == {'format': 'gtiff',
                                               'etm7': {'inputs': ['LE70900652008327EDC00'],
                                                        'products': ['sr']}})
        production_provider.load_ee_orders()
        reorder = Order.find(order.id)
        self.assertTrue(reorder.product_opts == {'format': 'gtiff',
                                               'etm7': {'inputs': ['LE70900652008327EDC00'],
                                                        'products': ['sr']},
                                               'tm5': {'inputs': ['LT50900652008327EDC00'],
                                                        'products': ['sr']}})

    @patch('api.external.lta.update_order_status', lta.update_order_status)
    def test_production_handle_failed_ee_updates(self):
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        for scene in order.scenes():
            scene.update('failed_lta_status_update', 'C')

        production_provider.handle_failed_ee_updates()

        scenes = Scene.where({'failed_lta_status_update IS NOT': None})
        self.assertTrue(len(scenes) == 0)

    @patch('api.providers.production.production_provider.ProductionProvider.handle_submitted_landsat_products',
           mock_production_provider.respond_true)
    @patch('api.providers.production.production_provider.ProductionProvider.handle_submitted_modis_products',
           mock_production_provider.respond_true)
    @patch('api.providers.production.production_provider.ProductionProvider.handle_submitted_plot_products',
           mock_production_provider.respond_true)
    def test_production_handle_submitted_products(self):
        self.assertTrue(production_provider.handle_submitted_products())

    @patch('api.providers.production.production_provider.ProductionProvider.mark_nlaps_unavailable',
           mock_production_provider.respond_true)
    @patch('api.providers.production.production_provider.ProductionProvider.update_landsat_product_status',
           mock_production_provider.respond_true)
    @patch('api.providers.production.production_provider.ProductionProvider.get_contactids_for_submitted_landsat_products',
           mock_production_provider.contact_ids_list)
    def test_production_handle_submitted_landsat_products(self):
        self.assertTrue(production_provider.handle_submitted_landsat_products())

    # !!! need to write test for nlaps.products_are_nlaps !!!
    @patch('api.external.nlaps.products_are_nlaps', nlaps.products_are_nlaps)
    @patch('api.providers.production.production_provider.ProductionProvider.set_products_unavailable',
           mock_production_provider.respond_true)
    def test_production_mark_nlaps_unavailable(self):
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        for scene in order.scenes():
            scene.status = 'submitted'
            scene.sensor_type = 'landsat'
            scene.save()
        self.assertTrue(production_provider.mark_nlaps_unavailable())

    @patch('api.external.lta.update_order_status', lta.update_order_status)
    def test_production_set_products_unavailable(self):
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        self.assertTrue(production_provider.set_products_unavailable(order.scenes(), "you want a reason?"))

    @patch('api.external.lta.order_scenes', lta.order_scenes)
    @patch('api.providers.production.production_provider.ProductionProvider.set_products_unavailable',
           mock_production_provider.respond_true)
    def test_production_update_landsat_product_status(self):
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        for scene in order.scenes():
            scene.status = 'submitted'
            scene.sensor_type = 'landsat'
            scene.save()
        self.assertTrue(production_provider.update_landsat_product_status(User.find(self.user_id).contactid))

    def test_production_get_contactids_for_submitted_landsat_products(self):
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        for scene in order.scenes():
            scene.status = 'submitted'
            scene.sensor_type = 'landsat'
            scene.save()
        response = production_provider.get_contactids_for_submitted_landsat_products()
        self.assertIsInstance(response, set)
        self.assertTrue(len(response) > 0)

    @patch('api.external.lpdaac.input_exists', lpdaac.input_exists_true)
    def test_production_handle_submitted_modis_products_input_exists(self):
        # handle oncache scenario
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        for scene in order.scenes():
            scene.status = 'submitted'
            scene.sensor_type = 'modis'
            scene.save()
            sid = scene.id
        self.assertTrue(production_provider.handle_submitted_modis_products())
        self.assertEquals(Scene.find(sid).status, "oncache")

    @patch('api.external.lpdaac.input_exists', lpdaac.input_exists_false)
    def test_production_handle_submitted_modis_products_input_missing(self):
        # handle unavailable scenario
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        for scene in order.scenes():
            scene.status = 'submitted'
            scene.sensor_type = 'modis'
            scene.save()
            sid = scene.id
        self.assertTrue(production_provider.handle_submitted_modis_products())
        self.assertEquals(Scene.find(sid).status, "unavailable")

    def test_production_handle_submitted_plot_products(self):
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        order.status = 'ordered'
        order.order_type = 'lpcs'
        order.save()
        plot_id = None
        for idx, scene in enumerate(order.scenes()):
            # at the moment, mock_order.generate_testing_order
            # creates 21 products for the order. divvy those
            # up between 'complete' and 'unavailable', setting
            # one aside as the 'plot' product
            if idx % 2 == 0:
                if idx == 0:
                    # need to define a plot product
                    scene.update('status', 'submitted')
                    scene.update('sensor_type', 'plot')
                    plot_id = scene.id
                else:
                    scene.update('status', 'complete')
            else:
                scene.update('status', 'unavailable')

        self.assertTrue(production_provider.handle_submitted_plot_products())
        self.assertEqual(Scene.find(plot_id).status, "oncache")

    @patch('os.path.getsize', lambda y: 999)
    def test_production_calc_scene_download_sizes(self):
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        scenes = order.scenes()
        Scene.bulk_update([s.id for s in scenes], {'status': 'complete', 'download_size': 0})
        self.assertTrue(production_provider.calc_scene_download_sizes())
        upscenes = Scene.where({'status': 'complete', 'download_size': 999})
        self.assertEqual(len(upscenes), len(scenes))

    @patch('api.providers.production.production_provider.ProductionProvider.update_order_if_complete',
           mock_production_provider.respond_true)
    def test_production_finalize_orders(self):
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        order.update('status', 'ordered')
        self.assertTrue(production_provider.finalize_orders())

    @patch('api.providers.production.production_provider.ProductionProvider.send_completion_email',
           mock_production_provider.respond_true)
    def test_production_update_order_if_complete(self):
        order = Order.find(self.mock_order.generate_testing_order(self.user_id))
        Scene.bulk_update([s.id for s in order.scenes()], {'status': 'retry'})
        order.order_source = 'espa'
        order.completion_email_sent = None
        order.save()
        self.assertTrue(production_provider.update_order_if_complete(order))

    def test_production_queue_products_success(self):
        names_tuple = self.mock_order.names_tuple(3, self.user_id)
        processing_loc = "get_products_to_process"
        job_name = 'jobname49'
        params = (names_tuple, processing_loc, job_name)
        response = api.queue_products(*params)
        self.assertTrue(response)

    def test_production_get_key(self):
        key = 'system_message_title'
        response = api.get_production_key(key)
        val = response[key]
        self.assertIsInstance(val, str)

    def test_get_production_key_invalid(self):
        bad_key = 'foobar'
        response = api.get_production_key(bad_key)
        self.assertEqual(response.keys(), ['msg'])

    @patch('api.external.hadoop.HadoopHandler.job_names_ids',
           hadoop.jobs_names_ids)
    def test_catch_orphaned_scenes(self):
        order_id = self.mock_order.generate_testing_order(self.user_id)
        # need scenes with statuses of 'queued'
        self.mock_order.update_scenes(order_id, 'status', ['queued'])
        response = production_provider.catch_orphaned_scenes()
        self.assertTrue(response)

        old_time = datetime.datetime.now() - datetime.timedelta(minutes=15)

        for s in Scene.where({'order_id': order_id}):
            self.assertTrue(s.reported_orphan is not None)
            s.reported_orphan = old_time
            s.save()

        response = production_provider.catch_orphaned_scenes()
        self.assertTrue(response)
        for s in Scene.where({'order_id': order_id}):
            self.assertTrue(s.orphaned)

    def test_convert_product_options(self):
        """
        Test the conversion procedure to make sure that the new format for orders converts
        to the old format
        """
        scenes = ['LE70480272012076EDC00', 'LC80500272013196LGN00',
                  'LT40480271983028PAC00', 'LT50490262009162PAC03']

        includes = ['include_sr', 'include_sr_toa',
                    'include_cfmask', 'include_sr_thermal']

        new_format = {u'etm7': {u'inputs': [u'LE70480272012076EDC00'],
                                u'products': [u'sr']},
                      u'olitirs8': {u'inputs': [u'LC80500272013196LGN00'],
                                    u'products': [u'toa']},
                      u'tm4': {u'inputs': [u'LT40480271983028PAC00'],
                               u'products': [u'cloud']},
                      u'tm5': {u'inputs': [u'LT50490262009162PAC03'],
                               u'products': [u'bt']},
                      u'format': u'gtiff',
                      u'image_extents': {u'east': -2265585.0,
                                         u'north': 3164805.0,
                                         u'south': 3014805.0,
                                         u'units': u'meters',
                                         u'west': -2415585.0},
                      u'note': u'CONUS_h1v1',

                      u'projection': {u'aea': {u'central_meridian': -96,
                                               u'datum': u'nad83',
                                               u'false_easting': 0,
                                               u'false_northing': 0,
                                               u'latitude_of_origin': 23,
                                               u'standard_parallel_1': 29.5,
                                               u'standard_parallel_2': 45.5}},
                      u'resampling_method': u'cc'}

        ruberic = {'central_meridian': -96,
                   'datum': u'nad83',
                   'false_easting': 0,
                   'false_northing': 0,
                   'image_extents': True,
                   'image_extents_units': u'meters',
                   'include_cfmask': False,
                   'include_customized_source_data': False,
                   'include_dswe': False,
                   'include_lst': False,
                   'include_solr_index': False,
                   'include_source_data': False,
                   'include_source_metadata': False,
                   'include_sr': False,
                   'include_sr_browse': False,
                   'include_sr_evi': False,
                   'include_sr_msavi': False,
                   'include_sr_nbr': False,
                   'include_sr_nbr2': False,
                   'include_sr_ndmi': False,
                   'include_sr_ndvi': False,
                   'include_sr_savi': False,
                   'include_sr_thermal': False,
                   'include_sr_toa': False,
                   'include_statistics': False,
                   'latitude_true_scale': None,
                   'longitude_pole': None,
                   'maxx': -2265585.0,
                   'maxy': 3164805.0,
                   'minx': -2415585.0,
                   'miny': 3014805.0,
                   'origin_lat': 23,
                   'output_format': u'gtiff',
                   'pixel_size': None,
                   'pixel_size_units': None,
                   'reproject': True,
                   'resample_method': 'cubic',
                   'resize': False,
                   'std_parallel_1': 29.5,
                   'std_parallel_2': 45.5,
                   'target_projection': u'aea',
                   'utm_north_south': None,
                   'utm_zone': None}

        for scene, include in zip(scenes, includes):
            ruberic[include] = True
            old_format = OptionsConversion.convert(new=new_format, scenes=[scene])

            self.assertDictEqual(ruberic, old_format)

            ruberic[include] = False