class BossIngestManagerCompleteTest(APITestCase): """ Test the completion process implemented by IngestManager. """ def setUp(self): """ Initialize the database """ # AWS region. self.region = 'us-east-1' dbsetup = SetupTestDB() self.user = dbsetup.create_super_user(username='******', email='*****@*****.**', password='******') dbsetup.set_user(self.user) self.client.force_login(self.user) dbsetup.insert_ingest_test_data() SetupTests() # Unit under test. self.ingest_mgr = IngestManager() def patch_ingest_mgr(self, name): """ Patch a method or attribute of self.ingest_manager. Allows patching w/o using with so there's not many levels of nested indentation. Args: name (str): Name of method or attribute to replace. Returns: (MagicMock): Mock or fake """ patch_wrapper = patch.object(self.ingest_mgr, name, autospec=True) magic_mock = patch_wrapper.start() # This ensures the patch is removed when the test is torn down. self.addCleanup(patch_wrapper.stop) return magic_mock def make_fake_sqs_queues(self): """ Patch the SQS queues used by the ingest manager. """ upload_q = MagicMock(spec=UploadQueue) upload_q.url = UPLOAD_QUEUE_URL upload_q.region_name = self.region upload_q.queue = MagicMock() get_upload_q = self.patch_ingest_mgr('get_ingest_job_upload_queue') get_upload_q.return_value = upload_q ingest_q = MagicMock(spec=IngestQueue) ingest_q.url = INGEST_QUEUE_URL ingest_q.region_name = self.region ingest_q.queue = MagicMock() get_ingest_q = self.patch_ingest_mgr('get_ingest_job_ingest_queue') get_ingest_q.return_value = ingest_q tile_index_q = MagicMock(spec=TileIndexQueue) tile_index_q.url = TILE_INDEX_QUEUE_URL tile_index_q.region_name = self.region tile_index_q.queue = MagicMock() get_tile_index_q = self.patch_ingest_mgr( 'get_ingest_job_tile_index_queue') get_tile_index_q.return_value = tile_index_q tile_error_q = MagicMock(spec=TileErrorQueue) tile_error_q.url = TILE_ERROR_QUEUE_URL tile_error_q.region_name = self.region tile_error_q.queue = MagicMock() get_tile_error_q = self.patch_ingest_mgr( 'get_ingest_job_tile_error_queue') get_tile_error_q.return_value = tile_error_q def make_ingest_job(self, **kwargs): """ Create an ingest job for use in a test Args: kwargs: Keyword args to override the test defaults for the ingest job. Returns: (IngestJob) """ data = { 'status': IngestJob.UPLOADING, 'creator': self.user, 'resolution': 0, 'x_start': 0, 'y_start': 0, 'z_start': 0, 't_start': 0, 'x_stop': 10, 'y_stop': 10, 'z_stop': 10, 't_stop': 1, 'tile_size_x': 1024, 'tile_size_y': 1024, 'tile_size_z': 16, 'tile_size_t': 1, 'wait_on_queues_ts': None } for key, value in kwargs.items(): data[key] = value job = IngestJob.objects.create(**data) job.save() return job @patch('bossingest.ingest_manager.timezone', autospec=True) def test_try_enter_wait_on_queue_state_success(self, fake_tz): timestamp = datetime.now(timezone.utc) fake_tz.now.return_value = timestamp job = self.make_ingest_job(status=IngestJob.WAIT_ON_QUEUES, wait_on_queues_ts=timestamp) self.patch_ingest_mgr('ensure_queues_empty') self.patch_ingest_mgr('_start_completion_activity') actual = self.ingest_mgr.try_enter_wait_on_queue_state(job) updated_job = self.ingest_mgr.get_ingest_job(job.id) self.assertEqual(IngestJob.WAIT_ON_QUEUES, updated_job.status) self.assertEqual(timestamp, updated_job.wait_on_queues_ts) exp = { 'job_status': IngestJob.WAIT_ON_QUEUES, 'wait_secs': WAIT_FOR_QUEUES_SECS } self.assertDictEqual(exp, actual) @patch('bossingest.ingest_manager.timezone', autospec=True) def test_try_enter_wait_on_queue_state_already_there(self, fake_tz): now_timestamp = datetime.now(timezone.utc) fake_tz.now.return_value = now_timestamp seconds_waiting = 100 # Time WAIT_ON_QUEUES entered. wait_timestamp = now_timestamp - timedelta(seconds=seconds_waiting) job = self.make_ingest_job(status=IngestJob.WAIT_ON_QUEUES, wait_on_queues_ts=wait_timestamp) self.patch_ingest_mgr('ensure_queues_empty') actual = self.ingest_mgr.try_enter_wait_on_queue_state(job) updated_job = self.ingest_mgr.get_ingest_job(job.id) self.assertEqual(IngestJob.WAIT_ON_QUEUES, updated_job.status) exp = { 'job_status': IngestJob.WAIT_ON_QUEUES, 'wait_secs': WAIT_FOR_QUEUES_SECS - seconds_waiting } self.assertDictEqual(exp, actual) def test_try_enter_wait_on_queue_state_should_fail_if_upload_queue_not_empty( self): job = self.make_ingest_job(status=IngestJob.UPLOADING) fake_ensure_q = self.patch_ingest_mgr('ensure_queues_empty') fake_ensure_q.side_effect = BossError(UPLOAD_QUEUE_NOT_EMPTY_ERR_MSG, ErrorCodes.BAD_REQUEST) with self.assertRaises(BossError): self.ingest_mgr.try_enter_wait_on_queue_state(job) updated_job = self.ingest_mgr.get_ingest_job(job.id) self.assertEqual(IngestJob.UPLOADING, updated_job.status) @patch('bossingest.ingest_manager.timezone', autospec=True) def test_try_start_completing_success_case(self, fake_tz): now_timestamp = datetime.now(timezone.utc) fake_tz.now.return_value = now_timestamp seconds_waiting = WAIT_FOR_QUEUES_SECS + 2 # Time WAIT_ON_QUEUES entered. wait_timestamp = now_timestamp - timedelta(seconds=seconds_waiting) job = self.make_ingest_job(status=IngestJob.WAIT_ON_QUEUES, wait_on_queues_ts=wait_timestamp) self.patch_ingest_mgr('ensure_queues_empty') self.patch_ingest_mgr('_start_completion_activity') actual = self.ingest_mgr.try_start_completing(job) updated_job = self.ingest_mgr.get_ingest_job(job.id) self.assertEqual(IngestJob.COMPLETING, updated_job.status) exp = {'job_status': IngestJob.COMPLETING, 'wait_secs': 0} self.assertDictEqual(exp, actual) def test_try_start_completing_should_fail_if_not_in_wait_on_queues_state( self): """ This method can only be called when the ingest job status is WAIT_ON_QUEUES. """ job = self.make_ingest_job(status=IngestJob.UPLOADING) self.patch_ingest_mgr('ensure_queues_empty') self.patch_ingest_mgr('_start_completion_activity') with self.assertRaises(BossError) as be: self.ingest_mgr.try_start_completing(job) actual = be.exception self.assertEqual(400, actual.status_code) self.assertEqual(ErrorCodes.BAD_REQUEST, actual.error_code) self.assertEqual(NOT_IN_WAIT_ON_QUEUES_STATE_ERR_MSG, actual.message) def test_try_start_completing_should_return_completing_if_already_completing( self): """Should fail if already completing.""" job = self.make_ingest_job(status=IngestJob.COMPLETING) self.patch_ingest_mgr('ensure_queues_empty') self.patch_ingest_mgr('_start_completion_activity') actual = self.ingest_mgr.try_start_completing(job) self.assertEqual(IngestJob.COMPLETING, actual['job_status']) @patch('bossingest.ingest_manager.timezone', autospec=True) def test_try_start_completing_should_fail_if_queue_wait_period_not_expired( self, fake_tz): now_timestamp = datetime.now(timezone.utc) fake_tz.now.return_value = now_timestamp seconds_waiting = 138 # Time WAIT_ON_QUEUES entered. wait_timestamp = now_timestamp - timedelta(seconds=seconds_waiting) job = self.make_ingest_job(status=IngestJob.WAIT_ON_QUEUES, wait_on_queues_ts=wait_timestamp) self.patch_ingest_mgr('ensure_queues_empty') self.patch_ingest_mgr('_start_completion_activity') actual = self.ingest_mgr.try_start_completing(job) exp = { 'job_status': IngestJob.WAIT_ON_QUEUES, 'wait_secs': WAIT_FOR_QUEUES_SECS - seconds_waiting } self.assertDictEqual(exp, actual) @patch('bossingest.ingest_manager.get_sqs_num_msgs', autospec=True) def test_try_start_completing_should_set_uploading_status_on_nonempty_upload_queue( self, fake_get_sqs_num_msgs): """If the upload queue isn't empty, the job status should be set to UPLOADING.""" job = self.make_ingest_job(status=IngestJob.WAIT_ON_QUEUES) fake_get_sqs_num_msgs.side_effect = make_fake_get_sqs_num_msgs([ (UPLOAD_QUEUE_URL, 1) ]) self.make_fake_sqs_queues() self.patch_ingest_mgr('_start_completion_activity') with self.assertRaises(BossError) as be: self.ingest_mgr.try_start_completing(job) actual = be.exception self.assertEqual(400, actual.status_code) self.assertEqual(ErrorCodes.BAD_REQUEST, actual.error_code) self.assertEqual(UPLOAD_QUEUE_NOT_EMPTY_ERR_MSG, actual.message) updated_job = self.ingest_mgr.get_ingest_job(job.id) self.assertEqual(IngestJob.UPLOADING, updated_job.status) @patch('bossingest.ingest_manager.get_sqs_num_msgs', autospec=True) def test_ensure_queues_empty_should_fail_if_upload_queue_not_empty( self, fake_get_sqs_num_msgs): """Should fail if the upload queue isn't empty.""" job = self.make_ingest_job(status=IngestJob.UPLOADING) fake_get_sqs_num_msgs.side_effect = make_fake_get_sqs_num_msgs([ (UPLOAD_QUEUE_URL, 1) ]) self.make_fake_sqs_queues() with self.assertRaises(BossError) as be: self.ingest_mgr.ensure_queues_empty(job) actual = be.exception self.assertEqual(400, actual.status_code) self.assertEqual(ErrorCodes.BAD_REQUEST, actual.error_code) self.assertEqual(UPLOAD_QUEUE_NOT_EMPTY_ERR_MSG, actual.message) @patch('bossingest.ingest_manager.get_sqs_num_msgs', autospec=True) def test_ensure_queues_empty_should_fail_if_ingest_queue_not_empty( self, fake_get_sqs_num_msgs): """Should fail if the ingest queue isn't empty.""" job = self.make_ingest_job(status=IngestJob.UPLOADING) fake_get_sqs_num_msgs.side_effect = make_fake_get_sqs_num_msgs([ (INGEST_QUEUE_URL, 1) ]) self.make_fake_sqs_queues() self.patch_ingest_mgr('lambda_connect_sqs') with self.assertRaises(BossError) as be: self.ingest_mgr.ensure_queues_empty(job) actual = be.exception self.assertEqual(400, actual.status_code) self.assertEqual(ErrorCodes.BAD_REQUEST, actual.error_code) self.assertEqual(INGEST_QUEUE_NOT_EMPTY_ERR_MSG, actual.message) @patch('bossingest.ingest_manager.get_sqs_num_msgs', autospec=True) def test_ensure_queues_empty_should_attach_ingest_lambda_if_ingest_queue_not_empty( self, fake_get_sqs_num_msgs): """Should fail if the ingest queue isn't empty.""" job = self.make_ingest_job(status=IngestJob.UPLOADING) fake_get_sqs_num_msgs.side_effect = make_fake_get_sqs_num_msgs([ (INGEST_QUEUE_URL, 1) ]) self.make_fake_sqs_queues() fake_lambda_connect = self.patch_ingest_mgr('lambda_connect_sqs') with self.assertRaises(BossError): self.ingest_mgr.ensure_queues_empty(job) self.assertEquals(fake_lambda_connect.call_args_list, [call(ANY, INGEST_LAMBDA)]) @patch('bossingest.ingest_manager.get_sqs_num_msgs', autospec=True) def test_ensure_queues_empty_should_fail_if_tile_index_queue_not_empty( self, fake_get_sqs_num_msgs): """Should fail if the tile index queue isn't empty.""" job = self.make_ingest_job(status=IngestJob.UPLOADING) fake_get_sqs_num_msgs.side_effect = make_fake_get_sqs_num_msgs([ (TILE_INDEX_QUEUE_URL, 1) ]) self.make_fake_sqs_queues() with self.assertRaises(BossError) as be: self.ingest_mgr.ensure_queues_empty(job) actual = be.exception self.assertEqual(400, actual.status_code) self.assertEqual(ErrorCodes.BAD_REQUEST, actual.error_code) self.assertEqual(TILE_INDEX_QUEUE_NOT_EMPTY_ERR_MSG, actual.message) def test_start_completion_activity_exits_if_not_tile_ingest(self): job = self.make_ingest_job(status=IngestJob.UPLOADING) job.ingest_type = IngestJob.VOLUMETRIC_INGEST self.assertIsNone(self.ingest_mgr._start_completion_activity(job))
def post(self, request, ingest_job_id): """ Signal an ingest job is complete and should be cleaned up by POSTing to this view Args: request: Django Rest framework Request object ingest_job_id: Ingest job id Returns: """ try: blog = bossLogger() ingest_mgmr = IngestManager() ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id) # Check if user is the ingest job creator or the sys admin if not self.is_user_or_admin(request, ingest_job): return BossHTTPError( "Only the creator or admin can complete an ingest job", ErrorCodes.INGEST_NOT_CREATOR) if ingest_job.status == IngestJob.PREPARING: # If status is Preparing. Deny return BossHTTPError( "You cannot complete a job that is still preparing. You must cancel instead.", ErrorCodes.BAD_REQUEST) elif ingest_job.status == IngestJob.UPLOADING: try: data = ingest_mgmr.try_enter_wait_on_queue_state( ingest_job) return Response(data=data, status=status.HTTP_202_ACCEPTED) except BossError as be: if (be.message == INGEST_QUEUE_NOT_EMPTY_ERR_MSG or be.message == TILE_INDEX_QUEUE_NOT_EMPTY_ERR_MSG): # If there are messages in the tile error queue, this # will have to be handled manually. Non-empty ingest # or tile index queues should resolve on their own. return Response(data={ 'wait_secs': WAIT_FOR_QUEUES_SECS, 'info': 'Internal queues not empty yet' }, status=status.HTTP_400_BAD_REQUEST) raise elif ingest_job.status == IngestJob.WAIT_ON_QUEUES: pass # Continue below. elif ingest_job.status == IngestJob.COMPLETE: # If status is already Complete, just return another 204 return Response(data={'job_status': ingest_job.status}, status=status.HTTP_204_NO_CONTENT) elif ingest_job.status == IngestJob.DELETED: # Job had already been cancelled return BossHTTPError("Ingest job has already been cancelled.", ErrorCodes.BAD_REQUEST) elif ingest_job.status == IngestJob.FAILED: # Job had failed return BossHTTPError( "Ingest job has failed during creation. You must Cancel instead.", ErrorCodes.BAD_REQUEST) elif ingest_job.status == IngestJob.COMPLETING: return Response(data={'job_status': ingest_job.status}, status=status.HTTP_202_ACCEPTED) # Check if user is the ingest job creator or the sys admin if not self.is_user_or_admin(request, ingest_job): return BossHTTPError( "Only the creator or admin can complete an ingest job", ErrorCodes.INGEST_NOT_CREATOR) # Try to start completing. try: data = ingest_mgmr.try_start_completing(ingest_job) if data['job_status'] == IngestJob.WAIT_ON_QUEUES: # Refuse complete requests until wait period expires. return Response(data=data, status=status.HTTP_400_BAD_REQUEST) except BossError as be: if (be.message == INGEST_QUEUE_NOT_EMPTY_ERR_MSG or be.message == TILE_INDEX_QUEUE_NOT_EMPTY_ERR_MSG or be.message == INGEST_QUEUE_NOT_EMPTY_ERR_MSG): return Response(data={ 'wait_secs': WAIT_FOR_QUEUES_SECS, 'info': 'Internal queues not empty yet' }, status=status.HTTP_400_BAD_REQUEST) raise blog.info("Completion process started for ingest Job {}".format( ingest_job_id)) return Response(data=data, status=status.HTTP_202_ACCEPTED) # TODO SH This is a quick fix to make sure the ingest-client does not run close option. # the clean up code commented out below, because it is not working correctly. # return Response(status=status.HTTP_204_NO_CONTENT) # if ingest_job.ingest_type == IngestJob.TILE_INGEST: # # Check if any messages remain in the ingest queue # ingest_queue = ingest_mgmr.get_ingest_job_ingest_queue(ingest_job) # num_messages_in_queue = int(ingest_queue.queue.attributes['ApproximateNumberOfMessages']) # # # Kick off extra lambdas just in case # if num_messages_in_queue: # blog.info("{} messages remaining in Ingest Queue".format(num_messages_in_queue)) # ingest_mgmr.invoke_ingest_lambda(ingest_job, num_messages_in_queue) # # # Give lambda a few seconds to fire things off # time.sleep(30) # # ingest_mgmr.cleanup_ingest_job(ingest_job, IngestJob.COMPLETE) # # elif ingest_job.ingest_type == IngestJob.VOLUMETRIC_INGEST: # ingest_mgmr.cleanup_ingest_job(ingest_job, IngestJob.COMPLETE) # # # ToDo: call cleanup method for volumetric ingests. Don't want # # to cleanup until after testing with real data. # #ingest_mgmr.cleanup_ingest_job(ingest_job, IngestJob.COMPLETE) # # blog.info("Complete successful") # return Response(status=status.HTTP_204_NO_CONTENT) except BossError as err: return err.to_http() except Exception as err: blog.error('Caught general exception: {}'.format(err)) return BossError("{}".format(err), ErrorCodes.BOSS_SYSTEM_ERROR).to_http()