class BossIngestManagerCompleteTest(APITestCase):
"""
Test the completion process implemented by IngestManager.
"""
def setUp(self):
"""
Initialize the database
"""
# AWS region.
self.region = 'us-east-1'
dbsetup = SetupTestDB()
self.user = dbsetup.create_super_user(username='******',
email='*****@*****.**',
password='******')
dbsetup.set_user(self.user)
self.client.force_login(self.user)
dbsetup.insert_ingest_test_data()
SetupTests()
# Unit under test.
self.ingest_mgr = IngestManager()
def patch_ingest_mgr(self, name):
"""
Patch a method or attribute of self.ingest_manager.
Allows patching w/o using with so there's not many levels of nested
indentation.
Args:
name (str): Name of method or attribute to replace.
Returns:
(MagicMock): Mock or fake
"""
patch_wrapper = patch.object(self.ingest_mgr, name, autospec=True)
magic_mock = patch_wrapper.start()
# This ensures the patch is removed when the test is torn down.
self.addCleanup(patch_wrapper.stop)
return magic_mock
def make_fake_sqs_queues(self):
"""
Patch the SQS queues used by the ingest manager.
"""
upload_q = MagicMock(spec=UploadQueue)
upload_q.url = UPLOAD_QUEUE_URL
upload_q.region_name = self.region
upload_q.queue = MagicMock()
get_upload_q = self.patch_ingest_mgr('get_ingest_job_upload_queue')
get_upload_q.return_value = upload_q
ingest_q = MagicMock(spec=IngestQueue)
ingest_q.url = INGEST_QUEUE_URL
ingest_q.region_name = self.region
ingest_q.queue = MagicMock()
get_ingest_q = self.patch_ingest_mgr('get_ingest_job_ingest_queue')
get_ingest_q.return_value = ingest_q
tile_index_q = MagicMock(spec=TileIndexQueue)
tile_index_q.url = TILE_INDEX_QUEUE_URL
tile_index_q.region_name = self.region
tile_index_q.queue = MagicMock()
get_tile_index_q = self.patch_ingest_mgr(
'get_ingest_job_tile_index_queue')
get_tile_index_q.return_value = tile_index_q
tile_error_q = MagicMock(spec=TileErrorQueue)
tile_error_q.url = TILE_ERROR_QUEUE_URL
tile_error_q.region_name = self.region
tile_error_q.queue = MagicMock()
get_tile_error_q = self.patch_ingest_mgr(
'get_ingest_job_tile_error_queue')
get_tile_error_q.return_value = tile_error_q
def make_ingest_job(self, **kwargs):
"""
Create an ingest job for use in a test
Args:
kwargs: Keyword args to override the test defaults for the ingest job.
Returns:
(IngestJob)
"""
data = {
'status': IngestJob.UPLOADING,
'creator': self.user,
'resolution': 0,
'x_start': 0,
'y_start': 0,
'z_start': 0,
't_start': 0,
'x_stop': 10,
'y_stop': 10,
'z_stop': 10,
't_stop': 1,
'tile_size_x': 1024,
'tile_size_y': 1024,
'tile_size_z': 16,
'tile_size_t': 1,
'wait_on_queues_ts': None
}
for key, value in kwargs.items():
data[key] = value
job = IngestJob.objects.create(**data)
job.save()
return job
@patch('bossingest.ingest_manager.timezone', autospec=True)
def test_try_enter_wait_on_queue_state_success(self, fake_tz):
timestamp = datetime.now(timezone.utc)
fake_tz.now.return_value = timestamp
job = self.make_ingest_job(status=IngestJob.WAIT_ON_QUEUES,
wait_on_queues_ts=timestamp)
self.patch_ingest_mgr('ensure_queues_empty')
self.patch_ingest_mgr('_start_completion_activity')
actual = self.ingest_mgr.try_enter_wait_on_queue_state(job)
updated_job = self.ingest_mgr.get_ingest_job(job.id)
self.assertEqual(IngestJob.WAIT_ON_QUEUES, updated_job.status)
self.assertEqual(timestamp, updated_job.wait_on_queues_ts)
exp = {
'job_status': IngestJob.WAIT_ON_QUEUES,
'wait_secs': WAIT_FOR_QUEUES_SECS
}
self.assertDictEqual(exp, actual)
@patch('bossingest.ingest_manager.timezone', autospec=True)
def test_try_enter_wait_on_queue_state_already_there(self, fake_tz):
now_timestamp = datetime.now(timezone.utc)
fake_tz.now.return_value = now_timestamp
seconds_waiting = 100
# Time WAIT_ON_QUEUES entered.
wait_timestamp = now_timestamp - timedelta(seconds=seconds_waiting)
job = self.make_ingest_job(status=IngestJob.WAIT_ON_QUEUES,
wait_on_queues_ts=wait_timestamp)
self.patch_ingest_mgr('ensure_queues_empty')
actual = self.ingest_mgr.try_enter_wait_on_queue_state(job)
updated_job = self.ingest_mgr.get_ingest_job(job.id)
self.assertEqual(IngestJob.WAIT_ON_QUEUES, updated_job.status)
exp = {
'job_status': IngestJob.WAIT_ON_QUEUES,
'wait_secs': WAIT_FOR_QUEUES_SECS - seconds_waiting
}
self.assertDictEqual(exp, actual)
def test_try_enter_wait_on_queue_state_should_fail_if_upload_queue_not_empty(
self):
job = self.make_ingest_job(status=IngestJob.UPLOADING)
fake_ensure_q = self.patch_ingest_mgr('ensure_queues_empty')
fake_ensure_q.side_effect = BossError(UPLOAD_QUEUE_NOT_EMPTY_ERR_MSG,
ErrorCodes.BAD_REQUEST)
with self.assertRaises(BossError):
self.ingest_mgr.try_enter_wait_on_queue_state(job)
updated_job = self.ingest_mgr.get_ingest_job(job.id)
self.assertEqual(IngestJob.UPLOADING, updated_job.status)
@patch('bossingest.ingest_manager.timezone', autospec=True)
def test_try_start_completing_success_case(self, fake_tz):
now_timestamp = datetime.now(timezone.utc)
fake_tz.now.return_value = now_timestamp
seconds_waiting = WAIT_FOR_QUEUES_SECS + 2
# Time WAIT_ON_QUEUES entered.
wait_timestamp = now_timestamp - timedelta(seconds=seconds_waiting)
job = self.make_ingest_job(status=IngestJob.WAIT_ON_QUEUES,
wait_on_queues_ts=wait_timestamp)
self.patch_ingest_mgr('ensure_queues_empty')
self.patch_ingest_mgr('_start_completion_activity')
actual = self.ingest_mgr.try_start_completing(job)
updated_job = self.ingest_mgr.get_ingest_job(job.id)
self.assertEqual(IngestJob.COMPLETING, updated_job.status)
exp = {'job_status': IngestJob.COMPLETING, 'wait_secs': 0}
self.assertDictEqual(exp, actual)
def test_try_start_completing_should_fail_if_not_in_wait_on_queues_state(
self):
"""
This method can only be called when the ingest job status is WAIT_ON_QUEUES.
"""
job = self.make_ingest_job(status=IngestJob.UPLOADING)
self.patch_ingest_mgr('ensure_queues_empty')
self.patch_ingest_mgr('_start_completion_activity')
with self.assertRaises(BossError) as be:
self.ingest_mgr.try_start_completing(job)
actual = be.exception
self.assertEqual(400, actual.status_code)
self.assertEqual(ErrorCodes.BAD_REQUEST, actual.error_code)
self.assertEqual(NOT_IN_WAIT_ON_QUEUES_STATE_ERR_MSG, actual.message)
def test_try_start_completing_should_return_completing_if_already_completing(
self):
"""Should fail if already completing."""
job = self.make_ingest_job(status=IngestJob.COMPLETING)
self.patch_ingest_mgr('ensure_queues_empty')
self.patch_ingest_mgr('_start_completion_activity')
actual = self.ingest_mgr.try_start_completing(job)
self.assertEqual(IngestJob.COMPLETING, actual['job_status'])
@patch('bossingest.ingest_manager.timezone', autospec=True)
def test_try_start_completing_should_fail_if_queue_wait_period_not_expired(
self, fake_tz):
now_timestamp = datetime.now(timezone.utc)
fake_tz.now.return_value = now_timestamp
seconds_waiting = 138
# Time WAIT_ON_QUEUES entered.
wait_timestamp = now_timestamp - timedelta(seconds=seconds_waiting)
job = self.make_ingest_job(status=IngestJob.WAIT_ON_QUEUES,
wait_on_queues_ts=wait_timestamp)
self.patch_ingest_mgr('ensure_queues_empty')
self.patch_ingest_mgr('_start_completion_activity')
actual = self.ingest_mgr.try_start_completing(job)
exp = {
'job_status': IngestJob.WAIT_ON_QUEUES,
'wait_secs': WAIT_FOR_QUEUES_SECS - seconds_waiting
}
self.assertDictEqual(exp, actual)
@patch('bossingest.ingest_manager.get_sqs_num_msgs', autospec=True)
def test_try_start_completing_should_set_uploading_status_on_nonempty_upload_queue(
self, fake_get_sqs_num_msgs):
"""If the upload queue isn't empty, the job status should be set to UPLOADING."""
job = self.make_ingest_job(status=IngestJob.WAIT_ON_QUEUES)
fake_get_sqs_num_msgs.side_effect = make_fake_get_sqs_num_msgs([
(UPLOAD_QUEUE_URL, 1)
])
self.make_fake_sqs_queues()
self.patch_ingest_mgr('_start_completion_activity')
with self.assertRaises(BossError) as be:
self.ingest_mgr.try_start_completing(job)
actual = be.exception
self.assertEqual(400, actual.status_code)
self.assertEqual(ErrorCodes.BAD_REQUEST, actual.error_code)
self.assertEqual(UPLOAD_QUEUE_NOT_EMPTY_ERR_MSG, actual.message)
updated_job = self.ingest_mgr.get_ingest_job(job.id)
self.assertEqual(IngestJob.UPLOADING, updated_job.status)
@patch('bossingest.ingest_manager.get_sqs_num_msgs', autospec=True)
def test_ensure_queues_empty_should_fail_if_upload_queue_not_empty(
self, fake_get_sqs_num_msgs):
"""Should fail if the upload queue isn't empty."""
job = self.make_ingest_job(status=IngestJob.UPLOADING)
fake_get_sqs_num_msgs.side_effect = make_fake_get_sqs_num_msgs([
(UPLOAD_QUEUE_URL, 1)
])
self.make_fake_sqs_queues()
with self.assertRaises(BossError) as be:
self.ingest_mgr.ensure_queues_empty(job)
actual = be.exception
self.assertEqual(400, actual.status_code)
self.assertEqual(ErrorCodes.BAD_REQUEST, actual.error_code)
self.assertEqual(UPLOAD_QUEUE_NOT_EMPTY_ERR_MSG, actual.message)
@patch('bossingest.ingest_manager.get_sqs_num_msgs', autospec=True)
def test_ensure_queues_empty_should_fail_if_ingest_queue_not_empty(
self, fake_get_sqs_num_msgs):
"""Should fail if the ingest queue isn't empty."""
job = self.make_ingest_job(status=IngestJob.UPLOADING)
fake_get_sqs_num_msgs.side_effect = make_fake_get_sqs_num_msgs([
(INGEST_QUEUE_URL, 1)
])
self.make_fake_sqs_queues()
self.patch_ingest_mgr('lambda_connect_sqs')
with self.assertRaises(BossError) as be:
self.ingest_mgr.ensure_queues_empty(job)
actual = be.exception
self.assertEqual(400, actual.status_code)
self.assertEqual(ErrorCodes.BAD_REQUEST, actual.error_code)
self.assertEqual(INGEST_QUEUE_NOT_EMPTY_ERR_MSG, actual.message)
@patch('bossingest.ingest_manager.get_sqs_num_msgs', autospec=True)
def test_ensure_queues_empty_should_attach_ingest_lambda_if_ingest_queue_not_empty(
self, fake_get_sqs_num_msgs):
"""Should fail if the ingest queue isn't empty."""
job = self.make_ingest_job(status=IngestJob.UPLOADING)
fake_get_sqs_num_msgs.side_effect = make_fake_get_sqs_num_msgs([
(INGEST_QUEUE_URL, 1)
])
self.make_fake_sqs_queues()
fake_lambda_connect = self.patch_ingest_mgr('lambda_connect_sqs')
with self.assertRaises(BossError):
self.ingest_mgr.ensure_queues_empty(job)
self.assertEquals(fake_lambda_connect.call_args_list,
[call(ANY, INGEST_LAMBDA)])
@patch('bossingest.ingest_manager.get_sqs_num_msgs', autospec=True)
def test_ensure_queues_empty_should_fail_if_tile_index_queue_not_empty(
self, fake_get_sqs_num_msgs):
"""Should fail if the tile index queue isn't empty."""
job = self.make_ingest_job(status=IngestJob.UPLOADING)
fake_get_sqs_num_msgs.side_effect = make_fake_get_sqs_num_msgs([
(TILE_INDEX_QUEUE_URL, 1)
])
self.make_fake_sqs_queues()
with self.assertRaises(BossError) as be:
self.ingest_mgr.ensure_queues_empty(job)
actual = be.exception
self.assertEqual(400, actual.status_code)
self.assertEqual(ErrorCodes.BAD_REQUEST, actual.error_code)
self.assertEqual(TILE_INDEX_QUEUE_NOT_EMPTY_ERR_MSG, actual.message)
def test_start_completion_activity_exits_if_not_tile_ingest(self):
job = self.make_ingest_job(status=IngestJob.UPLOADING)
job.ingest_type = IngestJob.VOLUMETRIC_INGEST
self.assertIsNone(self.ingest_mgr._start_completion_activity(job))
def post(self, request, ingest_job_id):
"""
Signal an ingest job is complete and should be cleaned up by POSTing to this view
Args:
request: Django Rest framework Request object
ingest_job_id: Ingest job id
Returns:
"""
try:
blog = bossLogger()
ingest_mgmr = IngestManager()
ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id)
# Check if user is the ingest job creator or the sys admin
if not self.is_user_or_admin(request, ingest_job):
return BossHTTPError(
"Only the creator or admin can complete an ingest job",
ErrorCodes.INGEST_NOT_CREATOR)
if ingest_job.status == IngestJob.PREPARING:
# If status is Preparing. Deny
return BossHTTPError(
"You cannot complete a job that is still preparing. You must cancel instead.",
ErrorCodes.BAD_REQUEST)
elif ingest_job.status == IngestJob.UPLOADING:
try:
data = ingest_mgmr.try_enter_wait_on_queue_state(
ingest_job)
return Response(data=data, status=status.HTTP_202_ACCEPTED)
except BossError as be:
if (be.message == INGEST_QUEUE_NOT_EMPTY_ERR_MSG or
be.message == TILE_INDEX_QUEUE_NOT_EMPTY_ERR_MSG):
# If there are messages in the tile error queue, this
# will have to be handled manually. Non-empty ingest
# or tile index queues should resolve on their own.
return Response(data={
'wait_secs': WAIT_FOR_QUEUES_SECS,
'info': 'Internal queues not empty yet'
},
status=status.HTTP_400_BAD_REQUEST)
raise
elif ingest_job.status == IngestJob.WAIT_ON_QUEUES:
pass
# Continue below.
elif ingest_job.status == IngestJob.COMPLETE:
# If status is already Complete, just return another 204
return Response(data={'job_status': ingest_job.status},
status=status.HTTP_204_NO_CONTENT)
elif ingest_job.status == IngestJob.DELETED:
# Job had already been cancelled
return BossHTTPError("Ingest job has already been cancelled.",
ErrorCodes.BAD_REQUEST)
elif ingest_job.status == IngestJob.FAILED:
# Job had failed
return BossHTTPError(
"Ingest job has failed during creation. You must Cancel instead.",
ErrorCodes.BAD_REQUEST)
elif ingest_job.status == IngestJob.COMPLETING:
return Response(data={'job_status': ingest_job.status},
status=status.HTTP_202_ACCEPTED)
# Check if user is the ingest job creator or the sys admin
if not self.is_user_or_admin(request, ingest_job):
return BossHTTPError(
"Only the creator or admin can complete an ingest job",
ErrorCodes.INGEST_NOT_CREATOR)
# Try to start completing.
try:
data = ingest_mgmr.try_start_completing(ingest_job)
if data['job_status'] == IngestJob.WAIT_ON_QUEUES:
# Refuse complete requests until wait period expires.
return Response(data=data,
status=status.HTTP_400_BAD_REQUEST)
except BossError as be:
if (be.message == INGEST_QUEUE_NOT_EMPTY_ERR_MSG
or be.message == TILE_INDEX_QUEUE_NOT_EMPTY_ERR_MSG
or be.message == INGEST_QUEUE_NOT_EMPTY_ERR_MSG):
return Response(data={
'wait_secs': WAIT_FOR_QUEUES_SECS,
'info': 'Internal queues not empty yet'
},
status=status.HTTP_400_BAD_REQUEST)
raise
blog.info("Completion process started for ingest Job {}".format(
ingest_job_id))
return Response(data=data, status=status.HTTP_202_ACCEPTED)
# TODO SH This is a quick fix to make sure the ingest-client does not run close option.
# the clean up code commented out below, because it is not working correctly.
# return Response(status=status.HTTP_204_NO_CONTENT)
# if ingest_job.ingest_type == IngestJob.TILE_INGEST:
# # Check if any messages remain in the ingest queue
# ingest_queue = ingest_mgmr.get_ingest_job_ingest_queue(ingest_job)
# num_messages_in_queue = int(ingest_queue.queue.attributes['ApproximateNumberOfMessages'])
#
# # Kick off extra lambdas just in case
# if num_messages_in_queue:
# blog.info("{} messages remaining in Ingest Queue".format(num_messages_in_queue))
# ingest_mgmr.invoke_ingest_lambda(ingest_job, num_messages_in_queue)
#
# # Give lambda a few seconds to fire things off
# time.sleep(30)
#
# ingest_mgmr.cleanup_ingest_job(ingest_job, IngestJob.COMPLETE)
#
# elif ingest_job.ingest_type == IngestJob.VOLUMETRIC_INGEST:
# ingest_mgmr.cleanup_ingest_job(ingest_job, IngestJob.COMPLETE)
#
# # ToDo: call cleanup method for volumetric ingests. Don't want
# # to cleanup until after testing with real data.
# #ingest_mgmr.cleanup_ingest_job(ingest_job, IngestJob.COMPLETE)
#
# blog.info("Complete successful")
# return Response(status=status.HTTP_204_NO_CONTENT)
except BossError as err:
return err.to_http()
except Exception as err:
blog.error('Caught general exception: {}'.format(err))
return BossError("{}".format(err),
ErrorCodes.BOSS_SYSTEM_ERROR).to_http()