def test_verify_ingest_job_not_ready(
self, fake_query_tile_ind, fake_patch_upl_q, fake_upload_q, fake_lookup_key, fake_get_region
):
"""Test false returned when chunks still remain in tile index"""
ingest_mgmr = IngestManager()
ingest_job = IngestJob()
ingest_job.status = IngestJob.UPLOADING
ingest_job.collection = 'test_coll'
ingest_job.experiment = 'test_exp'
ingest_job.channel = 'test_chan'
ingest_job.resolution = 0
ingest_job.id = 8
queue = MagicMock(spec=UploadQueue)
queue.queue = MagicMock()
fake_upload_q.return_value = queue
key = MagicMock()
key.lookup_key = '3&8&1'
fake_lookup_key.get_lookup_key.return_value = key
# Method under test.
actual = ingest_mgmr.verify_ingest_job(ingest_job)
self.assertFalse(actual)
self.assertEqual(IngestJob.UPLOADING, ingest_job.status)
def test_verify_ingest_job_good(self, fake_query_tile_ind, fake_get_region):
"""Test with no chunks left in tile index"""
ingest_mgmr = IngestManager()
ingest_job = IngestJob()
ingest_job.status = IngestJob.UPLOADING
with patch.object(ingest_job, 'save') as fake_save:
actual = ingest_mgmr.verify_ingest_job(ingest_job)
self.assertTrue(actual)
def post(self, request, ingest_job_id):
"""
Signal an ingest job is complete and should be cleaned up by POSTing to this view
Args:
request: Django Rest framework Request object
ingest_job_id: Ingest job id
Returns:
"""
try:
blog = BossLogger().logger
ingest_mgmr = IngestManager()
ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id)
if ingest_job.status == IngestJob.PREPARING:
# If status is Preparing. Deny
return BossHTTPError(
"You cannot complete a job that is still preparing. You must cancel instead.",
ErrorCodes.BAD_REQUEST)
elif ingest_job.status == IngestJob.UPLOADING:
# Check if user is the ingest job creator or the sys admin
if not self.is_user_or_admin(request, ingest_job):
return BossHTTPError(
"Only the creator or admin can start verification of an ingest job",
ErrorCodes.INGEST_NOT_CREATOR)
blog.info('Verifying ingest job {}'.format(ingest_job_id))
# Start verification process
if not ingest_mgmr.verify_ingest_job(ingest_job):
# Ingest not finished
return Response(status=status.HTTP_202_ACCEPTED)
# Verification successful, fall through to the complete process.
elif ingest_job.status == IngestJob.COMPLETE:
# If status is already Complete, just return another 204
return Response(status=status.HTTP_204_NO_CONTENT)
elif ingest_job.status == IngestJob.DELETED:
# Job had already been cancelled
return BossHTTPError("Ingest job has already been cancelled.",
ErrorCodes.BAD_REQUEST)
elif ingest_job.status == IngestJob.FAILED:
# Job had failed
return BossHTTPError(
"Ingest job has failed during creation. You must Cancel instead.",
ErrorCodes.BAD_REQUEST)
# Complete the job.
blog.info("Completing Ingest Job {}".format(ingest_job_id))
# Check if user is the ingest job creator or the sys admin
if not self.is_user_or_admin(request, ingest_job):
return BossHTTPError(
"Only the creator or admin can complete an ingest job",
ErrorCodes.INGEST_NOT_CREATOR)
if ingest_job.ingest_type == IngestJob.TILE_INGEST:
# Check if any messages remain in the ingest queue
ingest_queue = ingest_mgmr.get_ingest_job_ingest_queue(
ingest_job)
num_messages_in_queue = int(
ingest_queue.queue.
attributes['ApproximateNumberOfMessages'])
# Kick off extra lambdas just in case
if num_messages_in_queue:
blog.info("{} messages remaining in Ingest Queue".format(
num_messages_in_queue))
ingest_mgmr.invoke_ingest_lambda(ingest_job,
num_messages_in_queue)
# Give lambda a few seconds to fire things off
time.sleep(30)
ingest_mgmr.cleanup_ingest_job(ingest_job, IngestJob.COMPLETE)
elif ingest_job.ingest_type == IngestJob.VOLUMETRIC_INGEST:
ingest_mgmr.cleanup_ingest_job(ingest_job, IngestJob.COMPLETE)
# ToDo: call cleanup method for volumetric ingests. Don't want
# to cleanup until after testing with real data.
#ingest_mgmr.cleanup_ingest_job(ingest_job, IngestJob.COMPLETE)
blog.info("Complete successful")
return Response(status=status.HTTP_204_NO_CONTENT)
except BossError as err:
return err.to_http()
except Exception as err:
blog.error('Caught general exception: {}'.format(err))
return BossError("{}".format(err),
ErrorCodes.BOSS_SYSTEM_ERROR).to_http()