def delete(self, request, ingest_job_id):
"""
Args:
request:
ingest_job_id:
Returns:
"""
try:
ingest_mgmr = IngestManager()
ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id)
# Check permissions
if not self.is_user_or_admin(request, ingest_job):
return BossHTTPError(
"Only the creator or admin can cancel an ingest job",
ErrorCodes.INGEST_NOT_CREATOR)
# "DELETED" status is 3
ingest_mgmr.cleanup_ingest_job(ingest_job, 3)
blog = BossLogger().logger
blog.info("Deleted Ingest Job {}".format(ingest_job_id))
return Response(status=status.HTTP_204_NO_CONTENT)
except BossError as err:
return err.to_http()
def get(self, request, ingest_job_id):
"""
Get the status of an ingest_job and number of messages in the upload queue
Args:
request: Django Rest framework object
ingest_job_id: Ingest job id
Returns: Status of the job
"""
try:
ingest_mgmr = IngestManager()
ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id)
# Check if user is the ingest job creator or the sys admin
if not self.is_user_or_admin(request, ingest_job):
return BossHTTPError(
"Only the creator or admin can check the status of an ingest job",
ErrorCodes.INGEST_NOT_CREATOR)
if ingest_job.status == 3:
# Deleted Job
raise BossError(
"The job with id {} has been deleted".format(
ingest_job_id), ErrorCodes.INVALID_REQUEST)
else:
if ingest_job.status == 2:
# Job is Complete so queues are gone
num_messages_in_queue = 0
else:
upload_queue = ingest_mgmr.get_ingest_job_upload_queue(
ingest_job)
num_messages_in_queue = int(
upload_queue.queue.
attributes['ApproximateNumberOfMessages'])
if num_messages_in_queue < ingest_job.tile_count:
for n in range(9):
num_messages_in_queue += int(
upload_queue.queue.
attributes['ApproximateNumberOfMessages'])
num_messages_in_queue /= 10
data = {
"id": ingest_job.id,
"status": ingest_job.status,
"total_message_count": ingest_job.tile_count,
"current_message_count": int(num_messages_in_queue)
}
return Response(data, status=status.HTTP_200_OK)
except BossError as err:
return err.to_http()
except Exception as err:
return BossError("{}".format(err),
ErrorCodes.BOSS_SYSTEM_ERROR).to_http()
def get(self, request, ingest_job_id=None):
"""
Join a job with the specified job id or list all job ids if ingest_job_id is omitted
Args:
request: Django rest framework request object
ingest_job_id: Ingest job id
Returns:
Ingest job
"""
try:
if ingest_job_id is None:
# If the job ID is empty on a get, you are listing jobs
return self.list_ingest_jobs(request)
ingest_mgmr = IngestManager()
ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id)
# Check permissions
if not self.is_user_or_admin(request, ingest_job):
return BossHTTPError(
"Only the creator or admin can join an ingest job",
ErrorCodes.INGEST_NOT_CREATOR)
serializer = IngestJobListSerializer(ingest_job)
# Start setting up output
data = {'ingest_job': serializer.data}
if ingest_job.status == 3:
# The job has been deleted
raise BossError(
"The job with id {} has been deleted".format(
ingest_job_id), ErrorCodes.INVALID_REQUEST)
elif ingest_job.status == 2 or ingest_job.status == 4:
# Failed job or completed job
return Response(data, status=status.HTTP_200_OK)
elif ingest_job.status == 0:
# Job is still in progress
# check status of the step function
session = bossutils.aws.get_session()
if bossutils.aws.sfn_status(
session, ingest_job.step_function_arn) == 'SUCCEEDED':
# generate credentials
ingest_job.status = 1
ingest_job.save()
ingest_mgmr.generate_ingest_credentials(ingest_job)
elif bossutils.aws.sfn_status(
session, ingest_job.step_function_arn) == 'FAILED':
# This indicates an error in step function
raise BossError(
"Error generating ingest job messages"
" Delete the ingest job with id {} and try again.".
format(ingest_job_id), ErrorCodes.BOSS_SYSTEM_ERROR)
if ingest_job.status == 1:
data['ingest_job']['status'] = 1
ingest_creds = IngestCredentials()
data['credentials'] = ingest_creds.get_credentials(
ingest_job.id)
else:
data['credentials'] = None
data['tile_bucket_name'] = ingest_mgmr.get_tile_bucket()
data['KVIO_SETTINGS'] = settings.KVIO_SETTINGS
data['STATEIO_CONFIG'] = settings.STATEIO_CONFIG
data['OBJECTIO_CONFIG'] = settings.OBJECTIO_CONFIG
# add the lambda - Possibly remove this later
config = bossutils.configuration.BossConfig()
data['ingest_lambda'] = config["lambda"]["page_in_function"]
# Generate a "resource" for the ingest lambda function to be able to use SPDB cleanly
collection = Collection.objects.get(
name=data['ingest_job']["collection"])
experiment = Experiment.objects.get(
name=data['ingest_job']["experiment"], collection=collection)
coord_frame = experiment.coord_frame
channel = Channel.objects.get(name=data['ingest_job']["channel"],
experiment=experiment)
resource = {}
resource['boss_key'] = '{}&{}&{}'.format(
data['ingest_job']["collection"],
data['ingest_job']["experiment"],
data['ingest_job']["channel"])
resource['lookup_key'] = '{}&{}&{}'.format(collection.id,
experiment.id,
channel.id)
# The Lambda function needs certain resource properties to perform write ops. Set required things only.
# This is because S3 metadata is limited to 2kb, so we only set the bits of info needed, and in the lambda
# Function Populate the rest with dummy info
# IF YOU NEED ADDITIONAL DATA YOU MUST ADD IT HERE AND IN THE LAMBDA FUNCTION
resource['channel'] = {}
resource['channel']['type'] = channel.type
resource['channel']['datatype'] = channel.datatype
resource['channel']['base_resolution'] = channel.base_resolution
resource['experiment'] = {}
resource['experiment'][
'num_hierarchy_levels'] = experiment.num_hierarchy_levels
resource['experiment'][
'hierarchy_method'] = experiment.hierarchy_method
resource['coord_frame'] = {}
resource['coord_frame']['x_voxel_size'] = coord_frame.x_voxel_size
resource['coord_frame']['y_voxel_size'] = coord_frame.y_voxel_size
resource['coord_frame']['z_voxel_size'] = coord_frame.z_voxel_size
# Set resource
data['resource'] = resource
return Response(data, status=status.HTTP_200_OK)
except BossError as err:
return err.to_http()
except Exception as err:
return BossError("{}".format(err),
ErrorCodes.BOSS_SYSTEM_ERROR).to_http()
def post(self, request, ingest_job_id):
"""
Signal an ingest job is complete and should be cleaned up by POSTing to this view
Args:
request: Django Rest framework Request object
ingest_job_id: Ingest job id
Returns:
"""
try:
ingest_mgmr = IngestManager()
ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id)
if ingest_job.status == 0:
# If status is Preparing. Deny
return BossHTTPError(
"You cannot complete a job that is still preparing. You must cancel instead.",
ErrorCodes.BAD_REQUEST)
elif ingest_job.status == 1:
# If status is Uploading. Complete the job.
blog = BossLogger().logger
blog.info("Completing Ingest Job {}".format(ingest_job_id))
# Check if user is the ingest job creator or the sys admin
if not self.is_user_or_admin(request, ingest_job):
return BossHTTPError(
"Only the creator or admin can complete an ingest job",
ErrorCodes.INGEST_NOT_CREATOR)
# Check if any messages remain in the ingest queue
ingest_queue = ingest_mgmr.get_ingest_job_ingest_queue(
ingest_job)
num_messages_in_queue = int(
ingest_queue.queue.
attributes['ApproximateNumberOfMessages'])
# Kick off extra lambdas just in case
if num_messages_in_queue:
blog.info("{} messages remaining in Ingest Queue".format(
num_messages_in_queue))
ingest_mgmr.invoke_ingest_lambda(ingest_job,
num_messages_in_queue)
# Give lambda a few seconds to fire things off
time.sleep(30)
# "COMPLETE" status is 2
ingest_mgmr.cleanup_ingest_job(ingest_job, 2)
blog.info("Complete successful")
return Response(status=status.HTTP_204_NO_CONTENT)
elif ingest_job.status == 2:
# If status is already Complete, just return another 204
return Response(status=status.HTTP_204_NO_CONTENT)
elif ingest_job.status == 3:
# Job had already been cancelled
return BossHTTPError("Ingest job has already been cancelled.",
ErrorCodes.BAD_REQUEST)
elif ingest_job.status == 4:
# Job had failed
return BossHTTPError(
"Ingest job has failed during creation. You must Cancel instead.",
ErrorCodes.BAD_REQUEST)
except BossError as err:
return err.to_http()
class BossIngestManagerCompleteTest(APITestCase):
"""
Test the completion process implemented by IngestManager.
"""
def setUp(self):
"""
Initialize the database
"""
# AWS region.
self.region = 'us-east-1'
dbsetup = SetupTestDB()
self.user = dbsetup.create_super_user(username='******',
email='*****@*****.**',
password='******')
dbsetup.set_user(self.user)
self.client.force_login(self.user)
dbsetup.insert_ingest_test_data()
SetupTests()
# Unit under test.
self.ingest_mgr = IngestManager()
def patch_ingest_mgr(self, name):
"""
Patch a method or attribute of self.ingest_manager.
Allows patching w/o using with so there's not many levels of nested
indentation.
Args:
name (str): Name of method or attribute to replace.
Returns:
(MagicMock): Mock or fake
"""
patch_wrapper = patch.object(self.ingest_mgr, name, autospec=True)
magic_mock = patch_wrapper.start()
# This ensures the patch is removed when the test is torn down.
self.addCleanup(patch_wrapper.stop)
return magic_mock
def make_fake_sqs_queues(self):
"""
Patch the SQS queues used by the ingest manager.
"""
upload_q = MagicMock(spec=UploadQueue)
upload_q.url = UPLOAD_QUEUE_URL
upload_q.region_name = self.region
upload_q.queue = MagicMock()
get_upload_q = self.patch_ingest_mgr('get_ingest_job_upload_queue')
get_upload_q.return_value = upload_q
ingest_q = MagicMock(spec=IngestQueue)
ingest_q.url = INGEST_QUEUE_URL
ingest_q.region_name = self.region
ingest_q.queue = MagicMock()
get_ingest_q = self.patch_ingest_mgr('get_ingest_job_ingest_queue')
get_ingest_q.return_value = ingest_q
tile_index_q = MagicMock(spec=TileIndexQueue)
tile_index_q.url = TILE_INDEX_QUEUE_URL
tile_index_q.region_name = self.region
tile_index_q.queue = MagicMock()
get_tile_index_q = self.patch_ingest_mgr(
'get_ingest_job_tile_index_queue')
get_tile_index_q.return_value = tile_index_q
tile_error_q = MagicMock(spec=TileErrorQueue)
tile_error_q.url = TILE_ERROR_QUEUE_URL
tile_error_q.region_name = self.region
tile_error_q.queue = MagicMock()
get_tile_error_q = self.patch_ingest_mgr(
'get_ingest_job_tile_error_queue')
get_tile_error_q.return_value = tile_error_q
def make_ingest_job(self, **kwargs):
"""
Create an ingest job for use in a test
Args:
kwargs: Keyword args to override the test defaults for the ingest job.
Returns:
(IngestJob)
"""
data = {
'status': IngestJob.UPLOADING,
'creator': self.user,
'resolution': 0,
'x_start': 0,
'y_start': 0,
'z_start': 0,
't_start': 0,
'x_stop': 10,
'y_stop': 10,
'z_stop': 10,
't_stop': 1,
'tile_size_x': 1024,
'tile_size_y': 1024,
'tile_size_z': 16,
'tile_size_t': 1,
'wait_on_queues_ts': None
}
for key, value in kwargs.items():
data[key] = value
job = IngestJob.objects.create(**data)
job.save()
return job
@patch('bossingest.ingest_manager.timezone', autospec=True)
def test_try_enter_wait_on_queue_state_success(self, fake_tz):
timestamp = datetime.now(timezone.utc)
fake_tz.now.return_value = timestamp
job = self.make_ingest_job(status=IngestJob.WAIT_ON_QUEUES,
wait_on_queues_ts=timestamp)
self.patch_ingest_mgr('ensure_queues_empty')
self.patch_ingest_mgr('_start_completion_activity')
actual = self.ingest_mgr.try_enter_wait_on_queue_state(job)
updated_job = self.ingest_mgr.get_ingest_job(job.id)
self.assertEqual(IngestJob.WAIT_ON_QUEUES, updated_job.status)
self.assertEqual(timestamp, updated_job.wait_on_queues_ts)
exp = {
'job_status': IngestJob.WAIT_ON_QUEUES,
'wait_secs': WAIT_FOR_QUEUES_SECS
}
self.assertDictEqual(exp, actual)
@patch('bossingest.ingest_manager.timezone', autospec=True)
def test_try_enter_wait_on_queue_state_already_there(self, fake_tz):
now_timestamp = datetime.now(timezone.utc)
fake_tz.now.return_value = now_timestamp
seconds_waiting = 100
# Time WAIT_ON_QUEUES entered.
wait_timestamp = now_timestamp - timedelta(seconds=seconds_waiting)
job = self.make_ingest_job(status=IngestJob.WAIT_ON_QUEUES,
wait_on_queues_ts=wait_timestamp)
self.patch_ingest_mgr('ensure_queues_empty')
actual = self.ingest_mgr.try_enter_wait_on_queue_state(job)
updated_job = self.ingest_mgr.get_ingest_job(job.id)
self.assertEqual(IngestJob.WAIT_ON_QUEUES, updated_job.status)
exp = {
'job_status': IngestJob.WAIT_ON_QUEUES,
'wait_secs': WAIT_FOR_QUEUES_SECS - seconds_waiting
}
self.assertDictEqual(exp, actual)
def test_try_enter_wait_on_queue_state_should_fail_if_upload_queue_not_empty(
self):
job = self.make_ingest_job(status=IngestJob.UPLOADING)
fake_ensure_q = self.patch_ingest_mgr('ensure_queues_empty')
fake_ensure_q.side_effect = BossError(UPLOAD_QUEUE_NOT_EMPTY_ERR_MSG,
ErrorCodes.BAD_REQUEST)
with self.assertRaises(BossError):
self.ingest_mgr.try_enter_wait_on_queue_state(job)
updated_job = self.ingest_mgr.get_ingest_job(job.id)
self.assertEqual(IngestJob.UPLOADING, updated_job.status)
@patch('bossingest.ingest_manager.timezone', autospec=True)
def test_try_start_completing_success_case(self, fake_tz):
now_timestamp = datetime.now(timezone.utc)
fake_tz.now.return_value = now_timestamp
seconds_waiting = WAIT_FOR_QUEUES_SECS + 2
# Time WAIT_ON_QUEUES entered.
wait_timestamp = now_timestamp - timedelta(seconds=seconds_waiting)
job = self.make_ingest_job(status=IngestJob.WAIT_ON_QUEUES,
wait_on_queues_ts=wait_timestamp)
self.patch_ingest_mgr('ensure_queues_empty')
self.patch_ingest_mgr('_start_completion_activity')
actual = self.ingest_mgr.try_start_completing(job)
updated_job = self.ingest_mgr.get_ingest_job(job.id)
self.assertEqual(IngestJob.COMPLETING, updated_job.status)
exp = {'job_status': IngestJob.COMPLETING, 'wait_secs': 0}
self.assertDictEqual(exp, actual)
def test_try_start_completing_should_fail_if_not_in_wait_on_queues_state(
self):
"""
This method can only be called when the ingest job status is WAIT_ON_QUEUES.
"""
job = self.make_ingest_job(status=IngestJob.UPLOADING)
self.patch_ingest_mgr('ensure_queues_empty')
self.patch_ingest_mgr('_start_completion_activity')
with self.assertRaises(BossError) as be:
self.ingest_mgr.try_start_completing(job)
actual = be.exception
self.assertEqual(400, actual.status_code)
self.assertEqual(ErrorCodes.BAD_REQUEST, actual.error_code)
self.assertEqual(NOT_IN_WAIT_ON_QUEUES_STATE_ERR_MSG, actual.message)
def test_try_start_completing_should_return_completing_if_already_completing(
self):
"""Should fail if already completing."""
job = self.make_ingest_job(status=IngestJob.COMPLETING)
self.patch_ingest_mgr('ensure_queues_empty')
self.patch_ingest_mgr('_start_completion_activity')
actual = self.ingest_mgr.try_start_completing(job)
self.assertEqual(IngestJob.COMPLETING, actual['job_status'])
@patch('bossingest.ingest_manager.timezone', autospec=True)
def test_try_start_completing_should_fail_if_queue_wait_period_not_expired(
self, fake_tz):
now_timestamp = datetime.now(timezone.utc)
fake_tz.now.return_value = now_timestamp
seconds_waiting = 138
# Time WAIT_ON_QUEUES entered.
wait_timestamp = now_timestamp - timedelta(seconds=seconds_waiting)
job = self.make_ingest_job(status=IngestJob.WAIT_ON_QUEUES,
wait_on_queues_ts=wait_timestamp)
self.patch_ingest_mgr('ensure_queues_empty')
self.patch_ingest_mgr('_start_completion_activity')
actual = self.ingest_mgr.try_start_completing(job)
exp = {
'job_status': IngestJob.WAIT_ON_QUEUES,
'wait_secs': WAIT_FOR_QUEUES_SECS - seconds_waiting
}
self.assertDictEqual(exp, actual)
@patch('bossingest.ingest_manager.get_sqs_num_msgs', autospec=True)
def test_try_start_completing_should_set_uploading_status_on_nonempty_upload_queue(
self, fake_get_sqs_num_msgs):
"""If the upload queue isn't empty, the job status should be set to UPLOADING."""
job = self.make_ingest_job(status=IngestJob.WAIT_ON_QUEUES)
fake_get_sqs_num_msgs.side_effect = make_fake_get_sqs_num_msgs([
(UPLOAD_QUEUE_URL, 1)
])
self.make_fake_sqs_queues()
self.patch_ingest_mgr('_start_completion_activity')
with self.assertRaises(BossError) as be:
self.ingest_mgr.try_start_completing(job)
actual = be.exception
self.assertEqual(400, actual.status_code)
self.assertEqual(ErrorCodes.BAD_REQUEST, actual.error_code)
self.assertEqual(UPLOAD_QUEUE_NOT_EMPTY_ERR_MSG, actual.message)
updated_job = self.ingest_mgr.get_ingest_job(job.id)
self.assertEqual(IngestJob.UPLOADING, updated_job.status)
@patch('bossingest.ingest_manager.get_sqs_num_msgs', autospec=True)
def test_ensure_queues_empty_should_fail_if_upload_queue_not_empty(
self, fake_get_sqs_num_msgs):
"""Should fail if the upload queue isn't empty."""
job = self.make_ingest_job(status=IngestJob.UPLOADING)
fake_get_sqs_num_msgs.side_effect = make_fake_get_sqs_num_msgs([
(UPLOAD_QUEUE_URL, 1)
])
self.make_fake_sqs_queues()
with self.assertRaises(BossError) as be:
self.ingest_mgr.ensure_queues_empty(job)
actual = be.exception
self.assertEqual(400, actual.status_code)
self.assertEqual(ErrorCodes.BAD_REQUEST, actual.error_code)
self.assertEqual(UPLOAD_QUEUE_NOT_EMPTY_ERR_MSG, actual.message)
@patch('bossingest.ingest_manager.get_sqs_num_msgs', autospec=True)
def test_ensure_queues_empty_should_fail_if_ingest_queue_not_empty(
self, fake_get_sqs_num_msgs):
"""Should fail if the ingest queue isn't empty."""
job = self.make_ingest_job(status=IngestJob.UPLOADING)
fake_get_sqs_num_msgs.side_effect = make_fake_get_sqs_num_msgs([
(INGEST_QUEUE_URL, 1)
])
self.make_fake_sqs_queues()
self.patch_ingest_mgr('lambda_connect_sqs')
with self.assertRaises(BossError) as be:
self.ingest_mgr.ensure_queues_empty(job)
actual = be.exception
self.assertEqual(400, actual.status_code)
self.assertEqual(ErrorCodes.BAD_REQUEST, actual.error_code)
self.assertEqual(INGEST_QUEUE_NOT_EMPTY_ERR_MSG, actual.message)
@patch('bossingest.ingest_manager.get_sqs_num_msgs', autospec=True)
def test_ensure_queues_empty_should_attach_ingest_lambda_if_ingest_queue_not_empty(
self, fake_get_sqs_num_msgs):
"""Should fail if the ingest queue isn't empty."""
job = self.make_ingest_job(status=IngestJob.UPLOADING)
fake_get_sqs_num_msgs.side_effect = make_fake_get_sqs_num_msgs([
(INGEST_QUEUE_URL, 1)
])
self.make_fake_sqs_queues()
fake_lambda_connect = self.patch_ingest_mgr('lambda_connect_sqs')
with self.assertRaises(BossError):
self.ingest_mgr.ensure_queues_empty(job)
self.assertEquals(fake_lambda_connect.call_args_list,
[call(ANY, INGEST_LAMBDA)])
@patch('bossingest.ingest_manager.get_sqs_num_msgs', autospec=True)
def test_ensure_queues_empty_should_fail_if_tile_index_queue_not_empty(
self, fake_get_sqs_num_msgs):
"""Should fail if the tile index queue isn't empty."""
job = self.make_ingest_job(status=IngestJob.UPLOADING)
fake_get_sqs_num_msgs.side_effect = make_fake_get_sqs_num_msgs([
(TILE_INDEX_QUEUE_URL, 1)
])
self.make_fake_sqs_queues()
with self.assertRaises(BossError) as be:
self.ingest_mgr.ensure_queues_empty(job)
actual = be.exception
self.assertEqual(400, actual.status_code)
self.assertEqual(ErrorCodes.BAD_REQUEST, actual.error_code)
self.assertEqual(TILE_INDEX_QUEUE_NOT_EMPTY_ERR_MSG, actual.message)
def test_start_completion_activity_exits_if_not_tile_ingest(self):
job = self.make_ingest_job(status=IngestJob.UPLOADING)
job.ingest_type = IngestJob.VOLUMETRIC_INGEST
self.assertIsNone(self.ingest_mgr._start_completion_activity(job))
def post(self, request, ingest_job_id):
"""
Signal an ingest job is complete and should be cleaned up by POSTing to this view
Args:
request: Django Rest framework Request object
ingest_job_id: Ingest job id
Returns:
"""
try:
blog = BossLogger().logger
ingest_mgmr = IngestManager()
ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id)
if ingest_job.status == IngestJob.PREPARING:
# If status is Preparing. Deny
return BossHTTPError(
"You cannot complete a job that is still preparing. You must cancel instead.",
ErrorCodes.BAD_REQUEST)
elif ingest_job.status == IngestJob.UPLOADING:
# Check if user is the ingest job creator or the sys admin
if not self.is_user_or_admin(request, ingest_job):
return BossHTTPError(
"Only the creator or admin can start verification of an ingest job",
ErrorCodes.INGEST_NOT_CREATOR)
# Disable verification until it is reworked and always return
# success for now.
blog.info(
'Telling client job complete - completion/verificcation to be fixed later.'
)
return Response(status=status.HTTP_204_NO_CONTENT)
"""
blog.info('Verifying ingest job {}'.format(ingest_job_id))
# Start verification process
if not ingest_mgmr.verify_ingest_job(ingest_job):
# Ingest not finished
return Response(status=status.HTTP_202_ACCEPTED)
"""
# Verification successful, fall through to the complete process.
elif ingest_job.status == IngestJob.COMPLETE:
# If status is already Complete, just return another 204
return Response(status=status.HTTP_204_NO_CONTENT)
elif ingest_job.status == IngestJob.DELETED:
# Job had already been cancelled
return BossHTTPError("Ingest job has already been cancelled.",
ErrorCodes.BAD_REQUEST)
elif ingest_job.status == IngestJob.FAILED:
# Job had failed
return BossHTTPError(
"Ingest job has failed during creation. You must Cancel instead.",
ErrorCodes.BAD_REQUEST)
# Complete the job.
blog.info("Completing Ingest Job {}".format(ingest_job_id))
# Check if user is the ingest job creator or the sys admin
if not self.is_user_or_admin(request, ingest_job):
return BossHTTPError(
"Only the creator or admin can complete an ingest job",
ErrorCodes.INGEST_NOT_CREATOR)
# TODO SH This is a quick fix to make sure the ingest-client does not run close option.
# the clean up code commented out below, because it is not working correctly.
return Response(status=status.HTTP_204_NO_CONTENT)
# if ingest_job.ingest_type == IngestJob.TILE_INGEST:
# # Check if any messages remain in the ingest queue
# ingest_queue = ingest_mgmr.get_ingest_job_ingest_queue(ingest_job)
# num_messages_in_queue = int(ingest_queue.queue.attributes['ApproximateNumberOfMessages'])
#
# # Kick off extra lambdas just in case
# if num_messages_in_queue:
# blog.info("{} messages remaining in Ingest Queue".format(num_messages_in_queue))
# ingest_mgmr.invoke_ingest_lambda(ingest_job, num_messages_in_queue)
#
# # Give lambda a few seconds to fire things off
# time.sleep(30)
#
# ingest_mgmr.cleanup_ingest_job(ingest_job, IngestJob.COMPLETE)
#
# elif ingest_job.ingest_type == IngestJob.VOLUMETRIC_INGEST:
# ingest_mgmr.cleanup_ingest_job(ingest_job, IngestJob.COMPLETE)
#
# # ToDo: call cleanup method for volumetric ingests. Don't want
# # to cleanup until after testing with real data.
# #ingest_mgmr.cleanup_ingest_job(ingest_job, IngestJob.COMPLETE)
#
# blog.info("Complete successful")
# return Response(status=status.HTTP_204_NO_CONTENT)
except BossError as err:
return err.to_http()
except Exception as err:
blog.error('Caught general exception: {}'.format(err))
return BossError("{}".format(err),
ErrorCodes.BOSS_SYSTEM_ERROR).to_http()
def get(self, request, ingest_job_id):
"""
Args:
job_id:
Returns:
"""
try:
ingest_mgmr = IngestManager()
ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id)
serializer = IngestJobListSerializer(ingest_job)
print(serializer.data)
# Start setting up output
data = {}
data['ingest_job'] = serializer.data
if ingest_job.status == 3 or ingest_job.status == 2:
# Return the information for the deleted job/completed job
return Response(data, status=status.HTTP_200_OK)
elif ingest_job.status == 0:
# check if all message are in the upload queue
upload_queue = ingest_mgmr.get_ingest_job_upload_queue(
ingest_job)
if int(upload_queue.queue.
attributes['ApproximateNumberOfMessages']) == int(
ingest_job.tile_count):
#generate credentials
ingest_job.status = 1
ingest_job.save()
elif int(upload_queue.queue.
attributes['ApproximateNumberOfMessages']) > int(
ingest_job.tile_count):
# This indicates an error in the lambda
raise BossError(
"Error generating ingest job messages due to resources timing out ."
" Delete the ingest job with id {} and try again.".
format(ingest_job_id), ErrorCodes.BOSS_SYSTEM_ERROR)
if ingest_job.status == 1:
data['ingest_job']['status'] = 1
ingest_creds = IngestCredentials()
data['credentials'] = ingest_creds.get_credentials(
ingest_job.id)
else:
data['credentials'] = None
data['tile_bucket_name'] = ingest_mgmr.get_tile_bucket()
data['KVIO_SETTINGS'] = settings.KVIO_SETTINGS
data['STATEIO_CONFIG'] = settings.STATEIO_CONFIG
data['OBJECTIO_CONFIG'] = settings.OBJECTIO_CONFIG
# add the lambda - Possibly remove this later
config = bossutils.configuration.BossConfig()
data['ingest_lambda'] = config["lambda"]["page_in_function"]
# Generate a "resource" for the ingest lambda function to be able to use SPDB cleanly
collection = Collection.objects.get(
name=data['ingest_job']["collection"])
experiment = Experiment.objects.get(
name=data['ingest_job']["experiment"], collection=collection)
channel = Channel.objects.get(name=data['ingest_job']["channel"],
experiment=experiment)
resource = {}
resource['boss_key'] = '{}&{}&{}'.format(
data['ingest_job']["collection"],
data['ingest_job']["experiment"],
data['ingest_job']["channel"])
resource['lookup_key'] = '{}&{}&{}'.format(collection.id,
experiment.id,
channel.id)
resource['channel'] = {}
resource['channel']['name'] = channel.name
resource['channel']['description'] = ""
resource['channel']['type'] = channel.type
resource['channel']['datatype'] = channel.datatype
resource['channel']['base_resolution'] = channel.base_resolution
resource['channel']['sources'] = [
x.name for x in channel.sources.all()
]
resource['channel']['related'] = [
x.name for x in channel.related.all()
]
resource['channel'][
'default_time_sample'] = channel.default_time_sample
# Set resource
data['resource'] = resource
return Response(data, status=status.HTTP_200_OK)
except BossError as err:
return err.to_http()
except Exception as err:
return BossError("{}".format(err),
ErrorCodes.BOSS_SYSTEM_ERROR).to_http()
def post(self, request, ingest_job_id):
"""
Signal an ingest job is complete and should be cleaned up by POSTing to this view
Args:
request: Django Rest framework Request object
ingest_job_id: Ingest job id
Returns:
"""
try:
blog = bossLogger()
ingest_mgmr = IngestManager()
ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id)
# Check if user is the ingest job creator or the sys admin
if not self.is_user_or_admin(request, ingest_job):
return BossHTTPError(
"Only the creator or admin can complete an ingest job",
ErrorCodes.INGEST_NOT_CREATOR)
if ingest_job.status == IngestJob.PREPARING:
# If status is Preparing. Deny
return BossHTTPError(
"You cannot complete a job that is still preparing. You must cancel instead.",
ErrorCodes.BAD_REQUEST)
elif ingest_job.status == IngestJob.UPLOADING:
try:
data = ingest_mgmr.try_enter_wait_on_queue_state(
ingest_job)
return Response(data=data, status=status.HTTP_202_ACCEPTED)
except BossError as be:
if (be.message == INGEST_QUEUE_NOT_EMPTY_ERR_MSG or
be.message == TILE_INDEX_QUEUE_NOT_EMPTY_ERR_MSG):
# If there are messages in the tile error queue, this
# will have to be handled manually. Non-empty ingest
# or tile index queues should resolve on their own.
return Response(data={
'wait_secs': WAIT_FOR_QUEUES_SECS,
'info': 'Internal queues not empty yet'
},
status=status.HTTP_400_BAD_REQUEST)
raise
elif ingest_job.status == IngestJob.WAIT_ON_QUEUES:
pass
# Continue below.
elif ingest_job.status == IngestJob.COMPLETE:
# If status is already Complete, just return another 204
return Response(data={'job_status': ingest_job.status},
status=status.HTTP_204_NO_CONTENT)
elif ingest_job.status == IngestJob.DELETED:
# Job had already been cancelled
return BossHTTPError("Ingest job has already been cancelled.",
ErrorCodes.BAD_REQUEST)
elif ingest_job.status == IngestJob.FAILED:
# Job had failed
return BossHTTPError(
"Ingest job has failed during creation. You must Cancel instead.",
ErrorCodes.BAD_REQUEST)
elif ingest_job.status == IngestJob.COMPLETING:
return Response(data={'job_status': ingest_job.status},
status=status.HTTP_202_ACCEPTED)
# Check if user is the ingest job creator or the sys admin
if not self.is_user_or_admin(request, ingest_job):
return BossHTTPError(
"Only the creator or admin can complete an ingest job",
ErrorCodes.INGEST_NOT_CREATOR)
# Try to start completing.
try:
data = ingest_mgmr.try_start_completing(ingest_job)
if data['job_status'] == IngestJob.WAIT_ON_QUEUES:
# Refuse complete requests until wait period expires.
return Response(data=data,
status=status.HTTP_400_BAD_REQUEST)
except BossError as be:
if (be.message == INGEST_QUEUE_NOT_EMPTY_ERR_MSG
or be.message == TILE_INDEX_QUEUE_NOT_EMPTY_ERR_MSG
or be.message == INGEST_QUEUE_NOT_EMPTY_ERR_MSG):
return Response(data={
'wait_secs': WAIT_FOR_QUEUES_SECS,
'info': 'Internal queues not empty yet'
},
status=status.HTTP_400_BAD_REQUEST)
raise
blog.info("Completion process started for ingest Job {}".format(
ingest_job_id))
return Response(data=data, status=status.HTTP_202_ACCEPTED)
# TODO SH This is a quick fix to make sure the ingest-client does not run close option.
# the clean up code commented out below, because it is not working correctly.
# return Response(status=status.HTTP_204_NO_CONTENT)
# if ingest_job.ingest_type == IngestJob.TILE_INGEST:
# # Check if any messages remain in the ingest queue
# ingest_queue = ingest_mgmr.get_ingest_job_ingest_queue(ingest_job)
# num_messages_in_queue = int(ingest_queue.queue.attributes['ApproximateNumberOfMessages'])
#
# # Kick off extra lambdas just in case
# if num_messages_in_queue:
# blog.info("{} messages remaining in Ingest Queue".format(num_messages_in_queue))
# ingest_mgmr.invoke_ingest_lambda(ingest_job, num_messages_in_queue)
#
# # Give lambda a few seconds to fire things off
# time.sleep(30)
#
# ingest_mgmr.cleanup_ingest_job(ingest_job, IngestJob.COMPLETE)
#
# elif ingest_job.ingest_type == IngestJob.VOLUMETRIC_INGEST:
# ingest_mgmr.cleanup_ingest_job(ingest_job, IngestJob.COMPLETE)
#
# # ToDo: call cleanup method for volumetric ingests. Don't want
# # to cleanup until after testing with real data.
# #ingest_mgmr.cleanup_ingest_job(ingest_job, IngestJob.COMPLETE)
#
# blog.info("Complete successful")
# return Response(status=status.HTTP_204_NO_CONTENT)
except BossError as err:
return err.to_http()
except Exception as err:
blog.error('Caught general exception: {}'.format(err))
return BossError("{}".format(err),
ErrorCodes.BOSS_SYSTEM_ERROR).to_http()
def get(self, request, ingest_job_id=None):
"""
Join a job with the specified job id or list all job ids if ingest_job_id is omitted
Args:
request: Django rest framework request object
ingest_job_id: Ingest job id
Returns:
Ingest job
"""
try:
if ingest_job_id is None:
# If the job ID is empty on a get, you are listing jobs
return self.list_ingest_jobs(request)
ingest_mgmr = IngestManager()
ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id)
# Check permissions
if not self.is_user_or_admin(request, ingest_job):
return BossHTTPError(
"Only the creator or admin can join an ingest job",
ErrorCodes.INGEST_NOT_CREATOR)
serializer = IngestJobListSerializer(ingest_job)
# Start setting up output
data = {'ingest_job': serializer.data}
data['ingest_job']['tile_index_queue'] = None
if ingest_job.ingest_type == IngestJob.TILE_INGEST:
data['ingest_job'][
'tile_index_queue'] = ingest_mgmr.get_ingest_job_tile_index_queue(
ingest_job).url
if ingest_job.status == IngestJob.DELETED:
raise BossError(
"The job with id {} has been deleted".format(
ingest_job_id), ErrorCodes.INVALID_REQUEST)
elif ingest_job.status == IngestJob.COMPLETE or ingest_job.status == IngestJob.FAILED:
return Response(data, status=status.HTTP_200_OK)
elif ingest_job.status == IngestJob.PREPARING:
# check status of the step function
session = bossutils.aws.get_session()
if bossutils.aws.sfn_status(
session, ingest_job.step_function_arn) == 'SUCCEEDED':
# generate credentials
ingest_job.status = 1
ingest_job.save()
ingest_mgmr.generate_ingest_credentials(ingest_job)
elif bossutils.aws.sfn_status(
session, ingest_job.step_function_arn) == 'FAILED':
# This indicates an error in step function
raise BossError(
"Error generating ingest job messages"
" Delete the ingest job with id {} and try again.".
format(ingest_job_id), ErrorCodes.BOSS_SYSTEM_ERROR)
if ingest_job.status in [
IngestJob.UPLOADING, IngestJob.WAIT_ON_QUEUES,
IngestJob.COMPLETING
]:
data['ingest_job']['status'] = ingest_job.status
ingest_creds = IngestCredentials()
data['credentials'] = ingest_creds.get_credentials(
ingest_job.id)
else:
data['credentials'] = None
data['tile_bucket_name'] = ingest_mgmr.get_tile_bucket()
data['ingest_bucket_name'] = INGEST_BUCKET
data['KVIO_SETTINGS'] = settings.KVIO_SETTINGS
data['STATEIO_CONFIG'] = settings.STATEIO_CONFIG
data['OBJECTIO_CONFIG'] = settings.OBJECTIO_CONFIG
# Strip out un-needed data from OBJECTIO_CONFIG to save space when
# including in S3 metadata.
data['OBJECTIO_CONFIG'].pop('index_deadletter_queue', None)
data['OBJECTIO_CONFIG'].pop('index_cuboids_keys_queue', None)
# Set resource
data['resource'] = ingest_mgmr.get_resource_data(ingest_job_id)
# ingest_lambda is no longer required by the backend. The backend
# gets the name of the ingest lambda from boss-manage/lib/names.py.
# Keep providing it in case an older ingest client used (which
# still expects it).
data['ingest_lambda'] = 'deprecated'
return Response(data, status=status.HTTP_200_OK)
except BossError as err:
return err.to_http()
except Exception as err:
return BossError("{}".format(err),
ErrorCodes.BOSS_SYSTEM_ERROR).to_http()