def get(self, request, ingest_job_id=None):
"""
Join a job with the specified job id or list all job ids if ingest_job_id is omitted
Args:
request: Django rest framework request object
ingest_job_id: Ingest job id
Returns:
Ingest job
"""
try:
if ingest_job_id is None:
# If the job ID is empty on a get, you are listing jobs
return self.list_ingest_jobs(request)
ingest_mgmr = IngestManager()
ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id)
# Check permissions
if not self.is_user_or_admin(request, ingest_job):
return BossHTTPError(
"Only the creator or admin can join an ingest job",
ErrorCodes.INGEST_NOT_CREATOR)
serializer = IngestJobListSerializer(ingest_job)
# Start setting up output
data = {'ingest_job': serializer.data}
data['ingest_job']['tile_index_queue'] = None
if ingest_job.ingest_type == IngestJob.TILE_INGEST:
data['ingest_job'][
'tile_index_queue'] = ingest_mgmr.get_ingest_job_tile_index_queue(
ingest_job).url
if ingest_job.status == IngestJob.DELETED:
raise BossError(
"The job with id {} has been deleted".format(
ingest_job_id), ErrorCodes.INVALID_REQUEST)
elif ingest_job.status == IngestJob.COMPLETE or ingest_job.status == IngestJob.FAILED:
return Response(data, status=status.HTTP_200_OK)
elif ingest_job.status == IngestJob.PREPARING:
# check status of the step function
session = bossutils.aws.get_session()
if bossutils.aws.sfn_status(
session, ingest_job.step_function_arn) == 'SUCCEEDED':
# generate credentials
ingest_job.status = 1
ingest_job.save()
ingest_mgmr.generate_ingest_credentials(ingest_job)
elif bossutils.aws.sfn_status(
session, ingest_job.step_function_arn) == 'FAILED':
# This indicates an error in step function
raise BossError(
"Error generating ingest job messages"
" Delete the ingest job with id {} and try again.".
format(ingest_job_id), ErrorCodes.BOSS_SYSTEM_ERROR)
if ingest_job.status in [
IngestJob.UPLOADING, IngestJob.WAIT_ON_QUEUES,
IngestJob.COMPLETING
]:
data['ingest_job']['status'] = ingest_job.status
ingest_creds = IngestCredentials()
data['credentials'] = ingest_creds.get_credentials(
ingest_job.id)
else:
data['credentials'] = None
data['tile_bucket_name'] = ingest_mgmr.get_tile_bucket()
data['ingest_bucket_name'] = INGEST_BUCKET
data['KVIO_SETTINGS'] = settings.KVIO_SETTINGS
data['STATEIO_CONFIG'] = settings.STATEIO_CONFIG
data['OBJECTIO_CONFIG'] = settings.OBJECTIO_CONFIG
# Strip out un-needed data from OBJECTIO_CONFIG to save space when
# including in S3 metadata.
data['OBJECTIO_CONFIG'].pop('index_deadletter_queue', None)
data['OBJECTIO_CONFIG'].pop('index_cuboids_keys_queue', None)
# Set resource
data['resource'] = ingest_mgmr.get_resource_data(ingest_job_id)
# ingest_lambda is no longer required by the backend. The backend
# gets the name of the ingest lambda from boss-manage/lib/names.py.
# Keep providing it in case an older ingest client used (which
# still expects it).
data['ingest_lambda'] = 'deprecated'
return Response(data, status=status.HTTP_200_OK)
except BossError as err:
return err.to_http()
except Exception as err:
return BossError("{}".format(err),
ErrorCodes.BOSS_SYSTEM_ERROR).to_http()
class BossIngestManagerTest(APITestCase):
def setUp(self):
"""
Initialize the database
:return:
"""
dbsetup = SetupTestDB()
self.user = dbsetup.create_super_user(username='******', email='*****@*****.**', password='******')
dbsetup.set_user(self.user)
self.client.force_login(self.user)
dbsetup.insert_ingest_test_data()
setup = SetupTests()
# Get the config_data for v1 schema
config_data = setup.get_ingest_config_data_dict()
self.example_config_data = config_data
self.volumetric_config_data = setup.get_ingest_config_data_dict_volumetric()
# Unit under test.
self.ingest_mgr = IngestManager()
def test_validate_ingest(self):
"""Method to test validation method"""
#Validate schema and config file
response = self.ingest_mgr.validate_config_file(self.example_config_data)
assert (response is True)
#Validate properties
response = self.ingest_mgr.validate_properties()
assert (response is True)
def test_validate_config_file(self):
"""Method to test validation of a config file"""
self.ingest_mgr.validate_config_file(self.example_config_data)
assert(self.ingest_mgr.config is not None)
assert (self.ingest_mgr.config.config_data is not None)
def test_validate_properties(self):
"""Methos to test validation of properties of the config data"""
self.ingest_mgr.validate_config_file(self.example_config_data)
self.ingest_mgr.validate_properties()
assert (self.ingest_mgr.collection.name == 'my_col_1')
assert (self.ingest_mgr.experiment.name == 'my_exp_1')
assert (self.ingest_mgr.channel.name == 'my_ch_1')
def test_create_ingest_job(self):
"""Method to test creation of a ingest job from a config_data dict"""
self.ingest_mgr.validate_config_file(self.example_config_data)
self.ingest_mgr.validate_properties()
self.ingest_mgr.owner = self.user.pk
job = self.ingest_mgr.create_ingest_job()
assert (job.id is not None)
assert (job.ingest_type == IngestJob.TILE_INGEST)
assert (job.tile_size_x == 512)
assert (job.tile_size_y == 512)
assert (job.tile_size_z == 1)
assert (job.tile_size_t == 1)
def test_create_ingest_job_volumetric(self):
self.ingest_mgr.validate_config_file(self.volumetric_config_data)
self.ingest_mgr.validate_properties()
self.ingest_mgr.owner = self.user.pk
job = self.ingest_mgr.create_ingest_job()
assert (job.id is not None)
assert (job.ingest_type == IngestJob.VOLUMETRIC_INGEST)
assert (job.tile_size_x == 1024)
assert (job.tile_size_y == 1024)
assert (job.tile_size_z == 64)
assert (job.tile_size_t == 1)
def test_generate_upload_queue_args_tile_job(self):
"""Ensure ingest_type set properly"""
self.ingest_mgr.validate_config_file(self.example_config_data)
self.ingest_mgr.validate_properties()
self.ingest_mgr.owner = self.user.pk
job = self.ingest_mgr.create_ingest_job()
actual = self.ingest_mgr._generate_upload_queue_args(job)
assert actual['ingest_type'] == IngestJob.TILE_INGEST
assert actual['z_chunk_size'] == 16
def test_generate_upload_queue_args_volumetric_job(self):
"""Ensure ingest_type set properly"""
self.ingest_mgr.validate_config_file(self.volumetric_config_data)
self.ingest_mgr.validate_properties()
self.ingest_mgr.owner = self.user.pk
job = self.ingest_mgr.create_ingest_job()
actual = self.ingest_mgr._generate_upload_queue_args(job)
assert actual['ingest_type'] == IngestJob.VOLUMETRIC_INGEST
assert actual['z_chunk_size'] == 64
assert actual['ingest_queue'] is None
def test_tile_bucket_name(self):
""" Test get tile bucket name"""
tile_bucket_name = self.ingest_mgr.get_tile_bucket()
assert(tile_bucket_name is not None)
def test_get_resource_data(self):
"""Run the method and ensure keys set"""
self.ingest_mgr.validate_config_file(self.example_config_data)
self.ingest_mgr.validate_properties()
self.ingest_mgr.owner = self.user.pk
job = self.ingest_mgr.create_ingest_job()
actual = self.ingest_mgr.get_resource_data(job.id)
self.assertIn('boss_key', actual)
self.assertIn('lookup_key', actual)
self.assertIn('channel', actual)
self.assertIn('experiment', actual)
self.assertIn('coord_frame', actual)