def get(self, request, ingest_job_id=None):
"""
Join a job with the specified job id or list all job ids if ingest_job_id is omitted
Args:
request: Django rest framework request object
ingest_job_id: Ingest job id
Returns:
Ingest job
"""
try:
if ingest_job_id is None:
# If the job ID is empty on a get, you are listing jobs
return self.list_ingest_jobs(request)
ingest_mgmr = IngestManager()
ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id)
# Check permissions
if not self.is_user_or_admin(request, ingest_job):
return BossHTTPError(
"Only the creator or admin can join an ingest job",
ErrorCodes.INGEST_NOT_CREATOR)
serializer = IngestJobListSerializer(ingest_job)
# Start setting up output
data = {'ingest_job': serializer.data}
data['ingest_job']['tile_index_queue'] = None
if ingest_job.ingest_type == IngestJob.TILE_INGEST:
data['ingest_job'][
'tile_index_queue'] = ingest_mgmr.get_ingest_job_tile_index_queue(
ingest_job).url
if ingest_job.status == 3:
# The job has been deleted
raise BossError(
"The job with id {} has been deleted".format(
ingest_job_id), ErrorCodes.INVALID_REQUEST)
elif ingest_job.status == 2 or ingest_job.status == 4:
# Failed job or completed job
return Response(data, status=status.HTTP_200_OK)
elif ingest_job.status == 0:
# Job is still in progress
# check status of the step function
session = bossutils.aws.get_session()
if bossutils.aws.sfn_status(
session, ingest_job.step_function_arn) == 'SUCCEEDED':
# generate credentials
ingest_job.status = 1
ingest_job.save()
ingest_mgmr.generate_ingest_credentials(ingest_job)
elif bossutils.aws.sfn_status(
session, ingest_job.step_function_arn) == 'FAILED':
# This indicates an error in step function
raise BossError(
"Error generating ingest job messages"
" Delete the ingest job with id {} and try again.".
format(ingest_job_id), ErrorCodes.BOSS_SYSTEM_ERROR)
if ingest_job.status == 1:
data['ingest_job']['status'] = 1
ingest_creds = IngestCredentials()
data['credentials'] = ingest_creds.get_credentials(
ingest_job.id)
else:
data['credentials'] = None
data['tile_bucket_name'] = ingest_mgmr.get_tile_bucket()
data['ingest_bucket_name'] = INGEST_BUCKET
data['KVIO_SETTINGS'] = settings.KVIO_SETTINGS
data['STATEIO_CONFIG'] = settings.STATEIO_CONFIG
data['OBJECTIO_CONFIG'] = settings.OBJECTIO_CONFIG
# Strip out un-needed data from OBJECTIO_CONFIG to save space when
# including in S3 metadata.
data['OBJECTIO_CONFIG'].pop('index_deadletter_queue', None)
data['OBJECTIO_CONFIG'].pop('index_cuboids_keys_queue', None)
# add the lambda - Possibly remove this later
config = bossutils.configuration.BossConfig()
data['ingest_lambda'] = config["lambda"]["page_in_function"]
# Generate a "resource" for the ingest lambda function to be able to use SPDB cleanly
collection = Collection.objects.get(
name=data['ingest_job']["collection"])
experiment = Experiment.objects.get(
name=data['ingest_job']["experiment"], collection=collection)
coord_frame = experiment.coord_frame
channel = Channel.objects.get(name=data['ingest_job']["channel"],
experiment=experiment)
resource = {}
resource['boss_key'] = '{}&{}&{}'.format(
data['ingest_job']["collection"],
data['ingest_job']["experiment"],
data['ingest_job']["channel"])
resource['lookup_key'] = '{}&{}&{}'.format(collection.id,
experiment.id,
channel.id)
# The Lambda function needs certain resource properties to perform write ops. Set required things only.
# This is because S3 metadata is limited to 2kb, so we only set the bits of info needed, and in the lambda
# Function Populate the rest with dummy info
# IF YOU NEED ADDITIONAL DATA YOU MUST ADD IT HERE AND IN THE LAMBDA FUNCTION
resource['channel'] = {}
resource['channel']['type'] = channel.type
resource['channel']['datatype'] = channel.datatype
resource['channel']['base_resolution'] = channel.base_resolution
resource['experiment'] = {}
resource['experiment'][
'num_hierarchy_levels'] = experiment.num_hierarchy_levels
resource['experiment'][
'hierarchy_method'] = experiment.hierarchy_method
resource['coord_frame'] = {}
resource['coord_frame']['x_voxel_size'] = coord_frame.x_voxel_size
resource['coord_frame']['y_voxel_size'] = coord_frame.y_voxel_size
resource['coord_frame']['z_voxel_size'] = coord_frame.z_voxel_size
# Set resource
data['resource'] = resource
return Response(data, status=status.HTTP_200_OK)
except BossError as err:
return err.to_http()
except Exception as err:
return BossError("{}".format(err),
ErrorCodes.BOSS_SYSTEM_ERROR).to_http()
def get(self, request, ingest_job_id=None):
"""
Join a job with the specified job id or list all job ids if ingest_job_id is omitted
Args:
request: Django rest framework request object
ingest_job_id: Ingest job id
Returns:
Ingest job
"""
try:
if ingest_job_id is None:
# If the job ID is empty on a get, you are listing jobs
return self.list_ingest_jobs(request)
ingest_mgmr = IngestManager()
ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id)
# Check permissions
if not self.is_user_or_admin(request, ingest_job):
return BossHTTPError(
"Only the creator or admin can join an ingest job",
ErrorCodes.INGEST_NOT_CREATOR)
serializer = IngestJobListSerializer(ingest_job)
# Start setting up output
data = {'ingest_job': serializer.data}
data['ingest_job']['tile_index_queue'] = None
if ingest_job.ingest_type == IngestJob.TILE_INGEST:
data['ingest_job'][
'tile_index_queue'] = ingest_mgmr.get_ingest_job_tile_index_queue(
ingest_job).url
if ingest_job.status == IngestJob.DELETED:
raise BossError(
"The job with id {} has been deleted".format(
ingest_job_id), ErrorCodes.INVALID_REQUEST)
elif ingest_job.status == IngestJob.COMPLETE or ingest_job.status == IngestJob.FAILED:
return Response(data, status=status.HTTP_200_OK)
elif ingest_job.status == IngestJob.PREPARING:
# check status of the step function
session = bossutils.aws.get_session()
if bossutils.aws.sfn_status(
session, ingest_job.step_function_arn) == 'SUCCEEDED':
# generate credentials
ingest_job.status = 1
ingest_job.save()
ingest_mgmr.generate_ingest_credentials(ingest_job)
elif bossutils.aws.sfn_status(
session, ingest_job.step_function_arn) == 'FAILED':
# This indicates an error in step function
raise BossError(
"Error generating ingest job messages"
" Delete the ingest job with id {} and try again.".
format(ingest_job_id), ErrorCodes.BOSS_SYSTEM_ERROR)
if ingest_job.status in [
IngestJob.UPLOADING, IngestJob.WAIT_ON_QUEUES,
IngestJob.COMPLETING
]:
data['ingest_job']['status'] = ingest_job.status
ingest_creds = IngestCredentials()
data['credentials'] = ingest_creds.get_credentials(
ingest_job.id)
else:
data['credentials'] = None
data['tile_bucket_name'] = ingest_mgmr.get_tile_bucket()
data['ingest_bucket_name'] = INGEST_BUCKET
data['KVIO_SETTINGS'] = settings.KVIO_SETTINGS
data['STATEIO_CONFIG'] = settings.STATEIO_CONFIG
data['OBJECTIO_CONFIG'] = settings.OBJECTIO_CONFIG
# Strip out un-needed data from OBJECTIO_CONFIG to save space when
# including in S3 metadata.
data['OBJECTIO_CONFIG'].pop('index_deadletter_queue', None)
data['OBJECTIO_CONFIG'].pop('index_cuboids_keys_queue', None)
# Set resource
data['resource'] = ingest_mgmr.get_resource_data(ingest_job_id)
# ingest_lambda is no longer required by the backend. The backend
# gets the name of the ingest lambda from boss-manage/lib/names.py.
# Keep providing it in case an older ingest client used (which
# still expects it).
data['ingest_lambda'] = 'deprecated'
return Response(data, status=status.HTTP_200_OK)
except BossError as err:
return err.to_http()
except Exception as err:
return BossError("{}".format(err),
ErrorCodes.BOSS_SYSTEM_ERROR).to_http()