def post(self, request):
"""
Post a new config job and create a new ingest job
Args:
request: Django Rest framework Request object
ingest_config_data: COnfiguration data for the ingest job
Returns:
"""
ingest_config_data = request.data
try:
self.track_usage_data(ingest_config_data, request)
ingest_mgmr = IngestManager()
ingest_job = ingest_mgmr.setup_ingest(self.request.user.id,
ingest_config_data)
serializer = IngestJobListSerializer(ingest_job)
return Response(serializer.data, status=status.HTTP_201_CREATED)
except BossError as err:
return err.to_http()
def post(self, request):
"""
Post a new config job and create a new ingest job
Args:
ingest_config_data:
Returns:
"""
ingest_config_data = request.data
try:
ingest_mgmr = IngestManager()
ingest_job = ingest_mgmr.setup_ingest(self.request.user.id,
ingest_config_data)
serializer = IngestJobListSerializer(ingest_job)
return Response(serializer.data, status=status.HTTP_201_CREATED)
except BossError as err:
return err.to_http()
def get(self, request, ingest_job_id=None):
"""
Join a job with the specified job id or list all job ids if ingest_job_id is omitted
Args:
request: Django rest framework request object
ingest_job_id: Ingest job id
Returns:
Ingest job
"""
try:
if ingest_job_id is None:
# If the job ID is empty on a get, you are listing jobs
return self.list_ingest_jobs(request)
ingest_mgmr = IngestManager()
ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id)
# Check permissions
if not self.is_user_or_admin(request, ingest_job):
return BossHTTPError(
"Only the creator or admin can join an ingest job",
ErrorCodes.INGEST_NOT_CREATOR)
serializer = IngestJobListSerializer(ingest_job)
# Start setting up output
data = {'ingest_job': serializer.data}
if ingest_job.status == 3:
# The job has been deleted
raise BossError(
"The job with id {} has been deleted".format(
ingest_job_id), ErrorCodes.INVALID_REQUEST)
elif ingest_job.status == 2 or ingest_job.status == 4:
# Failed job or completed job
return Response(data, status=status.HTTP_200_OK)
elif ingest_job.status == 0:
# Job is still in progress
# check status of the step function
session = bossutils.aws.get_session()
if bossutils.aws.sfn_status(
session, ingest_job.step_function_arn) == 'SUCCEEDED':
# generate credentials
ingest_job.status = 1
ingest_job.save()
ingest_mgmr.generate_ingest_credentials(ingest_job)
elif bossutils.aws.sfn_status(
session, ingest_job.step_function_arn) == 'FAILED':
# This indicates an error in step function
raise BossError(
"Error generating ingest job messages"
" Delete the ingest job with id {} and try again.".
format(ingest_job_id), ErrorCodes.BOSS_SYSTEM_ERROR)
if ingest_job.status == 1:
data['ingest_job']['status'] = 1
ingest_creds = IngestCredentials()
data['credentials'] = ingest_creds.get_credentials(
ingest_job.id)
else:
data['credentials'] = None
data['tile_bucket_name'] = ingest_mgmr.get_tile_bucket()
data['KVIO_SETTINGS'] = settings.KVIO_SETTINGS
data['STATEIO_CONFIG'] = settings.STATEIO_CONFIG
data['OBJECTIO_CONFIG'] = settings.OBJECTIO_CONFIG
# add the lambda - Possibly remove this later
config = bossutils.configuration.BossConfig()
data['ingest_lambda'] = config["lambda"]["page_in_function"]
# Generate a "resource" for the ingest lambda function to be able to use SPDB cleanly
collection = Collection.objects.get(
name=data['ingest_job']["collection"])
experiment = Experiment.objects.get(
name=data['ingest_job']["experiment"], collection=collection)
coord_frame = experiment.coord_frame
channel = Channel.objects.get(name=data['ingest_job']["channel"],
experiment=experiment)
resource = {}
resource['boss_key'] = '{}&{}&{}'.format(
data['ingest_job']["collection"],
data['ingest_job']["experiment"],
data['ingest_job']["channel"])
resource['lookup_key'] = '{}&{}&{}'.format(collection.id,
experiment.id,
channel.id)
# The Lambda function needs certain resource properties to perform write ops. Set required things only.
# This is because S3 metadata is limited to 2kb, so we only set the bits of info needed, and in the lambda
# Function Populate the rest with dummy info
# IF YOU NEED ADDITIONAL DATA YOU MUST ADD IT HERE AND IN THE LAMBDA FUNCTION
resource['channel'] = {}
resource['channel']['type'] = channel.type
resource['channel']['datatype'] = channel.datatype
resource['channel']['base_resolution'] = channel.base_resolution
resource['experiment'] = {}
resource['experiment'][
'num_hierarchy_levels'] = experiment.num_hierarchy_levels
resource['experiment'][
'hierarchy_method'] = experiment.hierarchy_method
resource['coord_frame'] = {}
resource['coord_frame']['x_voxel_size'] = coord_frame.x_voxel_size
resource['coord_frame']['y_voxel_size'] = coord_frame.y_voxel_size
resource['coord_frame']['z_voxel_size'] = coord_frame.z_voxel_size
# Set resource
data['resource'] = resource
return Response(data, status=status.HTTP_200_OK)
except BossError as err:
return err.to_http()
except Exception as err:
return BossError("{}".format(err),
ErrorCodes.BOSS_SYSTEM_ERROR).to_http()
def post(self, request):
"""
Post a new config job and create a new ingest job
Args:
request: Django Rest framework Request object
ingest_config_data: COnfiguration data for the ingest job
Returns:
"""
ingest_config_data = request.data
# Add metrics to CloudWatch
extent = ingest_config_data['ingest_job']['extent']
tile_size = ingest_config_data['ingest_job']['tile_size']
database = ingest_config_data['database']
# Check that only permitted users are creating extra large ingests
try:
group = Group.objects.get(name=INGEST_GRP)
in_large_ingest_group = group.user_set.filter(
id=request.user.id).exists()
except Group.DoesNotExist:
# Just in case the group has not been created yet
in_large_ingest_group = False
if (not in_large_ingest_group) and \
((extent['x'][1] - extent['x'][0]) * \
(extent['y'][1] - extent['y'][0]) * \
(extent['z'][1] - extent['z'][0]) * \
(extent['t'][1] - extent['t'][0]) > settings.INGEST_MAX_SIZE):
return BossHTTPError(
"Large ingests require special permission to create. Contact system administrator.",
ErrorCodes.INVALID_STATE)
# Calculate the cost of the ingest
cost = (((extent['x'][1] - extent['x'][0]) / tile_size['x']) *
((extent['y'][1] - extent['y'][0]) / tile_size['y']) *
((extent['z'][1] - extent['z'][0]) / tile_size['z']) *
((extent['t'][1] - extent['t'][0]) / tile_size['t']) *
1.0625 # 1 lambda per tile + 1 lambda per 16 tiles (per cube)
* 1 # the cost per lambda
) # Calculating the cost of the lambda invocations
boss_config = bossutils.configuration.BossConfig()
dimensions = [
{
'Name': 'User',
'Value': request.user.username
},
{
'Name':
'Resource',
'Value':
'{}/{}/{}'.format(database['collection'],
database['experiment'], database['channel'])
},
{
'Name': 'Stack',
'Value': boss_config['system']['fqdn']
},
]
session = bossutils.aws.get_session()
client = session.client('cloudwatch')
client.put_metric_data(Namespace="BOSS/Ingest",
MetricData=[{
'MetricName': 'InvokeCount',
'Dimensions': dimensions,
'Value': 1.0,
'Unit': 'Count'
}, {
'MetricName': 'ComputeCost',
'Dimensions': dimensions,
'Value': cost,
'Unit': 'Count'
}])
try:
ingest_mgmr = IngestManager()
ingest_job = ingest_mgmr.setup_ingest(self.request.user.id,
ingest_config_data)
serializer = IngestJobListSerializer(ingest_job)
return Response(serializer.data, status=status.HTTP_201_CREATED)
except BossError as err:
return err.to_http()
def get(self, request, ingest_job_id):
"""
Args:
job_id:
Returns:
"""
try:
ingest_mgmr = IngestManager()
ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id)
serializer = IngestJobListSerializer(ingest_job)
print(serializer.data)
# Start setting up output
data = {}
data['ingest_job'] = serializer.data
if ingest_job.status == 3 or ingest_job.status == 2:
# Return the information for the deleted job/completed job
return Response(data, status=status.HTTP_200_OK)
elif ingest_job.status == 0:
# check if all message are in the upload queue
upload_queue = ingest_mgmr.get_ingest_job_upload_queue(
ingest_job)
if int(upload_queue.queue.
attributes['ApproximateNumberOfMessages']) == int(
ingest_job.tile_count):
#generate credentials
ingest_job.status = 1
ingest_job.save()
elif int(upload_queue.queue.
attributes['ApproximateNumberOfMessages']) > int(
ingest_job.tile_count):
# This indicates an error in the lambda
raise BossError(
"Error generating ingest job messages due to resources timing out ."
" Delete the ingest job with id {} and try again.".
format(ingest_job_id), ErrorCodes.BOSS_SYSTEM_ERROR)
if ingest_job.status == 1:
data['ingest_job']['status'] = 1
ingest_creds = IngestCredentials()
data['credentials'] = ingest_creds.get_credentials(
ingest_job.id)
else:
data['credentials'] = None
data['tile_bucket_name'] = ingest_mgmr.get_tile_bucket()
data['KVIO_SETTINGS'] = settings.KVIO_SETTINGS
data['STATEIO_CONFIG'] = settings.STATEIO_CONFIG
data['OBJECTIO_CONFIG'] = settings.OBJECTIO_CONFIG
# add the lambda - Possibly remove this later
config = bossutils.configuration.BossConfig()
data['ingest_lambda'] = config["lambda"]["page_in_function"]
# Generate a "resource" for the ingest lambda function to be able to use SPDB cleanly
collection = Collection.objects.get(
name=data['ingest_job']["collection"])
experiment = Experiment.objects.get(
name=data['ingest_job']["experiment"], collection=collection)
channel = Channel.objects.get(name=data['ingest_job']["channel"],
experiment=experiment)
resource = {}
resource['boss_key'] = '{}&{}&{}'.format(
data['ingest_job']["collection"],
data['ingest_job']["experiment"],
data['ingest_job']["channel"])
resource['lookup_key'] = '{}&{}&{}'.format(collection.id,
experiment.id,
channel.id)
resource['channel'] = {}
resource['channel']['name'] = channel.name
resource['channel']['description'] = ""
resource['channel']['type'] = channel.type
resource['channel']['datatype'] = channel.datatype
resource['channel']['base_resolution'] = channel.base_resolution
resource['channel']['sources'] = [
x.name for x in channel.sources.all()
]
resource['channel']['related'] = [
x.name for x in channel.related.all()
]
resource['channel'][
'default_time_sample'] = channel.default_time_sample
# Set resource
data['resource'] = resource
return Response(data, status=status.HTTP_200_OK)
except BossError as err:
return err.to_http()
except Exception as err:
return BossError("{}".format(err),
ErrorCodes.BOSS_SYSTEM_ERROR).to_http()
def get(self, request, ingest_job_id=None):
"""
Join a job with the specified job id or list all job ids if ingest_job_id is omitted
Args:
request: Django rest framework request object
ingest_job_id: Ingest job id
Returns:
Ingest job
"""
try:
if ingest_job_id is None:
# If the job ID is empty on a get, you are listing jobs
return self.list_ingest_jobs(request)
ingest_mgmr = IngestManager()
ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id)
# Check permissions
if not self.is_user_or_admin(request, ingest_job):
return BossHTTPError(
"Only the creator or admin can join an ingest job",
ErrorCodes.INGEST_NOT_CREATOR)
serializer = IngestJobListSerializer(ingest_job)
# Start setting up output
data = {'ingest_job': serializer.data}
data['ingest_job']['tile_index_queue'] = None
if ingest_job.ingest_type == IngestJob.TILE_INGEST:
data['ingest_job'][
'tile_index_queue'] = ingest_mgmr.get_ingest_job_tile_index_queue(
ingest_job).url
if ingest_job.status == IngestJob.DELETED:
raise BossError(
"The job with id {} has been deleted".format(
ingest_job_id), ErrorCodes.INVALID_REQUEST)
elif ingest_job.status == IngestJob.COMPLETE or ingest_job.status == IngestJob.FAILED:
return Response(data, status=status.HTTP_200_OK)
elif ingest_job.status == IngestJob.PREPARING:
# check status of the step function
session = bossutils.aws.get_session()
if bossutils.aws.sfn_status(
session, ingest_job.step_function_arn) == 'SUCCEEDED':
# generate credentials
ingest_job.status = 1
ingest_job.save()
ingest_mgmr.generate_ingest_credentials(ingest_job)
elif bossutils.aws.sfn_status(
session, ingest_job.step_function_arn) == 'FAILED':
# This indicates an error in step function
raise BossError(
"Error generating ingest job messages"
" Delete the ingest job with id {} and try again.".
format(ingest_job_id), ErrorCodes.BOSS_SYSTEM_ERROR)
if ingest_job.status in [
IngestJob.UPLOADING, IngestJob.WAIT_ON_QUEUES,
IngestJob.COMPLETING
]:
data['ingest_job']['status'] = ingest_job.status
ingest_creds = IngestCredentials()
data['credentials'] = ingest_creds.get_credentials(
ingest_job.id)
else:
data['credentials'] = None
data['tile_bucket_name'] = ingest_mgmr.get_tile_bucket()
data['ingest_bucket_name'] = INGEST_BUCKET
data['KVIO_SETTINGS'] = settings.KVIO_SETTINGS
data['STATEIO_CONFIG'] = settings.STATEIO_CONFIG
data['OBJECTIO_CONFIG'] = settings.OBJECTIO_CONFIG
# Strip out un-needed data from OBJECTIO_CONFIG to save space when
# including in S3 metadata.
data['OBJECTIO_CONFIG'].pop('index_deadletter_queue', None)
data['OBJECTIO_CONFIG'].pop('index_cuboids_keys_queue', None)
# Set resource
data['resource'] = ingest_mgmr.get_resource_data(ingest_job_id)
# ingest_lambda is no longer required by the backend. The backend
# gets the name of the ingest lambda from boss-manage/lib/names.py.
# Keep providing it in case an older ingest client used (which
# still expects it).
data['ingest_lambda'] = 'deprecated'
return Response(data, status=status.HTTP_200_OK)
except BossError as err:
return err.to_http()
except Exception as err:
return BossError("{}".format(err),
ErrorCodes.BOSS_SYSTEM_ERROR).to_http()