def post(self, request): """ Post a new config job and create a new ingest job Args: request: Django Rest framework Request object ingest_config_data: COnfiguration data for the ingest job Returns: """ ingest_config_data = request.data try: self.track_usage_data(ingest_config_data, request) ingest_mgmr = IngestManager() ingest_job = ingest_mgmr.setup_ingest(self.request.user.id, ingest_config_data) serializer = IngestJobListSerializer(ingest_job) return Response(serializer.data, status=status.HTTP_201_CREATED) except BossError as err: return err.to_http()
def post(self, request): """ Post a new config job and create a new ingest job Args: ingest_config_data: Returns: """ ingest_config_data = request.data try: ingest_mgmr = IngestManager() ingest_job = ingest_mgmr.setup_ingest(self.request.user.id, ingest_config_data) serializer = IngestJobListSerializer(ingest_job) return Response(serializer.data, status=status.HTTP_201_CREATED) except BossError as err: return err.to_http()
def get(self, request, ingest_job_id=None): """ Join a job with the specified job id or list all job ids if ingest_job_id is omitted Args: request: Django rest framework request object ingest_job_id: Ingest job id Returns: Ingest job """ try: if ingest_job_id is None: # If the job ID is empty on a get, you are listing jobs return self.list_ingest_jobs(request) ingest_mgmr = IngestManager() ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id) # Check permissions if not self.is_user_or_admin(request, ingest_job): return BossHTTPError( "Only the creator or admin can join an ingest job", ErrorCodes.INGEST_NOT_CREATOR) serializer = IngestJobListSerializer(ingest_job) # Start setting up output data = {'ingest_job': serializer.data} if ingest_job.status == 3: # The job has been deleted raise BossError( "The job with id {} has been deleted".format( ingest_job_id), ErrorCodes.INVALID_REQUEST) elif ingest_job.status == 2 or ingest_job.status == 4: # Failed job or completed job return Response(data, status=status.HTTP_200_OK) elif ingest_job.status == 0: # Job is still in progress # check status of the step function session = bossutils.aws.get_session() if bossutils.aws.sfn_status( session, ingest_job.step_function_arn) == 'SUCCEEDED': # generate credentials ingest_job.status = 1 ingest_job.save() ingest_mgmr.generate_ingest_credentials(ingest_job) elif bossutils.aws.sfn_status( session, ingest_job.step_function_arn) == 'FAILED': # This indicates an error in step function raise BossError( "Error generating ingest job messages" " Delete the ingest job with id {} and try again.". format(ingest_job_id), ErrorCodes.BOSS_SYSTEM_ERROR) if ingest_job.status == 1: data['ingest_job']['status'] = 1 ingest_creds = IngestCredentials() data['credentials'] = ingest_creds.get_credentials( ingest_job.id) else: data['credentials'] = None data['tile_bucket_name'] = ingest_mgmr.get_tile_bucket() data['KVIO_SETTINGS'] = settings.KVIO_SETTINGS data['STATEIO_CONFIG'] = settings.STATEIO_CONFIG data['OBJECTIO_CONFIG'] = settings.OBJECTIO_CONFIG # add the lambda - Possibly remove this later config = bossutils.configuration.BossConfig() data['ingest_lambda'] = config["lambda"]["page_in_function"] # Generate a "resource" for the ingest lambda function to be able to use SPDB cleanly collection = Collection.objects.get( name=data['ingest_job']["collection"]) experiment = Experiment.objects.get( name=data['ingest_job']["experiment"], collection=collection) coord_frame = experiment.coord_frame channel = Channel.objects.get(name=data['ingest_job']["channel"], experiment=experiment) resource = {} resource['boss_key'] = '{}&{}&{}'.format( data['ingest_job']["collection"], data['ingest_job']["experiment"], data['ingest_job']["channel"]) resource['lookup_key'] = '{}&{}&{}'.format(collection.id, experiment.id, channel.id) # The Lambda function needs certain resource properties to perform write ops. Set required things only. # This is because S3 metadata is limited to 2kb, so we only set the bits of info needed, and in the lambda # Function Populate the rest with dummy info # IF YOU NEED ADDITIONAL DATA YOU MUST ADD IT HERE AND IN THE LAMBDA FUNCTION resource['channel'] = {} resource['channel']['type'] = channel.type resource['channel']['datatype'] = channel.datatype resource['channel']['base_resolution'] = channel.base_resolution resource['experiment'] = {} resource['experiment'][ 'num_hierarchy_levels'] = experiment.num_hierarchy_levels resource['experiment'][ 'hierarchy_method'] = experiment.hierarchy_method resource['coord_frame'] = {} resource['coord_frame']['x_voxel_size'] = coord_frame.x_voxel_size resource['coord_frame']['y_voxel_size'] = coord_frame.y_voxel_size resource['coord_frame']['z_voxel_size'] = coord_frame.z_voxel_size # Set resource data['resource'] = resource return Response(data, status=status.HTTP_200_OK) except BossError as err: return err.to_http() except Exception as err: return BossError("{}".format(err), ErrorCodes.BOSS_SYSTEM_ERROR).to_http()
def post(self, request): """ Post a new config job and create a new ingest job Args: request: Django Rest framework Request object ingest_config_data: COnfiguration data for the ingest job Returns: """ ingest_config_data = request.data # Add metrics to CloudWatch extent = ingest_config_data['ingest_job']['extent'] tile_size = ingest_config_data['ingest_job']['tile_size'] database = ingest_config_data['database'] # Check that only permitted users are creating extra large ingests try: group = Group.objects.get(name=INGEST_GRP) in_large_ingest_group = group.user_set.filter( id=request.user.id).exists() except Group.DoesNotExist: # Just in case the group has not been created yet in_large_ingest_group = False if (not in_large_ingest_group) and \ ((extent['x'][1] - extent['x'][0]) * \ (extent['y'][1] - extent['y'][0]) * \ (extent['z'][1] - extent['z'][0]) * \ (extent['t'][1] - extent['t'][0]) > settings.INGEST_MAX_SIZE): return BossHTTPError( "Large ingests require special permission to create. Contact system administrator.", ErrorCodes.INVALID_STATE) # Calculate the cost of the ingest cost = (((extent['x'][1] - extent['x'][0]) / tile_size['x']) * ((extent['y'][1] - extent['y'][0]) / tile_size['y']) * ((extent['z'][1] - extent['z'][0]) / tile_size['z']) * ((extent['t'][1] - extent['t'][0]) / tile_size['t']) * 1.0625 # 1 lambda per tile + 1 lambda per 16 tiles (per cube) * 1 # the cost per lambda ) # Calculating the cost of the lambda invocations boss_config = bossutils.configuration.BossConfig() dimensions = [ { 'Name': 'User', 'Value': request.user.username }, { 'Name': 'Resource', 'Value': '{}/{}/{}'.format(database['collection'], database['experiment'], database['channel']) }, { 'Name': 'Stack', 'Value': boss_config['system']['fqdn'] }, ] session = bossutils.aws.get_session() client = session.client('cloudwatch') client.put_metric_data(Namespace="BOSS/Ingest", MetricData=[{ 'MetricName': 'InvokeCount', 'Dimensions': dimensions, 'Value': 1.0, 'Unit': 'Count' }, { 'MetricName': 'ComputeCost', 'Dimensions': dimensions, 'Value': cost, 'Unit': 'Count' }]) try: ingest_mgmr = IngestManager() ingest_job = ingest_mgmr.setup_ingest(self.request.user.id, ingest_config_data) serializer = IngestJobListSerializer(ingest_job) return Response(serializer.data, status=status.HTTP_201_CREATED) except BossError as err: return err.to_http()
def get(self, request, ingest_job_id): """ Args: job_id: Returns: """ try: ingest_mgmr = IngestManager() ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id) serializer = IngestJobListSerializer(ingest_job) print(serializer.data) # Start setting up output data = {} data['ingest_job'] = serializer.data if ingest_job.status == 3 or ingest_job.status == 2: # Return the information for the deleted job/completed job return Response(data, status=status.HTTP_200_OK) elif ingest_job.status == 0: # check if all message are in the upload queue upload_queue = ingest_mgmr.get_ingest_job_upload_queue( ingest_job) if int(upload_queue.queue. attributes['ApproximateNumberOfMessages']) == int( ingest_job.tile_count): #generate credentials ingest_job.status = 1 ingest_job.save() elif int(upload_queue.queue. attributes['ApproximateNumberOfMessages']) > int( ingest_job.tile_count): # This indicates an error in the lambda raise BossError( "Error generating ingest job messages due to resources timing out ." " Delete the ingest job with id {} and try again.". format(ingest_job_id), ErrorCodes.BOSS_SYSTEM_ERROR) if ingest_job.status == 1: data['ingest_job']['status'] = 1 ingest_creds = IngestCredentials() data['credentials'] = ingest_creds.get_credentials( ingest_job.id) else: data['credentials'] = None data['tile_bucket_name'] = ingest_mgmr.get_tile_bucket() data['KVIO_SETTINGS'] = settings.KVIO_SETTINGS data['STATEIO_CONFIG'] = settings.STATEIO_CONFIG data['OBJECTIO_CONFIG'] = settings.OBJECTIO_CONFIG # add the lambda - Possibly remove this later config = bossutils.configuration.BossConfig() data['ingest_lambda'] = config["lambda"]["page_in_function"] # Generate a "resource" for the ingest lambda function to be able to use SPDB cleanly collection = Collection.objects.get( name=data['ingest_job']["collection"]) experiment = Experiment.objects.get( name=data['ingest_job']["experiment"], collection=collection) channel = Channel.objects.get(name=data['ingest_job']["channel"], experiment=experiment) resource = {} resource['boss_key'] = '{}&{}&{}'.format( data['ingest_job']["collection"], data['ingest_job']["experiment"], data['ingest_job']["channel"]) resource['lookup_key'] = '{}&{}&{}'.format(collection.id, experiment.id, channel.id) resource['channel'] = {} resource['channel']['name'] = channel.name resource['channel']['description'] = "" resource['channel']['type'] = channel.type resource['channel']['datatype'] = channel.datatype resource['channel']['base_resolution'] = channel.base_resolution resource['channel']['sources'] = [ x.name for x in channel.sources.all() ] resource['channel']['related'] = [ x.name for x in channel.related.all() ] resource['channel'][ 'default_time_sample'] = channel.default_time_sample # Set resource data['resource'] = resource return Response(data, status=status.HTTP_200_OK) except BossError as err: return err.to_http() except Exception as err: return BossError("{}".format(err), ErrorCodes.BOSS_SYSTEM_ERROR).to_http()
def get(self, request, ingest_job_id=None): """ Join a job with the specified job id or list all job ids if ingest_job_id is omitted Args: request: Django rest framework request object ingest_job_id: Ingest job id Returns: Ingest job """ try: if ingest_job_id is None: # If the job ID is empty on a get, you are listing jobs return self.list_ingest_jobs(request) ingest_mgmr = IngestManager() ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id) # Check permissions if not self.is_user_or_admin(request, ingest_job): return BossHTTPError( "Only the creator or admin can join an ingest job", ErrorCodes.INGEST_NOT_CREATOR) serializer = IngestJobListSerializer(ingest_job) # Start setting up output data = {'ingest_job': serializer.data} data['ingest_job']['tile_index_queue'] = None if ingest_job.ingest_type == IngestJob.TILE_INGEST: data['ingest_job'][ 'tile_index_queue'] = ingest_mgmr.get_ingest_job_tile_index_queue( ingest_job).url if ingest_job.status == IngestJob.DELETED: raise BossError( "The job with id {} has been deleted".format( ingest_job_id), ErrorCodes.INVALID_REQUEST) elif ingest_job.status == IngestJob.COMPLETE or ingest_job.status == IngestJob.FAILED: return Response(data, status=status.HTTP_200_OK) elif ingest_job.status == IngestJob.PREPARING: # check status of the step function session = bossutils.aws.get_session() if bossutils.aws.sfn_status( session, ingest_job.step_function_arn) == 'SUCCEEDED': # generate credentials ingest_job.status = 1 ingest_job.save() ingest_mgmr.generate_ingest_credentials(ingest_job) elif bossutils.aws.sfn_status( session, ingest_job.step_function_arn) == 'FAILED': # This indicates an error in step function raise BossError( "Error generating ingest job messages" " Delete the ingest job with id {} and try again.". format(ingest_job_id), ErrorCodes.BOSS_SYSTEM_ERROR) if ingest_job.status in [ IngestJob.UPLOADING, IngestJob.WAIT_ON_QUEUES, IngestJob.COMPLETING ]: data['ingest_job']['status'] = ingest_job.status ingest_creds = IngestCredentials() data['credentials'] = ingest_creds.get_credentials( ingest_job.id) else: data['credentials'] = None data['tile_bucket_name'] = ingest_mgmr.get_tile_bucket() data['ingest_bucket_name'] = INGEST_BUCKET data['KVIO_SETTINGS'] = settings.KVIO_SETTINGS data['STATEIO_CONFIG'] = settings.STATEIO_CONFIG data['OBJECTIO_CONFIG'] = settings.OBJECTIO_CONFIG # Strip out un-needed data from OBJECTIO_CONFIG to save space when # including in S3 metadata. data['OBJECTIO_CONFIG'].pop('index_deadletter_queue', None) data['OBJECTIO_CONFIG'].pop('index_cuboids_keys_queue', None) # Set resource data['resource'] = ingest_mgmr.get_resource_data(ingest_job_id) # ingest_lambda is no longer required by the backend. The backend # gets the name of the ingest lambda from boss-manage/lib/names.py. # Keep providing it in case an older ingest client used (which # still expects it). data['ingest_lambda'] = 'deprecated' return Response(data, status=status.HTTP_200_OK) except BossError as err: return err.to_http() except Exception as err: return BossError("{}".format(err), ErrorCodes.BOSS_SYSTEM_ERROR).to_http()