def update_source_related_channels(channel, experiment, source_channels, related_channels): """ Update a list of source and related channels Args: related_channels: New list of related channels source_channels: New list of source channels experiment: Experiment for the current channel channel: Curren channel Returns: Updated Channel """ try: # update ist of sources # Get all the source cur_sources = channel.sources.all() # Get the list of sources to remove rm_sources = [ch for ch in cur_sources if ch not in source_channels] for source in rm_sources: channel.remove_source(source) # add new sources add_sources = [ch for ch in source_channels if ch not in cur_sources] for source_channel in add_sources: channel.add_source(source_channel) cur_related = channel.related.all() rm_related = [ch for ch in cur_related if ch not in related_channels] for related in rm_related: channel.related.remove(related) add_related = [ch for ch in related_channels if ch not in cur_related] for related_channel in add_related: channel.related.add(related_channel.pk) channel.save() return channel except Exception as err: channel.delete() raise BossError("Exception adding source/related channels.{}".format(err), ErrorCodes.INVALID_POST_ARGUMENT)
def get_ingest_job(self, ingest_job_id): """ Get the ingest job with the specific id Args: ingest_job_id: Id of the ingest job Returns: IngestJob : Data model with the ingest job if the id is valid Raises: BossError : If the ingets job id does not exist """ try: ingest_job = IngestJob.objects.get(id=ingest_job_id) return ingest_job except IngestJob.DoesNotExist: raise BossError( "The ingest job with id {} does not exist".format( str(ingest_job_id)), ErrorCodes.OBJECT_NOT_FOUND)
def enqueue_job(session, args, downsample_sqs): """Enqueue downsample job Args: session (boto3.session): args (dict): Arguments passed to the downsample step function via SQS downsample_sqs (str): URL of SQS queue Raises: (BossError): If failed to enqueue job. """ rows_updated = (Channel.objects.filter(id=args['channel_id']).exclude( downsample_status=Channel.DownsampleStatus.IN_PROGRESS).exclude( downsample_status=Channel.DownsampleStatus.QUEUED).update( downsample_status=Channel.DownsampleStatus.QUEUED)) if rows_updated == 0: raise BossError(DOWNSAMPLE_CANNOT_BE_QUEUED_ERR_MSG, ErrorCodes.BAD_REQUEST) _sqs_enqueue(session, args, downsample_sqs)
def _convert_string_to_ingest_job(self, s): """ Convert a string representation of ingest_type to int. Args: s (str): Returns: (int): IngestJob.TILE_INGEST | IngestJob.VOLUMETRIC_INGEST Raises: (BossError): If string is invalid. """ lowered = s.lower() if lowered == 'tile': return IngestJob.TILE_INGEST if lowered == 'volumetric': return IngestJob.VOLUMETRIC_INGEST raise BossError('Unknown ingest_type: {}'.format(s))
def create_ingest_job(self): """ Returns: """ ingest_job_serializer_data = { 'creator': self.owner, 'collection': self.collection.name, 'experiment': self.experiment.name, 'channel_layer': self.channel_layer.name, 'config_data': json.dumps(self.config.config_data), 'resolution': self.resolution, 'x_start': self.config.config_data["ingest_job"]["extent"]["x"][0], 'x_stop': self.config.config_data["ingest_job"]["extent"]["x"][1], 'y_start': self.config.config_data["ingest_job"]["extent"]["y"][0], 'y_stop': self.config.config_data["ingest_job"]["extent"]["y"][1], 'z_start': self.config.config_data["ingest_job"]["extent"]["z"][0], 'z_stop': self.config.config_data["ingest_job"]["extent"]["z"][1], 't_start': self.config.config_data["ingest_job"]["extent"]["t"][0], 't_stop': self.config.config_data["ingest_job"]["extent"]["t"][1], 'tile_size_x': self.config.config_data["ingest_job"]["tile_size"]["x"], 'tile_size_y': self.config.config_data["ingest_job"]["tile_size"]["y"], 'tile_size_z': self.config.config_data["ingest_job"]["tile_size"]["z"], 'tile_size_t': self.config.config_data["ingest_job"]["tile_size"]["t"], } serializer = IngestJobCreateSerializer(data=ingest_job_serializer_data) if serializer.is_valid(): ingest_job = serializer.save() return ingest_job else: raise BossError("{}".format(serializer.errors), ErrorCodes.SERIALIZATION_ERROR)
def test_complete_should_fail_if_queue_not_empty(self, ingest_mgr_creator): job_id = 28 ingest_job = MagicMock(spec=IngestJob) ingest_job.status = IngestJob.UPLOADING fake_ingest_mgr = MagicMock(spec=IngestManager) fake_ingest_mgr.get_ingest_job.return_value = ingest_job # This method tries to move from UPLOADING to WAIT_ON_QUEUES. fake_ingest_mgr.try_enter_wait_on_queue_state.side_effect = BossError( INGEST_QUEUE_NOT_EMPTY_ERR_MSG, ErrorCodes.BAD_REQUEST) # Provide the fake when the complete view instantiates an IngestManager. ingest_mgr_creator.return_value = fake_ingest_mgr testuser = User.objects.create_user(username='******') ingest_job.creator = testuser self.client.force_authenticate(user=testuser) url = '/{}/ingest/{}/complete'.format(version, job_id) resp = self.client.post(url, format='json') self.assertEqual(400, resp.status_code) actual = resp.json() self.assertIn('wait_secs', actual) self.assertIn('info', actual)
def delete_tiles(self, ingest_job): """ Delete all remaining tiles from the tile index database and tile bucket 5/24/2018 - This code depends on a GSI for the tile index. The GSI was removed because its key didn't shard well. Cleanup will now be handled by TTL policies applied to the tile bucket and the tile index. This method will be removed once that code is merged. Args: ingest_job: Ingest job model Returns: None Raises: BossError : For exceptions that happen while deleting the tiles and index """ try: # Get all the chunks for a job tiledb = BossTileIndexDB(ingest_job.collection + '&' + ingest_job.experiment) tilebucket = TileBucket(ingest_job.collection + '&' + ingest_job.experiment) chunks = list(tiledb.getTaskItems(ingest_job.id)) for chunk in chunks: # delete each tile in the chunk for key in chunk['tile_uploaded_map']: response = tilebucket.deleteObject(key) tiledb.deleteCuboid(chunk['chunk_key'], ingest_job.id) except Exception as e: raise BossError( "Exception while deleteing tiles for the ingest job {}. {}". format(ingest_job.id, e), ErrorCodes.BOSS_SYSTEM_ERROR)
def create_ingest_job(self): """ Create a new ingest job using the parameters in the ingest config data file Returns: IngestJob : Data model with the current ingest job Raises: BossError : For serialization errors that occur while creating a ingest job or if ingest_type is invalid """ ingest_job_serializer_data = { 'creator': self.owner, 'collection': self.collection.name, 'experiment': self.experiment.name, 'channel': self.channel.name, 'config_data': json.dumps(self.config.config_data), 'resolution': self.resolution, 'x_start': self.config.config_data["ingest_job"]["extent"]["x"][0], 'x_stop': self.config.config_data["ingest_job"]["extent"]["x"][1], 'y_start': self.config.config_data["ingest_job"]["extent"]["y"][0], 'y_stop': self.config.config_data["ingest_job"]["extent"]["y"][1], 'z_start': self.config.config_data["ingest_job"]["extent"]["z"][0], 'z_stop': self.config.config_data["ingest_job"]["extent"]["z"][1], 't_start': self.config.config_data["ingest_job"]["extent"]["t"][0], 't_stop': self.config.config_data["ingest_job"]["extent"]["t"][1], } if "ingest_type" in self.config.config_data["ingest_job"]: ingest_job_serializer_data[ "ingest_type"] = self._convert_string_to_ingest_job( self.config.config_data["ingest_job"]["ingest_type"]) else: ingest_job_serializer_data["ingest_type"] = IngestJob.TILE_INGEST if ingest_job_serializer_data["ingest_type"] == IngestJob.TILE_INGEST: ingest_job_serializer_data[ 'tile_size_x'] = self.config.config_data["ingest_job"][ "tile_size"]["x"] ingest_job_serializer_data[ 'tile_size_y'] = self.config.config_data["ingest_job"][ "tile_size"]["y"] #ingest_job_serializer_data['tile_size_z'] = self.config.config_data["ingest_job"]["tile_size"]["z"] ingest_job_serializer_data['tile_size_z'] = 1 ingest_job_serializer_data[ 'tile_size_t'] = self.config.config_data["ingest_job"][ "tile_size"]["t"] elif ingest_job_serializer_data[ "ingest_type"] == IngestJob.VOLUMETRIC_INGEST: ingest_job_serializer_data[ 'tile_size_x'] = self.config.config_data["ingest_job"][ "chunk_size"]["x"] ingest_job_serializer_data[ 'tile_size_y'] = self.config.config_data["ingest_job"][ "chunk_size"]["y"] ingest_job_serializer_data[ 'tile_size_z'] = self.config.config_data["ingest_job"][ "chunk_size"]["z"] ingest_job_serializer_data['tile_size_t'] = 1 else: raise BossError( 'Invalid ingest_type: {}'.format( ingest_job_serializer_data["ingest_type"]), ErrorCodes.UNABLE_TO_VALIDATE) serializer = IngestJobCreateSerializer(data=ingest_job_serializer_data) if serializer.is_valid(): ingest_job = serializer.save() return ingest_job else: raise BossError("{}".format(serializer.errors), ErrorCodes.SERIALIZATION_ERROR)
def get(self, request, ingest_job_id=None): """ Join a job with the specified job id or list all job ids if ingest_job_id is omitted Args: request: Django rest framework request object ingest_job_id: Ingest job id Returns: Ingest job """ try: if ingest_job_id is None: # If the job ID is empty on a get, you are listing jobs return self.list_ingest_jobs(request) ingest_mgmr = IngestManager() ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id) # Check permissions if not self.is_user_or_admin(request, ingest_job): return BossHTTPError( "Only the creator or admin can join an ingest job", ErrorCodes.INGEST_NOT_CREATOR) serializer = IngestJobListSerializer(ingest_job) # Start setting up output data = {'ingest_job': serializer.data} if ingest_job.status == 3: # The job has been deleted raise BossError( "The job with id {} has been deleted".format( ingest_job_id), ErrorCodes.INVALID_REQUEST) elif ingest_job.status == 2 or ingest_job.status == 4: # Failed job or completed job return Response(data, status=status.HTTP_200_OK) elif ingest_job.status == 0: # Job is still in progress # check status of the step function session = bossutils.aws.get_session() if bossutils.aws.sfn_status( session, ingest_job.step_function_arn) == 'SUCCEEDED': # generate credentials ingest_job.status = 1 ingest_job.save() ingest_mgmr.generate_ingest_credentials(ingest_job) elif bossutils.aws.sfn_status( session, ingest_job.step_function_arn) == 'FAILED': # This indicates an error in step function raise BossError( "Error generating ingest job messages" " Delete the ingest job with id {} and try again.". format(ingest_job_id), ErrorCodes.BOSS_SYSTEM_ERROR) if ingest_job.status == 1: data['ingest_job']['status'] = 1 ingest_creds = IngestCredentials() data['credentials'] = ingest_creds.get_credentials( ingest_job.id) else: data['credentials'] = None data['tile_bucket_name'] = ingest_mgmr.get_tile_bucket() data['KVIO_SETTINGS'] = settings.KVIO_SETTINGS data['STATEIO_CONFIG'] = settings.STATEIO_CONFIG data['OBJECTIO_CONFIG'] = settings.OBJECTIO_CONFIG # add the lambda - Possibly remove this later config = bossutils.configuration.BossConfig() data['ingest_lambda'] = config["lambda"]["page_in_function"] # Generate a "resource" for the ingest lambda function to be able to use SPDB cleanly collection = Collection.objects.get( name=data['ingest_job']["collection"]) experiment = Experiment.objects.get( name=data['ingest_job']["experiment"], collection=collection) coord_frame = experiment.coord_frame channel = Channel.objects.get(name=data['ingest_job']["channel"], experiment=experiment) resource = {} resource['boss_key'] = '{}&{}&{}'.format( data['ingest_job']["collection"], data['ingest_job']["experiment"], data['ingest_job']["channel"]) resource['lookup_key'] = '{}&{}&{}'.format(collection.id, experiment.id, channel.id) # The Lambda function needs certain resource properties to perform write ops. Set required things only. # This is because S3 metadata is limited to 2kb, so we only set the bits of info needed, and in the lambda # Function Populate the rest with dummy info # IF YOU NEED ADDITIONAL DATA YOU MUST ADD IT HERE AND IN THE LAMBDA FUNCTION resource['channel'] = {} resource['channel']['type'] = channel.type resource['channel']['datatype'] = channel.datatype resource['channel']['base_resolution'] = channel.base_resolution resource['experiment'] = {} resource['experiment'][ 'num_hierarchy_levels'] = experiment.num_hierarchy_levels resource['experiment'][ 'hierarchy_method'] = experiment.hierarchy_method resource['coord_frame'] = {} resource['coord_frame']['x_voxel_size'] = coord_frame.x_voxel_size resource['coord_frame']['y_voxel_size'] = coord_frame.y_voxel_size resource['coord_frame']['z_voxel_size'] = coord_frame.z_voxel_size # Set resource data['resource'] = resource return Response(data, status=status.HTTP_200_OK) except BossError as err: return err.to_http() except Exception as err: return BossError("{}".format(err), ErrorCodes.BOSS_SYSTEM_ERROR).to_http()
def post(self, request, ingest_job_id): """ Signal an ingest job is complete and should be cleaned up by POSTing to this view Args: request: Django Rest framework Request object ingest_job_id: Ingest job id Returns: """ try: blog = BossLogger().logger ingest_mgmr = IngestManager() ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id) if ingest_job.status == IngestJob.PREPARING: # If status is Preparing. Deny return BossHTTPError( "You cannot complete a job that is still preparing. You must cancel instead.", ErrorCodes.BAD_REQUEST) elif ingest_job.status == IngestJob.UPLOADING: # Check if user is the ingest job creator or the sys admin if not self.is_user_or_admin(request, ingest_job): return BossHTTPError( "Only the creator or admin can start verification of an ingest job", ErrorCodes.INGEST_NOT_CREATOR) # Disable verification until it is reworked and always return # success for now. blog.info( 'Telling client job complete - completion/verificcation to be fixed later.' ) return Response(status=status.HTTP_204_NO_CONTENT) """ blog.info('Verifying ingest job {}'.format(ingest_job_id)) # Start verification process if not ingest_mgmr.verify_ingest_job(ingest_job): # Ingest not finished return Response(status=status.HTTP_202_ACCEPTED) """ # Verification successful, fall through to the complete process. elif ingest_job.status == IngestJob.COMPLETE: # If status is already Complete, just return another 204 return Response(status=status.HTTP_204_NO_CONTENT) elif ingest_job.status == IngestJob.DELETED: # Job had already been cancelled return BossHTTPError("Ingest job has already been cancelled.", ErrorCodes.BAD_REQUEST) elif ingest_job.status == IngestJob.FAILED: # Job had failed return BossHTTPError( "Ingest job has failed during creation. You must Cancel instead.", ErrorCodes.BAD_REQUEST) # Complete the job. blog.info("Completing Ingest Job {}".format(ingest_job_id)) # Check if user is the ingest job creator or the sys admin if not self.is_user_or_admin(request, ingest_job): return BossHTTPError( "Only the creator or admin can complete an ingest job", ErrorCodes.INGEST_NOT_CREATOR) # TODO SH This is a quick fix to make sure the ingest-client does not run close option. # the clean up code commented out below, because it is not working correctly. return Response(status=status.HTTP_204_NO_CONTENT) # if ingest_job.ingest_type == IngestJob.TILE_INGEST: # # Check if any messages remain in the ingest queue # ingest_queue = ingest_mgmr.get_ingest_job_ingest_queue(ingest_job) # num_messages_in_queue = int(ingest_queue.queue.attributes['ApproximateNumberOfMessages']) # # # Kick off extra lambdas just in case # if num_messages_in_queue: # blog.info("{} messages remaining in Ingest Queue".format(num_messages_in_queue)) # ingest_mgmr.invoke_ingest_lambda(ingest_job, num_messages_in_queue) # # # Give lambda a few seconds to fire things off # time.sleep(30) # # ingest_mgmr.cleanup_ingest_job(ingest_job, IngestJob.COMPLETE) # # elif ingest_job.ingest_type == IngestJob.VOLUMETRIC_INGEST: # ingest_mgmr.cleanup_ingest_job(ingest_job, IngestJob.COMPLETE) # # # ToDo: call cleanup method for volumetric ingests. Don't want # # to cleanup until after testing with real data. # #ingest_mgmr.cleanup_ingest_job(ingest_job, IngestJob.COMPLETE) # # blog.info("Complete successful") # return Response(status=status.HTTP_204_NO_CONTENT) except BossError as err: return err.to_http() except Exception as err: blog.error('Caught general exception: {}'.format(err)) return BossError("{}".format(err), ErrorCodes.BOSS_SYSTEM_ERROR).to_http()
def generate_upload_tasks(self, job_id=None): """ Generate upload tasks for the ingest job. This creates once task for each tile that has to be uploaded in the ingest queue Args: job_id: Job id of the ingest queue. If not included this takes the current ingest job Returns: None Raises: BossError : if there is no valid ingest job """ if job_id is None and self.job is None: raise BossError( "Unable to generate upload tasks for the ingest service. Please specify a ingest job", ErrorCodes.UNABLE_TO_VALIDATE) elif job_id: # Using the job id to get the job try: ingest_job = IngestJob.objects.get(id=job_id) except IngestJob.DoesNotExist: raise BossError( "Ingest job with id {} does not exist".format(job_id), ErrorCodes.RESOURCE_NOT_FOUND) else: ingest_job = self.job # Generate upload tasks for the ingest job # Get the project information bosskey = ingest_job.collection + CONNECTER + ingest_job.experiment + CONNECTER + ingest_job.channel lookup_key = (LookUpKey.get_lookup_key(bosskey)).lookup_key [col_id, exp_id, ch_id] = lookup_key.split('&') project_info = [col_id, exp_id, ch_id] # Batch messages and write to file base_file_name = 'tasks_' + lookup_key + '_' + str(ingest_job.id) self.file_index = 0 # open file f = io.StringIO() header = { 'job_id': ingest_job.id, 'upload_queue_url': ingest_job.upload_queue, 'ingest_queue_url': ingest_job.ingest_queue } f.write(json.dumps(header)) f.write('\n') num_msg_per_file = 0 for time_step in range(ingest_job.t_start, ingest_job.t_stop, 1): # For each time step, compute the chunks and tile keys for z in range(ingest_job.z_start, ingest_job.z_stop, 16): for y in range(ingest_job.y_start, ingest_job.y_stop, ingest_job.tile_size_y): for x in range(ingest_job.x_start, ingest_job.x_stop, ingest_job.tile_size_x): # compute the chunk indices chunk_x = int(x / ingest_job.tile_size_x) chunk_y = int(y / ingest_job.tile_size_y) chunk_z = int(z / 16) # Compute the number of tiles in the chunk if ingest_job.z_stop - z >= 16: num_of_tiles = 16 else: num_of_tiles = ingest_job.z_stop - z # Generate the chunk key chunk_key = (BossBackend( self.config)).encode_chunk_key( num_of_tiles, project_info, ingest_job.resolution, chunk_x, chunk_y, chunk_z, time_step) self.num_of_chunks += 1 # get the tiles keys for this chunk for tile in range(z, z + num_of_tiles): # get the tile key tile_key = (BossBackend( self.config)).encode_tile_key( project_info, ingest_job.resolution, chunk_x, chunk_y, tile, time_step) self.count_of_tiles += 1 # Generate the upload task msg msg = chunk_key + ',' + tile_key + '\n' f.write(msg) num_msg_per_file += 1 # if there are 10 messages in the batch send it to the upload queue. if num_msg_per_file == MAX_NUM_MSG_PER_FILE: fname = base_file_name + '_' + str( self.file_index + 1) + '.txt' self.upload_task_file(fname, f.getvalue()) self.file_index += 1 f.close() # status = self.send_upload_message_batch(batch_msg) fname = base_file_name + '_' + str( self.file_index + 1) + '.txt' f = io.StringIO() header = { 'job_id': ingest_job.id, 'upload_queue_url': ingest_job.upload_queue, 'ingest_queue_url': ingest_job.ingest_queue } f.write(json.dumps(header)) f.write('\n') num_msg_per_file = 0 # Edge case: the last batch size maybe smaller than 10 if num_msg_per_file != 0: fname = base_file_name + '_' + str(self.file_index + 1) + '.txt' self.upload_task_file(fname, f.getvalue()) f.close() self.file_index += 1 num_msg_per_file = 0 # Update status self.job.tile_count = self.count_of_tiles self.job.save()
def get(self, request, collection, experiment, dataset, orientation, resolution, x_args, y_args, z_args, t_args=None): """ View to handle GET requests for a cuboid of data while providing all params :param request: DRF Request object :type request: rest_framework.request.Request :param collection: Unique Collection identifier, indicating which collection you want to access :param experiment: Experiment identifier, indicating which experiment you want to access :param dataset: Dataset identifier, indicating which channel or layer you want to access :param orientation: Image plane requested. Vaid options include xy,xz or yz :param resolution: Integer indicating the level in the resolution hierarchy (0 = native) :param x_args: Python style range indicating the X coordinates of where to post the cuboid (eg. 100:200) :param y_args: Python style range indicating the Y coordinates of where to post the cuboid (eg. 100:200) :param z_args: Python style range indicating the Z coordinates of where to post the cuboid (eg. 100:200) :return: """ # Process request and validate try: req = BossRequest(request) except BossError as err: return BossError.to_http() # Convert to Resource resource = spdb.project.BossResourceDjango(req) # Get bit depth try: self.bit_depth = resource.get_bit_depth() except ValueError: return BossHTTPError("Datatype does not match channel/layer", ErrorCodes.DATATYPE_DOES_NOT_MATCH) # Make sure cutout request is under 1GB UNCOMPRESSED total_bytes = req.get_x_span() * req.get_y_span() * req.get_z_span( ) * len(req.get_time()) * (self.bit_depth / 8) if total_bytes > settings.CUTOUT_MAX_SIZE: return BossHTTPError( "Cutout request is over 1GB when uncompressed. Reduce cutout dimensions.", ErrorCodes.REQUEST_TOO_LARGE) # Get interface to SPDB cache cache = spdb.spatialdb.SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG, settings.OBJECTIO_CONFIG) # Get the params to pull data out of the cache corner = (req.get_x_start(), req.get_y_start(), req.get_z_start()) extent = (req.get_x_span(), req.get_y_span(), req.get_z_span()) # Do a cutout as specified data = cache.cutout( resource, corner, extent, req.get_resolution(), [req.get_time().start, req.get_time().stop]) # Covert the cutout back to an image and return it if orientation == 'xy': img = data.xy_image() elif orientation == 'yz': img = data.yz_image() elif orientation == 'xz': img = data.xz_image() else: return BossHTTPError("Invalid orientation: {}".format(orientation), ErrorCodes.INVALID_CUTOUT_ARGS) return Response(img)
def setup_ingest(self, creator, config_data): """ Setup the ingest job. This is the primary method for the ingest manager. It creates the ingest job and queues required for the ingest. It also uploads the messages for the ingest Args: creator: The validated user from the request to create the ingest jon config_data : Config data to create the ingest job Returns: IngestJob : data model containing the ingest job Raises: BossError : For all exceptions that happen """ # Validate config data and schema self.owner = creator try: valid_schema = self.validate_config_file(config_data) valid_prop = self.validate_properties() if valid_schema is True and valid_prop is True: # create the django model for the job self.job = self.create_ingest_job() # create the additional resources needed for the ingest # initialize the ndingest project for use with the library proj_class = BossIngestProj.load() self.nd_proj = proj_class(self.collection.name, self.experiment.name, self.channel.name, self.resolution, self.job.id) # Create the upload queue upload_queue = self.create_upload_queue() self.job.upload_queue = upload_queue.url # Create the ingest queue ingest_queue = self.create_ingest_queue() self.job.ingest_queue = ingest_queue.url # Call the step function to populate the queue. self.job.step_function_arn = self.populate_upload_queue() # Compute # of tiles in the job x_extent = self.job.x_stop - self.job.x_start y_extent = self.job.y_stop - self.job.y_start z_extent = self.job.z_stop - self.job.z_start t_extent = self.job.t_stop - self.job.t_start num_tiles_in_x = math.ceil(x_extent / self.job.tile_size_x) num_tiles_in_y = math.ceil(y_extent / self.job.tile_size_y) num_tiles_in_z = math.ceil(z_extent / self.job.tile_size_z) num_tiles_in_t = math.ceil(t_extent / self.job.tile_size_t) self.job.tile_count = num_tiles_in_x * num_tiles_in_y * num_tiles_in_z * num_tiles_in_t self.job.save() # tile_bucket = TileBucket(self.job.collection + '&' + self.job.experiment) # self.create_ingest_credentials(upload_queue, tile_bucket) except BossError as err: raise BossError(err.message, err.error_code) except Exception as e: raise BossError( "Unable to create the upload and ingest queue.{}".format(e), ErrorCodes.BOSS_SYSTEM_ERROR) return self.job
def generate_upload_tasks(self, job_id=None): """ Args: job_id: Returns: """ if job_id is None and self.job is None: raise BossError( "Unable to generate upload tasks for the ingest service. Please specify a ingest job", ErrorCodes.UNABLE_TO_VALIDATE) elif job_id: # Using the job id to get the job try: ingest_job = IngestJob.objects.get(id=job_id) except IngestJob.DoesNotExist: raise BossError( "Ingest job with id {} does not exist".format(job_id), ErrorCodes.RESOURCE_NOT_FOUND) else: ingest_job = self.job # Generate upload tasks for the ingest job # Get the project information bosskey = ingest_job.collection + CONNECTER + ingest_job.experiment + CONNECTER + ingest_job.channel_layer lookup_key = (LookUpKey.get_lookup_key(bosskey)).lookup_key [col_id, exp_id, ch_id] = lookup_key.split('&') project_info = [col_id, exp_id, ch_id] for time_step in range(ingest_job.t_start, ingest_job.t_stop, 1): # For each time step, compute the chunks and tile keys for z in range(ingest_job.z_start, ingest_job.z_stop, 16): for y in range(ingest_job.y_start, ingest_job.y_stop, ingest_job.tile_size_y): for x in range(ingest_job.x_start, ingest_job.x_stop, ingest_job.tile_size_x): # compute the chunk indices chunk_x = int(x / ingest_job.tile_size_x) chunk_y = int(y / ingest_job.tile_size_y) chunk_z = int(z / 16) # Compute the number of tiles in the chunk if ingest_job.z_stop - z >= 16: num_of_tiles = 16 else: num_of_tiles = ingest_job.z_stop - z # Generate the chunk key chunk_key = (BossBackend( self.config)).encode_chunk_key( num_of_tiles, project_info, ingest_job.resolution, chunk_x, chunk_y, chunk_z, time_step) # get the tiles keys for this chunk for tile in range(0, num_of_tiles): # get the tile key tile_key = (BossBackend( self.config)).encode_tile_key( project_info, ingest_job.resolution, chunk_x, chunk_y, tile, time_step) # Generate the upload task msg msg = self.create_upload_task_message( ingest_job.id, chunk_key, tile_key, ingest_job.upload_queue, ingest_job.ingest_queue) # Upload the message self.send_upload_task_message(msg)
def get(self, request, ingest_job_id=None): """ Join a job with the specified job id or list all job ids if ingest_job_id is omitted Args: request: Django rest framework request object ingest_job_id: Ingest job id Returns: Ingest job """ try: if ingest_job_id is None: # If the job ID is empty on a get, you are listing jobs return self.list_ingest_jobs(request) ingest_mgmr = IngestManager() ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id) # Check permissions if not self.is_user_or_admin(request, ingest_job): return BossHTTPError( "Only the creator or admin can join an ingest job", ErrorCodes.INGEST_NOT_CREATOR) serializer = IngestJobListSerializer(ingest_job) # Start setting up output data = {'ingest_job': serializer.data} data['ingest_job']['tile_index_queue'] = None if ingest_job.ingest_type == IngestJob.TILE_INGEST: data['ingest_job'][ 'tile_index_queue'] = ingest_mgmr.get_ingest_job_tile_index_queue( ingest_job).url if ingest_job.status == IngestJob.DELETED: raise BossError( "The job with id {} has been deleted".format( ingest_job_id), ErrorCodes.INVALID_REQUEST) elif ingest_job.status == IngestJob.COMPLETE or ingest_job.status == IngestJob.FAILED: return Response(data, status=status.HTTP_200_OK) elif ingest_job.status == IngestJob.PREPARING: # check status of the step function session = bossutils.aws.get_session() if bossutils.aws.sfn_status( session, ingest_job.step_function_arn) == 'SUCCEEDED': # generate credentials ingest_job.status = 1 ingest_job.save() ingest_mgmr.generate_ingest_credentials(ingest_job) elif bossutils.aws.sfn_status( session, ingest_job.step_function_arn) == 'FAILED': # This indicates an error in step function raise BossError( "Error generating ingest job messages" " Delete the ingest job with id {} and try again.". format(ingest_job_id), ErrorCodes.BOSS_SYSTEM_ERROR) if ingest_job.status in [ IngestJob.UPLOADING, IngestJob.WAIT_ON_QUEUES, IngestJob.COMPLETING ]: data['ingest_job']['status'] = ingest_job.status ingest_creds = IngestCredentials() data['credentials'] = ingest_creds.get_credentials( ingest_job.id) else: data['credentials'] = None data['tile_bucket_name'] = ingest_mgmr.get_tile_bucket() data['ingest_bucket_name'] = INGEST_BUCKET data['KVIO_SETTINGS'] = settings.KVIO_SETTINGS data['STATEIO_CONFIG'] = settings.STATEIO_CONFIG data['OBJECTIO_CONFIG'] = settings.OBJECTIO_CONFIG # Strip out un-needed data from OBJECTIO_CONFIG to save space when # including in S3 metadata. data['OBJECTIO_CONFIG'].pop('index_deadletter_queue', None) data['OBJECTIO_CONFIG'].pop('index_cuboids_keys_queue', None) # Set resource data['resource'] = ingest_mgmr.get_resource_data(ingest_job_id) # ingest_lambda is no longer required by the backend. The backend # gets the name of the ingest lambda from boss-manage/lib/names.py. # Keep providing it in case an older ingest client used (which # still expects it). data['ingest_lambda'] = 'deprecated' return Response(data, status=status.HTTP_200_OK) except BossError as err: return err.to_http() except Exception as err: return BossError("{}".format(err), ErrorCodes.BOSS_SYSTEM_ERROR).to_http()
def try_start_completing(self, ingest_job): """ Tries to start completion process. It is assumed that the ingest job status is currently WAIT_ON_QUEUES. If ingest_job status can be set to COMPLETING, then this process "wins" and starts the completion process. Args: ingest_job: Ingest job model Returns: (dict): { status: (job status str), wait_secs: (int) - # seconds client should wait } Raises: (BossError): If completion process cannot be started or is already in process. """ completing_success = { 'job_status': IngestJob.COMPLETING, 'wait_secs': 0 } if ingest_job.status == IngestJob.COMPLETING: return completing_success try: self.ensure_queues_empty(ingest_job) except BossError as be: # Ensure state goes back to UPLOADING if the upload queue isn't # empty. if be.message == UPLOAD_QUEUE_NOT_EMPTY_ERR_MSG: ingest_job.status = IngestJob.UPLOADING ingest_job.save() raise if ingest_job.status != IngestJob.WAIT_ON_QUEUES: raise BossError(NOT_IN_WAIT_ON_QUEUES_STATE_ERR_MSG, ErrorCodes.BAD_REQUEST) wait_remaining = self.calculate_remaining_queue_wait(ingest_job) if wait_remaining > 0: return { 'job_status': IngestJob.WAIT_ON_QUEUES, 'wait_secs': wait_remaining } rows_updated = (IngestJob.objects .exclude(status=IngestJob.COMPLETING) .filter(id=ingest_job.id) .update(status=IngestJob.COMPLETING) ) # If successfully set status to COMPLETING, kick off the completion # process. Otherwise, completion already started. if rows_updated > 0: self._start_completion_activity(ingest_job) log = bossLogger() log.info(f"Started completion step function for job: {ingest_job.id}") return completing_success
def track_usage_data(self, ingest_config_data, request): """ Set up usage tracking of this ingest. Args: ingest_config_data (dict): Ingest job config. Raises: (BossError): if user doesn't have permission for a large ingest. """ # need to get bytes per pixel to caculate ingest in total bytes try: ingest_job = ingest_config_data['ingest_job'] theCollection = Collection.objects.get( name=ingest_config_data['database']['collection']) theExperiment = Experiment.objects.get( name=ingest_config_data['database']['experiment']) theChannel = Channel.objects.get( name=ingest_config_data['database']['channel']) bytesPerPixel = int(theChannel.datatype.replace("uint", "")) / 8 # Add metrics to CloudWatch extent = ingest_job['extent'] database = ingest_config_data['database'] # Check that only permitted users are creating extra large ingests try: group = Group.objects.get(name=INGEST_GRP) in_large_ingest_group = group.user_set.filter( id=request.user.id).exists() except Group.DoesNotExist: # Just in case the group has not been created yet in_large_ingest_group = False if (not in_large_ingest_group) and \ ((extent['x'][1] - extent['x'][0]) * \ (extent['y'][1] - extent['y'][0]) * \ (extent['z'][1] - extent['z'][0]) * \ (extent['t'][1] - extent['t'][0]) > settings.INGEST_MAX_SIZE): raise BossError( "Large ingests require special permission to create. Contact system administrator.", ErrorCodes.INVALID_STATE) # Calculate the cost of the ingest in pixels costInPixels = ((extent['x'][1] - extent['x'][0]) * (extent['y'][1] - extent['y'][0]) * (extent['z'][1] - extent['z'][0]) * (extent['t'][1] - extent['t'][0])) cost = costInPixels * bytesPerPixel BossThrottle().check('ingest', ThrottleMetric.METRIC_TYPE_INGRESS, request.user, cost, ThrottleMetric.METRIC_UNITS_BYTES) boss_config = bossutils.configuration.BossConfig() dimensions = [ { 'Name': 'User', 'Value': request.user.username or "public" }, { 'Name': 'Resource', 'Value': '{}/{}/{}'.format(database['collection'], database['experiment'], database['channel']) }, { 'Name': 'Stack', 'Value': boss_config['system']['fqdn'] }, ] session = bossutils.aws.get_session() client = session.client('cloudwatch') try: client.put_metric_data(Namespace="BOSS/Ingest", MetricData=[{ 'MetricName': 'InvokeCount', 'Dimensions': dimensions, 'Value': 1.0, 'Unit': 'Count' }, { 'MetricName': 'IngressCost', 'Dimensions': dimensions, 'Value': cost, 'Unit': 'Bytes' }]) except Exception as e: log = bossLogger() log.exception('Error during put_metric_data: {}'.format(e)) log.exception('Allowing bossDB to continue after logging') except BossError as err: return err.to_http() except Exception as err: return BossError("{}".format(err), ErrorCodes.BOSS_SYSTEM_ERROR).to_http()
def get(self, request, ingest_job_id): """ Args: job_id: Returns: """ try: ingest_mgmr = IngestManager() ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id) serializer = IngestJobListSerializer(ingest_job) print(serializer.data) # Start setting up output data = {} data['ingest_job'] = serializer.data if ingest_job.status == 3 or ingest_job.status == 2: # Return the information for the deleted job/completed job return Response(data, status=status.HTTP_200_OK) elif ingest_job.status == 0: # check if all message are in the upload queue upload_queue = ingest_mgmr.get_ingest_job_upload_queue( ingest_job) if int(upload_queue.queue. attributes['ApproximateNumberOfMessages']) == int( ingest_job.tile_count): #generate credentials ingest_job.status = 1 ingest_job.save() elif int(upload_queue.queue. attributes['ApproximateNumberOfMessages']) > int( ingest_job.tile_count): # This indicates an error in the lambda raise BossError( "Error generating ingest job messages due to resources timing out ." " Delete the ingest job with id {} and try again.". format(ingest_job_id), ErrorCodes.BOSS_SYSTEM_ERROR) if ingest_job.status == 1: data['ingest_job']['status'] = 1 ingest_creds = IngestCredentials() data['credentials'] = ingest_creds.get_credentials( ingest_job.id) else: data['credentials'] = None data['tile_bucket_name'] = ingest_mgmr.get_tile_bucket() data['KVIO_SETTINGS'] = settings.KVIO_SETTINGS data['STATEIO_CONFIG'] = settings.STATEIO_CONFIG data['OBJECTIO_CONFIG'] = settings.OBJECTIO_CONFIG # add the lambda - Possibly remove this later config = bossutils.configuration.BossConfig() data['ingest_lambda'] = config["lambda"]["page_in_function"] # Generate a "resource" for the ingest lambda function to be able to use SPDB cleanly collection = Collection.objects.get( name=data['ingest_job']["collection"]) experiment = Experiment.objects.get( name=data['ingest_job']["experiment"], collection=collection) channel = Channel.objects.get(name=data['ingest_job']["channel"], experiment=experiment) resource = {} resource['boss_key'] = '{}&{}&{}'.format( data['ingest_job']["collection"], data['ingest_job']["experiment"], data['ingest_job']["channel"]) resource['lookup_key'] = '{}&{}&{}'.format(collection.id, experiment.id, channel.id) resource['channel'] = {} resource['channel']['name'] = channel.name resource['channel']['description'] = "" resource['channel']['type'] = channel.type resource['channel']['datatype'] = channel.datatype resource['channel']['base_resolution'] = channel.base_resolution resource['channel']['sources'] = [ x.name for x in channel.sources.all() ] resource['channel']['related'] = [ x.name for x in channel.related.all() ] resource['channel'][ 'default_time_sample'] = channel.default_time_sample # Set resource data['resource'] = resource return Response(data, status=status.HTTP_200_OK) except BossError as err: return err.to_http() except Exception as err: return BossError("{}".format(err), ErrorCodes.BOSS_SYSTEM_ERROR).to_http()
def get(self, request, ingest_job_id): """ Get the status of an ingest_job and number of messages in the upload queue Args: request: Django Rest framework object ingest_job_id: Ingest job id Returns: Status of the job """ try: ingest_mgmr = IngestManager() ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id) # Check if user is the ingest job creator or the sys admin if not self.is_user_or_admin(request, ingest_job): return BossHTTPError( "Only the creator or admin can check the status of an ingest job", ErrorCodes.INGEST_NOT_CREATOR) if ingest_job.status == IngestJob.DELETED: # Deleted Job raise BossError( "The job with id {} has been deleted".format( ingest_job_id), ErrorCodes.INVALID_REQUEST) else: if ingest_job.status == IngestJob.COMPLETE: # Job is Complete so queues are gone num_messages_in_queue = 0 else: try: upload_queue = ingest_mgmr.get_ingest_job_upload_queue( ingest_job) num_messages_in_queue = int( upload_queue.queue. attributes['ApproximateNumberOfMessages']) if num_messages_in_queue < ingest_job.tile_count: for n in range(9): num_messages_in_queue += int( upload_queue.queue. attributes['ApproximateNumberOfMessages']) num_messages_in_queue /= 10 except Exception: if ingest_job.status != IngestJob.COMPLETING: raise # Probably threw because queues were deleted while # completing ingest. num_messages_in_queue = 0 data = { "id": ingest_job.id, "status": ingest_job.status, "total_message_count": ingest_job.tile_count, "current_message_count": int(num_messages_in_queue) } return Response(data, status=status.HTTP_200_OK) except BossError as err: return err.to_http() except Exception as err: return BossError("{}".format(err), ErrorCodes.BOSS_SYSTEM_ERROR).to_http()
def post(self, request, ingest_job_id): """ Signal an ingest job is complete and should be cleaned up by POSTing to this view Args: request: Django Rest framework Request object ingest_job_id: Ingest job id Returns: """ try: blog = bossLogger() ingest_mgmr = IngestManager() ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id) # Check if user is the ingest job creator or the sys admin if not self.is_user_or_admin(request, ingest_job): return BossHTTPError( "Only the creator or admin can complete an ingest job", ErrorCodes.INGEST_NOT_CREATOR) if ingest_job.status == IngestJob.PREPARING: # If status is Preparing. Deny return BossHTTPError( "You cannot complete a job that is still preparing. You must cancel instead.", ErrorCodes.BAD_REQUEST) elif ingest_job.status == IngestJob.UPLOADING: try: data = ingest_mgmr.try_enter_wait_on_queue_state( ingest_job) return Response(data=data, status=status.HTTP_202_ACCEPTED) except BossError as be: if (be.message == INGEST_QUEUE_NOT_EMPTY_ERR_MSG or be.message == TILE_INDEX_QUEUE_NOT_EMPTY_ERR_MSG): # If there are messages in the tile error queue, this # will have to be handled manually. Non-empty ingest # or tile index queues should resolve on their own. return Response(data={ 'wait_secs': WAIT_FOR_QUEUES_SECS, 'info': 'Internal queues not empty yet' }, status=status.HTTP_400_BAD_REQUEST) raise elif ingest_job.status == IngestJob.WAIT_ON_QUEUES: pass # Continue below. elif ingest_job.status == IngestJob.COMPLETE: # If status is already Complete, just return another 204 return Response(data={'job_status': ingest_job.status}, status=status.HTTP_204_NO_CONTENT) elif ingest_job.status == IngestJob.DELETED: # Job had already been cancelled return BossHTTPError("Ingest job has already been cancelled.", ErrorCodes.BAD_REQUEST) elif ingest_job.status == IngestJob.FAILED: # Job had failed return BossHTTPError( "Ingest job has failed during creation. You must Cancel instead.", ErrorCodes.BAD_REQUEST) elif ingest_job.status == IngestJob.COMPLETING: return Response(data={'job_status': ingest_job.status}, status=status.HTTP_202_ACCEPTED) # Check if user is the ingest job creator or the sys admin if not self.is_user_or_admin(request, ingest_job): return BossHTTPError( "Only the creator or admin can complete an ingest job", ErrorCodes.INGEST_NOT_CREATOR) # Try to start completing. try: data = ingest_mgmr.try_start_completing(ingest_job) if data['job_status'] == IngestJob.WAIT_ON_QUEUES: # Refuse complete requests until wait period expires. return Response(data=data, status=status.HTTP_400_BAD_REQUEST) except BossError as be: if (be.message == INGEST_QUEUE_NOT_EMPTY_ERR_MSG or be.message == TILE_INDEX_QUEUE_NOT_EMPTY_ERR_MSG or be.message == INGEST_QUEUE_NOT_EMPTY_ERR_MSG): return Response(data={ 'wait_secs': WAIT_FOR_QUEUES_SECS, 'info': 'Internal queues not empty yet' }, status=status.HTTP_400_BAD_REQUEST) raise blog.info("Completion process started for ingest Job {}".format( ingest_job_id)) return Response(data=data, status=status.HTTP_202_ACCEPTED) # TODO SH This is a quick fix to make sure the ingest-client does not run close option. # the clean up code commented out below, because it is not working correctly. # return Response(status=status.HTTP_204_NO_CONTENT) # if ingest_job.ingest_type == IngestJob.TILE_INGEST: # # Check if any messages remain in the ingest queue # ingest_queue = ingest_mgmr.get_ingest_job_ingest_queue(ingest_job) # num_messages_in_queue = int(ingest_queue.queue.attributes['ApproximateNumberOfMessages']) # # # Kick off extra lambdas just in case # if num_messages_in_queue: # blog.info("{} messages remaining in Ingest Queue".format(num_messages_in_queue)) # ingest_mgmr.invoke_ingest_lambda(ingest_job, num_messages_in_queue) # # # Give lambda a few seconds to fire things off # time.sleep(30) # # ingest_mgmr.cleanup_ingest_job(ingest_job, IngestJob.COMPLETE) # # elif ingest_job.ingest_type == IngestJob.VOLUMETRIC_INGEST: # ingest_mgmr.cleanup_ingest_job(ingest_job, IngestJob.COMPLETE) # # # ToDo: call cleanup method for volumetric ingests. Don't want # # to cleanup until after testing with real data. # #ingest_mgmr.cleanup_ingest_job(ingest_job, IngestJob.COMPLETE) # # blog.info("Complete successful") # return Response(status=status.HTTP_204_NO_CONTENT) except BossError as err: return err.to_http() except Exception as err: blog.error('Caught general exception: {}'.format(err)) return BossError("{}".format(err), ErrorCodes.BOSS_SYSTEM_ERROR).to_http()