def get(self, request, ingest_job_id=None): """ Join a job with the specified job id or list all job ids if ingest_job_id is omitted Args: request: Django rest framework request object ingest_job_id: Ingest job id Returns: Ingest job """ try: if ingest_job_id is None: # If the job ID is empty on a get, you are listing jobs return self.list_ingest_jobs(request) ingest_mgmr = IngestManager() ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id) # Check permissions if not self.is_user_or_admin(request, ingest_job): return BossHTTPError( "Only the creator or admin can join an ingest job", ErrorCodes.INGEST_NOT_CREATOR) serializer = IngestJobListSerializer(ingest_job) # Start setting up output data = {'ingest_job': serializer.data} if ingest_job.status == 3: # The job has been deleted raise BossError( "The job with id {} has been deleted".format( ingest_job_id), ErrorCodes.INVALID_REQUEST) elif ingest_job.status == 2 or ingest_job.status == 4: # Failed job or completed job return Response(data, status=status.HTTP_200_OK) elif ingest_job.status == 0: # Job is still in progress # check status of the step function session = bossutils.aws.get_session() if bossutils.aws.sfn_status( session, ingest_job.step_function_arn) == 'SUCCEEDED': # generate credentials ingest_job.status = 1 ingest_job.save() ingest_mgmr.generate_ingest_credentials(ingest_job) elif bossutils.aws.sfn_status( session, ingest_job.step_function_arn) == 'FAILED': # This indicates an error in step function raise BossError( "Error generating ingest job messages" " Delete the ingest job with id {} and try again.". format(ingest_job_id), ErrorCodes.BOSS_SYSTEM_ERROR) if ingest_job.status == 1: data['ingest_job']['status'] = 1 ingest_creds = IngestCredentials() data['credentials'] = ingest_creds.get_credentials( ingest_job.id) else: data['credentials'] = None data['tile_bucket_name'] = ingest_mgmr.get_tile_bucket() data['KVIO_SETTINGS'] = settings.KVIO_SETTINGS data['STATEIO_CONFIG'] = settings.STATEIO_CONFIG data['OBJECTIO_CONFIG'] = settings.OBJECTIO_CONFIG # add the lambda - Possibly remove this later config = bossutils.configuration.BossConfig() data['ingest_lambda'] = config["lambda"]["page_in_function"] # Generate a "resource" for the ingest lambda function to be able to use SPDB cleanly collection = Collection.objects.get( name=data['ingest_job']["collection"]) experiment = Experiment.objects.get( name=data['ingest_job']["experiment"], collection=collection) coord_frame = experiment.coord_frame channel = Channel.objects.get(name=data['ingest_job']["channel"], experiment=experiment) resource = {} resource['boss_key'] = '{}&{}&{}'.format( data['ingest_job']["collection"], data['ingest_job']["experiment"], data['ingest_job']["channel"]) resource['lookup_key'] = '{}&{}&{}'.format(collection.id, experiment.id, channel.id) # The Lambda function needs certain resource properties to perform write ops. Set required things only. # This is because S3 metadata is limited to 2kb, so we only set the bits of info needed, and in the lambda # Function Populate the rest with dummy info # IF YOU NEED ADDITIONAL DATA YOU MUST ADD IT HERE AND IN THE LAMBDA FUNCTION resource['channel'] = {} resource['channel']['type'] = channel.type resource['channel']['datatype'] = channel.datatype resource['channel']['base_resolution'] = channel.base_resolution resource['experiment'] = {} resource['experiment'][ 'num_hierarchy_levels'] = experiment.num_hierarchy_levels resource['experiment'][ 'hierarchy_method'] = experiment.hierarchy_method resource['coord_frame'] = {} resource['coord_frame']['x_voxel_size'] = coord_frame.x_voxel_size resource['coord_frame']['y_voxel_size'] = coord_frame.y_voxel_size resource['coord_frame']['z_voxel_size'] = coord_frame.z_voxel_size # Set resource data['resource'] = resource return Response(data, status=status.HTTP_200_OK) except BossError as err: return err.to_http() except Exception as err: return BossError("{}".format(err), ErrorCodes.BOSS_SYSTEM_ERROR).to_http()
def test_tile_bucket_name(self): """ Test get tile bucket name""" ingest_mgmr = IngestManager() tile_bucket_name = ingest_mgmr.get_tile_bucket() assert(tile_bucket_name is not None)
def get(self, request, ingest_job_id=None): """ Join a job with the specified job id or list all job ids if ingest_job_id is omitted Args: request: Django rest framework request object ingest_job_id: Ingest job id Returns: Ingest job """ try: if ingest_job_id is None: # If the job ID is empty on a get, you are listing jobs return self.list_ingest_jobs(request) ingest_mgmr = IngestManager() ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id) # Check permissions if not self.is_user_or_admin(request, ingest_job): return BossHTTPError( "Only the creator or admin can join an ingest job", ErrorCodes.INGEST_NOT_CREATOR) serializer = IngestJobListSerializer(ingest_job) # Start setting up output data = {'ingest_job': serializer.data} data['ingest_job']['tile_index_queue'] = None if ingest_job.ingest_type == IngestJob.TILE_INGEST: data['ingest_job'][ 'tile_index_queue'] = ingest_mgmr.get_ingest_job_tile_index_queue( ingest_job).url if ingest_job.status == IngestJob.DELETED: raise BossError( "The job with id {} has been deleted".format( ingest_job_id), ErrorCodes.INVALID_REQUEST) elif ingest_job.status == IngestJob.COMPLETE or ingest_job.status == IngestJob.FAILED: return Response(data, status=status.HTTP_200_OK) elif ingest_job.status == IngestJob.PREPARING: # check status of the step function session = bossutils.aws.get_session() if bossutils.aws.sfn_status( session, ingest_job.step_function_arn) == 'SUCCEEDED': # generate credentials ingest_job.status = 1 ingest_job.save() ingest_mgmr.generate_ingest_credentials(ingest_job) elif bossutils.aws.sfn_status( session, ingest_job.step_function_arn) == 'FAILED': # This indicates an error in step function raise BossError( "Error generating ingest job messages" " Delete the ingest job with id {} and try again.". format(ingest_job_id), ErrorCodes.BOSS_SYSTEM_ERROR) if ingest_job.status in [ IngestJob.UPLOADING, IngestJob.WAIT_ON_QUEUES, IngestJob.COMPLETING ]: data['ingest_job']['status'] = ingest_job.status ingest_creds = IngestCredentials() data['credentials'] = ingest_creds.get_credentials( ingest_job.id) else: data['credentials'] = None data['tile_bucket_name'] = ingest_mgmr.get_tile_bucket() data['ingest_bucket_name'] = INGEST_BUCKET data['KVIO_SETTINGS'] = settings.KVIO_SETTINGS data['STATEIO_CONFIG'] = settings.STATEIO_CONFIG data['OBJECTIO_CONFIG'] = settings.OBJECTIO_CONFIG # Strip out un-needed data from OBJECTIO_CONFIG to save space when # including in S3 metadata. data['OBJECTIO_CONFIG'].pop('index_deadletter_queue', None) data['OBJECTIO_CONFIG'].pop('index_cuboids_keys_queue', None) # Set resource data['resource'] = ingest_mgmr.get_resource_data(ingest_job_id) # ingest_lambda is no longer required by the backend. The backend # gets the name of the ingest lambda from boss-manage/lib/names.py. # Keep providing it in case an older ingest client used (which # still expects it). data['ingest_lambda'] = 'deprecated' return Response(data, status=status.HTTP_200_OK) except BossError as err: return err.to_http() except Exception as err: return BossError("{}".format(err), ErrorCodes.BOSS_SYSTEM_ERROR).to_http()
def get(self, request, ingest_job_id): """ Args: job_id: Returns: """ try: ingest_mgmr = IngestManager() ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id) serializer = IngestJobListSerializer(ingest_job) print(serializer.data) # Start setting up output data = {} data['ingest_job'] = serializer.data if ingest_job.status == 3 or ingest_job.status == 2: # Return the information for the deleted job/completed job return Response(data, status=status.HTTP_200_OK) elif ingest_job.status == 0: # check if all message are in the upload queue upload_queue = ingest_mgmr.get_ingest_job_upload_queue( ingest_job) if int(upload_queue.queue. attributes['ApproximateNumberOfMessages']) == int( ingest_job.tile_count): #generate credentials ingest_job.status = 1 ingest_job.save() elif int(upload_queue.queue. attributes['ApproximateNumberOfMessages']) > int( ingest_job.tile_count): # This indicates an error in the lambda raise BossError( "Error generating ingest job messages due to resources timing out ." " Delete the ingest job with id {} and try again.". format(ingest_job_id), ErrorCodes.BOSS_SYSTEM_ERROR) if ingest_job.status == 1: data['ingest_job']['status'] = 1 ingest_creds = IngestCredentials() data['credentials'] = ingest_creds.get_credentials( ingest_job.id) else: data['credentials'] = None data['tile_bucket_name'] = ingest_mgmr.get_tile_bucket() data['KVIO_SETTINGS'] = settings.KVIO_SETTINGS data['STATEIO_CONFIG'] = settings.STATEIO_CONFIG data['OBJECTIO_CONFIG'] = settings.OBJECTIO_CONFIG # add the lambda - Possibly remove this later config = bossutils.configuration.BossConfig() data['ingest_lambda'] = config["lambda"]["page_in_function"] # Generate a "resource" for the ingest lambda function to be able to use SPDB cleanly collection = Collection.objects.get( name=data['ingest_job']["collection"]) experiment = Experiment.objects.get( name=data['ingest_job']["experiment"], collection=collection) channel = Channel.objects.get(name=data['ingest_job']["channel"], experiment=experiment) resource = {} resource['boss_key'] = '{}&{}&{}'.format( data['ingest_job']["collection"], data['ingest_job']["experiment"], data['ingest_job']["channel"]) resource['lookup_key'] = '{}&{}&{}'.format(collection.id, experiment.id, channel.id) resource['channel'] = {} resource['channel']['name'] = channel.name resource['channel']['description'] = "" resource['channel']['type'] = channel.type resource['channel']['datatype'] = channel.datatype resource['channel']['base_resolution'] = channel.base_resolution resource['channel']['sources'] = [ x.name for x in channel.sources.all() ] resource['channel']['related'] = [ x.name for x in channel.related.all() ] resource['channel'][ 'default_time_sample'] = channel.default_time_sample # Set resource data['resource'] = resource return Response(data, status=status.HTTP_200_OK) except BossError as err: return err.to_http() except Exception as err: return BossError("{}".format(err), ErrorCodes.BOSS_SYSTEM_ERROR).to_http()
class BossIngestManagerTest(APITestCase): def setUp(self): """ Initialize the database :return: """ dbsetup = SetupTestDB() self.user = dbsetup.create_super_user(username='******', email='*****@*****.**', password='******') dbsetup.set_user(self.user) self.client.force_login(self.user) dbsetup.insert_ingest_test_data() setup = SetupTests() # Get the config_data for v1 schema config_data = setup.get_ingest_config_data_dict() self.example_config_data = config_data self.volumetric_config_data = setup.get_ingest_config_data_dict_volumetric() # Unit under test. self.ingest_mgr = IngestManager() def test_validate_ingest(self): """Method to test validation method""" #Validate schema and config file response = self.ingest_mgr.validate_config_file(self.example_config_data) assert (response is True) #Validate properties response = self.ingest_mgr.validate_properties() assert (response is True) def test_validate_config_file(self): """Method to test validation of a config file""" self.ingest_mgr.validate_config_file(self.example_config_data) assert(self.ingest_mgr.config is not None) assert (self.ingest_mgr.config.config_data is not None) def test_validate_properties(self): """Methos to test validation of properties of the config data""" self.ingest_mgr.validate_config_file(self.example_config_data) self.ingest_mgr.validate_properties() assert (self.ingest_mgr.collection.name == 'my_col_1') assert (self.ingest_mgr.experiment.name == 'my_exp_1') assert (self.ingest_mgr.channel.name == 'my_ch_1') def test_create_ingest_job(self): """Method to test creation of a ingest job from a config_data dict""" self.ingest_mgr.validate_config_file(self.example_config_data) self.ingest_mgr.validate_properties() self.ingest_mgr.owner = self.user.pk job = self.ingest_mgr.create_ingest_job() assert (job.id is not None) assert (job.ingest_type == IngestJob.TILE_INGEST) assert (job.tile_size_x == 512) assert (job.tile_size_y == 512) assert (job.tile_size_z == 1) assert (job.tile_size_t == 1) def test_create_ingest_job_volumetric(self): self.ingest_mgr.validate_config_file(self.volumetric_config_data) self.ingest_mgr.validate_properties() self.ingest_mgr.owner = self.user.pk job = self.ingest_mgr.create_ingest_job() assert (job.id is not None) assert (job.ingest_type == IngestJob.VOLUMETRIC_INGEST) assert (job.tile_size_x == 1024) assert (job.tile_size_y == 1024) assert (job.tile_size_z == 64) assert (job.tile_size_t == 1) def test_generate_upload_queue_args_tile_job(self): """Ensure ingest_type set properly""" self.ingest_mgr.validate_config_file(self.example_config_data) self.ingest_mgr.validate_properties() self.ingest_mgr.owner = self.user.pk job = self.ingest_mgr.create_ingest_job() actual = self.ingest_mgr._generate_upload_queue_args(job) assert actual['ingest_type'] == IngestJob.TILE_INGEST assert actual['z_chunk_size'] == 16 def test_generate_upload_queue_args_volumetric_job(self): """Ensure ingest_type set properly""" self.ingest_mgr.validate_config_file(self.volumetric_config_data) self.ingest_mgr.validate_properties() self.ingest_mgr.owner = self.user.pk job = self.ingest_mgr.create_ingest_job() actual = self.ingest_mgr._generate_upload_queue_args(job) assert actual['ingest_type'] == IngestJob.VOLUMETRIC_INGEST assert actual['z_chunk_size'] == 64 assert actual['ingest_queue'] is None def test_tile_bucket_name(self): """ Test get tile bucket name""" tile_bucket_name = self.ingest_mgr.get_tile_bucket() assert(tile_bucket_name is not None) def test_get_resource_data(self): """Run the method and ensure keys set""" self.ingest_mgr.validate_config_file(self.example_config_data) self.ingest_mgr.validate_properties() self.ingest_mgr.owner = self.user.pk job = self.ingest_mgr.create_ingest_job() actual = self.ingest_mgr.get_resource_data(job.id) self.assertIn('boss_key', actual) self.assertIn('lookup_key', actual) self.assertIn('channel', actual) self.assertIn('experiment', actual) self.assertIn('coord_frame', actual)