def get(self, request, ingest_job_id=None): """ Join a job with the specified job id or list all job ids if ingest_job_id is omitted Args: request: Django rest framework request object ingest_job_id: Ingest job id Returns: Ingest job """ try: if ingest_job_id is None: # If the job ID is empty on a get, you are listing jobs return self.list_ingest_jobs(request) ingest_mgmr = IngestManager() ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id) # Check permissions if not self.is_user_or_admin(request, ingest_job): return BossHTTPError( "Only the creator or admin can join an ingest job", ErrorCodes.INGEST_NOT_CREATOR) serializer = IngestJobListSerializer(ingest_job) # Start setting up output data = {'ingest_job': serializer.data} data['ingest_job']['tile_index_queue'] = None if ingest_job.ingest_type == IngestJob.TILE_INGEST: data['ingest_job'][ 'tile_index_queue'] = ingest_mgmr.get_ingest_job_tile_index_queue( ingest_job).url if ingest_job.status == IngestJob.DELETED: raise BossError( "The job with id {} has been deleted".format( ingest_job_id), ErrorCodes.INVALID_REQUEST) elif ingest_job.status == IngestJob.COMPLETE or ingest_job.status == IngestJob.FAILED: return Response(data, status=status.HTTP_200_OK) elif ingest_job.status == IngestJob.PREPARING: # check status of the step function session = bossutils.aws.get_session() if bossutils.aws.sfn_status( session, ingest_job.step_function_arn) == 'SUCCEEDED': # generate credentials ingest_job.status = 1 ingest_job.save() ingest_mgmr.generate_ingest_credentials(ingest_job) elif bossutils.aws.sfn_status( session, ingest_job.step_function_arn) == 'FAILED': # This indicates an error in step function raise BossError( "Error generating ingest job messages" " Delete the ingest job with id {} and try again.". format(ingest_job_id), ErrorCodes.BOSS_SYSTEM_ERROR) if ingest_job.status in [ IngestJob.UPLOADING, IngestJob.WAIT_ON_QUEUES, IngestJob.COMPLETING ]: data['ingest_job']['status'] = ingest_job.status ingest_creds = IngestCredentials() data['credentials'] = ingest_creds.get_credentials( ingest_job.id) else: data['credentials'] = None data['tile_bucket_name'] = ingest_mgmr.get_tile_bucket() data['ingest_bucket_name'] = INGEST_BUCKET data['KVIO_SETTINGS'] = settings.KVIO_SETTINGS data['STATEIO_CONFIG'] = settings.STATEIO_CONFIG data['OBJECTIO_CONFIG'] = settings.OBJECTIO_CONFIG # Strip out un-needed data from OBJECTIO_CONFIG to save space when # including in S3 metadata. data['OBJECTIO_CONFIG'].pop('index_deadletter_queue', None) data['OBJECTIO_CONFIG'].pop('index_cuboids_keys_queue', None) # Set resource data['resource'] = ingest_mgmr.get_resource_data(ingest_job_id) # ingest_lambda is no longer required by the backend. The backend # gets the name of the ingest lambda from boss-manage/lib/names.py. # Keep providing it in case an older ingest client used (which # still expects it). data['ingest_lambda'] = 'deprecated' return Response(data, status=status.HTTP_200_OK) except BossError as err: return err.to_http() except Exception as err: return BossError("{}".format(err), ErrorCodes.BOSS_SYSTEM_ERROR).to_http()
class BossIngestManagerTest(APITestCase): def setUp(self): """ Initialize the database :return: """ dbsetup = SetupTestDB() self.user = dbsetup.create_super_user(username='******', email='*****@*****.**', password='******') dbsetup.set_user(self.user) self.client.force_login(self.user) dbsetup.insert_ingest_test_data() setup = SetupTests() # Get the config_data for v1 schema config_data = setup.get_ingest_config_data_dict() self.example_config_data = config_data self.volumetric_config_data = setup.get_ingest_config_data_dict_volumetric() # Unit under test. self.ingest_mgr = IngestManager() def test_validate_ingest(self): """Method to test validation method""" #Validate schema and config file response = self.ingest_mgr.validate_config_file(self.example_config_data) assert (response is True) #Validate properties response = self.ingest_mgr.validate_properties() assert (response is True) def test_validate_config_file(self): """Method to test validation of a config file""" self.ingest_mgr.validate_config_file(self.example_config_data) assert(self.ingest_mgr.config is not None) assert (self.ingest_mgr.config.config_data is not None) def test_validate_properties(self): """Methos to test validation of properties of the config data""" self.ingest_mgr.validate_config_file(self.example_config_data) self.ingest_mgr.validate_properties() assert (self.ingest_mgr.collection.name == 'my_col_1') assert (self.ingest_mgr.experiment.name == 'my_exp_1') assert (self.ingest_mgr.channel.name == 'my_ch_1') def test_create_ingest_job(self): """Method to test creation of a ingest job from a config_data dict""" self.ingest_mgr.validate_config_file(self.example_config_data) self.ingest_mgr.validate_properties() self.ingest_mgr.owner = self.user.pk job = self.ingest_mgr.create_ingest_job() assert (job.id is not None) assert (job.ingest_type == IngestJob.TILE_INGEST) assert (job.tile_size_x == 512) assert (job.tile_size_y == 512) assert (job.tile_size_z == 1) assert (job.tile_size_t == 1) def test_create_ingest_job_volumetric(self): self.ingest_mgr.validate_config_file(self.volumetric_config_data) self.ingest_mgr.validate_properties() self.ingest_mgr.owner = self.user.pk job = self.ingest_mgr.create_ingest_job() assert (job.id is not None) assert (job.ingest_type == IngestJob.VOLUMETRIC_INGEST) assert (job.tile_size_x == 1024) assert (job.tile_size_y == 1024) assert (job.tile_size_z == 64) assert (job.tile_size_t == 1) def test_generate_upload_queue_args_tile_job(self): """Ensure ingest_type set properly""" self.ingest_mgr.validate_config_file(self.example_config_data) self.ingest_mgr.validate_properties() self.ingest_mgr.owner = self.user.pk job = self.ingest_mgr.create_ingest_job() actual = self.ingest_mgr._generate_upload_queue_args(job) assert actual['ingest_type'] == IngestJob.TILE_INGEST assert actual['z_chunk_size'] == 16 def test_generate_upload_queue_args_volumetric_job(self): """Ensure ingest_type set properly""" self.ingest_mgr.validate_config_file(self.volumetric_config_data) self.ingest_mgr.validate_properties() self.ingest_mgr.owner = self.user.pk job = self.ingest_mgr.create_ingest_job() actual = self.ingest_mgr._generate_upload_queue_args(job) assert actual['ingest_type'] == IngestJob.VOLUMETRIC_INGEST assert actual['z_chunk_size'] == 64 assert actual['ingest_queue'] is None def test_tile_bucket_name(self): """ Test get tile bucket name""" tile_bucket_name = self.ingest_mgr.get_tile_bucket() assert(tile_bucket_name is not None) def test_get_resource_data(self): """Run the method and ensure keys set""" self.ingest_mgr.validate_config_file(self.example_config_data) self.ingest_mgr.validate_properties() self.ingest_mgr.owner = self.user.pk job = self.ingest_mgr.create_ingest_job() actual = self.ingest_mgr.get_resource_data(job.id) self.assertIn('boss_key', actual) self.assertIn('lookup_key', actual) self.assertIn('channel', actual) self.assertIn('experiment', actual) self.assertIn('coord_frame', actual)