def test_create_ingest_job(self):
"""Method to test creation o a ingest job from a config_data dict"""
ingest_mgmr = IngestManager()
ingest_mgmr.validate_config_file(self.example_config_data)
ingest_mgmr.validate_properties()
ingest_mgmr.owner = self.user.id
job = ingest_mgmr.create_ingest_job()
assert (job.id is not None)
def test_validate_properties(self):
"""Methos to test validation of properties of the config data"""
ingest_mgmr = IngestManager()
ingest_mgmr.validate_config_file(self.example_config_data)
ingest_mgmr.validate_properties()
assert (ingest_mgmr.collection.name == 'my_col_1')
assert (ingest_mgmr.experiment.name == 'my_exp_1')
assert (ingest_mgmr.channel.name == 'my_ch_1')
def test_create_ingest_credentials(self):
""""""
ingest_mgmr = IngestManager()
ingest_mgmr = IngestManager()
ingest_mgmr.validate_config_file(self.example_config_data)
ingest_mgmr.validate_properties()
ingest_mgmr.owner = self.user.pk
job = ingest_mgmr.create_ingest_job()
ingest_mgmr.job = job
def test_generate_upload_queue_args_tile_job(self):
"""Ensure ingest_type set properly"""
ingest_mgmr = IngestManager()
ingest_mgmr.validate_config_file(self.example_config_data)
ingest_mgmr.validate_properties()
ingest_mgmr.owner = self.user.pk
job = ingest_mgmr.create_ingest_job()
actual = ingest_mgmr._generate_upload_queue_args(job)
assert actual['ingest_type'] == IngestJob.TILE_INGEST
assert actual['z_chunk_size'] == 16
def test_generate_upload_queue_args_volumetric_job(self):
"""Ensure ingest_type set properly"""
ingest_mgmr = IngestManager()
ingest_mgmr.validate_config_file(self.volumetric_config_data)
ingest_mgmr.validate_properties()
ingest_mgmr.owner = self.user.pk
job = ingest_mgmr.create_ingest_job()
actual = ingest_mgmr._generate_upload_queue_args(job)
assert actual['ingest_type'] == IngestJob.VOLUMETRIC_INGEST
assert actual['z_chunk_size'] == 64
assert actual['ingest_queue'] is None
def test_create_ingest_job_volumetric(self):
ingest_mgmr = IngestManager()
ingest_mgmr.validate_config_file(self.volumetric_config_data)
ingest_mgmr.validate_properties()
ingest_mgmr.owner = self.user.pk
job = ingest_mgmr.create_ingest_job()
assert (job.id is not None)
assert (job.ingest_type == IngestJob.VOLUMETRIC_INGEST)
assert (job.tile_size_x == 1024)
assert (job.tile_size_y == 1024)
assert (job.tile_size_z == 64)
assert (job.tile_size_t == 1)
def test_create_ingest_job(self):
"""Method to test creation o a ingest job from a config_data dict"""
ingest_mgmr = IngestManager()
ingest_mgmr.validate_config_file(self.example_config_data)
ingest_mgmr.validate_properties()
ingest_mgmr.owner = self.user.pk
job = ingest_mgmr.create_ingest_job()
assert (job.id is not None)
assert (job.ingest_type == IngestJob.TILE_INGEST)
assert (job.tile_size_x == 512)
assert (job.tile_size_y == 512)
assert (job.tile_size_z == 1)
assert (job.tile_size_t == 1)
def test_generate_upload_tasks(self):
"""
Test that the correct number of messages are being uploaded
"""
ingest_mgmr = IngestManager()
ingest_mgmr.validate_config_file(self.example_config_data)
ingest_mgmr.validate_properties()
ingest_mgmr.owner = self.user.pk
job = ingest_mgmr.create_ingest_job()
ingest_mgmr.job = job
with patch.object(IngestManager, 'upload_task_file') as mock_method:
ingest_mgmr.generate_upload_tasks(job.id)
self.assertEquals(ingest_mgmr.file_index, 1)
self.assertEquals(ingest_mgmr.num_of_chunks, 48)
self.assertEquals(ingest_mgmr.count_of_tiles, 640)
def test_validate_ingest(self):
"""Method to test validation method"""
#Validate schema and config file
ingest_mgmr = IngestManager()
response = ingest_mgmr.validate_config_file(self.example_config_data)
assert (response is True)
#Validate properties
response = ingest_mgmr.validate_properties()
assert (response is True)
def test_upload_tile_index_table(self):
""""""
ingest_mgmr = IngestManager()
ingest_mgmr.validate_config_file(self.example_config_data)
ingest_mgmr.validate_properties()
ingest_mgmr.owner = self.user.pk
ingest_job = ingest_mgmr.create_ingest_job()
assert (ingest_job.id is not None)
# Get the chunks in this job
# Get the project information
bosskey = ingest_job.collection + '&' + ingest_job.experiment + '&' + ingest_job.channel_layer
lookup_key = (LookUpKey.get_lookup_key(bosskey)).lookup_key
[col_id, exp_id, ch_id] = lookup_key.split('&')
project_info = [col_id, exp_id, ch_id]
proj_name = ingest_job.collection + '&' + ingest_job.experiment
tile_index_db = BossTileIndexDB(proj_name)
tilebucket = TileBucket(str(col_id) + '&' + str(exp_id))
for time_step in range(ingest_job.t_start, ingest_job.t_stop, 1):
# For each time step, compute the chunks and tile keys
for z in range(ingest_job.z_start, ingest_job.z_stop, 16):
for y in range(ingest_job.y_start, ingest_job.y_stop,
ingest_job.tile_size_y):
for x in range(ingest_job.x_start, ingest_job.x_stop,
ingest_job.tile_size_x):
# compute the chunk indices
chunk_x = int(x / ingest_job.tile_size_x)
chunk_y = int(y / ingest_job.tile_size_y)
chunk_z = int(z / 16)
# Compute the number of tiles in the chunk
if ingest_job.z_stop - z >= 16:
num_of_tiles = 16
else:
num_of_tiles = ingest_job.z_stop - z
# Generate the chunk key
chunk_key = (BossBackend(
ingest_mgmr.config)).encode_chunk_key(
num_of_tiles, project_info,
ingest_job.resolution, chunk_x, chunk_y,
chunk_z, time_step)
# Upload the chunk to the tile index db
tile_index_db.createCuboidEntry(
chunk_key, ingest_job.id)
key_map = {}
for tile in range(0, num_of_tiles):
# get the object key and upload it
#tile_key = tilebucket.encodeObjectKey(ch_id, ingest_job.resolution,
# chunk_x, chunk_y, tile, time_step)
tile_key = 'fakekey' + str(tile)
tile_index_db.markTileAsUploaded(
chunk_key, tile_key)
# for each chunk key, delete entries from the tile_bucket
# Check if data has been uploaded
chunks = list(tile_index_db.getTaskItems(ingest_job.id))
assert (len(chunks) != 0)
ingest_mgmr.delete_tiles(ingest_job)
chunks = list(tile_index_db.getTaskItems(ingest_job.id))
assert (len(chunks) == 0)
class BossIngestManagerTest(APITestCase):
def setUp(self):
"""
Initialize the database
:return:
"""
dbsetup = SetupTestDB()
self.user = dbsetup.create_super_user(username='******', email='*****@*****.**', password='******')
dbsetup.set_user(self.user)
self.client.force_login(self.user)
dbsetup.insert_ingest_test_data()
setup = SetupTests()
# Get the config_data for v1 schema
config_data = setup.get_ingest_config_data_dict()
self.example_config_data = config_data
self.volumetric_config_data = setup.get_ingest_config_data_dict_volumetric()
# Unit under test.
self.ingest_mgr = IngestManager()
def test_validate_ingest(self):
"""Method to test validation method"""
#Validate schema and config file
response = self.ingest_mgr.validate_config_file(self.example_config_data)
assert (response is True)
#Validate properties
response = self.ingest_mgr.validate_properties()
assert (response is True)
def test_validate_config_file(self):
"""Method to test validation of a config file"""
self.ingest_mgr.validate_config_file(self.example_config_data)
assert(self.ingest_mgr.config is not None)
assert (self.ingest_mgr.config.config_data is not None)
def test_validate_properties(self):
"""Methos to test validation of properties of the config data"""
self.ingest_mgr.validate_config_file(self.example_config_data)
self.ingest_mgr.validate_properties()
assert (self.ingest_mgr.collection.name == 'my_col_1')
assert (self.ingest_mgr.experiment.name == 'my_exp_1')
assert (self.ingest_mgr.channel.name == 'my_ch_1')
def test_create_ingest_job(self):
"""Method to test creation of a ingest job from a config_data dict"""
self.ingest_mgr.validate_config_file(self.example_config_data)
self.ingest_mgr.validate_properties()
self.ingest_mgr.owner = self.user.pk
job = self.ingest_mgr.create_ingest_job()
assert (job.id is not None)
assert (job.ingest_type == IngestJob.TILE_INGEST)
assert (job.tile_size_x == 512)
assert (job.tile_size_y == 512)
assert (job.tile_size_z == 1)
assert (job.tile_size_t == 1)
def test_create_ingest_job_volumetric(self):
self.ingest_mgr.validate_config_file(self.volumetric_config_data)
self.ingest_mgr.validate_properties()
self.ingest_mgr.owner = self.user.pk
job = self.ingest_mgr.create_ingest_job()
assert (job.id is not None)
assert (job.ingest_type == IngestJob.VOLUMETRIC_INGEST)
assert (job.tile_size_x == 1024)
assert (job.tile_size_y == 1024)
assert (job.tile_size_z == 64)
assert (job.tile_size_t == 1)
def test_generate_upload_queue_args_tile_job(self):
"""Ensure ingest_type set properly"""
self.ingest_mgr.validate_config_file(self.example_config_data)
self.ingest_mgr.validate_properties()
self.ingest_mgr.owner = self.user.pk
job = self.ingest_mgr.create_ingest_job()
actual = self.ingest_mgr._generate_upload_queue_args(job)
assert actual['ingest_type'] == IngestJob.TILE_INGEST
assert actual['z_chunk_size'] == 16
def test_generate_upload_queue_args_volumetric_job(self):
"""Ensure ingest_type set properly"""
self.ingest_mgr.validate_config_file(self.volumetric_config_data)
self.ingest_mgr.validate_properties()
self.ingest_mgr.owner = self.user.pk
job = self.ingest_mgr.create_ingest_job()
actual = self.ingest_mgr._generate_upload_queue_args(job)
assert actual['ingest_type'] == IngestJob.VOLUMETRIC_INGEST
assert actual['z_chunk_size'] == 64
assert actual['ingest_queue'] is None
def test_tile_bucket_name(self):
""" Test get tile bucket name"""
tile_bucket_name = self.ingest_mgr.get_tile_bucket()
assert(tile_bucket_name is not None)
def test_get_resource_data(self):
"""Run the method and ensure keys set"""
self.ingest_mgr.validate_config_file(self.example_config_data)
self.ingest_mgr.validate_properties()
self.ingest_mgr.owner = self.user.pk
job = self.ingest_mgr.create_ingest_job()
actual = self.ingest_mgr.get_resource_data(job.id)
self.assertIn('boss_key', actual)
self.assertIn('lookup_key', actual)
self.assertIn('channel', actual)
self.assertIn('experiment', actual)
self.assertIn('coord_frame', actual)