def test_basic_resource_get_bit_depth(self): """Test basic get bit depth interface Returns: None """ setup_data = get_image_dict() resource = BossResourceBasic(setup_data) assert resource.get_bit_depth() == 8
def test_basic_resource_numpy_data_type(self): """Test basic get bit depth interface Returns: None """ setup_data = get_image_dict() resource = BossResourceBasic(setup_data) assert resource.get_numpy_data_type() == np.uint8
def test_basic_resource_get_lookup_key(self): """Test basic get lookup key interface Returns: None """ setup_data = get_image_dict() resource = BossResourceBasic(setup_data) assert resource.get_lookup_key() == setup_data['lookup_key']
def test_basic_resource_get_data_type(self): """Test basic get datatype interface Returns: None """ setup_data = get_image_dict() resource = BossResourceBasic(setup_data) assert resource.get_data_type() == setup_data['channel']['datatype']
def test_basic_resource_get_iso_level_anisotropic(self): """Test get iso level anisotropic Returns: None """ setup_data = get_anno_dict() resource = BossResourceBasic(setup_data) self.assertEqual(resource.get_isotropic_level(), 3)
def test_basic_resource_get_iso_level_isotropic(self): """Test get iso level isotropic Returns: None """ setup_data = get_anno_dict() setup_data["experiment"]['hierarchy_method'] = "isotropic" resource = BossResourceBasic(setup_data) self.assertEqual(resource.get_isotropic_level(), 0)
def test_basic_resource_get_downsampled_voxel_dims_anisotropic(self): """Test downsample voxel dims anisotropic Returns: None """ setup_data = get_anno_dict() resource = BossResourceBasic(setup_data) voxel_dims = resource.get_downsampled_voxel_dims() self.assertEqual(len(voxel_dims), setup_data["experiment"]['num_hierarchy_levels']) self.assertEqual(voxel_dims[0], [4, 4, 35]) self.assertEqual(voxel_dims[4], [64, 64, 35])
def test_basic_resource_get_downsampled_extent_dims_anisotropic(self): """Test downsample extent anisotropic Returns: None """ setup_data = get_anno_dict() resource = BossResourceBasic(setup_data) extent_dims = resource.get_downsampled_extent_dims() self.assertEqual(len(extent_dims), setup_data["experiment"]['num_hierarchy_levels']) self.assertEqual(extent_dims[0], [2000, 5000, 200]) self.assertEqual(extent_dims[4], [125, 313, 200])
def test_basic_resource_col(self): """Test basic get collection interface Returns: None """ setup_data = get_image_dict() resource = BossResourceBasic(setup_data) col = resource.get_collection() assert col.name == setup_data['collection']['name'] assert col.description == setup_data['collection']['description']
def test_is_downsampled(self): """Test is downsampled method Returns: None """ setup_data = get_image_dict() resource = BossResourceBasic(setup_data) self.assertFalse(resource.is_downsampled()) setup_data['channel']['downsample_status'] = "DOWNSAMPLED" resource = BossResourceBasic(setup_data) self.assertTrue(resource.is_downsampled())
def test_basic_resource_get_downsampled_voxel_dims_anisotropic_iso(self): """Test downsample voxel dims anisotropic with iso flag Returns: None """ setup_data = get_anno_dict() resource = BossResourceBasic(setup_data) voxel_dims = resource.get_downsampled_voxel_dims(iso=True) self.assertEqual(len(voxel_dims), setup_data["experiment"]['num_hierarchy_levels']) self.assertEqual(voxel_dims[0], [4, 4, 35]) self.assertEqual(voxel_dims[1], [8, 8, 35]) self.assertEqual(voxel_dims[2], [16, 16, 35]) self.assertEqual(voxel_dims[3], [32, 32, 35]) self.assertEqual(voxel_dims[4], [64, 64, 70]) self.assertEqual(voxel_dims[5], [128, 128, 140])
def test_basic_resource_get_downsampled_voxel_dims_isotropic(self): """Test downsample voxel dims isotropic Returns: None """ setup_data = get_anno_dict() setup_data['coord_frame']['x_voxel_size'] = 6 setup_data['coord_frame']['y_voxel_size'] = 6 setup_data['coord_frame']['z_voxel_size'] = 6 setup_data['experiment']['hierarchy_method'] = "isotropic" resource = BossResourceBasic(setup_data) voxel_dims = resource.get_downsampled_voxel_dims() self.assertEqual(len(voxel_dims), setup_data["experiment"]['num_hierarchy_levels']) self.assertEqual(voxel_dims[0], [6, 6, 6]) self.assertEqual(voxel_dims[3], [48, 48, 48])
def test_basic_resource_experiment(self): """Test basic get experiment interface Returns: None """ setup_data = get_image_dict() resource = BossResourceBasic(setup_data) exp = resource.get_experiment() assert exp.name == setup_data['experiment']['name'] assert exp.description == setup_data['experiment']['description'] assert exp.num_hierarchy_levels == setup_data['experiment']['num_hierarchy_levels'] assert exp.hierarchy_method == setup_data['experiment']['hierarchy_method'] assert exp.num_time_samples == setup_data['experiment']['num_time_samples'] assert exp.time_step == setup_data['experiment']['time_step'] assert exp.time_step_unit == setup_data['experiment']['time_step_unit']
def test_basic_resource_get_downsampled_extent_dims_isotropic(self): """Test downsample extent isotropic Returns: None """ setup_data = get_anno_dict() setup_data['coord_frame']['x_voxel_size'] = 6 setup_data['coord_frame']['y_voxel_size'] = 6 setup_data['coord_frame']['z_voxel_size'] = 6 setup_data['experiment']['hierarchy_method'] = "isotropic" resource = BossResourceBasic(setup_data) extent_dims = resource.get_downsampled_extent_dims() self.assertEqual(len(extent_dims), setup_data["experiment"]['num_hierarchy_levels']) self.assertEqual(extent_dims[0], [2000, 5000, 200]) self.assertEqual(extent_dims[1], [1000, 2500, 100]) self.assertEqual(extent_dims[3], [250, 625, 25])
def test_basic_resource_annotation_no_time(self): """Test basic get layer interface Returns: None """ setup_data = get_anno_dict() resource = BossResourceBasic(setup_data) channel = resource.get_channel() assert channel.is_image() is False assert channel.name == setup_data['channel']['name'] assert channel.description == setup_data['channel']['description'] assert channel.datatype == setup_data['channel']['datatype'] assert channel.base_resolution == setup_data['channel']['base_resolution'] assert channel.sources == setup_data['channel']['sources'] assert channel.related == setup_data['channel']['related'] assert channel.default_time_sample == setup_data['channel']['default_time_sample']
def setUpClass(cls): """Setup the redis client at the start of the test""" cls.data = get_image_dict() cls.resource = BossResourceBasic(cls.data) cls.state_client = redis.StrictRedis(host='localhost', port=6379, db=1, decode_responses=False) cls.config_data = {"state_client": cls.state_client}
def test_basic_resource_to_dict(self): """Test basic to dict serialization method Returns: None """ setup_data = get_image_dict() resource = BossResourceBasic(setup_data) data = resource.to_dict() assert data['channel'] == setup_data['channel'] assert data['collection'] == setup_data['collection'] assert data['experiment'] == setup_data['experiment'] assert data['lookup_key'] == '4&3&2' assert data['boss_key'] == 'col1&exp1&ch1'
def test_factory(self): """Test the Cube factory in Cube""" data = get_image_dict() resource = BossResourceBasic(data) c = Cube.create_cube(resource, [30, 20, 13], [0, 15]) assert isinstance(c, ImageCube8) is True assert c.cube_size == [13, 20, 30] assert c.is_time_series is True assert c.time_range == [0, 15]
def setUpClass(cls): """Setup the redis client at the start of the test""" cls.data = get_image_dict() cls.resource = BossResourceBasic(cls.data) cls.config = configuration.BossConfig() cls.cache_client = redis.StrictRedis(host=cls.config["aws"]["cache"], port=6379, db=1, decode_responses=False) cls.config_data = {"cache_client": cls.cache_client, "read_timeout": 86400}
def test_basic_resource_channel_with_related(self): """Test basic get channel interface Returns: None """ setup_data = get_image_dict() setup_data['channel']['related'] = ["ch_2", "ch_3"] resource = BossResourceBasic(setup_data) channel = resource.get_channel() assert channel.is_image() is True assert channel.name == setup_data['channel']['name'] assert channel.description == setup_data['channel']['description'] assert channel.datatype == setup_data['channel']['datatype'] assert channel.base_resolution == setup_data['channel']['base_resolution'] assert channel.sources == setup_data['channel']['sources'] assert channel.related == setup_data['channel']['related'] assert channel.default_time_sample == setup_data['channel']['default_time_sample']
def test_basic_resource_channel_no_time(self): """Test basic get channel interface Returns: None """ setup_data = get_image_dict() resource = BossResourceBasic(setup_data) channel = resource.get_channel() assert channel.is_image() is True assert channel.name == setup_data['channel']['name'] assert channel.description == setup_data['channel']['description'] assert channel.datatype == setup_data['channel']['datatype'] assert channel.base_resolution == setup_data['channel']['base_resolution'] assert channel.sources == setup_data['channel']['sources'] assert channel.related == setup_data['channel']['related'] assert channel.default_time_sample == setup_data['channel']['default_time_sample'] assert channel.downsample_status == setup_data['channel']['downsample_status']
def test_factory_no_time(self): """Test the Cube factory in Cube""" data = get_anno_dict() resource = BossResourceBasic(data) c = Cube.create_cube(resource, [30, 20, 13]) assert isinstance(c, AnnotateCube64) is True assert c.cube_size == [13, 20, 30] assert c.is_time_series is False assert c.time_range == [0, 1]
def test_factory_no_time(self): """Test the Cube factory in Cube""" data = get_image_dict() data['channel']['datatype'] = 'uint16' resource = BossResourceBasic(data) c = Cube.create_cube(resource, [30, 20, 13]) assert isinstance(c, ImageCube16) is True assert c.cube_size == [13, 20, 30] assert c.is_time_series is False assert c.time_range == [0, 1]
def test_get_tight_bounding_box_single_cuboid(self): """ Get the tight bounding box for an object that exists within a single cuboid. """ resolution = 0 [x_cube_dim, y_cube_dim, z_cube_dim] = CUBOIDSIZE[resolution] id = 33333 id_as_str = '33333' # Customize resource with so it writes to its own channel and uses a # coord frame large enough to encompass the data written. This is # important for proper loose bounding box calculations. data = get_anno_dict(boss_key='col1&exp1&ch50', lookup_key='1&1&50') data['coord_frame']['x_stop'] = 10000 data['coord_frame']['y_stop'] = 10000 data['coord_frame']['z_stop'] = 10000 resource = BossResourceBasic(data) time_sample = 0 version = 0 x_rng = [0, x_cube_dim] y_rng = [0, y_cube_dim] z_rng = [0, z_cube_dim] t_rng = [0, 1] cube_dim_tuple = (self.x_dim, self.y_dim, self.z_dim) cube1 = Cube.create_cube(resource, [self.x_dim, self.y_dim, self.z_dim]) cube1.zeros() cube1.data[0][14][500][104] = id cube1.data[0][15][501][105] = id cube1.data[0][15][502][104] = id cube1.data[0][14][503][105] = id pos1 = [10*self.x_dim, 15*self.y_dim, 2*self.z_dim] cube1.morton_id = XYZMorton(pos1) sp = SpatialDB(self.kvio_config, self.state_config, self.object_store_config) sp.write_cuboid(resource, pos1, resolution, cube1.data, time_sample_start=0) # Make sure cube write complete and correct. actual_cube = sp.cutout(resource, pos1, cube_dim_tuple, resolution) np.testing.assert_array_equal(cube1.data, actual_cube.data) # Method under test. actual = sp.get_bounding_box(resource, resolution, id_as_str, bb_type='tight') expected = { 'x_range': [pos1[0]+104, pos1[0]+106], 'y_range': [pos1[1]+500, pos1[1]+504], 'z_range': [pos1[2]+14, pos1[2]+16], 't_range': t_rng } self.assertEqual(expected, actual)
def test_update_id_indices_replaces_existing_entry_in_cuboid_index(self): """ Test calling update_id_indices() replaces existing id set in the s3 cuboid index. Id set should be replaced because the entire cuboid is rewritten to s3 before this method is called. Thus, the ids in the cuboid data are the only ids that should exist in the index for that cuboid. """ bytes = np.zeros(10, dtype='uint64') bytes[1] = 20 bytes[2] = 20 bytes[5] = 55 bytes[8] = 1000 bytes[9] = 55 key = 'hash_coll_exp_chan_key_existing' version = 0 resource = BossResourceBasic(data=get_anno_dict()) resolution = 1 # Place initial ids for cuboid. self.obj_ind.update_id_indices(resource, resolution, [key], [bytes], version) new_bytes = np.zeros(4, dtype='uint64') new_bytes[0] = 1000 new_bytes[1] = 4444 new_bytes[3] = 55 # Test adding one new id to the index. self.obj_ind.update_id_indices(resource, resolution, [key], [new_bytes], version) response = self.dynamodb.get_item( TableName=self.object_store_config["s3_index_table"], Key={ 'object-key': { 'S': key }, 'version-node': { 'N': "{}".format(version) } }, ConsistentRead=True, ReturnConsumedCapacity='NONE') self.assertIn('Item', response) self.assertIn('id-set', response['Item']) self.assertIn('NS', response['Item']['id-set']) # Id 20 should no longer be present. expected = ['55', '1000', '4444'] self.assertCountEqual(expected, response['Item']['id-set']['NS'])
def test_basic_resource_experiment(self): """Test basic get experiment interface Returns: None """ setup_data = get_image_dict() resource = BossResourceBasic(setup_data) exp = resource.get_experiment() assert exp.name == setup_data['experiment']['name'] assert exp.description == setup_data['experiment']['description'] assert exp.num_hierarchy_levels == setup_data['experiment'][ 'num_hierarchy_levels'] assert exp.hierarchy_method == setup_data['experiment'][ 'hierarchy_method'] assert exp.num_time_samples == setup_data['experiment'][ 'num_time_samples'] assert exp.time_step == setup_data['experiment']['time_step'] assert exp.time_step_unit == setup_data['experiment']['time_step_unit']
def setUpClass(cls): """Setup the redis client at the start of the test""" cls.data = get_image_dict() cls.resource = BossResourceBasic(cls.data) cls.config = load_test_config_file() cls.state_client = redis.StrictRedis( host=cls.config["aws"]["cache-state"], port=6379, db=1, decode_responses=False) cls.config_data = {"state_client": cls.state_client}
def test_get_cuboids(self): resource = BossResourceBasic(data=get_anno_dict()) id = 22222 bytes = np.zeros(10, dtype='uint64') bytes[1] = id resolution = 1 key = AWSObjectStore.generate_object_key(resource, resolution, 0, 56) version = 0 resource = BossResourceBasic(data=get_anno_dict()) new_bytes = np.zeros(4, dtype='uint64') new_bytes[0] = id # Pre-existing id. new_key = AWSObjectStore.generate_object_key(resource, resolution, 0, 59) self.obj_ind.update_id_indices(resource, resolution, [key, new_key], [bytes, new_bytes], version) # Method under test. actual = self.obj_ind.get_cuboids(resource, resolution, id) expected = [key, new_key] self.assertCountEqual(expected, actual)
def test_legacy_cuboids_in_id_index(self): """Tet to verify that legacy and "new" cuboid indices in the ID index table both work Returns: """ bytes = np.zeros(10, dtype='uint64') bytes[1] = 222 bytes[2] = 222 bytes[5] = 555 bytes[8] = 1001 expected_ids = ['222', '555', '1001', '12345'] version = 0 resource = BossResourceBasic(data=get_anno_dict()) resolution = 1 time_sample = 0 morton_id = 2000 object_key = AWSObjectStore.generate_object_key( resource, resolution, time_sample, morton_id) # Write a legacy index self.dynamodb.update_item( TableName=self.object_store_config["id_index_table"], Key={ 'channel-id-key': { 'S': self.obj_ind.generate_channel_id_key( resource, resolution, 12345) }, 'version': { 'N': "{}".format(version) } }, UpdateExpression='ADD #cuboidset :objkey', ExpressionAttributeNames={'#cuboidset': 'cuboid-set'}, ExpressionAttributeValues={':objkey': { 'SS': [object_key] }}, ReturnConsumedCapacity='NONE') # Add new index values self.obj_ind.update_id_indices(resource, resolution, [object_key], [bytes], version) # Confirm each id has the object_key in its cuboid-set attribute. for id in expected_ids: cuboid_object_keys = self.obj_ind.get_cuboids( resource, resolution, id) self.assertEqual(cuboid_object_keys[0], object_key)
def test_basic_resource_coord_frame(self): """Test basic get coordinate frame interface Returns: None """ setup_data = get_image_dict() resource = BossResourceBasic(setup_data) coord = resource.get_coord_frame() assert coord.name == setup_data['coord_frame']['name'] assert coord.description == setup_data['coord_frame']['description'] assert coord.x_start == setup_data['coord_frame']['x_start'] assert coord.x_stop == setup_data['coord_frame']['x_stop'] assert coord.y_start == setup_data['coord_frame']['y_start'] assert coord.y_stop == setup_data['coord_frame']['y_stop'] assert coord.z_start == setup_data['coord_frame']['z_start'] assert coord.z_stop == setup_data['coord_frame']['z_stop'] assert coord.x_voxel_size == setup_data['coord_frame']['x_voxel_size'] assert coord.y_voxel_size == setup_data['coord_frame']['y_voxel_size'] assert coord.z_voxel_size == setup_data['coord_frame']['z_voxel_size'] assert coord.voxel_unit == setup_data['coord_frame']['voxel_unit']
def test_basic_resource_channel_cloudvolume_uint16(self): """Test basic get channel interface Returns: None """ setup_data = get_image_dict(datatype="uint16", storage_type='cloudvol') resource = BossResourceBasic(setup_data) channel = resource.get_channel() assert channel.is_image() is True assert channel.is_cloudvolume() is True assert channel.name == setup_data['channel']['name'] assert channel.description == setup_data['channel']['description'] assert channel.datatype == setup_data['channel']['datatype'] assert channel.base_resolution == setup_data['channel']['base_resolution'] assert channel.sources == setup_data['channel']['sources'] assert channel.related == setup_data['channel']['related'] assert channel.default_time_sample == setup_data['channel']['default_time_sample'] assert channel.downsample_status == setup_data['channel']['downsample_status'] assert channel.storage_type == setup_data['channel']['storage_type'] assert channel.bucket == setup_data['channel']['bucket'] assert channel.cv_path == setup_data['channel']['cv_path']
def setUp(self): # Suppress ResourceWarning messages about unclosed connections. warnings.simplefilter('ignore') self.dead_letter = DeadLetterDaemon('foo') self.dead_letter.set_spatialdb( SpatialDB(self.kvio_config, self.state_config, self.object_store_config)) self.setup_helper = SetupTests() self.data = self.setup_helper.get_image8_dict() self.resource = BossResourceBasic(self.data) # Make the daemon look at the flush queue so we don't need to create # a deadletter queue for testing. self.dead_letter.dead_letter_queue = ( self.object_store_config['s3_flush_queue'])
def test_generate_object_keys_iso_isotropic(self): """Test to create object key when asking for isotropic data, in an isotropic channel""" data = self.setup_helper.get_image8_dict() data['experiment']['hierarchy_method'] = "isotropic" data['coord_frame']['z_voxel_size'] = 4 resource = BossResourceBasic(data) os = AWSObjectStore(self.object_store_config) object_keys = os.generate_object_key(resource, 0, 2, 56, iso=True) assert object_keys == '631424bf68302b683a0be521101c192b&4&3&2&0&2&56' object_keys = os.generate_object_key(resource, 3, 2, 56, iso=True) assert object_keys == 'cf934dccf1764290fd3db83b9b46b07b&4&3&2&3&2&56' object_keys = os.generate_object_key(resource, 5, 2, 56, iso=True) assert object_keys == '831adead1bc05b24d0799206ee9fe832&4&3&2&5&2&56'
def test_update_id_indices_new_entry_for_id_index(self): """ Test adding new ids to the id index. """ bytes = np.zeros(10, dtype='uint64') bytes[1] = 20 bytes[2] = 20 bytes[5] = 55 bytes[8] = 1000 bytes[9] = 55 expected_ids = ['20', '55', '1000'] version = 0 resource = BossResourceBasic(data=get_anno_dict()) resolution = 1 time_sample = 0 morton_id = 20 object_key = AWSObjectStore.generate_object_key( resource, resolution, time_sample, morton_id) # Method under test. self.obj_ind.update_id_indices(resource, resolution, [object_key], [bytes], version) # Confirm each id has the object_key in its cuboid-set attribute. for id in expected_ids: key = self.obj_ind.generate_channel_id_key(resource, resolution, id) response = self.dynamodb.get_item( TableName=self.object_store_config["id_index_table"], Key={ 'channel-id-key': { 'S': key }, 'version': { 'N': "{}".format(version) } }, ConsistentRead=True, ReturnConsumedCapacity='NONE') self.assertIn('Item', response) self.assertIn('cuboid-set', response['Item']) self.assertIn('SS', response['Item']['cuboid-set']) self.assertIn( object_key.split("&")[-1], response['Item']['cuboid-set']['SS'])
def setUp(self): """ Copy params from the Layer setUpClass """ # Setup Data #self.data = self.layer.setup_helper.get_anno64_dict() self.data = get_anno_dict() # Make the coord frame extra large for this test suite. self.data['coord_frame']['x_stop'] = 10000 self.data['coord_frame']['y_stop'] = 10000 self.data['coord_frame']['z_stop'] = 10000 self.resource = BossResourceBasic(self.data) # Setup config self.kvio_config = self.layer.kvio_config self.state_config = self.layer.state_config self.object_store_config = self.layer.object_store_config
def setUpParams(self): self.setup_helper = SetupTests() # Don't use mock Amazon resources. self.setup_helper.mock = False self.data = self.setup_helper.get_image8_dict() self.resource = BossResourceBasic(self.data) self.config = configuration.BossConfig() # kvio settings, 1 is the test DB. self.kvio_config = { "cache_host": self.config['aws']['cache'], "cache_db": 1, "read_timeout": 86400 } # state settings, 1 is the test DB. self.state_config = { "cache_state_host": self.config['aws']['cache-state'], "cache_state_db": 1 } # object store settings _, self.domain = self.config['aws']['cuboid_bucket'].split('.', 1) self.s3_flush_queue_name = "intTest.S3FlushQueue.{}".format( self.domain).replace('.', '-') self.object_store_config = { "s3_flush_queue": '', # This will get updated after the queue is created. "cuboid_bucket": "intTest{}.{}".format(random.randint(0, 9999), self.config['aws']['cuboid_bucket']), "page_in_lambda_function": self.config['lambda']['page_in_function'], "page_out_lambda_function": self.config['lambda']['flush_function'], "s3_index_table": "intTest.{}".format(self.config['aws']['s3-index-table']), "id_index_table": "intTest.{}".format(self.config['aws']['id-index-table']), "id_count_table": "intTest.{}".format(self.config['aws']['id-count-table']) }
def test_get_cuboids(self): id = 22222 bytes = np.zeros(10, dtype='uint64') bytes[1] = id key = 'hash_coll_exp_chan_key_cuboids' version = 0 resource = BossResourceBasic(data=get_anno_dict()) resolution = 1 new_bytes = np.zeros(4, dtype='uint64') new_bytes[0] = id # Pre-existing id. new_key = 'hash_coll_exp_chan_key_cuboids2' self.obj_ind.update_id_indices(resource, resolution, [key, new_key], [bytes, new_bytes], version) # Method under test. actual = self.obj_ind.get_cuboids(resource, resolution, id) expected = [key, new_key] self.assertCountEqual(expected, actual)
def setUpClass(cls): """ Create a diction of configuration values for the test resource. """ # Create resource cls.setup_helper = SetupTests() cls.data = cls.setup_helper.get_image8_dict() cls.resource = BossResourceBasic(cls.data) # Load config cls.config = configuration.BossConfig() cls.object_store_config = {"s3_flush_queue": 'https://mytestqueue.com', "cuboid_bucket": "test_bucket", "page_in_lambda_function": "page_in.test.boss", "page_out_lambda_function": "page_out.test.boss", "s3_index_table": "test_table", "id_index_table": "test_id_table", "id_count_table": "test_count_table", } # Create AWS Resources needed for tests cls.setup_helper.start_mocking() cls.setup_helper.create_index_table(cls.object_store_config["s3_index_table"], cls.setup_helper.DYNAMODB_SCHEMA) cls.setup_helper.create_cuboid_bucket(cls.object_store_config["cuboid_bucket"])
def test_too_many_ids_in_cuboid(self): """ Test error handling when a cuboid has more unique ids than DynamoDB can support. """ version = 0 resolution = 0 time_sample = 0 resource = BossResourceBasic(data=get_anno_dict()) mortonid = XYZMorton([0, 0, 0]) obj_keys = [ AWSObjectStore.generate_object_key(resource, resolution, time_sample, mortonid) ] cubes = [ np.random.randint(2000000, size=(16, 512, 512), dtype='uint64') ] # If too many ids, the index is skipped, logged, and False is returned to the caller. result = self.obj_ind.update_id_indices(resource, resolution, obj_keys, cubes, version) self.assertFalse(result)
def test_too_many_ids_in_cuboid(self): """ Test error handling when a cuboid has more unique ids than DynamoDB can support. """ version = 0 resolution = 0 time_sample = 0 resource = BossResourceBasic(data=get_anno_dict()) mortonid = XYZMorton([0, 0, 0]) obj_keys = [ self.obj_store.generate_object_key(resource, resolution, time_sample, mortonid) ] cubes = [ np.random.randint(2000000, size=(16, 512, 512), dtype='uint64') ] with self.assertRaises(SpdbError) as ex: self.obj_ind.update_id_indices(resource, resolution, obj_keys, cubes, version) self.assertEqual(ErrorCodes.OBJECT_STORE_ERROR, ex.exception.error_code)
def setUp(self, fake_get_region): """ Set everything up for testing """ # setup resources fake_get_region.return_value = 'us-east-1' self.setup_helper = SetupTests() self.setup_helper.mock = True self.data = self.setup_helper.get_image16_dict() self.resource = BossResourceBasic(self.data) # kvio settings self.cache_client = redis.StrictRedis(host='https://mytestcache.com', port=6379, db=1, decode_responses=False) self.kvio_config = {"cache_client": self.cache_client, "read_timeout": 86400} # state settings self.state_client = redis.StrictRedis(host='https://mytestcache2.com', port=6379, db=1, decode_responses=False) self.state_config = {"state_client": self.state_client} # object store settings self.object_store_config = {"s3_flush_queue": 'https://mytestqueue.com', "cuboid_bucket": "test_bucket", "page_in_lambda_function": "page_in.test.boss", "page_out_lambda_function": "page_out.test.boss", "s3_index_table": "test_table", "id_index_table": "test_id_table", "id_count_table": "test_count_table", } # Create AWS Resources needed for tests self.setup_helper.start_mocking() with patch('spdb.spatialdb.test.setup.get_region') as fake_get_region: fake_get_region.return_value = 'us-east-1' self.setup_helper.create_index_table(self.object_store_config["s3_index_table"], self.setup_helper.DYNAMODB_SCHEMA) self.setup_helper.create_cuboid_bucket(self.object_store_config["cuboid_bucket"])
def test_cutout_no_time_multi_unaligned_hit_iso_above(self): """Test write_cuboid and cutout methods - no time - multi - unaligned - hit - isotropic, above iso fork""" data = self.data data["channel"]["base_resolution"] = 5 resource = BossResourceBasic(data) # Generate random data cube1 = Cube.create_cube(resource, [400, 400, 8]) cube1.random() cube1.morton_id = 0 sp = SpatialDB(self.kvio_config, self.state_config, self.object_store_config) sp.write_cuboid(resource, (200, 600, 3), 5, cube1.data, iso=True) cube2 = sp.cutout(resource, (200, 600, 3), (400, 400, 8), 5, iso=True) np.testing.assert_array_equal(cube1.data, cube2.data) # do it again...should be in cache cube2 = sp.cutout(resource, (200, 600, 3), (400, 400, 8), 5, iso=True) np.testing.assert_array_equal(cube1.data, cube2.data)
def setUp(self): # Get data from nose2 layer based setup # Setup Data self.data = self.layer.setup_helper.get_image8_dict() self.resource = BossResourceBasic(self.data) # Setup config self.kvio_config = self.layer.kvio_config self.state_config = self.layer.state_config self.object_store_config = self.layer.object_store_config client = redis.StrictRedis(host=self.kvio_config['cache_host'], port=6379, db=1, decode_responses=False) client.flushdb() client = redis.StrictRedis(host=self.state_config['cache_state_host'], port=6379, db=1, decode_responses=False) client.flushdb()
def test_update_id_indices_new_entry_in_cuboid_index(self): """ Test adding ids to new cuboids in the s3 cuboid index. """ bytes = np.zeros(10, dtype='uint64') bytes[1] = 20 bytes[2] = 20 bytes[5] = 55 bytes[8] = 1000 bytes[9] = 55 expected = ['20', '55', '1000'] key = 'hash_coll_exp_chan_key' version = 0 resource = BossResourceBasic(data=get_anno_dict()) resolution = 1 # Method under test. self.obj_ind.update_id_indices(resource, resolution, [key], [bytes], version) response = self.dynamodb.get_item( TableName=self.object_store_config["s3_index_table"], Key={ 'object-key': { 'S': key }, 'version-node': { 'N': "{}".format(version) } }, ConsistentRead=True, ReturnConsumedCapacity='NONE') self.assertIn('Item', response) self.assertIn('id-set', response['Item']) self.assertIn('NS', response['Item']['id-set']) self.assertCountEqual(expected, response['Item']['id-set']['NS'])
def test_reserve_id_wrong_type(self): img_data = self.setup_helper.get_image8_dict() img_resource = BossResourceBasic(img_data) with self.assertRaises(SpdbError): start_id = self.obj_ind.reserve_ids(img_resource, 10)
def test_basic_resource_from_json_annotation(self): """Test basic to json deserialization method Returns: None """ setup_data = get_anno_dict() resource1 = BossResourceBasic(setup_data) resource2 = BossResourceBasic() resource2.from_json(resource1.to_json()) # Check Collection col = resource2.get_collection() assert col.name == setup_data['collection']['name'] assert col.description == setup_data['collection']['description'] # Check coord frame coord = resource2.get_coord_frame() assert coord.name == setup_data['coord_frame']['name'] assert coord.description == setup_data['coord_frame']['description'] assert coord.x_start == setup_data['coord_frame']['x_start'] assert coord.x_stop == setup_data['coord_frame']['x_stop'] assert coord.y_start == setup_data['coord_frame']['y_start'] assert coord.y_stop == setup_data['coord_frame']['y_stop'] assert coord.z_start == setup_data['coord_frame']['z_start'] assert coord.z_stop == setup_data['coord_frame']['z_stop'] assert coord.x_voxel_size == setup_data['coord_frame']['x_voxel_size'] assert coord.y_voxel_size == setup_data['coord_frame']['y_voxel_size'] assert coord.z_voxel_size == setup_data['coord_frame']['z_voxel_size'] assert coord.voxel_unit == setup_data['coord_frame']['voxel_unit'] # Check exp exp = resource2.get_experiment() assert exp.name == setup_data['experiment']['name'] assert exp.description == setup_data['experiment']['description'] assert exp.num_hierarchy_levels == setup_data['experiment']['num_hierarchy_levels'] assert exp.hierarchy_method == setup_data['experiment']['hierarchy_method'] assert exp.num_time_samples == setup_data['experiment']['num_time_samples'] assert exp.time_step == setup_data['experiment']['time_step'] assert exp.time_step_unit == setup_data['experiment']['time_step_unit'] # Check channel channel = resource2.get_channel() assert channel.is_image() is False assert channel.name == setup_data['channel']['name'] assert channel.description == setup_data['channel']['description'] assert channel.datatype == setup_data['channel']['datatype'] assert channel.base_resolution == setup_data['channel']['base_resolution'] assert channel.sources == setup_data['channel']['sources'] assert channel.related == setup_data['channel']['related'] assert channel.default_time_sample == setup_data['channel']['default_time_sample'] # check keys assert resource2.get_lookup_key() == setup_data['lookup_key'] assert resource2.get_boss_key() == setup_data['boss_key']
# Load the message body flush_msg_data = json.loads(flush_msg_data['Body']) print("Message: {}".format(flush_msg_data)) # Setup SPDB instance sp = SpatialDB(flush_msg_data["config"]["kv_config"], flush_msg_data["config"]["state_config"], flush_msg_data["config"]["object_store_config"]) # Get the write-cuboid key to flush write_cuboid_key = flush_msg_data['write_cuboid_key'] print("Flushing {} to S3".format(write_cuboid_key)) # Create resource instance resource = BossResourceBasic() resource.from_dict(flush_msg_data["resource"]) else: # Nothing to flush. Exit. print("No flush message available") sys.exit(0) # Check if cuboid is in S3 object_keys = sp.objectio.write_cuboid_to_object_keys([write_cuboid_key]) cache_key = sp.kvio.write_cuboid_key_to_cache_key(write_cuboid_key) exist_keys, missing_keys = sp.objectio.cuboids_exist(cache_key) print("write key: {}".format(write_cuboid_key)) print("object key: {}".format(object_keys[0])) print("cache key: {}".format(cache_key))
def handler(event, context): # Load settings SETTINGS = BossSettings.load() # Used as a guard against trying to delete the SQS message when lambda is # triggered by SQS. sqs_triggered = 'Records' in event and len(event['Records']) > 0 if sqs_triggered : # Lambda invoked by an SQS trigger. msg_data = json.loads(event['Records'][0]['body']) # Load the project info from the chunk key you are processing chunk_key = msg_data['chunk_key'] proj_info = BossIngestProj.fromSupercuboidKey(chunk_key) proj_info.job_id = msg_data['ingest_job'] else: # Standard async invoke of this lambda. # Load the project info from the chunk key you are processing proj_info = BossIngestProj.fromSupercuboidKey(event["chunk_key"]) proj_info.job_id = event["ingest_job"] # Get message from SQS ingest queue, try for ~2 seconds rx_cnt = 0 msg_data = None msg_id = None msg_rx_handle = None while rx_cnt < 6: ingest_queue = IngestQueue(proj_info) msg = [x for x in ingest_queue.receiveMessage()] if msg: msg = msg[0] print("MESSAGE: {}".format(msg)) print(len(msg)) msg_id = msg[0] msg_rx_handle = msg[1] msg_data = json.loads(msg[2]) print("MESSAGE DATA: {}".format(msg_data)) break else: rx_cnt += 1 print("No message found. Try {} of 6".format(rx_cnt)) time.sleep(1) if not msg_id: # No tiles ready to ingest. print("No ingest message available") return # Get the chunk key of the tiles to ingest. chunk_key = msg_data['chunk_key'] tile_error_queue = TileErrorQueue(proj_info) print("Ingesting Chunk {}".format(chunk_key)) tiles_in_chunk = int(chunk_key.split('&')[1]) # Setup SPDB instance sp = SpatialDB(msg_data['parameters']["KVIO_SETTINGS"], msg_data['parameters']["STATEIO_CONFIG"], msg_data['parameters']["OBJECTIO_CONFIG"]) # Get tile list from Tile Index Table tile_index_db = BossTileIndexDB(proj_info.project_name) # tile_index_result (dict): keys are S3 object keys of the tiles comprising the chunk. tile_index_result = tile_index_db.getCuboid(msg_data["chunk_key"], int(msg_data["ingest_job"])) if tile_index_result is None: # If chunk_key is gone, another lambda uploaded the cuboids and deleted the chunk_key afterwards. if not sqs_triggered: # Remove message so it's not redelivered. ingest_queue.deleteMessage(msg_id, msg_rx_handle) print("Aborting due to chunk key missing from tile index table") return # Sort the tile keys print("Tile Keys: {}".format(tile_index_result["tile_uploaded_map"])) tile_key_list = [x.rsplit("&", 2) for x in tile_index_result["tile_uploaded_map"].keys()] if len(tile_key_list) < tiles_in_chunk: print("Not a full set of 16 tiles. Assuming it has handled already, tiles: {}".format(len(tile_key_list))) if not sqs_triggered: ingest_queue.deleteMessage(msg_id, msg_rx_handle) return tile_key_list = sorted(tile_key_list, key=lambda x: int(x[1])) tile_key_list = ["&".join(x) for x in tile_key_list] print("Sorted Tile Keys: {}".format(tile_key_list)) # Augment Resource JSON data so it will instantiate properly that was pruned due to S3 metadata size limits resource_dict = msg_data['parameters']['resource'] _, exp_name, ch_name = resource_dict["boss_key"].split("&") resource_dict["channel"]["name"] = ch_name resource_dict["channel"]["description"] = "" resource_dict["channel"]["sources"] = [] resource_dict["channel"]["related"] = [] resource_dict["channel"]["default_time_sample"] = 0 resource_dict["channel"]["downsample_status"] = "NOT_DOWNSAMPLED" resource_dict["experiment"]["name"] = exp_name resource_dict["experiment"]["description"] = "" resource_dict["experiment"]["num_time_samples"] = 1 resource_dict["experiment"]["time_step"] = None resource_dict["experiment"]["time_step_unit"] = None resource_dict["coord_frame"]["name"] = "cf" resource_dict["coord_frame"]["name"] = "" resource_dict["coord_frame"]["x_start"] = 0 resource_dict["coord_frame"]["x_stop"] = 100000 resource_dict["coord_frame"]["y_start"] = 0 resource_dict["coord_frame"]["y_stop"] = 100000 resource_dict["coord_frame"]["z_start"] = 0 resource_dict["coord_frame"]["z_stop"] = 100000 resource_dict["coord_frame"]["voxel_unit"] = "nanometers" # Setup the resource resource = BossResourceBasic() resource.from_dict(resource_dict) dtype = resource.get_numpy_data_type() # read all tiles from bucket into a slab tile_bucket = TileBucket(proj_info.project_name) data = [] num_z_slices = 0 for tile_key in tile_key_list: try: image_data, message_id, receipt_handle, metadata = tile_bucket.getObjectByKey(tile_key) except KeyError: print('Key: {} not found in tile bucket, assuming redelivered SQS message and aborting.'.format( tile_key)) if not sqs_triggered: # Remove message so it's not redelivered. ingest_queue.deleteMessage(msg_id, msg_rx_handle) print("Aborting due to missing tile in bucket") return image_bytes = BytesIO(image_data) image_size = image_bytes.getbuffer().nbytes # Get tiles size from metadata, need to shape black tile if actual tile is corrupt. if 'x_size' in metadata: tile_size_x = metadata['x_size'] else: print('MetadataMissing: x_size not in tile metadata: using 1024.') tile_size_x = 1024 if 'y_size' in metadata: tile_size_y = metadata['y_size'] else: print('MetadataMissing: y_size not in tile metadata: using 1024.') tile_size_y = 1024 if image_size == 0: print('TileError: Zero length tile, using black instead: {}'.format(tile_key)) error_msg = 'Zero length tile' enqueue_tile_error(tile_error_queue, tile_key, chunk_key, error_msg) tile_img = np.zeros((tile_size_x, tile_size_y), dtype=dtype) else: try: tile_img = np.asarray(Image.open(image_bytes), dtype=dtype) except TypeError as te: print('TileError: Incomplete tile, using black instead (tile_size_in_bytes, tile_key): {}, {}' .format(image_size, tile_key)) error_msg = 'Incomplete tile' enqueue_tile_error(tile_error_queue, tile_key, chunk_key, error_msg) tile_img = np.zeros((tile_size_x, tile_size_y), dtype=dtype) except OSError as oe: print('TileError: OSError, using black instead (tile_size_in_bytes, tile_key): {}, {} ErrorMessage: {}' .format(image_size, tile_key, oe)) error_msg = 'OSError: {}'.format(oe) enqueue_tile_error(tile_error_queue, tile_key, chunk_key, error_msg) tile_img = np.zeros((tile_size_x, tile_size_y), dtype=dtype) data.append(tile_img) num_z_slices += 1 # Make 3D array of image data. It should be in XYZ at this point chunk_data = np.array(data) del data tile_dims = chunk_data.shape # Break into Cube instances print("Tile Dims: {}".format(tile_dims)) print("Num Z Slices: {}".format(num_z_slices)) num_x_cuboids = int(math.ceil(tile_dims[2] / CUBOIDSIZE[proj_info.resolution][0])) num_y_cuboids = int(math.ceil(tile_dims[1] / CUBOIDSIZE[proj_info.resolution][1])) print("Num X Cuboids: {}".format(num_x_cuboids)) print("Num Y Cuboids: {}".format(num_y_cuboids)) chunk_key_parts = BossUtil.decode_chunk_key(chunk_key) t_index = chunk_key_parts['t_index'] for x_idx in range(0, num_x_cuboids): for y_idx in range(0, num_y_cuboids): # TODO: check time series support cube = Cube.create_cube(resource, CUBOIDSIZE[proj_info.resolution]) cube.zeros() # Compute Morton ID # TODO: verify Morton indices correct! print(chunk_key_parts) morton_x_ind = x_idx + (chunk_key_parts["x_index"] * num_x_cuboids) morton_y_ind = y_idx + (chunk_key_parts["y_index"] * num_y_cuboids) print("Morton X: {}".format(morton_x_ind)) print("Morton Y: {}".format(morton_y_ind)) morton_index = XYZMorton([morton_x_ind, morton_y_ind, int(chunk_key_parts['z_index'])]) # Insert sub-region from chunk_data into cuboid x_start = x_idx * CUBOIDSIZE[proj_info.resolution][0] x_end = x_start + CUBOIDSIZE[proj_info.resolution][0] x_end = min(x_end, tile_dims[2]) y_start = y_idx * CUBOIDSIZE[proj_info.resolution][1] y_end = y_start + CUBOIDSIZE[proj_info.resolution][1] y_end = min(y_end, tile_dims[1]) z_end = CUBOIDSIZE[proj_info.resolution][2] # TODO: get sub-array w/o making a copy. print("Yrange: {}".format(y_end - y_start)) print("Xrange: {}".format(x_end - x_start)) print("X start: {}".format(x_start)) print("X stop: {}".format(x_end)) cube.data[0, 0:num_z_slices, 0:(y_end - y_start), 0:(x_end - x_start)] = chunk_data[0:num_z_slices, y_start:y_end, x_start:x_end] # Create object key object_key = sp.objectio.generate_object_key(resource, proj_info.resolution, t_index, morton_index) print("Object Key: {}".format(object_key)) # Put object in S3 sp.objectio.put_objects([object_key], [cube.to_blosc()]) # Add object to index sp.objectio.add_cuboid_to_index(object_key, ingest_job=int(msg_data["ingest_job"])) # Update id indices if this is an annotation channel # We no longer index during ingest. #if resource.data['channel']['type'] == 'annotation': # try: # sp.objectio.update_id_indices( # resource, proj_info.resolution, [object_key], [cube.data]) # except SpdbError as ex: # sns_client = boto3.client('sns') # topic_arn = msg_data['parameters']["OBJECTIO_CONFIG"]["prod_mailing_list"] # msg = 'During ingest:\n{}\nCollection: {}\nExperiment: {}\n Channel: {}\n'.format( # ex.message, # resource.data['collection']['name'], # resource.data['experiment']['name'], # resource.data['channel']['name']) # sns_client.publish( # TopicArn=topic_arn, # Subject='Object services misuse', # Message=msg) lambda_client = boto3.client('lambda', region_name=SETTINGS.REGION_NAME) names = AWSNames.create_from_lambda_name(context.function_name) delete_tiles_data = { 'tile_key_list': tile_key_list, 'region': SETTINGS.REGION_NAME, 'bucket': tile_bucket.bucket.name } # Delete tiles from tile bucket. lambda_client.invoke( FunctionName=names.delete_tile_objs_lambda, InvocationType='Event', Payload=json.dumps(delete_tiles_data).encode() ) delete_tile_entry_data = { 'tile_index': tile_index_db.table.name, 'region': SETTINGS.REGION_NAME, 'chunk_key': chunk_key, 'task_id': msg_data['ingest_job'] } # Delete entry from tile index. lambda_client.invoke( FunctionName=names.delete_tile_index_entry_lambda, InvocationType='Event', Payload=json.dumps(delete_tile_entry_data).encode() ) if not sqs_triggered: # Delete message since it was processed successfully ingest_queue.deleteMessage(msg_id, msg_rx_handle)