def test_get_ids_in_region_single_cube(self): """Test single cuboid using DynamoDB index.""" cube_dim_tuple = (self.x_dim, self.y_dim, self.z_dim) cube1 = Cube.create_cube(self.resource, [self.x_dim, self.y_dim, self.z_dim]) cube1.zeros() cube1.data[0][0][40][0] = 55555 cube1.data[0][0][50][0] = 66666000000000 pos1 = [2*self.x_dim, 3*self.y_dim, 2*self.z_dim] cube1.morton_id = XYZMorton(pos1) sp = SpatialDB(self.kvio_config, self.state_config, self.object_store_config) resolution = 0 sp.write_cuboid(self.resource, pos1, resolution, cube1.data, time_sample_start=0) # Make sure cube write complete and correct. actual_cube = sp.cutout(self.resource, pos1, cube_dim_tuple, resolution) np.testing.assert_array_equal(cube1.data, actual_cube.data) corner = (2*self.x_dim, 3*self.y_dim, 2*self.z_dim) extent = (self.x_dim, self.y_dim, self.z_dim) t_range = [0, 1] version = 0 expected = ['55555', '66666000000000'] # Method under test. actual = sp.get_ids_in_region( self.resource, resolution, corner, extent, t_range, version) self.assertIn('ids', actual) self.assertCountEqual(expected, actual['ids'])
def test_page_in_multi_cuboids_y_dir(self): # Generate random data cube1 = Cube.create_cube(self.resource, [self.x_dim, self.y_dim * 2, self.z_dim]) cube1.random() cube1.morton_id = 0 sp = SpatialDB(self.kvio_config, self.state_config, self.object_store_config) sp.write_cuboid(self.resource, (0, 0, 0), 0, cube1.data) cube2 = sp.cutout(self.resource, (0, 0, 0), (self.x_dim, self.y_dim * 2, self.z_dim), 0) np.testing.assert_array_equal(cube1.data, cube2.data) # Make sure data is the same np.testing.assert_array_equal(cube1.data, cube2.data) # Delete everything in the cache sp.kvio.cache_client.flushdb() # Force use of lambda function. sp.read_lambda_threshold = 0 # Get the data again, which should trigger lambda page in. cube3 = sp.cutout(self.resource, (0, 0, 0), (self.x_dim, self.y_dim * 2, self.z_dim), 0) # Make sure the data is the same np.testing.assert_array_equal(cube1.data, cube3.data)
def test_cutout_no_time_single_aligned_miss(self): """Test the get_cubes method - no time - single - miss""" # Generate random data cube1 = Cube.create_cube(self.resource, [self.x_dim, self.y_dim, self.z_dim]) cube1.random() cube1.morton_id = 0 sp = SpatialDB(self.kvio_config, self.state_config, self.object_store_config) sp.write_cuboid(self.resource, (1, 0, 0), 0, cube1.data) cube2 = sp.cutout(self.resource, (1, 0, 0), (self.x_dim, self.y_dim, self.z_dim), 0) # Make sure data is the same np.testing.assert_array_equal(cube1.data, cube2.data) # Delete everything in the cache sp.kvio.cache_client.flushdb() # Get the data again cube3 = sp.cutout(self.resource, (1, 0, 0), (self.x_dim, self.y_dim, self.z_dim), 0) # Make sure the data is the same np.testing.assert_array_equal(cube1.data, cube2.data) np.testing.assert_array_equal(cube1.data, cube3.data)
def test_page_in_multi_cuboids_y_dir(self): # Generate random data cube1 = Cube.create_cube(self.resource, [self.x_dim, self.y_dim * 2, self.z_dim]) cube1.random() cube1.morton_id = 0 sp = SpatialDB(self.kvio_config, self.state_config, self.object_store_config) sp.write_cuboid(self.resource, (0, 0, 0), 0, cube1.data) cube2 = sp.cutout(self.resource, (0, 0, 0), (self.x_dim, self.y_dim * 2, self.z_dim), 0) np.testing.assert_array_equal(cube1.data, cube2.data) # Make sure data is the same np.testing.assert_array_equal(cube1.data, cube2.data) # Delete everything in the cache sp.kvio.cache_client.flushdb() # Force use of lambda function. sp.read_lambda_threshold = 0 # Get the data again, which should trigger lambda page in. cube3 = sp.cutout(self.resource, (0, 0, 0), (self.x_dim, self.y_dim * 2, self.z_dim), 0) # Make sure the data is the same np.testing.assert_array_equal(cube1.data, cube3.data)
def test_filtered_cutout(self): time_axis = [1] cube_dim = [self.x_dim, self.y_dim, self.z_dim] cube_dim_tuple = (self.x_dim, self.y_dim, self.z_dim) cube1 = Cube.create_cube(self.resource, cube_dim) cube1.data = np.ones(time_axis + [cube_dim[2], cube_dim[1], cube_dim[0]], dtype='uint64') cube1.morton_id = 0 corner = (0, 0, 0) expected = np.zeros(time_axis + [cube_dim[2], cube_dim[1], cube_dim[0]], dtype='uint64') # Will filter by these ids. id1 = 55555 id2 = 66666 cube1.data[0][0][40][0] = id1 cube1.data[0][0][50][0] = id2 expected[0][0][40][0] = id1 expected[0][0][50][0] = id2 sp = SpatialDB(self.kvio_config, self.state_config, self.object_store_config) resolution = 0 sp.write_cuboid(self.resource, corner, resolution, cube1.data, time_sample_start=0) # Make sure cube written correctly. actual_cube = sp.cutout(self.resource, corner, cube_dim_tuple, resolution) np.testing.assert_array_equal(cube1.data, actual_cube.data) # Method under test. actual_filtered = sp.cutout(self.resource, corner, cube_dim_tuple, resolution, filter_ids=[id1, id2]) np.testing.assert_array_equal(expected, actual_filtered.data)
def test_cutout_iso_not_present(self): """Test write_cuboid and cutout methods with iso option, testing iso is stored in parallel""" data = self.data data["channel"]["base_resolution"] = 5 resource = BossResourceBasic(data) # Generate random data cube1 = Cube.create_cube(resource, [400, 400, 8]) cube1.random() cube1.morton_id = 0 cubez = Cube.create_cube(resource, [400, 400, 8]) cubez.zeros() cubez.morton_id = 0 # Write at 5, not iso, and verify sp = SpatialDB(self.kvio_config, self.state_config, self.object_store_config) sp.write_cuboid(resource, (200, 600, 3), 5, cube1.data, iso=False) cube2 = sp.cutout(resource, (200, 600, 3), (400, 400, 8), 5, iso=False) np.testing.assert_array_equal(cube1.data, cube2.data) # Get at res 5 iso, which should be blank cube2 = sp.cutout(resource, (200, 600, 3), (400, 400, 8), 5, iso=True) np.testing.assert_array_equal(cubez.data, cube2.data)
def test_write_cuboid_off_base_res(self, fake_get_region): """Test writing a cuboid to not the base resolution""" # Generate random data cube1 = Cube.create_cube(self.resource, [self.x_dim, self.y_dim, self.z_dim]) cube1.random() cube1.morton_id = 0 db = SpatialDB(self.kvio_config, self.state_config, self.object_store_config) # populate dummy data with self.assertRaises(SpdbError): db.write_cuboid(self.resource, (0, 0, 0), 5, cube1.data, time_sample_start=0)
def test_get_tight_bounding_box_single_cuboid(self): """ Get the tight bounding box for an object that exists within a single cuboid. """ resolution = 0 [x_cube_dim, y_cube_dim, z_cube_dim] = CUBOIDSIZE[resolution] id = 33333 id_as_str = '33333' # Customize resource with so it writes to its own channel and uses a # coord frame large enough to encompass the data written. This is # important for proper loose bounding box calculations. data = get_anno_dict(boss_key='col1&exp1&ch50', lookup_key='1&1&50') data['coord_frame']['x_stop'] = 10000 data['coord_frame']['y_stop'] = 10000 data['coord_frame']['z_stop'] = 10000 resource = BossResourceBasic(data) time_sample = 0 version = 0 x_rng = [0, x_cube_dim] y_rng = [0, y_cube_dim] z_rng = [0, z_cube_dim] t_rng = [0, 1] cube_dim_tuple = (self.x_dim, self.y_dim, self.z_dim) cube1 = Cube.create_cube(resource, [self.x_dim, self.y_dim, self.z_dim]) cube1.zeros() cube1.data[0][14][500][104] = id cube1.data[0][15][501][105] = id cube1.data[0][15][502][104] = id cube1.data[0][14][503][105] = id pos1 = [10*self.x_dim, 15*self.y_dim, 2*self.z_dim] cube1.morton_id = XYZMorton(pos1) sp = SpatialDB(self.kvio_config, self.state_config, self.object_store_config) sp.write_cuboid(resource, pos1, resolution, cube1.data, time_sample_start=0) # Make sure cube write complete and correct. actual_cube = sp.cutout(resource, pos1, cube_dim_tuple, resolution) np.testing.assert_array_equal(cube1.data, actual_cube.data) # Method under test. actual = sp.get_bounding_box(resource, resolution, id_as_str, bb_type='tight') expected = { 'x_range': [pos1[0]+104, pos1[0]+106], 'y_range': [pos1[1]+500, pos1[1]+504], 'z_range': [pos1[2]+14, pos1[2]+16], 't_range': t_rng } self.assertEqual(expected, actual)
def test_get_ids_in_region_multiple_cubes_and_x_partials(self): """ Region has some full cuboids and some partial cuboids along the x axis. """ cube_dim_tuple = (self.x_dim, self.y_dim, self.z_dim) cube1 = Cube.create_cube(self.resource, [self.x_dim, self.y_dim, self.z_dim]) cube1.zeros() cube1.data[0][0][40][105] = 55555 cube1.data[0][0][50][105] = 66666 pos1 = [7*self.x_dim, 5*self.y_dim, 2*self.z_dim] cube1.morton_id = XYZMorton(pos1) cube2 = Cube.create_cube(self.resource, [self.x_dim, self.y_dim, self.z_dim]) cube2.zeros() cube2.data[0][0][40][105] = 55555 cube2.data[0][0][50][105] = 77777 pos2 = [8*self.x_dim, 5*self.y_dim, 2*self.z_dim] cube2.morton_id = XYZMorton(pos2) cube3 = Cube.create_cube(self.resource, [self.x_dim, self.y_dim, self.z_dim]) cube3.zeros() cube3.data[0][0][0][105] = 88888 pos3 = [9*self.x_dim, 5*self.y_dim, 2*self.z_dim] cube3.morton_id = XYZMorton(pos3) sp = SpatialDB(self.kvio_config, self.state_config, self.object_store_config) resolution = 0 sp.write_cuboid(self.resource, pos1, resolution, cube1.data, time_sample_start=0) sp.write_cuboid(self.resource, pos2, resolution, cube2.data, time_sample_start=0) sp.write_cuboid(self.resource, pos3, resolution, cube3.data, time_sample_start=0) # Make sure cube write complete and correct. actual_cube = sp.cutout(self.resource, pos1, cube_dim_tuple, resolution) np.testing.assert_array_equal(cube1.data, actual_cube.data) actual_cube = sp.cutout(self.resource, pos2, cube_dim_tuple, resolution) np.testing.assert_array_equal(cube2.data, actual_cube.data) actual_cube = sp.cutout(self.resource, pos3, cube_dim_tuple, resolution) np.testing.assert_array_equal(cube3.data, actual_cube.data) corner = (7*self.x_dim+100, 5*self.y_dim, 2*self.z_dim) extent = (2*self.x_dim+self.x_dim//2, self.y_dim, self.z_dim) t_range = [0, 1] version = 0 expected = ['55555', '66666', '77777', '88888'] # Method under test. actual = sp.get_ids_in_region( self.resource, resolution, corner, extent, t_range, version) self.assertIn('ids', actual) self.assertCountEqual(expected, actual['ids'])
def test_cutout_no_time_single_unaligned_hit(self): """Test the get_cubes method - no time - single - unaligned - hit""" # Generate random data cube1 = Cube.create_cube(self.resource, [self.x_dim, self.y_dim, self.z_dim]) cube1.random() cube1.morton_id = 0 sp = SpatialDB(self.kvio_config, self.state_config, self.object_store_config) sp.write_cuboid(self.resource, (600, 0, 0), 0, cube1.data) cube2 = sp.cutout(self.resource, (600, 0, 0), (self.x_dim, self.y_dim, 16), 0) np.testing.assert_array_equal(cube1.data, cube2.data)
def test_cutout_time_offset_single_aligned_hit(self): """Test the get_cubes method - w/ time - single - hit""" # Generate random data cube1 = Cube.create_cube(self.resource, [self.x_dim, self.y_dim, self.z_dim], time_range=[0, 3]) cube1.random() cube1.morton_id = 0 sp = SpatialDB(self.kvio_config, self.state_config, self.object_store_config) sp.write_cuboid(self.resource, (0, 0, 0), 0, cube1.data, time_sample_start=6) cube2 = sp.cutout(self.resource, (0, 0, 0), (self.x_dim, self.y_dim, self.z_dim), 0, time_sample_range=[6, 9]) np.testing.assert_array_equal(cube1.data, cube2.data)
def test_filtered_cutout_bad_id_list(self): time_axis = [1] cube_dim = [self.x_dim, self.y_dim, self.z_dim] cube_dim_tuple = (self.x_dim, self.y_dim, self.z_dim) cube1 = Cube.create_cube(self.resource, cube_dim) cube1.data = np.ones(time_axis + [cube_dim[2], cube_dim[1], cube_dim[0]], dtype='uint64') cube1.morton_id = 0 corner = (6*self.x_dim, 6*self.y_dim, 2*self.z_dim) sp = SpatialDB(self.kvio_config, self.state_config, self.object_store_config) resolution = 0 sp.write_cuboid(self.resource, corner, resolution, cube1.data, time_sample_start=0) # Method under test. with self.assertRaises(SpdbError): sp.cutout(self.resource, corner, cube_dim_tuple, resolution, filter_ids=['foo', 55555])
def test_cutout_no_time_multi_unaligned_hit(self): """Test the get_cubes method - no time - multi - unaligned - hit""" # Generate random data cube1 = Cube.create_cube(self.resource, [400, 400, 8]) cube1.random() cube1.morton_id = 0 sp = SpatialDB(self.kvio_config, self.state_config, self.object_store_config) sp.write_cuboid(self.resource, (200, 600, 3), 0, cube1.data) cube2 = sp.cutout(self.resource, (200, 600, 3), (400, 400, 8), 0) np.testing.assert_array_equal(cube1.data, cube2.data) # do it again...shoudl be in cache cube2 = sp.cutout(self.resource, (200, 600, 3), (400, 400, 8), 0) np.testing.assert_array_equal(cube1.data, cube2.data)
def test_cutout_no_time_multi_unaligned_hit_iso_below(self): """Test write_cuboid and cutout methods - no time - multi - unaligned - hit - isotropic, below iso fork""" # Generate random data cube1 = Cube.create_cube(self.resource, [400, 400, 8]) cube1.random() cube1.morton_id = 0 sp = SpatialDB(self.kvio_config, self.state_config, self.object_store_config) sp.write_cuboid(self.resource, (200, 600, 3), 0, cube1.data, iso=True) cube2 = sp.cutout(self.resource, (200, 600, 3), (400, 400, 8), 0, iso=True) np.testing.assert_array_equal(cube1.data, cube2.data) # do it again...should be in cache cube2 = sp.cutout(self.resource, (200, 600, 3), (400, 400, 8), 0, iso=True) np.testing.assert_array_equal(cube1.data, cube2.data)
def test_get_ids_in_region_multiple_partial_cubes(self): """ Region cuboid aligned in x, but doesn't span full cuboids in the y and z. """ cube1 = Cube.create_cube(self.resource, [self.x_dim, self.y_dim, self.z_dim]) cube1.zeros() cube1.data[0][0][40][0] = 55555 cube1.data[0][0][50][0] = 66666 pos1 = [4*self.x_dim, 4*self.y_dim, 2*self.z_dim] cube1.morton_id = XYZMorton(pos1) cube2 = Cube.create_cube(self.resource, [self.x_dim, self.y_dim, self.z_dim]) cube2.zeros() cube2.data[0][0][40][0] = 55555 cube2.data[0][0][50][0] = 77777 pos2 = [5*self.x_dim, 4*self.y_dim, 2*self.z_dim] cube2.morton_id = XYZMorton(pos2) sp = SpatialDB(self.kvio_config, self.state_config, self.object_store_config) resolution = 0 sp.write_cuboid(self.resource, pos1, resolution, cube1.data, time_sample_start=0) sp.write_cuboid(self.resource, pos2, resolution, cube2.data, time_sample_start=0) # Not verifying writes here because get_ids_in_region() should be doing # cutouts due to the region not containing full cuboids. corner = (4*self.x_dim, 4*self.y_dim, 2*self.z_dim) extent = (2*self.x_dim, 60, 10) t_range = [0, 1] version = 0 expected = ['55555', '66666', '77777'] # Method under test. actual = sp.get_ids_in_region( self.resource, resolution, corner, extent, t_range, version) self.assertIn('ids', actual) self.assertCountEqual(expected, actual['ids'])
def test_cutout_no_time_multi_unaligned_hit_iso_above(self): """Test write_cuboid and cutout methods - no time - multi - unaligned - hit - isotropic, above iso fork""" data = self.data data["channel"]["base_resolution"] = 5 resource = BossResourceBasic(data) # Generate random data cube1 = Cube.create_cube(resource, [400, 400, 8]) cube1.random() cube1.morton_id = 0 sp = SpatialDB(self.kvio_config, self.state_config, self.object_store_config) sp.write_cuboid(resource, (200, 600, 3), 5, cube1.data, iso=True) cube2 = sp.cutout(resource, (200, 600, 3), (400, 400, 8), 5, iso=True) np.testing.assert_array_equal(cube1.data, cube2.data) # do it again...should be in cache cube2 = sp.cutout(resource, (200, 600, 3), (400, 400, 8), 5, iso=True) np.testing.assert_array_equal(cube1.data, cube2.data)
def test_cutout_iso_below_fork(self): """Test write_cuboid and cutout methods with iso option, testing iso is equal below the res fork""" # Generate random data cube1 = Cube.create_cube(self.resource, [400, 400, 8]) cube1.random() cube1.morton_id = 0 cubez = Cube.create_cube(self.resource, [400, 400, 8]) cubez.zeros() cubez.morton_id = 0 # Write at 5, not iso, and verify sp = SpatialDB(self.kvio_config, self.state_config, self.object_store_config) sp.write_cuboid(self.resource, (200, 600, 3), 0, cube1.data, iso=False) cube2 = sp.cutout(self.resource, (200, 600, 3), (400, 400, 8), 0, iso=False) np.testing.assert_array_equal(cube1.data, cube2.data) # Get at res 5 iso, which should be equal to non-iso call cube2 = sp.cutout(self.resource, (200, 600, 3), (400, 400, 8), 0, iso=True) np.testing.assert_array_equal(cube1.data, cube2.data)
def test_cutout_no_time_single_aligned_existing_hit(self): """Test the get_cubes method - no time - aligned - existing data - miss""" # Generate random data cube1 = Cube.create_cube(self.resource, [self.x_dim, self.y_dim, self.z_dim]) cube1.random() sp = SpatialDB(self.kvio_config, self.state_config, self.object_store_config) sp.write_cuboid(self.resource, (0, 0, 0), 0, cube1.data) cube2 = sp.cutout(self.resource, (0, 0, 0), (self.x_dim, self.y_dim, self.z_dim), 0) np.testing.assert_array_equal(cube1.data, cube2.data) del cube1 del cube2 # now write to cuboid again cube3 = Cube.create_cube(self.resource, [self.x_dim, self.y_dim, self.z_dim]) cube3.random() sp.write_cuboid(self.resource, (0, 0, 0), 0, cube3.data) cube4 = sp.cutout(self.resource, (0, 0, 0), (self.x_dim, self.y_dim, self.z_dim), 0) np.testing.assert_array_equal(cube3.data, cube4.data)
class IntegrationTestCacheMissDaemon(unittest.TestCase): layer = AWSSetupLayer def setUp(self): # Get data from nose2 layer based setup # Setup Data self.data = self.layer.setup_helper.get_image8_dict() self.resource = BossResourceBasic(self.data) # Setup config self.kvio_config = self.layer.kvio_config self.state_config = self.layer.state_config self.object_store_config = self.layer.object_store_config client = redis.StrictRedis(host=self.kvio_config['cache_host'], port=6379, db=1, decode_responses=False) client.flushdb() client = redis.StrictRedis(host=self.state_config['cache_state_host'], port=6379, db=1, decode_responses=False) client.flushdb() self.cache_miss = CacheMissDaemon('foo') self.sp = SpatialDB(self.kvio_config, self.state_config, self.object_store_config) self.cache_miss.set_spatialdb(self.sp) def tearDown(self): """Clean kv store in between tests""" client = redis.StrictRedis(host=self.kvio_config['cache_host'], port=6379, db=1, decode_responses=False) client.flushdb() client = redis.StrictRedis(host=self.state_config['cache_state_host'], port=6379, db=1, decode_responses=False) client.flushdb() def test_add_to_prefetch(self): cuboid_dims = CUBOIDSIZE[0] # Cuboid dimensions. x_dim = cuboid_dims[0] y_dim = cuboid_dims[1] z_dim = cuboid_dims[2] cube = Cube.create_cube(self.resource, [x_dim, y_dim, z_dim]) cube.random() cube_above = Cube.create_cube(self.resource, [x_dim, y_dim, z_dim]) cube_above.random() cube_below = Cube.create_cube(self.resource, [x_dim, y_dim, z_dim]) cube_below.random() # Write 3 cuboids that are stacked vertically. self.sp.write_cuboid(self.resource, (0, 0, 0), 0, cube_below.data) self.sp.write_cuboid(self.resource, (0, 0, z_dim), 0, cube.data) self.sp.write_cuboid(self.resource, (0, 0, z_dim * 2), 0, cube_above.data) cube.morton_id = ndlib.XYZMorton([0, 0, z_dim // z_dim]) cube_below.morton_id = ndlib.XYZMorton([0, 0, 0]) cube_above.morton_id = ndlib.XYZMorton([0, 0, z_dim * 2 // z_dim]) print('mortons: {}, {}, {}'.format( cube_below.morton_id, cube.morton_id, cube_above.morton_id)) cube_below_cache_key, cube_cache_key, cube_above_cache_key = self.sp.kvio.generate_cached_cuboid_keys( self.resource, 0, [0], [cube_below.morton_id, cube.morton_id, cube_above.morton_id]) # Make sure cuboids saved. cube_act = self.sp.cutout(self.resource, (0, 0, 0), (x_dim, y_dim, z_dim), 0) np.testing.assert_array_equal(cube_below.data, cube_act.data) cube_act = self.sp.cutout(self.resource, (0, 0, z_dim), (x_dim, y_dim, z_dim), 0) np.testing.assert_array_equal(cube.data, cube_act.data) cube_act = self.sp.cutout(self.resource, (0, 0, z_dim * 2), (x_dim, y_dim, z_dim), 0) np.testing.assert_array_equal(cube_above.data, cube_act.data) # Clear cache so we can get a cache miss. self.sp.kvio.cache_client.flushdb() # Also clear CACHE-MISS before running testing. self.sp.cache_state.status_client.flushdb() # Get middle cube again. This should trigger a cache miss. cube_act = self.sp.cutout(self.resource, (0, 0, z_dim), (x_dim, y_dim, z_dim), 0) # Confirm there is a cache miss. misses = self.sp.cache_state.status_client.lrange('CACHE-MISS', 0, 10) print('misses:') print(misses) miss_actual = self.sp.cache_state.status_client.lindex('CACHE-MISS', 0) self.assertEqual(cube_cache_key, str(miss_actual, 'utf-8')) # This is the system under test. self.cache_miss.process() # Confirm PRE-FETCH has the object keys for the cube above and below. fetch_actual1 = self.sp.cache_state.status_client.lindex('PRE-FETCH', 0) fetch_actual2 = self.sp.cache_state.status_client.lindex('PRE-FETCH', 1) obj_keys = self.sp.objectio.cached_cuboid_to_object_keys( [cube_above_cache_key, cube_below_cache_key]) self.assertEqual(obj_keys[0], str(fetch_actual1, 'utf-8')) self.assertEqual(obj_keys[1], str(fetch_actual2, 'utf-8'))
class IntegrationTestPrefetchDaemon(unittest.TestCase): layer = AWSSetupLayer def setUp(self): # Setup Data self.data = self.layer.setup_helper.get_image8_dict() self.resource = BossResourceBasic(self.data) # Setup config self.kvio_config = self.layer.kvio_config self.state_config = self.layer.state_config self.object_store_config = self.layer.object_store_config client = redis.StrictRedis(host=self.kvio_config['cache_host'], port=6379, db=1, decode_responses=False) client.flushdb() client = redis.StrictRedis(host=self.state_config['cache_state_host'], port=6379, db=1, decode_responses=False) client.flushdb() # Suppress ResourceWarning messages about unclosed connections. warnings.simplefilter('ignore') self.prefetch = PrefetchDaemon('foo') self.sp = SpatialDB(self.kvio_config, self.state_config, self.object_store_config) self.prefetch.set_spatialdb(self.sp) def tearDown(self): """Clean kv store in between tests""" client = redis.StrictRedis(host=self.kvio_config['cache_host'], port=6379, db=1, decode_responses=False) client.flushdb() client = redis.StrictRedis(host=self.state_config['cache_state_host'], port=6379, db=1, decode_responses=False) client.flushdb() def test_add_to_prefetch(self): cuboid_dims = CUBOIDSIZE[0] # Cuboid dimensions. x_dim = cuboid_dims[0] y_dim = cuboid_dims[1] z_dim = cuboid_dims[2] cube_above = Cube.create_cube(self.resource, [x_dim, y_dim, z_dim]) cube_above.random() # Write cuboid that are stacked vertically. self.sp.write_cuboid(self.resource, (0, 0, z_dim * 2), 0, cube_above.data) cube_above.morton_id = ndlib.XYZMorton([0, 0, z_dim * 2 // z_dim]) cube_above_cache_key = self.sp.kvio.generate_cached_cuboid_keys( self.resource, 0, [0], [cube_above.morton_id]) # Make sure cuboid saved. cube_act = self.sp.cutout(self.resource, (0, 0, z_dim * 2), (x_dim, y_dim, z_dim), 0) np.testing.assert_array_equal(cube_above.data, cube_act.data) # Clear cache so we can test prefetch. self.sp.kvio.cache_client.flushdb() # Also clear cache state before running test. self.sp.cache_state.status_client.flushdb() obj_keys = self.sp.objectio.cached_cuboid_to_object_keys( cube_above_cache_key) # Place a cuboid in the pretch queue. self.sp.cache_state.status_client.rpush('PRE-FETCH', obj_keys[0]) # This is the system under test. self.prefetch.process() # Wait for cube to be prefetched. i = 0 while not self.sp.kvio.cube_exists(cube_above_cache_key[0]) and i < 30: time.sleep(1) i += 1 # Confirm cuboid now in cache. self.assertTrue(self.sp.kvio.cube_exists(cube_above_cache_key[0])) cube_act = self.sp.cutout( self.resource, (0, 0, z_dim * 2), (x_dim, y_dim, z_dim), 0) np.testing.assert_array_equal(cube_above.data, cube_act.data)
class IntegrationTestPrefetchDaemon(unittest.TestCase): layer = AWSSetupLayer def setUp(self): # Setup Data self.data = self.layer.setup_helper.get_image8_dict() self.resource = BossResourceBasic(self.data) # Setup config self.kvio_config = self.layer.kvio_config self.state_config = self.layer.state_config self.object_store_config = self.layer.object_store_config client = redis.StrictRedis(host=self.kvio_config['cache_host'], port=6379, db=1, decode_responses=False) client.flushdb() client = redis.StrictRedis(host=self.state_config['cache_state_host'], port=6379, db=1, decode_responses=False) client.flushdb() # Suppress ResourceWarning messages about unclosed connections. warnings.simplefilter('ignore') self.prefetch = PrefetchDaemon('foo') self.sp = SpatialDB(self.kvio_config, self.state_config, self.object_store_config) self.prefetch.set_spatialdb(self.sp) def tearDown(self): """Clean kv store in between tests""" client = redis.StrictRedis(host=self.kvio_config['cache_host'], port=6379, db=1, decode_responses=False) client.flushdb() client = redis.StrictRedis(host=self.state_config['cache_state_host'], port=6379, db=1, decode_responses=False) client.flushdb() def test_add_to_prefetch(self): cuboid_dims = CUBOIDSIZE[0] # Cuboid dimensions. x_dim = cuboid_dims[0] y_dim = cuboid_dims[1] z_dim = cuboid_dims[2] cube_above = Cube.create_cube(self.resource, [x_dim, y_dim, z_dim]) cube_above.random() # Write cuboid that are stacked vertically. self.sp.write_cuboid(self.resource, (0, 0, z_dim * 2), 0, cube_above.data) cube_above.morton_id = ndlib.XYZMorton([0, 0, z_dim * 2 // z_dim]) cube_above_cache_key = self.sp.kvio.generate_cached_cuboid_keys( self.resource, 0, [0], [cube_above.morton_id]) # Make sure cuboid saved. cube_act = self.sp.cutout(self.resource, (0, 0, z_dim * 2), (x_dim, y_dim, z_dim), 0) np.testing.assert_array_equal(cube_above.data, cube_act.data) # Clear cache so we can test prefetch. self.sp.kvio.cache_client.flushdb() # Also clear cache state before running test. self.sp.cache_state.status_client.flushdb() obj_keys = self.sp.objectio.cached_cuboid_to_object_keys( cube_above_cache_key) # Place a cuboid in the pretch queue. self.sp.cache_state.status_client.rpush('PRE-FETCH', obj_keys[0]) # This is the system under test. self.prefetch.process() # Wait for cube to be prefetched. i = 0 while not self.sp.kvio.cube_exists(cube_above_cache_key[0]) and i < 30: time.sleep(1) i += 1 # Confirm cuboid now in cache. self.assertTrue(self.sp.kvio.cube_exists(cube_above_cache_key[0])) cube_act = self.sp.cutout(self.resource, (0, 0, z_dim * 2), (x_dim, y_dim, z_dim), 0) np.testing.assert_array_equal(cube_above.data, cube_act.data)