def test_is_dirty(self):
"""Test cube delete method"""
resolution = 1
rkv = RedisKVIO(self.config_data)
# Clean up data
self.cache_client.flushdb()
data1 = np.random.randint(50, size=[10, 15, 5])
data_packed1 = blosc.pack_array(data1)
data = [data_packed1]
# Add item to cached-cuboid
morton_id = [112]
keys = rkv.generate_cached_cuboid_keys(self.resource, 2, [0],
morton_id)
rkv.put_cubes(keys, data)
result = rkv.is_dirty(keys)
assert not result[0]
# Fake a write
write_cuboid_key = "WRITE-CUBOID&{}".format(keys[0].split('&', 1)[1])
write_cuboid_key, time_sample, morton = write_cuboid_key.rsplit('&', 2)
rkv.insert_cube_in_write_buffer(write_cuboid_key, time_sample, morton,
data[0])
result = rkv.is_dirty(keys)
assert result[0]
def test_get_cubes(self):
"""Test adding cubes to the cache"""
resolution = 1
rkv = RedisKVIO(self.config_data)
# Clean up data
self.cache_client.flushdb()
data1 = np.random.randint(50, size=[10, 15, 5])
data2 = np.random.randint(50, size=[10, 15, 5])
data3 = np.random.randint(50, size=[10, 15, 5])
data_packed1 = blosc.pack_array(data1)
data_packed2 = blosc.pack_array(data2)
data_packed3 = blosc.pack_array(data3)
data = [data_packed1, data_packed2, data_packed3]
# Add items
morton_id = [112, 125, 516]
keys = rkv.generate_cached_cuboid_keys(self.resource, 2, [0],
morton_id)
rkv.put_cubes(keys, data)
# Get cube
cubes = rkv.get_cubes(keys)
assert len(cubes) == 3
for m, c, d in zip(morton_id, cubes, data):
assert c[0] == m
assert c[1] == 0
data_retrieved = blosc.unpack_array(c[2])
np.testing.assert_array_equal(data_retrieved,
blosc.unpack_array(d))
def test_delete_cube(self):
"""Test cube delete method"""
resolution = 1
rkv = RedisKVIO(self.config_data)
# Clean up data
self.cache_client.flushdb()
data1 = np.random.randint(50, size=[10, 15, 5])
data_packed1 = blosc.pack_array(data1)
data = [data_packed1]
# Add items
morton_id = [112]
keys = rkv.generate_cached_cuboid_keys(self.resource, 2, [0],
morton_id)
result = rkv.cube_exists(keys[0])
assert not result
rkv.put_cubes(keys, data)
result = rkv.cube_exists(keys[0])
assert result
rkv.delete_cube(keys[0])
result = rkv.cube_exists(keys[0])
assert not result
def test_is_dirty(self):
"""Test cube delete method"""
resolution = 1
rkv = RedisKVIO(self.config_data)
# Clean up data
self.cache_client.flushdb()
data1 = np.random.randint(50, size=[10, 15, 5])
data_packed1 = blosc.pack_array(data1)
data = [data_packed1]
# Add item to cached-cuboid
morton_id = [112]
keys = rkv.generate_cached_cuboid_keys(self.resource, 2, [0], morton_id)
rkv.put_cubes(keys, data)
result = rkv.is_dirty(keys)
assert not result[0]
# Fake a write
write_cuboid_key = "WRITE-CUBOID&{}".format(keys[0].split('&', 1)[1])
write_cuboid_key, time_sample, morton = write_cuboid_key.rsplit('&', 2)
rkv.insert_cube_in_write_buffer(write_cuboid_key, time_sample, morton, data[0])
result = rkv.is_dirty(keys)
assert result[0]
def test_put_cubes(self):
"""Test adding cubes to the cache"""
resolution = 1
rkv = RedisKVIO(self.config_data)
# Clean up data
self.cache_client.flushdb()
data1 = np.random.randint(50, size=[10, 15, 5])
data2 = np.random.randint(50, size=[10, 15, 5])
data3 = np.random.randint(50, size=[10, 15, 5])
data_packed1 = blosc.pack_array(data1)
data_packed2 = blosc.pack_array(data2)
data_packed3 = blosc.pack_array(data3)
data = [data_packed1, data_packed2, data_packed3]
# Make sure there are no cuboids in the cache
keys = self.cache_client.keys('CACHED-CUBOID&{}&{}*'.format(
self.resource.get_lookup_key(), resolution))
assert not keys
# Add items
keys = rkv.generate_cached_cuboid_keys(self.resource, 2, [0],
[123, 124, 126])
rkv.put_cubes(keys, data)
db_keys = self.cache_client.keys('CACHED-CUBOID*')
db_keys = [x.decode() for x in db_keys]
assert len(set(keys)) == 3
for k, d in zip(keys, db_keys):
assert k in db_keys
def test_delete_cube(self):
"""Test cube delete method"""
resolution = 1
rkv = RedisKVIO(self.config_data)
# Clean up data
self.cache_client.flushdb()
data1 = np.random.randint(50, size=[10, 15, 5])
data_packed1 = blosc.pack_array(data1)
data = [data_packed1]
# Add items
morton_id = [112]
keys = rkv.generate_cached_cuboid_keys(self.resource, 2, [0], morton_id)
result = rkv.cube_exists(keys[0])
assert not result
rkv.put_cubes(keys, data)
result = rkv.cube_exists(keys[0])
assert result
rkv.delete_cube(keys[0])
result = rkv.cube_exists(keys[0])
assert not result
def test_get_cubes(self):
"""Test adding cubes to the cache"""
resolution = 1
rkv = RedisKVIO(self.config_data)
# Clean up data
self.cache_client.flushdb()
data1 = np.random.randint(50, size=[10, 15, 5])
data2 = np.random.randint(50, size=[10, 15, 5])
data3 = np.random.randint(50, size=[10, 15, 5])
data_packed1 = blosc.pack_array(data1)
data_packed2 = blosc.pack_array(data2)
data_packed3 = blosc.pack_array(data3)
data = [data_packed1, data_packed2, data_packed3]
# Add items
morton_id = [112, 125, 516]
keys = rkv.generate_cached_cuboid_keys(self.resource, 2, [0], morton_id)
rkv.put_cubes(keys, data)
# Get cube
cubes = rkv.get_cubes(keys)
assert len(cubes) == 3
for m, c, d in zip(morton_id, cubes, data):
assert c[0] == m
assert c[1] == 0
data_retrieved = blosc.unpack_array(c[2])
np.testing.assert_array_equal(data_retrieved, blosc.unpack_array(d))
def test_put_cubes(self):
"""Test adding cubes to the cache"""
resolution = 1
rkv = RedisKVIO(self.config_data)
# Clean up data
self.cache_client.flushdb()
data1 = np.random.randint(50, size=[10, 15, 5])
data2 = np.random.randint(50, size=[10, 15, 5])
data3 = np.random.randint(50, size=[10, 15, 5])
data_packed1 = blosc.pack_array(data1)
data_packed2 = blosc.pack_array(data2)
data_packed3 = blosc.pack_array(data3)
data = [data_packed1, data_packed2, data_packed3]
# Make sure there are no cuboids in the cache
keys = self.cache_client.keys('CACHED-CUBOID&{}&{}*'.format(self.resource.get_lookup_key(), resolution))
assert not keys
# Add items
keys = rkv.generate_cached_cuboid_keys(self.resource, 2, [0], [123, 124, 126])
rkv.put_cubes(keys, data)
db_keys = self.cache_client.keys('CACHED-CUBOID*')
db_keys = [x.decode() for x in db_keys]
assert len(set(keys)) == 3
for k, d in zip(keys, db_keys):
assert k in db_keys
def test_param_constructor(self):
"""Re-run a testing using the parameter based constructor"""
config = {
"cache_host": self.config["aws"]["cache"],
"cache_db": 1,
"read_timeout": 86400
}
rkv = RedisKVIO(config)
# Clean up data
self.cache_client.flushdb()
data1 = np.random.randint(50, size=[10, 15, 5])
data2 = np.random.randint(50, size=[10, 15, 5])
data3 = np.random.randint(50, size=[10, 15, 5])
data_packed1 = blosc.pack_array(data1)
data_packed2 = blosc.pack_array(data2)
data_packed3 = blosc.pack_array(data3)
data = [data_packed1, data_packed2, data_packed3]
# Add items
morton_id = [112, 125, 516]
keys = rkv.generate_cached_cuboid_keys(self.resource, 2, [0],
morton_id)
rkv.put_cubes(keys, data)
# Get cube
cubes = rkv.get_cubes(keys)
cube = [x for x in cubes]
assert len(cube) == 3
for m, c, d in zip(morton_id, cube, data):
assert c[0] == m
assert c[1] == 0
data_retrieved = blosc.unpack_array(c[2])
np.testing.assert_array_equal(data_retrieved,
blosc.unpack_array(d))
def test_param_constructor(self):
"""Re-run a testing using the parameter based constructor"""
config = {
"cache_host": self.config["aws"]["cache"],
"cache_db": 1,
"read_timeout": 86400
}
rkv = RedisKVIO(config)
# Clean up data
self.cache_client.flushdb()
data1 = np.random.randint(50, size=[10, 15, 5])
data2 = np.random.randint(50, size=[10, 15, 5])
data3 = np.random.randint(50, size=[10, 15, 5])
data_packed1 = blosc.pack_array(data1)
data_packed2 = blosc.pack_array(data2)
data_packed3 = blosc.pack_array(data3)
data = [data_packed1, data_packed2, data_packed3]
# Add items
morton_id = [112, 125, 516]
keys = rkv.generate_cached_cuboid_keys(self.resource, 2, [0], morton_id)
rkv.put_cubes(keys, data)
# Get cube
cubes = rkv.get_cubes(keys)
cube = [x for x in cubes]
assert len(cube) == 3
for m, c, d in zip(morton_id, cube, data):
assert c[0] == m
assert c[1] == 0
data_retrieved = blosc.unpack_array(c[2])
np.testing.assert_array_equal(data_retrieved, blosc.unpack_array(d))
