def test_save_cache_remove_from_index(self) -> None:
# Test that the cache is updated appropriately on a removal.
cache_element = DataMemoryElement()
self.assertTrue(cache_element.is_empty())
i = LinearHashIndex(cache_element)
# noinspection PyTypeChecker
i.build_index([
[0, 1, 0], # 2
[0, 1, 1], # 3
[1, 0, 0], # 4
[1, 1, 0]
]) # 6
self.assertFalse(cache_element.is_empty())
self.assertSetEqual(
set(numpy.load(BytesIO(cache_element.get_bytes()))), {2, 3, 4, 6})
# noinspection PyTypeChecker
i.remove_from_index([
[0, 1, 1], # 3
[1, 0, 0]
]) # 4
self.assertFalse(cache_element.is_empty())
self.assertSetEqual(
set(numpy.load(BytesIO(cache_element.get_bytes()))), {2, 6})
def test_set_bytes(self) -> None:
bytes_a = b"test bytes first set"
bytes_b = b"the second set of bytes"
e = DataMemoryElement(bytes_a)
self.assertEqual(e.get_bytes(), bytes_a)
e.set_bytes(bytes_b)
self.assertEqual(e.get_bytes(), bytes_b)
def test_set_bytes_when_readonly(self) -> None:
bytes_a = b"test bytes first set"
bytes_b = b"the second set of bytes"
e = DataMemoryElement(bytes_a, readonly=True)
self.assertEqual(e.get_bytes(), bytes_a)
self.assertRaises(ReadOnlyError, e.set_bytes, bytes_b)
self.assertEqual(e.get_bytes(), bytes_a)
def test_save_cache_build_index(self) -> None:
cache_element = DataMemoryElement()
self.assertTrue(cache_element.is_empty())
i = LinearHashIndex(cache_element)
# noinspection PyTypeChecker
i.build_index([[0, 1, 0], [1, 0, 0], [0, 1, 1], [0, 0, 1]])
self.assertFalse(cache_element.is_empty())
# Check byte content
expected_cache = {1, 2, 3, 4}
actual_cache = set(numpy.load(BytesIO(cache_element.get_bytes())))
self.assertSetEqual(expected_cache, actual_cache)
def test_add_many_with_caching(self) -> None:
"""
Test that adding many reflects in cache.
"""
d = {
'a': 'b',
'foo': None,
0: 89,
}
c = DataMemoryElement()
s = MemoryKeyValueStore(c)
self.assertEqual(s._table, {})
self.assertEqual(c.get_bytes(), b"")
s.add_many(d)
self.assertEqual(s._table, d)
self.assertEqual(
pickle.loads(c.get_bytes()),
d
)
def test_remove_many_with_cache(self) -> None:
starting_table = {
0: 0,
1: 1,
2: 2,
}
c = DataMemoryElement(pickle.dumps(starting_table))
s = MemoryKeyValueStore(c)
self.assertDictEqual(s._table, starting_table)
s.remove_many([0, 2])
self.assertDictEqual(pickle.loads(c.get_bytes()), {1: 1})
def test_cacheing_with_map(self) -> None:
expected_cache = DataMemoryElement()
expected_map = {
0: 'a',
75: 'b',
124769: 'c',
}
dms = DataMemorySet(expected_cache)
dms._element_map = expected_map
dms.cache()
self.assertFalse(expected_cache.is_empty())
self.assertEqual(pickle.loads(expected_cache.get_bytes()), expected_map)
def test_remove_with_cache(self) -> None:
"""
Test that removal correctly updates the cache element.
"""
existing_data = {
0: 1,
'a': 'b',
}
c = DataMemoryElement(pickle.dumps(existing_data))
s = MemoryKeyValueStore(c)
self.assertDictEqual(s._table, existing_data)
s.remove('a')
self.assertDictEqual(s._table, {0: 1})
self.assertDictEqual(pickle.loads(c.get_bytes()),
{0: 1})
def test_add_with_caching(self) -> None:
"""
Test that we can add key-value pairs and they reflect in the cache
element.
"""
c = DataMemoryElement()
s = MemoryKeyValueStore(c)
expected_cache_dict = {'a': 'b', 'foo': None, 0: 89}
s.add('a', 'b')
s.add('foo', None)
s.add(0, 89)
self.assertEqual(
pickle.loads(c.get_bytes()),
expected_cache_dict
)
def test_from_config(self) -> None:
# Configured cache with some picked bytes
# Then convert to "string" (decode -> unicode) for python version used.
expected_table = dict(a=1, b=2, c=3)
expected_cache = DataMemoryElement(bytes=pickle.dumps(expected_table))
expected_cache_json_str = \
expected_cache.get_bytes().decode(BYTES_CONFIG_ENCODING)
dme_key = 'smqtk_dataprovider.impls.data_element.memory.DataMemoryElement'
inst = MemoryDescriptorSet.from_config({
'cache_element': {
'type': dme_key,
dme_key: {
'bytes': expected_cache_json_str
}
}
})
self.assertEqual(inst.cache_element, expected_cache)
self.assertEqual(inst._table, expected_table)
def test_get_bytes_some_bytes(self) -> None:
expected_bytes = b'some bytes'
e = DataMemoryElement(expected_bytes)
self.assertEqual(e.get_bytes(), expected_bytes)
def test_get_bytes_empty_bytes(self) -> None:
e = DataMemoryElement(b'')
self.assertEqual(e.get_bytes(), b'')
def test_get_bytes_none_bytes(self) -> None:
e = DataMemoryElement()
self.assertEqual(e.get_bytes(), b"")