def test_set_bytes(self):
bytes_a = six.b('test bytes first set')
bytes_b = six.b('the second set of bytes')
e = DataMemoryElement(bytes_a)
ntools.assert_equal(e.get_bytes(), bytes_a)
e.set_bytes(bytes_b)
ntools.assert_equal(e.get_bytes(), bytes_b)
def test_set_bytes_when_readonly(self):
bytes_a = 'test bytes first set'
bytes_b = 'the second set of bytes'
e = DataMemoryElement(bytes_a, readonly=True)
ntools.assert_equal(e.get_bytes(), bytes_a)
ntools.assert_raises(ReadOnlyError, e.set_bytes, bytes_b)
ntools.assert_equal(e.get_bytes(), bytes_a)
def test_set_bytes(self):
bytes_a = six.b('test bytes first set')
bytes_b = six.b('the second set of bytes')
e = DataMemoryElement(bytes_a)
self.assertEqual(e.get_bytes(), bytes_a)
e.set_bytes(bytes_b)
self.assertEqual(e.get_bytes(), bytes_b)
def test_save_cache_remove_from_index(self):
# Test that the cache is updated appropriately on a removal.
cache_element = DataMemoryElement()
self.assertTrue(cache_element.is_empty())
i = LinearHashIndex(cache_element)
# noinspection PyTypeChecker
i.build_index([
[0, 1, 0], # 2
[0, 1, 1], # 3
[1, 0, 0], # 4
[1, 1, 0]
]) # 6
self.assertFalse(cache_element.is_empty())
self.assertSetEqual(
set(numpy.load(BytesIO(cache_element.get_bytes()))), {2, 3, 4, 6})
# noinspection PyTypeChecker
i.remove_from_index([
[0, 1, 1], # 3
[1, 0, 0]
]) # 4
self.assertFalse(cache_element.is_empty())
self.assertSetEqual(
set(numpy.load(BytesIO(cache_element.get_bytes()))), {2, 6})
def test_save_cache_remove_from_index(self):
# Test that the cache is updated appropriately on a removal.
cache_element = DataMemoryElement()
self.assertTrue(cache_element.is_empty())
i = LinearHashIndex(cache_element)
# noinspection PyTypeChecker
i.build_index([[0, 1, 0], # 2
[0, 1, 1], # 3
[1, 0, 0], # 4
[1, 1, 0]]) # 6
self.assertFalse(cache_element.is_empty())
self.assertSetEqual(
set(numpy.load(BytesIO(cache_element.get_bytes()))),
{2, 3, 4, 6}
)
# noinspection PyTypeChecker
i.remove_from_index([[0, 1, 1], # 3
[1, 0, 0]]) # 4
self.assertFalse(cache_element.is_empty())
self.assertSetEqual(
set(numpy.load(BytesIO(cache_element.get_bytes()))),
{2, 6}
)
def test_set_bytes_when_readonly(self):
bytes_a = six.b('test bytes first set')
bytes_b = six.b('the second set of bytes')
e = DataMemoryElement(bytes_a, readonly=True)
self.assertEqual(e.get_bytes(), bytes_a)
self.assertRaises(ReadOnlyError, e.set_bytes, bytes_b)
self.assertEqual(e.get_bytes(), bytes_a)
def test_set_bytes_when_readonly(self):
bytes_a = six.b('test bytes first set')
bytes_b = six.b('the second set of bytes')
e = DataMemoryElement(bytes_a, readonly=True)
self.assertEqual(e.get_bytes(), bytes_a)
self.assertRaises(
ReadOnlyError,
e.set_bytes,
bytes_b
)
self.assertEqual(e.get_bytes(), bytes_a)
def test_add_many_with_caching(self):
d = {
'a': 'b',
'foo': None,
0: 89,
}
c = DataMemoryElement()
s = MemoryKeyValueStore(c)
self.assertEqual(s._table, {})
self.assertEqual(c.get_bytes(), six.b(""))
s.add_many(d)
self.assertEqual(s._table, d)
self.assertEqual(pickle.loads(c.get_bytes()), d)
def test_save_cache(self):
cache_element = DataMemoryElement()
nose.tools.assert_true(cache_element.is_empty())
i = LinearHashIndex(cache_element)
# noinspection PyTypeChecker
i.build_index([[0, 1, 0], [1, 0, 0], [0, 1, 1], [0, 0, 1]])
nose.tools.assert_false(cache_element.is_empty())
nose.tools.assert_true(len(cache_element.get_bytes()) > 0)
def test_add_with_caching_no_cache(self):
c = DataMemoryElement()
s = MemoryKeyValueStore(c)
expected_cache_dict = {'a': 'b', 'foo': None}
s.add('a', 'b', False)
# No caching means there should be nothign there yet
nose.tools.assert_equal(c.get_bytes(), six.b(""))
s.add('foo', None, True)
# With caching, meaning the state should be cached here, which includes
# everything added previously, including the a:b pair.
nose.tools.assert_equal(pickle.loads(c.get_bytes()),
expected_cache_dict)
s.add(0, 89, False)
nose.tools.assert_equal(pickle.loads(c.get_bytes()),
expected_cache_dict)
def test_add_with_caching(self):
c = DataMemoryElement()
s = MemoryKeyValueStore(c)
expected_cache_dict = {'a': 'b', 'foo': None, 0: 89}
s.add('a', 'b')
s.add('foo', None)
s.add(0, 89)
nose.tools.assert_equal(pickle.loads(c.get_bytes()),
expected_cache_dict)
def test_add_many_with_caching(self):
"""
Test that adding many reflects in cache.
"""
d = {
'a': 'b',
'foo': None,
0: 89,
}
c = DataMemoryElement()
s = MemoryKeyValueStore(c)
self.assertEqual(s._table, {})
self.assertEqual(c.get_bytes(), six.b(""))
s.add_many(d)
self.assertEqual(s._table, d)
self.assertEqual(
pickle.loads(c.get_bytes()),
d
)
def test_from_config(self):
# Configured cache with some picked bytes
expected_table = dict(a=1, b=2, c=3)
expected_cache = DataMemoryElement(bytes=pickle.dumps(expected_table))
inst = MemoryDescriptorIndex.from_config({
'cache_element': {
'type': 'DataMemoryElement',
'DataMemoryElement': {'bytes': expected_cache.get_bytes()}
}
})
self.assertEqual(inst.cache_element, expected_cache)
self.assertEqual(inst._table, expected_table)
def test_save_cache_build_index(self):
cache_element = DataMemoryElement()
self.assertTrue(cache_element.is_empty())
i = LinearHashIndex(cache_element)
# noinspection PyTypeChecker
i.build_index([[0, 1, 0], [1, 0, 0], [0, 1, 1], [0, 0, 1]])
self.assertFalse(cache_element.is_empty())
# Check byte content
expected_cache = {1, 2, 3, 4}
actual_cache = set(numpy.load(BytesIO(cache_element.get_bytes())))
self.assertSetEqual(expected_cache, actual_cache)
def test_remove_many_with_cache(self):
starting_table = {
0: 0,
1: 1,
2: 2,
}
c = DataMemoryElement(pickle.dumps(starting_table))
s = MemoryKeyValueStore(c)
self.assertDictEqual(s._table, starting_table)
s.remove_many([0, 2])
self.assertDictEqual(pickle.loads(c.get_bytes()), {1: 1})
def test_remove_many_with_cache(self):
starting_table = {
0: 0,
1: 1,
2: 2,
}
c = DataMemoryElement(pickle.dumps(starting_table))
s = MemoryKeyValueStore(c)
self.assertDictEqual(s._table, starting_table)
s.remove_many([0, 2])
self.assertDictEqual(pickle.loads(c.get_bytes()), {1: 1})
def test_cacheing_with_map(self):
expected_cache = DataMemoryElement()
expected_map = {
0: 'a',
75: 'b',
124769: 'c',
}
dms = DataMemorySet(expected_cache)
dms._element_map = expected_map
dms.cache()
self.assertFalse(expected_cache.is_empty())
self.assertEqual(pickle.loads(expected_cache.get_bytes()), expected_map)
def test_add_with_caching(self):
"""
Test that we can add key-value pairs and they reflect in the cache
element.
"""
c = DataMemoryElement()
s = MemoryKeyValueStore(c)
expected_cache_dict = {'a': 'b', 'foo': None, 0: 89}
s.add('a', 'b')
s.add('foo', None)
s.add(0, 89)
self.assertEqual(pickle.loads(c.get_bytes()), expected_cache_dict)
def test_from_config(self):
# Configured cache with some picked bytes
expected_table = dict(a=1, b=2, c=3)
expected_cache = DataMemoryElement(bytes=pickle.dumps(expected_table))
inst = MemoryDescriptorIndex.from_config({
'cache_element': {
'type': 'DataMemoryElement',
'DataMemoryElement': {
'bytes': expected_cache.get_bytes()
}
}
})
ntools.assert_equal(inst.cache_element, expected_cache)
ntools.assert_equal(inst._table, expected_table)
def test_save_cache_build_index(self):
cache_element = DataMemoryElement()
self.assertTrue(cache_element.is_empty())
i = LinearHashIndex(cache_element)
# noinspection PyTypeChecker
i.build_index([[0, 1, 0],
[1, 0, 0],
[0, 1, 1],
[0, 0, 1]])
self.assertFalse(cache_element.is_empty())
# Check byte content
expected_cache = {1, 2, 3, 4}
actual_cache = set(numpy.load(BytesIO(cache_element.get_bytes())))
self.assertSetEqual(expected_cache, actual_cache)
def test_remove_with_cache(self):
"""
Test that removal correctly updates the cache element.
"""
existing_data = {
0: 1,
'a': 'b',
}
c = DataMemoryElement(pickle.dumps(existing_data))
s = MemoryKeyValueStore(c)
self.assertDictEqual(s._table, existing_data)
s.remove('a')
self.assertDictEqual(s._table, {0: 1})
self.assertDictEqual(pickle.loads(c.get_bytes()), {0: 1})
def test_from_config(self):
# Configured cache with some picked bytes
expected_table = dict(a=1, b=2, c=3)
expected_cache = DataMemoryElement(bytes=pickle.dumps(expected_table))
expected_cache_json_str = \
expected_cache.get_bytes().decode(BYTES_CONFIG_ENCODING)
inst = MemoryDescriptorIndex.from_config({
'cache_element': {
'type': 'DataMemoryElement',
'DataMemoryElement': {
'bytes': expected_cache_json_str
}
}
})
self.assertEqual(inst.cache_element, expected_cache)
self.assertEqual(inst._table, expected_table)
def test_add_with_caching(self):
"""
Test that we can add key-value pairs and they reflect in the cache
element.
"""
c = DataMemoryElement()
s = MemoryKeyValueStore(c)
expected_cache_dict = {'a': 'b', 'foo': None, 0: 89}
s.add('a', 'b')
s.add('foo', None)
s.add(0, 89)
self.assertEqual(
pickle.loads(c.get_bytes()),
expected_cache_dict
)
def test_remove_with_cache(self):
"""
Test that removal correctly updates the cache element.
"""
existing_data = {
0: 1,
'a': 'b',
}
c = DataMemoryElement(pickle.dumps(existing_data))
s = MemoryKeyValueStore(c)
self.assertDictEqual(s._table, existing_data)
s.remove('a')
self.assertDictEqual(s._table, {0: 1})
self.assertDictEqual(pickle.loads(c.get_bytes()),
{0: 1})
def test_from_config(self):
# Configured cache with some picked bytes
# Then convert to "string" (decode -> unicode) for python version used.
expected_table = dict(a=1, b=2, c=3)
expected_cache = DataMemoryElement(bytes=pickle.dumps(expected_table))
expected_cache_json_str = \
expected_cache.get_bytes().decode(BYTES_CONFIG_ENCODING)
dme_key = 'smqtk.representation.data_element.memory_element.DataMemoryElement'
inst = MemoryDescriptorSet.from_config({
'cache_element': {
'type': dme_key,
dme_key: {
'bytes': expected_cache_json_str
}
}
})
self.assertEqual(inst.cache_element, expected_cache)
self.assertEqual(inst._table, expected_table)
def test_get_bytes_some_bytes(self):
expected_bytes = 'some bytes'
e = DataMemoryElement(expected_bytes)
self.assertEqual(e.get_bytes(), expected_bytes)
def test_get_bytes_some_bytes(self):
expected_bytes = 'some bytes'
e = DataMemoryElement(expected_bytes)
ntools.assert_equal(e.get_bytes(), expected_bytes)
def test_get_bytes_empty_bytes(self):
e = DataMemoryElement('')
self.assertEqual(e.get_bytes(), six.b(''))
def test_get_bytes_empty_bytes(self):
e = DataMemoryElement('')
ntools.assert_equal(e.get_bytes(), six.b(''))
def test_get_bytes_none_bytes(self):
e = DataMemoryElement()
self.assertEqual(e.get_bytes(), six.b(''))
def test_get_bytes_empty_bytes(self):
e = DataMemoryElement(b'')
self.assertEqual(e.get_bytes(), b'')
def test_get_bytes_none_bytes(self):
e = DataMemoryElement()
ntools.assert_equal(e.get_bytes(), '')
def test_get_bytes_some_bytes(self):
expected_bytes = b'some bytes'
e = DataMemoryElement(expected_bytes)
self.assertEqual(e.get_bytes(), expected_bytes)
