def test_save_cache_remove_from_index(self):
# Test that the cache is updated appropriately on a removal.
cache_element = DataMemoryElement()
self.assertTrue(cache_element.is_empty())
i = LinearHashIndex(cache_element)
# noinspection PyTypeChecker
i.build_index([[0, 1, 0], # 2
[0, 1, 1], # 3
[1, 0, 0], # 4
[1, 1, 0]]) # 6
self.assertFalse(cache_element.is_empty())
self.assertSetEqual(
set(numpy.load(BytesIO(cache_element.get_bytes()))),
{2, 3, 4, 6}
)
# noinspection PyTypeChecker
i.remove_from_index([[0, 1, 1], # 3
[1, 0, 0]]) # 4
self.assertFalse(cache_element.is_empty())
self.assertSetEqual(
set(numpy.load(BytesIO(cache_element.get_bytes()))),
{2, 6}
)
def test_added_descriptor_table_caching(self):
cache_elem = DataMemoryElement(readonly=False)
descrs = [random_descriptor() for _ in range(3)]
expected_table = dict((r.uuid(), r) for r in descrs)
i = MemoryDescriptorSet(cache_elem)
self.assertTrue(cache_elem.is_empty())
# Should add descriptors to table, caching to writable element.
i.add_many_descriptors(descrs)
self.assertFalse(cache_elem.is_empty())
self.assertEqual(pickle.loads(i.cache_element.get_bytes()),
expected_table)
# Changing the internal table (remove, add) it should reflect in
# cache
new_d = random_descriptor()
expected_table[new_d.uuid()] = new_d
i.add_descriptor(new_d)
self.assertEqual(pickle.loads(i.cache_element.get_bytes()),
expected_table)
rm_d = list(expected_table.values())[0]
del expected_table[rm_d.uuid()]
i.remove_descriptor(rm_d.uuid())
self.assertEqual(pickle.loads(i.cache_element.get_bytes()),
expected_table)
def test_added_descriptor_table_caching(self):
cache_elem = DataMemoryElement(readonly=False)
descrs = [random_descriptor() for _ in range(3)]
expected_table = dict((r.uuid(), r) for r in descrs)
i = MemoryDescriptorIndex(cache_elem)
self.assertTrue(cache_elem.is_empty())
# Should add descriptors to table, caching to writable element.
i.add_many_descriptors(descrs)
self.assertFalse(cache_elem.is_empty())
self.assertEqual(pickle.loads(i.cache_element.get_bytes()),
expected_table)
# Changing the internal table (remove, add) it should reflect in
# cache
new_d = random_descriptor()
expected_table[new_d.uuid()] = new_d
i.add_descriptor(new_d)
self.assertEqual(pickle.loads(i.cache_element.get_bytes()),
expected_table)
rm_d = list(expected_table.values())[0]
del expected_table[rm_d.uuid()]
i.remove_descriptor(rm_d.uuid())
self.assertEqual(pickle.loads(i.cache_element.get_bytes()),
expected_table)
def test_save_cache_remove_from_index(self):
# Test that the cache is updated appropriately on a removal.
cache_element = DataMemoryElement()
self.assertTrue(cache_element.is_empty())
i = LinearHashIndex(cache_element)
# noinspection PyTypeChecker
i.build_index([
[0, 1, 0], # 2
[0, 1, 1], # 3
[1, 0, 0], # 4
[1, 1, 0]
]) # 6
self.assertFalse(cache_element.is_empty())
self.assertSetEqual(
set(numpy.load(BytesIO(cache_element.get_bytes()))), {2, 3, 4, 6})
# noinspection PyTypeChecker
i.remove_from_index([
[0, 1, 1], # 3
[1, 0, 0]
]) # 4
self.assertFalse(cache_element.is_empty())
self.assertSetEqual(
set(numpy.load(BytesIO(cache_element.get_bytes()))), {2, 6})
def test_save_cache(self):
cache_element = DataMemoryElement()
nose.tools.assert_true(cache_element.is_empty())
i = LinearHashIndex(cache_element)
# noinspection PyTypeChecker
i.build_index([[0, 1, 0], [1, 0, 0], [0, 1, 1], [0, 0, 1]])
nose.tools.assert_false(cache_element.is_empty())
nose.tools.assert_true(len(cache_element.get_bytes()) > 0)
def test_save_cache_build_index(self):
cache_element = DataMemoryElement()
self.assertTrue(cache_element.is_empty())
i = LinearHashIndex(cache_element)
# noinspection PyTypeChecker
i.build_index([[0, 1, 0], [1, 0, 0], [0, 1, 1], [0, 0, 1]])
self.assertFalse(cache_element.is_empty())
# Check byte content
expected_cache = {1, 2, 3, 4}
actual_cache = set(numpy.load(BytesIO(cache_element.get_bytes())))
self.assertSetEqual(expected_cache, actual_cache)
def test_save_cache_build_index(self):
cache_element = DataMemoryElement()
self.assertTrue(cache_element.is_empty())
i = LinearHashIndex(cache_element)
# noinspection PyTypeChecker
i.build_index([[0, 1, 0],
[1, 0, 0],
[0, 1, 1],
[0, 0, 1]])
self.assertFalse(cache_element.is_empty())
# Check byte content
expected_cache = {1, 2, 3, 4}
actual_cache = set(numpy.load(BytesIO(cache_element.get_bytes())))
self.assertSetEqual(expected_cache, actual_cache)
def test_build_index_with_cache(self):
cache_element = DataMemoryElement()
i = LinearHashIndex(cache_element)
# noinspection PyTypeChecker
i.build_index([[0, 1, 0], [1, 0, 0], [0, 1, 1], [0, 0, 1]])
nose.tools.assert_equal(i.index, {1, 2, 3, 4})
nose.tools.assert_false(cache_element.is_empty())
def test_remove_from_index_last_element_with_cache(self):
"""
Test removing final element also clears the cache element.
"""
c = DataMemoryElement()
bt = SkLearnBallTreeHashIndex(cache_element=c, random_seed=0)
index = np.ndarray((1, 256), bool)
index[0] = int_to_bit_vector_large(1, 256)
bt.build_index(index)
self.assertEqual(bt.count(), 1)
self.assertFalse(c.is_empty())
bt.remove_from_index(index)
self.assertEqual(bt.count(), 0)
self.assertTrue(c.is_empty())
def test_load_as_matrix_empty_data(self):
"""
Test that we catch and do not load an empty data element.
"""
empty_de = DataMemoryElement(readonly=True, content_type='image/png')
assert empty_de.is_empty()
msg = "GdalImageReader cannot load 0-sized data"
with pytest.raises(ValueError, match=msg):
GdalImageReader().load_as_matrix(empty_de)
def test_build_index_with_cache(self):
cache_element = DataMemoryElement()
i = LinearHashIndex(cache_element)
# noinspection PyTypeChecker
i.build_index([[0, 1, 0],
[1, 0, 0],
[0, 1, 1],
[0, 0, 1]])
self.assertEqual(i.index, {1, 2, 3, 4})
self.assertFalse(cache_element.is_empty())
def test_cacheing_with_map(self):
expected_cache = DataMemoryElement()
expected_map = {
0: 'a',
75: 'b',
124769: 'c',
}
dms = DataMemorySet(expected_cache)
dms._element_map = expected_map
dms.cache()
self.assertFalse(expected_cache.is_empty())
self.assertEqual(pickle.loads(expected_cache.get_bytes()), expected_map)
def test_is_empty_zero_bytes(self):
e = DataMemoryElement(b'')
self.assertTrue(e.is_empty())
def test_is_empty_nonzero_bytes(self):
e = DataMemoryElement('some bytes')
ntools.assert_false(e.is_empty())
def test_is_empty_zero_bytes(self):
e = DataMemoryElement('')
ntools.assert_true(e.is_empty())
def test_is_empty_nonzero_bytes(self):
e = DataMemoryElement(b'some bytes')
self.assertFalse(e.is_empty())
def test_is_empty_nonzero_bytes(self):
e = DataMemoryElement('some bytes')
self.assertFalse(e.is_empty())
def test_is_empty_zero_bytes(self):
e = DataMemoryElement('')
self.assertTrue(e.is_empty())
