def testSimple(self):
""" check basic functions by adding one chunk to cache """
cc = LruCache(mem_target=1000 * 1000 * 10)
cc.consistencyCheck()
self.assertEqual(len(cc), 0)
self.assertEqual(cc.dump_lru(), "->\n<-\n")
id = createObjId("chunks")
try:
# only dict objects can be added
cc[id] = list(range(20))
self.assertTrue(False)
except TypeError:
pass # expected
arr = np.empty((16, 16), dtype='i4')
id = createObjId("datasets")
try:
cc[id] = arr
self.assertTrue(False)
except ValueError:
pass # expected - not a chunk id
rand_id = createObjId("chunks")
np_arr = np.random.random(
(500, 500)) # smaller than our chunk cache size
cc[rand_id] = np_arr # add to cache
cc.consistencyCheck()
self.assertEqual(len(cc), 1)
self.assertTrue(rand_id in cc)
lru_str = "->" + rand_id + "\n<-" + rand_id + "\n"
mem_tgt = cc.memTarget
self.assertEqual(mem_tgt, 1000 * 1000 * 10)
mem_used = cc.memUsed
self.assertEqual(mem_used, 500 * 500 * 8)
mem_dirty = cc.memDirty
self.assertEqual(mem_dirty, 0)
mem_per = cc.cacheUtilizationPercent
self.assertEqual(mem_per, 20) # have used 20% of target memory
# try adding the same id to the cache again
cc[rand_id] = np_arr
cc.consistencyCheck()
self.assertEqual(len(cc), 1)
self.assertTrue(rand_id in cc)
# try out the dirty flags
self.assertFalse(cc.isDirty(rand_id))
self.assertEqual(cc.dirtyCount, 0)
cc.setDirty(rand_id)
cc.consistencyCheck()
self.assertTrue(cc.isDirty(rand_id))
self.assertEqual(cc.dirtyCount, 1)
self.assertEqual(cc.dump_lru(), lru_str)
cc.consistencyCheck()
cc.clearDirty(rand_id)
cc.consistencyCheck()
self.assertFalse(cc.isDirty(rand_id))
self.assertEqual(cc.dirtyCount, 0)
# chunk should not have been evicted from cache
self.assertEqual(len(cc), 1)
self.assertTrue(rand_id in cc)
# delete from cache
del cc[rand_id]
cc.consistencyCheck()
# check cache is empty
self.assertEqual(len(cc), 0)
self.assertFalse(rand_id in cc)
mem_tgt = cc.memTarget
self.assertEqual(mem_tgt, 1000 * 1000 * 10)
mem_used = cc.memUsed
self.assertEqual(mem_used, 0)
mem_dirty = cc.memDirty
self.assertEqual(mem_dirty, 0)
mem_per = cc.cacheUtilizationPercent
self.assertEqual(mem_per, 0) # no memory used
def testMetaDataCache(self):
""" check metadata cache functionality """
cc = LruCache(mem_target=1024 * 10, chunk_cache=False)
cc.consistencyCheck()
self.assertEqual(len(cc), 0)
self.assertEqual(cc.dump_lru(), "->\n<-\n")
id = createObjId("datasets")
try:
# only numpy arrays an be added
cc[id] = np.zeros((3, 4))
self.assertTrue(False)
except TypeError:
pass # expected
data = {"x": 123, "y": 456}
arr = np.zeros((10, ))
id = createObjId("chunks")
try:
cc[id] = arr
self.assertTrue(False)
except TypeError:
pass # expected - not a dict
rand_id = createObjId("groups")
data = {"foo": "bar"}
cc[rand_id] = data # add to cache
cc.consistencyCheck()
self.assertEqual(len(cc), 1)
self.assertTrue(rand_id in cc)
lru_str = "->" + rand_id + "\n<-" + rand_id + "\n"
mem_tgt = cc.memTarget
self.assertEqual(mem_tgt, 1024 * 10)
mem_used = cc.memUsed
self.assertEqual(mem_used, 1024) # not based on actual size
mem_per = cc.cacheUtilizationPercent
self.assertEqual(mem_per, 10) # have used 10% of target memory
# try out the dirty flags
self.assertFalse(cc.isDirty(rand_id))
self.assertEqual(cc.dirtyCount, 0)
cc.setDirty(rand_id)
cc.consistencyCheck()
self.assertTrue(cc.isDirty(rand_id))
self.assertEqual(cc.dirtyCount, 1)
self.assertEqual(cc.dump_lru(), lru_str)
cc.clearDirty(rand_id)
cc.consistencyCheck()
self.assertFalse(cc.isDirty(rand_id))
self.assertEqual(cc.dirtyCount, 0)
# chunk should not have been evicted from cache
self.assertEqual(len(cc), 1)
self.assertTrue(rand_id in cc)
# delete from cache
del cc[rand_id]
cc.consistencyCheck()
# check cache is empty
self.assertEqual(len(cc), 0)
self.assertFalse(rand_id in cc)
mem_tgt = cc.memTarget
self.assertEqual(mem_tgt, 1024 * 10)
mem_used = cc.memUsed
self.assertEqual(mem_used, 0)
mem_per = cc.cacheUtilizationPercent
self.assertEqual(mem_per, 0) # no memory used