def test_caching_off(self, zarr_version):
# setup store
store = CountingDict() if zarr_version == 2 else CountingDictV3()
attrs_key = '.zattrs' if zarr_version == 2 else 'meta/root/attrs'
assert 0 == store.counter['__getitem__', attrs_key]
assert 0 == store.counter['__setitem__', attrs_key]
if zarr_version == 2:
store[attrs_key] = json.dumps(dict(foo='xxx',
bar=42)).encode('ascii')
else:
store[attrs_key] = json.dumps(
dict(attributes=dict(foo='xxx', bar=42))).encode('ascii')
assert 0 == store.counter['__getitem__', attrs_key]
assert 1 == store.counter['__setitem__', attrs_key]
# setup attributes
a = self.init_attributes(store, cache=False, zarr_version=zarr_version)
# test __getitem__
assert a['foo'] == 'xxx'
assert 1 == store.counter['__getitem__', attrs_key]
assert a['bar'] == 42
assert 2 == store.counter['__getitem__', attrs_key]
assert a['foo'] == 'xxx'
assert 3 == store.counter['__getitem__', attrs_key]
# test __setitem__
a['foo'] = 'yyy'
get_cnt = 4 if zarr_version == 2 else 5
assert get_cnt == store.counter['__getitem__', attrs_key]
assert 2 == store.counter['__setitem__', attrs_key]
assert a['foo'] == 'yyy'
get_cnt = 5 if zarr_version == 2 else 6
assert get_cnt == store.counter['__getitem__', attrs_key]
assert 2 == store.counter['__setitem__', attrs_key]
# test update()
a.update(foo='zzz', bar=84)
get_cnt = 6 if zarr_version == 2 else 8
assert get_cnt == store.counter['__getitem__', attrs_key]
assert 3 == store.counter['__setitem__', attrs_key]
assert a['foo'] == 'zzz'
assert a['bar'] == 84
get_cnt = 8 if zarr_version == 2 else 10
assert get_cnt == store.counter['__getitem__', attrs_key]
assert 3 == store.counter['__setitem__', attrs_key]
# test __contains__
assert 'foo' in a
get_cnt = 9 if zarr_version == 2 else 11
assert get_cnt == store.counter['__getitem__', attrs_key]
assert 3 == store.counter['__setitem__', attrs_key]
assert 'spam' not in a
get_cnt = 10 if zarr_version == 2 else 12
assert get_cnt == store.counter['__getitem__', attrs_key]
assert 3 == store.counter['__setitem__', attrs_key]
def test_cache_keys(self):
# setup
store = CountingDict()
store['foo'] = b'xxx'
store['bar'] = b'yyy'
assert 0 == store.counter['__contains__', 'foo']
assert 0 == store.counter['__iter__']
assert 0 == store.counter['keys']
cache = LRUStoreCache(store, max_size=None)
# keys should be cached on first call
keys = sorted(cache.keys())
assert keys == ['bar', 'foo']
assert 1 == store.counter['keys']
# keys should now be cached
assert keys == sorted(cache.keys())
assert 1 == store.counter['keys']
assert 'foo' in cache
assert 0 == store.counter['__contains__', 'foo']
assert keys == sorted(cache)
assert 0 == store.counter['__iter__']
assert 1 == store.counter['keys']
# cache should be cleared if store is modified - crude but simple for now
cache['baz'] = b'zzz'
keys = sorted(cache.keys())
assert keys == ['bar', 'baz', 'foo']
assert 2 == store.counter['keys']
# keys should now be cached
assert keys == sorted(cache.keys())
assert 2 == store.counter['keys']
# manually invalidate keys
cache.invalidate_keys()
keys = sorted(cache.keys())
assert keys == ['bar', 'baz', 'foo']
assert 3 == store.counter['keys']
assert 0 == store.counter['__contains__', 'foo']
assert 0 == store.counter['__iter__']
cache.invalidate_keys()
keys = sorted(cache)
assert keys == ['bar', 'baz', 'foo']
assert 4 == store.counter['keys']
assert 0 == store.counter['__contains__', 'foo']
assert 0 == store.counter['__iter__']
cache.invalidate_keys()
assert 'foo' in cache
assert 5 == store.counter['keys']
assert 0 == store.counter['__contains__', 'foo']
assert 0 == store.counter['__iter__']
# check these would get counted if called directly
assert 'foo' in store
assert 1 == store.counter['__contains__', 'foo']
assert keys == sorted(store)
assert 1 == store.counter['__iter__']
def test_caching_off(self):
# setup store
store = CountingDict()
assert 0 == store.counter['__getitem__', 'attrs']
assert 0 == store.counter['__setitem__', 'attrs']
store['attrs'] = json.dumps(dict(foo='xxx', bar=42)).encode('ascii')
assert 0 == store.counter['__getitem__', 'attrs']
assert 1 == store.counter['__setitem__', 'attrs']
# setup attributes
a = self.init_attributes(store, cache=False)
# test __getitem__
assert a['foo'] == 'xxx'
assert 1 == store.counter['__getitem__', 'attrs']
assert a['bar'] == 42
assert 2 == store.counter['__getitem__', 'attrs']
assert a['foo'] == 'xxx'
assert 3 == store.counter['__getitem__', 'attrs']
# test __setitem__
a['foo'] = 'yyy'
assert 4 == store.counter['__getitem__', 'attrs']
assert 2 == store.counter['__setitem__', 'attrs']
assert a['foo'] == 'yyy'
assert 5 == store.counter['__getitem__', 'attrs']
assert 2 == store.counter['__setitem__', 'attrs']
# test update()
a.update(foo='zzz', bar=84)
assert 6 == store.counter['__getitem__', 'attrs']
assert 3 == store.counter['__setitem__', 'attrs']
assert a['foo'] == 'zzz'
assert a['bar'] == 84
assert 8 == store.counter['__getitem__', 'attrs']
assert 3 == store.counter['__setitem__', 'attrs']
# test __contains__
assert 'foo' in a
assert 9 == store.counter['__getitem__', 'attrs']
assert 3 == store.counter['__setitem__', 'attrs']
assert 'spam' not in a
assert 10 == store.counter['__getitem__', 'attrs']
assert 3 == store.counter['__setitem__', 'attrs']
def test_cache_values_with_max_size(self):
# setup store
store = CountingDict()
store['foo'] = b'xxx'
store['bar'] = b'yyy'
assert 0 == store.counter['__getitem__', 'foo']
assert 0 == store.counter['__getitem__', 'bar']
# setup cache - can only hold one item
cache = LRUStoreCache(store, max_size=5)
assert 0 == cache.hits
assert 0 == cache.misses
# test first 'foo' __getitem__, cache miss
assert b'xxx' == cache['foo']
assert 1 == store.counter['__getitem__', 'foo']
assert 0 == cache.hits
assert 1 == cache.misses
# test second 'foo' __getitem__, cache hit
assert b'xxx' == cache['foo']
assert 1 == store.counter['__getitem__', 'foo']
assert 1 == cache.hits
assert 1 == cache.misses
# test first 'bar' __getitem__, cache miss
assert b'yyy' == cache['bar']
assert 1 == store.counter['__getitem__', 'bar']
assert 1 == cache.hits
assert 2 == cache.misses
# test second 'bar' __getitem__, cache hit
assert b'yyy' == cache['bar']
assert 1 == store.counter['__getitem__', 'bar']
assert 2 == cache.hits
assert 2 == cache.misses
# test 'foo' __getitem__, should have been evicted, cache miss
assert b'xxx' == cache['foo']
assert 2 == store.counter['__getitem__', 'foo']
assert 2 == cache.hits
assert 3 == cache.misses
# test 'bar' __getitem__, should have been evicted, cache miss
assert b'yyy' == cache['bar']
assert 2 == store.counter['__getitem__', 'bar']
assert 2 == cache.hits
assert 4 == cache.misses
# setup store
store = CountingDict()
store['foo'] = b'xxx'
store['bar'] = b'yyy'
assert 0 == store.counter['__getitem__', 'foo']
assert 0 == store.counter['__getitem__', 'bar']
# setup cache - can hold two items
cache = LRUStoreCache(store, max_size=6)
assert 0 == cache.hits
assert 0 == cache.misses
# test first 'foo' __getitem__, cache miss
assert b'xxx' == cache['foo']
assert 1 == store.counter['__getitem__', 'foo']
assert 0 == cache.hits
assert 1 == cache.misses
# test second 'foo' __getitem__, cache hit
assert b'xxx' == cache['foo']
assert 1 == store.counter['__getitem__', 'foo']
assert 1 == cache.hits
assert 1 == cache.misses
# test first 'bar' __getitem__, cache miss
assert b'yyy' == cache['bar']
assert 1 == store.counter['__getitem__', 'bar']
assert 1 == cache.hits
assert 2 == cache.misses
# test second 'bar' __getitem__, cache hit
assert b'yyy' == cache['bar']
assert 1 == store.counter['__getitem__', 'bar']
assert 2 == cache.hits
assert 2 == cache.misses
# test 'foo' __getitem__, should still be cached
assert b'xxx' == cache['foo']
assert 1 == store.counter['__getitem__', 'foo']
assert 3 == cache.hits
assert 2 == cache.misses
# test 'bar' __getitem__, should still be cached
assert b'yyy' == cache['bar']
assert 1 == store.counter['__getitem__', 'bar']
assert 4 == cache.hits
assert 2 == cache.misses
def test_cache_values_no_max_size(self):
# setup store
store = CountingDict()
store['foo'] = b'xxx'
store['bar'] = b'yyy'
assert 0 == store.counter['__getitem__', 'foo']
assert 1 == store.counter['__setitem__', 'foo']
assert 0 == store.counter['__getitem__', 'bar']
assert 1 == store.counter['__setitem__', 'bar']
# setup cache
cache = LRUStoreCache(store, max_size=None)
assert 0 == cache.hits
assert 0 == cache.misses
# test first __getitem__, cache miss
assert b'xxx' == cache['foo']
assert 1 == store.counter['__getitem__', 'foo']
assert 1 == store.counter['__setitem__', 'foo']
assert 0 == cache.hits
assert 1 == cache.misses
# test second __getitem__, cache hit
assert b'xxx' == cache['foo']
assert 1 == store.counter['__getitem__', 'foo']
assert 1 == store.counter['__setitem__', 'foo']
assert 1 == cache.hits
assert 1 == cache.misses
# test __setitem__, __getitem__
cache['foo'] = b'zzz'
assert 1 == store.counter['__getitem__', 'foo']
assert 2 == store.counter['__setitem__', 'foo']
# should be a cache hit
assert b'zzz' == cache['foo']
assert 1 == store.counter['__getitem__', 'foo']
assert 2 == store.counter['__setitem__', 'foo']
assert 2 == cache.hits
assert 1 == cache.misses
# manually invalidate all cached values
cache.invalidate_values()
assert b'zzz' == cache['foo']
assert 2 == store.counter['__getitem__', 'foo']
assert 2 == store.counter['__setitem__', 'foo']
cache.invalidate()
assert b'zzz' == cache['foo']
assert 3 == store.counter['__getitem__', 'foo']
assert 2 == store.counter['__setitem__', 'foo']
# test __delitem__
del cache['foo']
with pytest.raises(KeyError):
# noinspection PyStatementEffect
cache['foo']
with pytest.raises(KeyError):
# noinspection PyStatementEffect
store['foo']
# verify other keys untouched
assert 0 == store.counter['__getitem__', 'bar']
assert 1 == store.counter['__setitem__', 'bar']
def test_caching_on(self):
# caching is turned on by default
# setup store
store = CountingDict()
assert 0 == store.counter['__getitem__', 'attrs']
assert 0 == store.counter['__setitem__', 'attrs']
store['attrs'] = json.dumps(dict(foo='xxx', bar=42)).encode('ascii')
assert 0 == store.counter['__getitem__', 'attrs']
assert 1 == store.counter['__setitem__', 'attrs']
# setup attributes
a = self.init_attributes(store)
# test __getitem__ causes all attributes to be cached
assert a['foo'] == 'xxx'
assert 1 == store.counter['__getitem__', 'attrs']
assert a['bar'] == 42
assert 1 == store.counter['__getitem__', 'attrs']
assert a['foo'] == 'xxx'
assert 1 == store.counter['__getitem__', 'attrs']
# test __setitem__ updates the cache
a['foo'] = 'yyy'
assert 2 == store.counter['__getitem__', 'attrs']
assert 2 == store.counter['__setitem__', 'attrs']
assert a['foo'] == 'yyy'
assert 2 == store.counter['__getitem__', 'attrs']
assert 2 == store.counter['__setitem__', 'attrs']
# test update() updates the cache
a.update(foo='zzz', bar=84)
assert 3 == store.counter['__getitem__', 'attrs']
assert 3 == store.counter['__setitem__', 'attrs']
assert a['foo'] == 'zzz'
assert a['bar'] == 84
assert 3 == store.counter['__getitem__', 'attrs']
assert 3 == store.counter['__setitem__', 'attrs']
# test __contains__ uses the cache
assert 'foo' in a
assert 3 == store.counter['__getitem__', 'attrs']
assert 3 == store.counter['__setitem__', 'attrs']
assert 'spam' not in a
assert 3 == store.counter['__getitem__', 'attrs']
assert 3 == store.counter['__setitem__', 'attrs']
# test __delitem__ updates the cache
del a['bar']
assert 4 == store.counter['__getitem__', 'attrs']
assert 4 == store.counter['__setitem__', 'attrs']
assert 'bar' not in a
assert 4 == store.counter['__getitem__', 'attrs']
assert 4 == store.counter['__setitem__', 'attrs']
# test refresh()
store['attrs'] = json.dumps(dict(foo='xxx', bar=42)).encode('ascii')
assert 4 == store.counter['__getitem__', 'attrs']
a.refresh()
assert 5 == store.counter['__getitem__', 'attrs']
assert a['foo'] == 'xxx'
assert 5 == store.counter['__getitem__', 'attrs']
assert a['bar'] == 42
assert 5 == store.counter['__getitem__', 'attrs']
def test_caching_on(self, zarr_version):
# caching is turned on by default
# setup store
store = CountingDict() if zarr_version == 2 else CountingDictV3()
attrs_key = '.zattrs' if zarr_version == 2 else 'meta/root/attrs'
assert 0 == store.counter['__getitem__', attrs_key]
assert 0 == store.counter['__setitem__', attrs_key]
if zarr_version == 2:
store[attrs_key] = json.dumps(dict(foo='xxx',
bar=42)).encode('ascii')
else:
store[attrs_key] = json.dumps(
dict(attributes=dict(foo='xxx', bar=42))).encode('ascii')
assert 0 == store.counter['__getitem__', attrs_key]
assert 1 == store.counter['__setitem__', attrs_key]
# setup attributes
a = self.init_attributes(store, zarr_version=zarr_version)
# test __getitem__ causes all attributes to be cached
assert a['foo'] == 'xxx'
assert 1 == store.counter['__getitem__', attrs_key]
assert a['bar'] == 42
assert 1 == store.counter['__getitem__', attrs_key]
assert a['foo'] == 'xxx'
assert 1 == store.counter['__getitem__', attrs_key]
# test __setitem__ updates the cache
a['foo'] = 'yyy'
get_cnt = 2 if zarr_version == 2 else 3
assert get_cnt == store.counter['__getitem__', attrs_key]
assert 2 == store.counter['__setitem__', attrs_key]
assert a['foo'] == 'yyy'
assert get_cnt == store.counter['__getitem__', attrs_key]
assert 2 == store.counter['__setitem__', attrs_key]
# test update() updates the cache
a.update(foo='zzz', bar=84)
get_cnt = 3 if zarr_version == 2 else 5
assert get_cnt == store.counter['__getitem__', attrs_key]
assert 3 == store.counter['__setitem__', attrs_key]
assert a['foo'] == 'zzz'
assert a['bar'] == 84
assert get_cnt == store.counter['__getitem__', attrs_key]
assert 3 == store.counter['__setitem__', attrs_key]
# test __contains__ uses the cache
assert 'foo' in a
assert get_cnt == store.counter['__getitem__', attrs_key]
assert 3 == store.counter['__setitem__', attrs_key]
assert 'spam' not in a
assert get_cnt == store.counter['__getitem__', attrs_key]
assert 3 == store.counter['__setitem__', attrs_key]
# test __delitem__ updates the cache
del a['bar']
get_cnt = 4 if zarr_version == 2 else 7
assert get_cnt == store.counter['__getitem__', attrs_key]
assert 4 == store.counter['__setitem__', attrs_key]
assert 'bar' not in a
assert get_cnt == store.counter['__getitem__', attrs_key]
assert 4 == store.counter['__setitem__', attrs_key]
# test refresh()
if zarr_version == 2:
store[attrs_key] = json.dumps(dict(foo='xxx',
bar=42)).encode('ascii')
else:
store[attrs_key] = json.dumps(
dict(attributes=dict(foo='xxx', bar=42))).encode('ascii')
assert get_cnt == store.counter['__getitem__', attrs_key]
a.refresh()
get_cnt = 5 if zarr_version == 2 else 8
assert get_cnt == store.counter['__getitem__', attrs_key]
assert a['foo'] == 'xxx'
assert get_cnt == store.counter['__getitem__', attrs_key]
assert a['bar'] == 42
assert get_cnt == store.counter['__getitem__', attrs_key]