async def test_subscriber_invalidates(redis_container, dummy_guillotina, loop):
await cache.close_redis_pool()
trns = mocks.MockTransaction(mocks.MockTransactionManager())
trns.added = trns.deleted = {}
content = create_content()
trns.modified = {content._p_oid: content}
rcache = RedisCache(trns, loop=loop)
await rcache.clear()
await rcache.set('foobar', oid=content._p_oid)
assert serialize.loads(
await rcache._redis.get(
CACHE_PREFIX + 'root-' + content._p_oid)) == "foobar"
assert rcache._memory_cache.get(
'root-' + content._p_oid) == 'foobar'
assert await rcache.get(oid=content._p_oid) == 'foobar'
assert 'root-' + content._p_oid in rcache._memory_cache
await rcache._redis.publish(
app_settings['redis']['updates_channel'], serialize.dumps({
'tid': 32423,
'keys': ['root-' + content._p_oid]
}))
await asyncio.sleep(1) # should be enough for pub/sub to finish
assert 'root-' + content._p_oid not in rcache._memory_cache
await cache.close_redis_pool()
async def test_fill_cache_doesnt_cache_large_objects(guillotina_main,
mocked_cache_set):
tm = mocks.MockTransactionManager()
txn = Transaction(tm)
cache = BasicCache(txn)
# Mock storing an object that is over the limit
obj = Mock()
obj.__uuid__ = "foo"
obj.__serial__ = "bar"
obj.__name__ = "ba"
obj.__of__ = "bi"
pickled = MagicMock()
pickled.__len__.return_value = BasicCache.max_cache_record_size + 10
await cache.store_object(obj, pickled)
# Call fill_cache and check that caching was skipped
await cache.fill_cache()
cache.set.assert_not_called()
# Now add smaller object and check that is cached
pickled.__len__.return_value = 1
await cache.store_object(obj, pickled)
await cache.fill_cache()
cache.set.assert_called_once()
async def test_subscriber_ignores_trsn_on_invalidate(
redis_container, dummy_guillotina, loop):
await cache.close_redis_pool()
trns = mocks.MockTransaction(mocks.MockTransactionManager())
trns.added = trns.deleted = {}
content = create_content()
trns.modified = {content._p_oid: content}
rcache = RedisCache(trns, loop=loop)
await rcache.clear()
await rcache.set('foobar', oid=content._p_oid)
assert serialize.loads(
await rcache._redis.get(
CACHE_PREFIX + 'root-' + content._p_oid)) == "foobar"
assert rcache._memory_cache.get('root-' + content._p_oid) == 'foobar'
assert await rcache.get(oid=content._p_oid) == 'foobar'
assert 'root-' + content._p_oid in rcache._memory_cache
utility = getUtility(IRedisChannelUtility)
utility.ignore_tid(5555)
await rcache._redis.publish(
app_settings['redis']['updates_channel'], serialize.dumps({
'tid': 5555,
'keys': ['root-' + content._p_oid]
}))
await asyncio.sleep(1) # should be enough for pub/sub to finish
# should still be there because we set to ignore this tid
assert 'root-' + content._p_oid in rcache._memory_cache
# tid should also now be removed from ignored list
assert 5555 not in utility._ignored_tids
await cache.close_redis_pool()
async def test_do_not_cache_large_object(guillotina_main, loop):
tm = mocks.MockTransactionManager()
storage = tm._storage
txn = Transaction(tm)
cache = BasicCache(txn)
txn._cache = cache
ob = create_content()
ob.foobar = "X" * cache.max_cache_record_size # push size above cache threshold
storage.store(None, None, None, ob, txn)
loaded = await txn.get(ob.__uuid__)
assert id(loaded) != id(ob)
assert loaded.__uuid__ == ob.__uuid__
async def test_cache_set(guillotina_main, loop):
util = get_utility(ICacheUtility)
assert util.initialized
trns = mocks.MockTransaction(mocks.MockTransactionManager())
trns.added = trns.deleted = {}
rcache = BasicCache(trns)
await rcache.clear()
await rcache.set("bar", oid="foo")
# but also in memory
assert util._memory_cache.get("root-foo") == "bar"
# and api matches..
assert await rcache.get(oid="foo") == "bar"
async def test_cache_clear(guillotina_main, loop):
util = get_utility(ICacheUtility)
trns = mocks.MockTransaction(mocks.MockTransactionManager())
trns.added = trns.deleted = {}
rcache = BasicCache(trns)
await rcache.clear()
await rcache.set("bar", oid="foo")
assert util._memory_cache.get("root-foo") == "bar"
assert await rcache.get(oid="foo") == "bar"
await rcache.clear()
assert await rcache.get(oid="foo") is None
async def test_do_not_cache_large_object(dummy_guillotina):
tm = mocks.MockTransactionManager()
storage = tm._storage
txn = Transaction(tm)
cache = MemoryCache(storage, txn)
txn._cache = cache
ob = create_content()
ob.foobar = 'X' * cache.max_cache_record_size # push size above cache threshold
storage.store(ob)
loaded = await txn.get(ob._p_oid)
assert id(loaded) != id(ob)
assert loaded._p_oid == ob._p_oid
assert len(cache._actions) == 0
async def cleanup(aps):
conn = await aps.open()
tm = mocks.MockTransactionManager(aps)
txn = mocks.MockTransaction(tm)
txn._db_conn = conn
await aps.start_transaction(txn)
conn = txn._db_conn
await conn.execute("DROP TABLE IF EXISTS objects;")
await conn.execute("DROP TABLE IF EXISTS blobs;")
if DATABASE == "postgres":
await conn.execute("ALTER SEQUENCE tid_sequence RESTART WITH 1")
await txn._db_txn.commit()
await aps.pool.release(conn)
await aps.finalize()
async def test_invalidate_object(guillotina_main, loop):
util = get_utility(ICacheUtility)
trns = mocks.MockTransaction(mocks.MockTransactionManager())
trns.added = trns.deleted = {}
content = create_content()
trns.modified = {content.__uuid__: content}
rcache = BasicCache(trns)
await rcache.clear()
await rcache.set("foobar", oid=content.__uuid__)
assert util._memory_cache.get("root-" + content.__uuid__) == "foobar"
assert await rcache.get(oid=content.__uuid__) == "foobar"
await rcache.close(invalidate=True)
assert await rcache.get(oid=content.__uuid__) is None
async def test_cache_delete(guillotina_main, loop):
util = get_utility(ICacheUtility)
trns = mocks.MockTransaction(mocks.MockTransactionManager())
trns.added = trns.deleted = {}
rcache = BasicCache(trns)
await rcache.clear()
await rcache.set('bar', oid='foo')
# make sure it is in redis
assert util._memory_cache.get('root-foo') == 'bar'
assert await rcache.get(oid='foo') == 'bar'
# now delete
await rcache.delete('root-foo')
assert await rcache.get(oid='foo') is None
async def test_cache_object(dummy_guillotina):
tm = mocks.MockTransactionManager()
storage = tm._storage
txn = Transaction(tm)
cache = MemoryCache(storage, txn)
txn._cache = cache
ob = create_content()
storage.store(ob)
loaded = await txn.get(ob._p_oid)
assert id(loaded) != id(ob)
assert loaded._p_oid == ob._p_oid
assert cache._actions[0]['action'] == 'stored'
# and load from cache
await txn.get(ob._p_oid)
assert cache._actions[-1]['action'] == 'loaded'
async def test_get_size_of_item(dummy_guillotina, loop):
await cache.close_redis_pool()
trns = mocks.MockTransaction(mocks.MockTransactionManager())
trns.added = trns.deleted = {}
rcache = RedisCache(trns)
from guillotina_rediscache.cache_strategy import _default_size
import sys
assert rcache.get_size(dict(a=1)) == _default_size
assert rcache.get_size(1) == sys.getsizeof(1)
assert rcache.get_size(dict(state=b'x'*10)) == 10
item = [
'x'*10, 'x'*10, 'x'*10
]
assert rcache.get_size(item) == sys.getsizeof('x'*10) * 3
async def test_cache_set(redis_container, dummy_guillotina, loop):
await cache.close_redis_pool()
trns = mocks.MockTransaction(mocks.MockTransactionManager())
trns.added = trns.deleted = {}
rcache = RedisCache(trns, loop=loop)
await rcache.clear()
await rcache.set('bar', oid='foo')
# make sure it is in redis
val = await rcache._redis.get(CACHE_PREFIX + 'root-foo')
assert serialize.loads(val) == "bar"
# but also in memory
assert rcache._memory_cache.get('root-foo') == 'bar'
# and api matches..
assert await rcache.get(oid='foo') == 'bar'
await cache.close_redis_pool()
async def test_cache_object(guillotina_main, loop):
tm = mocks.MockTransactionManager()
storage = tm._storage
txn = Transaction(tm)
cache = BasicCache(txn)
txn._cache = cache
ob = create_content()
storage.store(None, None, None, ob, txn)
loaded = await txn.get(ob.__uuid__)
assert id(loaded) != id(ob)
assert loaded.__uuid__ == ob.__uuid__
assert cache._hits == 0
assert cache._misses == 1
# and load from cache
await txn.get(ob.__uuid__)
assert cache._hits == 1
async def test_cache_clear(redis_container, dummy_guillotina, loop):
await cache.close_redis_pool()
trns = mocks.MockTransaction(mocks.MockTransactionManager())
trns.added = trns.deleted = {}
rcache = RedisCache(trns, loop=loop)
await rcache.clear()
await rcache.set('bar', oid='foo')
# make sure it is in redis
assert serialize.loads(await
rcache._redis.get(CACHE_PREFIX + 'foo')) == "bar"
assert rcache._memory_cache.get('foo') == 'bar'
assert await rcache.get(oid='foo') == 'bar'
await rcache.clear()
assert await rcache.get(oid='foo') is None
await cache.close_redis_pool()
async def test_subscriber_ignores_trsn_on_invalidate(redis_container,
guillotina_main):
util = get_utility(ICacheUtility)
await util.initialize()
assert util.initialized
assert util._obj_driver is not None
assert util._subscriber is not None
trns = mocks.MockTransaction(mocks.MockTransactionManager())
trns.added = trns.deleted = {}
content = create_content()
trns.modified = {content.__uuid__: content}
rcache = BasicCache(trns)
await rcache.clear()
driver = await resolve_dotted_name("guillotina.contrib.redis").get_driver()
await rcache.set("foobar", oid=content.__uuid__)
assert serialize.loads(await driver.get(CACHE_PREFIX + "root-" +
content.__uuid__)) == "foobar"
assert util._memory_cache.get("root-" + content.__uuid__) == "foobar"
assert await rcache.get(oid=content.__uuid__) == "foobar"
assert "root-" + content.__uuid__ in util._memory_cache
util.ignore_tid(5555)
await driver.publish(
app_settings["cache"]["updates_channel"],
pickle.dumps({
"data":
serialize.dumps({
"tid": 5555,
"keys": ["root-" + content.__uuid__]
}),
"ruid":
"nonce"
}),
)
await asyncio.sleep(1) # should be enough for pub/sub to finish
# should still be there because we set to ignore this tid
assert "root-" + content.__uuid__ in util._memory_cache
# tid should also now be removed from ignored list
assert 5555 not in util._ignored_tids
async def test_invalidate_object(redis_container, dummy_guillotina, loop):
await cache.close_redis_pool()
trns = mocks.MockTransaction(mocks.MockTransactionManager())
trns.added = trns.deleted = {}
content = create_content()
trns.modified = {content._p_oid: content}
rcache = RedisCache(trns, loop=loop)
await rcache.clear()
await rcache.set('foobar', oid=content._p_oid)
assert serialize.loads(await rcache._redis.get(CACHE_PREFIX +
content._p_oid)) == "foobar"
assert rcache._memory_cache.get(content._p_oid) == 'foobar'
assert await rcache.get(oid=content._p_oid) == 'foobar'
await rcache.close(invalidate=True)
assert await rcache.get(oid=content._p_oid) is None
await cache.close_redis_pool()
async def test_cache_object_from_child(guillotina_main, loop):
tm = mocks.MockTransactionManager()
storage = tm._storage
txn = Transaction(tm)
cache = BasicCache(txn)
txn._cache = cache
ob = create_content()
parent = create_content()
ob.__parent__ = parent
storage.store(None, None, None, parent, txn)
storage.store(None, None, None, ob, txn)
loaded = await txn.get_child(parent, ob.id)
assert cache._hits == 0
loaded = await txn.get_child(parent, ob.id)
assert cache._hits == 1
assert id(loaded) != id(ob)
assert loaded.__uuid__ == ob.__uuid__
async def test_set_multiple_cache_keys_size(redis_container, guillotina_main):
util = get_utility(ICacheUtility)
await util.initialize()
assert util.initialized
assert util._obj_driver is not None
trns = mocks.MockTransaction(mocks.MockTransactionManager())
trns.added = trns.deleted = {}
rcache = BasicCache(trns)
await rcache.clear()
await rcache.set({"state": "foobar"},
keyset=[{
"oid": "foo"
}, {
"container": "foobar",
"id": "foobar"
}])
assert util._memory_cache.get_memory() == 6
await util.finalize(None)
async def test_cache_object_from_child(dummy_guillotina):
tm = mocks.MockTransactionManager()
storage = tm._storage
txn = Transaction(tm)
cache = MemoryCache(storage, txn)
txn._cache = cache
ob = create_content()
parent = create_content()
ob.__parent__ = parent
storage.store(parent)
storage.store(ob)
loaded = await txn.get_child(parent, ob.id)
assert len(cache._actions) == 1
assert cache._actions[0]['action'] == 'stored'
loaded = await txn.get_child(parent, ob.id)
assert cache._actions[-1]['action'] == 'loaded'
assert id(loaded) != id(ob)
assert loaded._p_oid == ob._p_oid
async def test_cache_set(redis_container, guillotina_main):
util = get_utility(ICacheUtility)
await util.initialize()
assert util.initialized
assert util._obj_driver is not None
trns = mocks.MockTransaction(mocks.MockTransactionManager())
trns.added = trns.deleted = {}
rcache = BasicCache(trns)
await rcache.clear()
await rcache.set("bar", oid="foo")
# make sure it is in redis
driver = await resolve_dotted_name("guillotina.contrib.redis").get_driver()
val = await driver.get(CACHE_PREFIX + "root-foo")
assert serialize.loads(val) == "bar"
# but also in memory
assert util._memory_cache.get("root-foo") == "bar"
# and api matches..
assert await rcache.get(oid="foo") == "bar"
await util.finalize(None)
async def test_subscriber_invalidates(redis_container, guillotina_main, loop):
util = get_utility(ICacheUtility)
await util.initialize()
assert util.initialized
assert util._obj_driver is not None
assert util._subscriber is not None
trns = mocks.MockTransaction(mocks.MockTransactionManager())
trns.added = trns.deleted = {}
content = create_content()
trns.modified = {content.__uuid__: content}
rcache = BasicCache(trns)
await rcache.clear()
await rcache.set('foobar', oid=content.__uuid__)
driver = await resolve_dotted_name('guillotina.contrib.redis').get_driver()
assert serialize.loads(await driver.get(CACHE_PREFIX + 'root-' +
content.__uuid__)) == "foobar"
assert util._memory_cache.get('root-' + content.__uuid__) == 'foobar'
assert await rcache.get(oid=content.__uuid__) == 'foobar'
assert 'root-' + content.__uuid__ in util._memory_cache
await driver.publish(
app_settings['cache']['updates_channel'],
pickle.dumps({
'data':
serialize.dumps({
'tid': 32423,
'keys': ['root-' + content.__uuid__]
}),
'ruid':
'nonce'
}))
await asyncio.sleep(1) # should be enough for pub/sub to finish
assert 'root-' + content.__uuid__ not in util._memory_cache
async def test_cache_clear(redis_container, guillotina_main, loop):
util = get_utility(ICacheUtility)
await util.initialize()
assert util.initialized
assert util._obj_driver is not None
trns = mocks.MockTransaction(mocks.MockTransactionManager())
trns.added = trns.deleted = {}
rcache = BasicCache(trns)
await rcache.clear()
await rcache.set('bar', oid='foo')
# make sure it is in redis
driver = await resolve_dotted_name('guillotina.contrib.redis').get_driver()
assert serialize.loads(await
driver.get(CACHE_PREFIX + 'root-foo')) == "bar"
assert util._memory_cache.get('root-foo') == 'bar'
assert await rcache.get(oid='foo') == 'bar'
await rcache.clear()
assert await rcache.get(oid='foo') is None
await util.finalize(None)
async def test_invalidate_object(redis_container, guillotina_main, loop):
util = get_utility(ICacheUtility)
await util.initialize()
assert util.initialized
assert util._obj_driver is not None
assert util._subscriber is not None
trns = mocks.MockTransaction(mocks.MockTransactionManager())
trns.added = trns.deleted = {}
content = create_content()
trns.modified = {content.__uuid__: content}
rcache = BasicCache(trns)
await rcache.clear()
await rcache.set('foobar', oid=content.__uuid__)
driver = await resolve_dotted_name('guillotina.contrib.redis').get_driver()
assert serialize.loads(await driver.get(CACHE_PREFIX + 'root-' +
content.__uuid__)) == "foobar"
assert util._memory_cache.get('root-' + content.__uuid__) == 'foobar'
assert await rcache.get(oid=content.__uuid__) == 'foobar'
await rcache.close(invalidate=True)
assert await rcache.get(oid=content.__uuid__) is None
def _make_strategy():
storage = mocks.MockStorage(transaction_strategy='lock')
trns = mocks.MockTransaction(mocks.MockTransactionManager(storage=storage))
return trns._strategy
async def test_no_tid_created_for_reads(dummy_request, loop):
tm = mocks.MockTransactionManager()
trns = Transaction(tm, loop=loop, read_only=True)
await trns.tpc_begin()
assert trns._tid is None
async def test_tid_created_for_writes(dummy_request, loop):
tm = mocks.MockTransactionManager()
trns = Transaction(tm, loop=loop)
await trns.tpc_begin()
assert trns._tid == 1
async def test_no_tid_created_for_reads(dummy_request, loop):
dummy_request._db_write_enabled = False
tm = mocks.MockTransactionManager()
trns = Transaction(tm, dummy_request, loop=loop)
await trns.tpc_begin()
assert trns._tid is None
