async def test_subscriber_invalidates(redis_container, dummy_guillotina, loop): await cache.close_redis_pool() trns = mocks.MockTransaction(mocks.MockTransactionManager()) trns.added = trns.deleted = {} content = create_content() trns.modified = {content._p_oid: content} rcache = RedisCache(trns, loop=loop) await rcache.clear() await rcache.set('foobar', oid=content._p_oid) assert serialize.loads( await rcache._redis.get( CACHE_PREFIX + 'root-' + content._p_oid)) == "foobar" assert rcache._memory_cache.get( 'root-' + content._p_oid) == 'foobar' assert await rcache.get(oid=content._p_oid) == 'foobar' assert 'root-' + content._p_oid in rcache._memory_cache await rcache._redis.publish( app_settings['redis']['updates_channel'], serialize.dumps({ 'tid': 32423, 'keys': ['root-' + content._p_oid] })) await asyncio.sleep(1) # should be enough for pub/sub to finish assert 'root-' + content._p_oid not in rcache._memory_cache await cache.close_redis_pool()
async def test_fill_cache_doesnt_cache_large_objects(guillotina_main, mocked_cache_set): tm = mocks.MockTransactionManager() txn = Transaction(tm) cache = BasicCache(txn) # Mock storing an object that is over the limit obj = Mock() obj.__uuid__ = "foo" obj.__serial__ = "bar" obj.__name__ = "ba" obj.__of__ = "bi" pickled = MagicMock() pickled.__len__.return_value = BasicCache.max_cache_record_size + 10 await cache.store_object(obj, pickled) # Call fill_cache and check that caching was skipped await cache.fill_cache() cache.set.assert_not_called() # Now add smaller object and check that is cached pickled.__len__.return_value = 1 await cache.store_object(obj, pickled) await cache.fill_cache() cache.set.assert_called_once()
async def test_subscriber_ignores_trsn_on_invalidate( redis_container, dummy_guillotina, loop): await cache.close_redis_pool() trns = mocks.MockTransaction(mocks.MockTransactionManager()) trns.added = trns.deleted = {} content = create_content() trns.modified = {content._p_oid: content} rcache = RedisCache(trns, loop=loop) await rcache.clear() await rcache.set('foobar', oid=content._p_oid) assert serialize.loads( await rcache._redis.get( CACHE_PREFIX + 'root-' + content._p_oid)) == "foobar" assert rcache._memory_cache.get('root-' + content._p_oid) == 'foobar' assert await rcache.get(oid=content._p_oid) == 'foobar' assert 'root-' + content._p_oid in rcache._memory_cache utility = getUtility(IRedisChannelUtility) utility.ignore_tid(5555) await rcache._redis.publish( app_settings['redis']['updates_channel'], serialize.dumps({ 'tid': 5555, 'keys': ['root-' + content._p_oid] })) await asyncio.sleep(1) # should be enough for pub/sub to finish # should still be there because we set to ignore this tid assert 'root-' + content._p_oid in rcache._memory_cache # tid should also now be removed from ignored list assert 5555 not in utility._ignored_tids await cache.close_redis_pool()
async def test_do_not_cache_large_object(guillotina_main, loop): tm = mocks.MockTransactionManager() storage = tm._storage txn = Transaction(tm) cache = BasicCache(txn) txn._cache = cache ob = create_content() ob.foobar = "X" * cache.max_cache_record_size # push size above cache threshold storage.store(None, None, None, ob, txn) loaded = await txn.get(ob.__uuid__) assert id(loaded) != id(ob) assert loaded.__uuid__ == ob.__uuid__
async def test_cache_set(guillotina_main, loop): util = get_utility(ICacheUtility) assert util.initialized trns = mocks.MockTransaction(mocks.MockTransactionManager()) trns.added = trns.deleted = {} rcache = BasicCache(trns) await rcache.clear() await rcache.set("bar", oid="foo") # but also in memory assert util._memory_cache.get("root-foo") == "bar" # and api matches.. assert await rcache.get(oid="foo") == "bar"
async def test_cache_clear(guillotina_main, loop): util = get_utility(ICacheUtility) trns = mocks.MockTransaction(mocks.MockTransactionManager()) trns.added = trns.deleted = {} rcache = BasicCache(trns) await rcache.clear() await rcache.set("bar", oid="foo") assert util._memory_cache.get("root-foo") == "bar" assert await rcache.get(oid="foo") == "bar" await rcache.clear() assert await rcache.get(oid="foo") is None
async def test_do_not_cache_large_object(dummy_guillotina): tm = mocks.MockTransactionManager() storage = tm._storage txn = Transaction(tm) cache = MemoryCache(storage, txn) txn._cache = cache ob = create_content() ob.foobar = 'X' * cache.max_cache_record_size # push size above cache threshold storage.store(ob) loaded = await txn.get(ob._p_oid) assert id(loaded) != id(ob) assert loaded._p_oid == ob._p_oid assert len(cache._actions) == 0
async def cleanup(aps): conn = await aps.open() tm = mocks.MockTransactionManager(aps) txn = mocks.MockTransaction(tm) txn._db_conn = conn await aps.start_transaction(txn) conn = txn._db_conn await conn.execute("DROP TABLE IF EXISTS objects;") await conn.execute("DROP TABLE IF EXISTS blobs;") if DATABASE == "postgres": await conn.execute("ALTER SEQUENCE tid_sequence RESTART WITH 1") await txn._db_txn.commit() await aps.pool.release(conn) await aps.finalize()
async def test_invalidate_object(guillotina_main, loop): util = get_utility(ICacheUtility) trns = mocks.MockTransaction(mocks.MockTransactionManager()) trns.added = trns.deleted = {} content = create_content() trns.modified = {content.__uuid__: content} rcache = BasicCache(trns) await rcache.clear() await rcache.set("foobar", oid=content.__uuid__) assert util._memory_cache.get("root-" + content.__uuid__) == "foobar" assert await rcache.get(oid=content.__uuid__) == "foobar" await rcache.close(invalidate=True) assert await rcache.get(oid=content.__uuid__) is None
async def test_cache_delete(guillotina_main, loop): util = get_utility(ICacheUtility) trns = mocks.MockTransaction(mocks.MockTransactionManager()) trns.added = trns.deleted = {} rcache = BasicCache(trns) await rcache.clear() await rcache.set('bar', oid='foo') # make sure it is in redis assert util._memory_cache.get('root-foo') == 'bar' assert await rcache.get(oid='foo') == 'bar' # now delete await rcache.delete('root-foo') assert await rcache.get(oid='foo') is None
async def test_cache_object(dummy_guillotina): tm = mocks.MockTransactionManager() storage = tm._storage txn = Transaction(tm) cache = MemoryCache(storage, txn) txn._cache = cache ob = create_content() storage.store(ob) loaded = await txn.get(ob._p_oid) assert id(loaded) != id(ob) assert loaded._p_oid == ob._p_oid assert cache._actions[0]['action'] == 'stored' # and load from cache await txn.get(ob._p_oid) assert cache._actions[-1]['action'] == 'loaded'
async def test_get_size_of_item(dummy_guillotina, loop): await cache.close_redis_pool() trns = mocks.MockTransaction(mocks.MockTransactionManager()) trns.added = trns.deleted = {} rcache = RedisCache(trns) from guillotina_rediscache.cache_strategy import _default_size import sys assert rcache.get_size(dict(a=1)) == _default_size assert rcache.get_size(1) == sys.getsizeof(1) assert rcache.get_size(dict(state=b'x'*10)) == 10 item = [ 'x'*10, 'x'*10, 'x'*10 ] assert rcache.get_size(item) == sys.getsizeof('x'*10) * 3
async def test_cache_set(redis_container, dummy_guillotina, loop): await cache.close_redis_pool() trns = mocks.MockTransaction(mocks.MockTransactionManager()) trns.added = trns.deleted = {} rcache = RedisCache(trns, loop=loop) await rcache.clear() await rcache.set('bar', oid='foo') # make sure it is in redis val = await rcache._redis.get(CACHE_PREFIX + 'root-foo') assert serialize.loads(val) == "bar" # but also in memory assert rcache._memory_cache.get('root-foo') == 'bar' # and api matches.. assert await rcache.get(oid='foo') == 'bar' await cache.close_redis_pool()
async def test_cache_object(guillotina_main, loop): tm = mocks.MockTransactionManager() storage = tm._storage txn = Transaction(tm) cache = BasicCache(txn) txn._cache = cache ob = create_content() storage.store(None, None, None, ob, txn) loaded = await txn.get(ob.__uuid__) assert id(loaded) != id(ob) assert loaded.__uuid__ == ob.__uuid__ assert cache._hits == 0 assert cache._misses == 1 # and load from cache await txn.get(ob.__uuid__) assert cache._hits == 1
async def test_cache_clear(redis_container, dummy_guillotina, loop): await cache.close_redis_pool() trns = mocks.MockTransaction(mocks.MockTransactionManager()) trns.added = trns.deleted = {} rcache = RedisCache(trns, loop=loop) await rcache.clear() await rcache.set('bar', oid='foo') # make sure it is in redis assert serialize.loads(await rcache._redis.get(CACHE_PREFIX + 'foo')) == "bar" assert rcache._memory_cache.get('foo') == 'bar' assert await rcache.get(oid='foo') == 'bar' await rcache.clear() assert await rcache.get(oid='foo') is None await cache.close_redis_pool()
async def test_subscriber_ignores_trsn_on_invalidate(redis_container, guillotina_main): util = get_utility(ICacheUtility) await util.initialize() assert util.initialized assert util._obj_driver is not None assert util._subscriber is not None trns = mocks.MockTransaction(mocks.MockTransactionManager()) trns.added = trns.deleted = {} content = create_content() trns.modified = {content.__uuid__: content} rcache = BasicCache(trns) await rcache.clear() driver = await resolve_dotted_name("guillotina.contrib.redis").get_driver() await rcache.set("foobar", oid=content.__uuid__) assert serialize.loads(await driver.get(CACHE_PREFIX + "root-" + content.__uuid__)) == "foobar" assert util._memory_cache.get("root-" + content.__uuid__) == "foobar" assert await rcache.get(oid=content.__uuid__) == "foobar" assert "root-" + content.__uuid__ in util._memory_cache util.ignore_tid(5555) await driver.publish( app_settings["cache"]["updates_channel"], pickle.dumps({ "data": serialize.dumps({ "tid": 5555, "keys": ["root-" + content.__uuid__] }), "ruid": "nonce" }), ) await asyncio.sleep(1) # should be enough for pub/sub to finish # should still be there because we set to ignore this tid assert "root-" + content.__uuid__ in util._memory_cache # tid should also now be removed from ignored list assert 5555 not in util._ignored_tids
async def test_invalidate_object(redis_container, dummy_guillotina, loop): await cache.close_redis_pool() trns = mocks.MockTransaction(mocks.MockTransactionManager()) trns.added = trns.deleted = {} content = create_content() trns.modified = {content._p_oid: content} rcache = RedisCache(trns, loop=loop) await rcache.clear() await rcache.set('foobar', oid=content._p_oid) assert serialize.loads(await rcache._redis.get(CACHE_PREFIX + content._p_oid)) == "foobar" assert rcache._memory_cache.get(content._p_oid) == 'foobar' assert await rcache.get(oid=content._p_oid) == 'foobar' await rcache.close(invalidate=True) assert await rcache.get(oid=content._p_oid) is None await cache.close_redis_pool()
async def test_cache_object_from_child(guillotina_main, loop): tm = mocks.MockTransactionManager() storage = tm._storage txn = Transaction(tm) cache = BasicCache(txn) txn._cache = cache ob = create_content() parent = create_content() ob.__parent__ = parent storage.store(None, None, None, parent, txn) storage.store(None, None, None, ob, txn) loaded = await txn.get_child(parent, ob.id) assert cache._hits == 0 loaded = await txn.get_child(parent, ob.id) assert cache._hits == 1 assert id(loaded) != id(ob) assert loaded.__uuid__ == ob.__uuid__
async def test_set_multiple_cache_keys_size(redis_container, guillotina_main): util = get_utility(ICacheUtility) await util.initialize() assert util.initialized assert util._obj_driver is not None trns = mocks.MockTransaction(mocks.MockTransactionManager()) trns.added = trns.deleted = {} rcache = BasicCache(trns) await rcache.clear() await rcache.set({"state": "foobar"}, keyset=[{ "oid": "foo" }, { "container": "foobar", "id": "foobar" }]) assert util._memory_cache.get_memory() == 6 await util.finalize(None)
async def test_cache_object_from_child(dummy_guillotina): tm = mocks.MockTransactionManager() storage = tm._storage txn = Transaction(tm) cache = MemoryCache(storage, txn) txn._cache = cache ob = create_content() parent = create_content() ob.__parent__ = parent storage.store(parent) storage.store(ob) loaded = await txn.get_child(parent, ob.id) assert len(cache._actions) == 1 assert cache._actions[0]['action'] == 'stored' loaded = await txn.get_child(parent, ob.id) assert cache._actions[-1]['action'] == 'loaded' assert id(loaded) != id(ob) assert loaded._p_oid == ob._p_oid
async def test_cache_set(redis_container, guillotina_main): util = get_utility(ICacheUtility) await util.initialize() assert util.initialized assert util._obj_driver is not None trns = mocks.MockTransaction(mocks.MockTransactionManager()) trns.added = trns.deleted = {} rcache = BasicCache(trns) await rcache.clear() await rcache.set("bar", oid="foo") # make sure it is in redis driver = await resolve_dotted_name("guillotina.contrib.redis").get_driver() val = await driver.get(CACHE_PREFIX + "root-foo") assert serialize.loads(val) == "bar" # but also in memory assert util._memory_cache.get("root-foo") == "bar" # and api matches.. assert await rcache.get(oid="foo") == "bar" await util.finalize(None)
async def test_subscriber_invalidates(redis_container, guillotina_main, loop): util = get_utility(ICacheUtility) await util.initialize() assert util.initialized assert util._obj_driver is not None assert util._subscriber is not None trns = mocks.MockTransaction(mocks.MockTransactionManager()) trns.added = trns.deleted = {} content = create_content() trns.modified = {content.__uuid__: content} rcache = BasicCache(trns) await rcache.clear() await rcache.set('foobar', oid=content.__uuid__) driver = await resolve_dotted_name('guillotina.contrib.redis').get_driver() assert serialize.loads(await driver.get(CACHE_PREFIX + 'root-' + content.__uuid__)) == "foobar" assert util._memory_cache.get('root-' + content.__uuid__) == 'foobar' assert await rcache.get(oid=content.__uuid__) == 'foobar' assert 'root-' + content.__uuid__ in util._memory_cache await driver.publish( app_settings['cache']['updates_channel'], pickle.dumps({ 'data': serialize.dumps({ 'tid': 32423, 'keys': ['root-' + content.__uuid__] }), 'ruid': 'nonce' })) await asyncio.sleep(1) # should be enough for pub/sub to finish assert 'root-' + content.__uuid__ not in util._memory_cache
async def test_cache_clear(redis_container, guillotina_main, loop): util = get_utility(ICacheUtility) await util.initialize() assert util.initialized assert util._obj_driver is not None trns = mocks.MockTransaction(mocks.MockTransactionManager()) trns.added = trns.deleted = {} rcache = BasicCache(trns) await rcache.clear() await rcache.set('bar', oid='foo') # make sure it is in redis driver = await resolve_dotted_name('guillotina.contrib.redis').get_driver() assert serialize.loads(await driver.get(CACHE_PREFIX + 'root-foo')) == "bar" assert util._memory_cache.get('root-foo') == 'bar' assert await rcache.get(oid='foo') == 'bar' await rcache.clear() assert await rcache.get(oid='foo') is None await util.finalize(None)
async def test_invalidate_object(redis_container, guillotina_main, loop): util = get_utility(ICacheUtility) await util.initialize() assert util.initialized assert util._obj_driver is not None assert util._subscriber is not None trns = mocks.MockTransaction(mocks.MockTransactionManager()) trns.added = trns.deleted = {} content = create_content() trns.modified = {content.__uuid__: content} rcache = BasicCache(trns) await rcache.clear() await rcache.set('foobar', oid=content.__uuid__) driver = await resolve_dotted_name('guillotina.contrib.redis').get_driver() assert serialize.loads(await driver.get(CACHE_PREFIX + 'root-' + content.__uuid__)) == "foobar" assert util._memory_cache.get('root-' + content.__uuid__) == 'foobar' assert await rcache.get(oid=content.__uuid__) == 'foobar' await rcache.close(invalidate=True) assert await rcache.get(oid=content.__uuid__) is None
def _make_strategy(): storage = mocks.MockStorage(transaction_strategy='lock') trns = mocks.MockTransaction(mocks.MockTransactionManager(storage=storage)) return trns._strategy
async def test_no_tid_created_for_reads(dummy_request, loop): tm = mocks.MockTransactionManager() trns = Transaction(tm, loop=loop, read_only=True) await trns.tpc_begin() assert trns._tid is None
async def test_tid_created_for_writes(dummy_request, loop): tm = mocks.MockTransactionManager() trns = Transaction(tm, loop=loop) await trns.tpc_begin() assert trns._tid == 1
async def test_no_tid_created_for_reads(dummy_request, loop): dummy_request._db_write_enabled = False tm = mocks.MockTransactionManager() trns = Transaction(tm, dummy_request, loop=loop) await trns.tpc_begin() assert trns._tid is None