def test_start_stop(self):
"""
Test.
"""
self.memory_cache = MemoryCache()
self.assertTrue(self.memory_cache._is_started)
self.redis_cache = RedisCache()
self.assertTrue(self.redis_cache._is_started)
self.high_cache = HighCacheEx(memory_cache=self.memory_cache,
redis_cache=self.redis_cache)
self.high_cache.stop_cache()
self.assertFalse(self.redis_cache._is_started)
self.assertFalse(self.memory_cache._is_started)
self.high_cache.start_cache()
self.assertTrue(self.redis_cache._is_started)
self.assertTrue(self.memory_cache._is_started)
self.high_cache.stop_cache()
self.assertFalse(self.redis_cache._is_started)
self.assertFalse(self.memory_cache._is_started)
self.high_cache = None
self.redis_cache = None
self.memory_cache = None
def test_basic_eviction_with_get_lru(self):
"""
Test
:return:
"""
# Alloc
self.mem_cache = MemoryCache(cb_evict=self.eviction_callback)
# Put
self.mem_cache.put("keyA", b"valA", 60000)
self.mem_cache.put("keyB", b"valB", 60000)
# A was put first => must be kicked, but we GET IT => B must be kicked
self.assertEqual(self.mem_cache.get("keyA"), b"valA")
# Evict LRU
self.mem_cache._evict_key_lru()
# A was used more recently => B must be kicked
self.assertIsNone(self.mem_cache.get("keyB"))
self.assertEqual(self.mem_cache.get("keyA"), b"valA")
self.assertEqual(self.evict_count, 1)
self.assertEqual(self.evict_last_key, "keyB")
self.assertEqual(self.evict_last_value, b"valB")
# Stop
self.mem_cache.stop_cache()
self.mem_cache = None
def test_basic_eviction_with_get_ttl(self):
"""
Test
:return:
"""
# Alloc
self.mem_cache = MemoryCache(cb_evict=self.eviction_callback)
# Put
self.mem_cache.put("keyA", b"valA", 60000)
self.mem_cache.put("keyB", b"valB", 500)
logger.info("ms cur=%s", SolBase.mscurrent())
logger.info("A : %s", self.mem_cache.get_raw("keyA"))
logger.info("B : %s", self.mem_cache.get_raw("keyB"))
# Wait a bit
SolBase.sleep(600)
logger.info("ms after sleep=%s", SolBase.mscurrent())
# A : must be present
# B : must be evicted (TTL elapsed)
self.assertEqual(self.mem_cache.get("keyA"), b"valA")
self.assertIsNone(self.mem_cache.get("keyB"))
self.assertEqual(self.evict_count, 1)
self.assertEqual(self.evict_last_key, "keyB")
self.assertEqual(self.evict_last_value, b"valB")
self.assertEqual(Meters.aig("mcs.cache_evict_ttl_get"), 1)
# Stop
self.mem_cache.stop_cache()
self.mem_cache = None
def test_basic_eviction_max_capacity_lru(self):
"""
Test
:return:
"""
# Alloc
self.mem_cache = MemoryCache(max_item=3,
cb_evict=self.eviction_callback)
# Put
self.mem_cache.put("keyA", b"valA", 60000)
self.mem_cache.put("keyB", b"valB", 60000)
self.mem_cache.put("keyC", b"valC", 60000)
# Use A and C => B becomes the older to be used
self.mem_cache.get("keyA")
self.mem_cache.get("keyC")
# We are maxed (3 items)
# Add D => B must be kicked
self.mem_cache.put("keyD", b"valD", 60000)
self.assertEqual(self.mem_cache.get("keyA"), b"valA")
self.assertEqual(self.mem_cache.get("keyC"), b"valC")
self.assertEqual(self.mem_cache.get("keyD"), b"valD")
self.assertIsNone(self.mem_cache.get("keyB"))
self.assertEqual(self.evict_count, 1)
self.assertEqual(self.evict_last_key, "keyB")
self.assertEqual(self.evict_last_value, b"valB")
self.assertEqual(Meters.aig("mcs.cache_evict_lru_put"), 1)
# Stop
self.mem_cache.stop_cache()
self.mem_cache = None
def test_init_no_stat(self):
"""
Test.
"""
# Alloc
m = MemoryCache()
self.assertIsNotNone(m)
def test_size_limit_on_data_size_first(self):
"""
Test.
"""
# Alloc
self.mem_cache = MemoryCache(
max_bytes=5 * 10 * 2,
max_single_item_bytes=6 * 2,
purge_min_bytes=5 * 5 * 2,
purge_min_count=2,
max_item=sys.maxsize,
)
# Put 10 items
for i in range(10, 20):
self.mem_cache.put("key" + str(i),
SolBase.unicode_to_binary("val%s" % i, "utf-8"),
60000)
logger.info("Cache=%s", self.mem_cache)
# Must have all of them
for i in range(10, 20):
self.assertEqual(self.mem_cache.get("key" + str(i)),
SolBase.unicode_to_binary("val%s" % i, "utf-8"))
self.assertEqual(len(self.mem_cache._hash_key), 10)
self.assertEqual(self.mem_cache._current_data_bytes.get(), 5 * 10 * 2)
# Then add a new one : we will over size the cache
self.mem_cache.put("key" + str(99),
SolBase.unicode_to_binary("val99", "utf-8"), 60000)
# We must have evicted AT least :
# 5*5 + 5 bytes => 5 items + the item added => 6 items
# 2 items minimum
# So 6 items : 10 to 15 must be evicted
logger.info("Hash = %s", self.mem_cache._hash_key)
for i in range(10, 16):
self.assertIsNone(self.mem_cache.get("key" + str(i)))
for i in range(16, 20):
self.assertEqual(self.mem_cache.get("key" + str(i)),
SolBase.unicode_to_binary("val%s" % i, "utf-8"))
self.assertEqual(self.mem_cache.get("key" + str(99)),
SolBase.unicode_to_binary("val99", "utf-8"))
self.assertEqual(len(self.mem_cache._hash_key), 5)
self.assertEqual(len(self.mem_cache._hash_context), 5)
self.assertEqual(self.mem_cache._current_data_bytes.get(), 5 * 5 * 2)
# Try add a big one this time : must not be done (over limit)
self.mem_cache.put("BIGDATA", b"aaaaaaaaaaaaaaaaaaaa", 60000)
self.assertIsNone(self.mem_cache.get("BIGDATA"))
# Stop
self.mem_cache.stop_cache()
self.mem_cache = None
def test_basic_eviction_with_watchdog_ttl(self):
"""
Test
:return:
"""
# Go
self.mem_cache = MemoryCache(watchdog_interval_ms=1000,
cb_watchdog=self.watchdog_callback,
cb_evict=self.eviction_callback)
# Put
self.mem_cache.put("keyA", b"valA", 60000)
self.mem_cache.put("keyB", b"valB", 500)
self.mem_cache.put("keyC", b"valC", 500)
self.mem_cache.put("keyD", b"valD", 60000)
logger.info("ms cur=%s", SolBase.mscurrent())
logger.info("A : %s", self.mem_cache.get_raw("keyA"))
logger.info("B : %s", self.mem_cache.get_raw("keyB"))
logger.info("C : %s", self.mem_cache.get_raw("keyC"))
logger.info("D : %s", self.mem_cache.get_raw("keyD"))
# Wait a bit
ms_start = SolBase.mscurrent()
while SolBase.msdiff(ms_start) < (1000.0 * 2.0):
if self.callback_call > 0:
break
else:
SolBase.sleep(10)
logger.info("ms after wait=%s", SolBase.mscurrent())
logger.info("_hash_key = %s", self.mem_cache._hash_key)
logger.info("_hash_context = %s", self.mem_cache._hash_context)
# A : must be present
# B : must be evicted (TTL elapsed, by watchdog)
self.assertEqual(self.callback_call, 1)
self.assertEqual(self.callback_evicted, 2)
self.assertFalse("valB" in self.mem_cache._hash_key)
self.assertFalse("valB" in self.mem_cache._hash_context)
self.assertFalse("valC" in self.mem_cache._hash_key)
self.assertFalse("valC" in self.mem_cache._hash_context)
self.assertIsNone(self.mem_cache.get("keyB"))
self.assertIsNone(self.mem_cache.get("keyC"))
self.assertEqual(self.mem_cache.get("keyA"), b"valA")
self.assertEqual(self.mem_cache.get("keyD"), b"valD")
self.assertEqual(self.evict_count, 2)
self.assertTrue(self.evict_last_key == "keyB"
or self.evict_last_key == "keyC")
self.assertTrue(self.evict_last_value == b"valB"
or self.evict_last_value == b"valC")
self.assertEqual(Meters.aig("mcs.cache_evict_ttl_watchdog"), 2)
# Stop
self.mem_cache.stop_cache()
self.mem_cache = None
def test_watchdog_schedule(self):
"""
Test
:return:
"""
# Continue callback loop
self.callback_return = True
# Go
self.mem_cache = MemoryCache(watchdog_interval_ms=500,
cb_watchdog=self.watchdog_callback)
# Put
self.mem_cache.put("keyA", b"valA", 60000)
self.mem_cache.put("keyD", b"valD", 60000)
# Wait a bit
ms_start = SolBase.mscurrent()
while SolBase.msdiff(ms_start) < (500.0 * 5.0):
if self.callback_call >= 1:
logger.info("Run 1 exit")
break
else:
SolBase.sleep(1)
logger.info("Run 1 done")
self.assertEqual(self.callback_call, 1)
self.assertEqual(self.callback_evicted, 0)
# Wait a bit
ms_start = SolBase.mscurrent()
while SolBase.msdiff(ms_start) < (500.0 * 5.0):
if self.callback_call >= 2:
logger.info("Run 2 exit")
break
else:
SolBase.sleep(1)
logger.info("Run 2 done")
self.assertGreaterEqual(self.callback_call, 2)
self.assertEqual(self.callback_evicted, 0)
# Stop
self.mem_cache.stop_cache()
self.mem_cache = None
# Wait a bit
SolBase.sleep(500 * 2)
# Nothing must happen
self.assertEqual(self.callback_call, 2)
self.assertEqual(self.callback_evicted, 0)
def test_size_limit_cache_clear(self):
"""
Test.
"""
# Alloc
self.mem_cache = MemoryCache(
max_bytes=5 * 10 * 2,
max_single_item_bytes=6 * 2,
purge_min_bytes=1 * 5,
purge_min_count=100,
max_item=sys.maxsize,
)
# Put 10 items
for i in range(10, 20):
self.mem_cache.put("key" + str(i),
SolBase.unicode_to_binary("val%s" % i, "utf-8"),
60000)
logger.info("Cache=%s", self.mem_cache)
# Must have all of them
for i in range(10, 20):
self.assertEqual(self.mem_cache.get("key" + str(i)),
SolBase.unicode_to_binary("val%s" % i, "utf-8"))
self.assertEqual(len(self.mem_cache._hash_key), 10)
self.assertEqual(self.mem_cache._current_data_bytes.get(),
5 * len(self.mem_cache._hash_key) * 2)
# Then add a new one : we will over size the cache
self.mem_cache.put("key" + str(99),
SolBase.unicode_to_binary("val99", "utf-8"), 60000)
# We must have evicted AT least :
# 1*5 + 5 bytes => 1 items + the item added => 2 items
# 100 items minimum
# So : All items must be kicked, will remain only the new one
logger.info("Hash = %s", self.mem_cache._hash_key)
self.assertEqual(self.mem_cache.get("key" + str(99)),
SolBase.unicode_to_binary("val99", "utf-8"))
self.assertEqual(len(self.mem_cache._hash_key), 1)
self.assertEqual(len(self.mem_cache._hash_context), 1)
self.assertEqual(self.mem_cache._current_data_bytes.get(),
5 * len(self.mem_cache._hash_key) * 2)
# Stop
self.mem_cache.stop_cache()
self.mem_cache = None
def test_size_limit_max_item_count(self):
"""
Test.
"""
# Alloc
self.mem_cache = MemoryCache(max_bytes=sys.maxsize,
max_single_item_bytes=6 * 2,
purge_min_bytes=1 * 5,
purge_min_count=0,
max_item=10)
# Put 10 items
for i in range(10, 20):
self.mem_cache.put("key" + str(i),
SolBase.unicode_to_binary("val%s" % i, "utf-8"),
60000)
logger.info("Cache=%s", self.mem_cache)
# Must have all of them
for i in range(10, 20):
self.assertEqual(self.mem_cache.get("key" + str(i)),
SolBase.unicode_to_binary("val%s" % i, "utf-8"))
self.assertEqual(len(self.mem_cache._hash_key), 10)
self.assertEqual(self.mem_cache._current_data_bytes.get(),
5 * len(self.mem_cache._hash_key) * 2)
# Then add a new one : we will over size the cache
self.mem_cache.put("key" + str(99),
SolBase.unicode_to_binary("val99", "utf-8"), 60000)
# We must have evicted only first added
logger.info("Hash = %s", self.mem_cache._hash_key)
self.assertIsNone(self.mem_cache.get("key" + str(10)))
for i in range(11, 20):
self.assertEqual(self.mem_cache.get("key" + str(i)),
SolBase.unicode_to_binary("val%s" % i, "utf-8"))
self.assertEqual(self.mem_cache.get("key" + str(99)),
SolBase.unicode_to_binary("val99", "utf-8"))
self.assertEqual(self.mem_cache._current_data_bytes.get(),
5 * len(self.mem_cache._hash_key) * 2)
# Stop
self.mem_cache.stop_cache()
self.mem_cache = None
def test_size_handling(self):
"""
Test.
"""
# Alloc
self.mem_cache = MemoryCache()
# Put
self.mem_cache.put("A", b"A1", 60000)
self.assertEqual(self.mem_cache._current_data_bytes.get(), 1 + 2)
# Put again, not the same data
self.mem_cache.put("A", b"A11", 60000)
self.assertEqual(self.mem_cache._current_data_bytes.get(), 1 + 3)
# Put again, not the same data
self.mem_cache.put("A", b"A", 60000)
self.assertEqual(self.mem_cache._current_data_bytes.get(), 1 + 1)
# Stop
self.mem_cache.stop_cache()
self.mem_cache = None
def test_start_stop(self):
"""
Test.
"""
# Alloc
self.mem_cache = MemoryCache()
self.assertTrue(self.mem_cache._is_started)
# Stop
self.mem_cache.stop_cache()
self.assertFalse(self.mem_cache._is_started)
# Start
self.mem_cache.start_cache()
self.assertTrue(self.mem_cache._is_started)
# Stop
self.mem_cache.stop_cache()
self.assertFalse(self.mem_cache._is_started)
# Over
self.mem_cache = None
def test_basic(self):
"""
Test.
"""
# Alloc
self.memory_cache = MemoryCache()
self.redis_cache = RedisCache()
self.high_cache = HighCacheEx(memory_cache=self.memory_cache,
redis_cache=self.redis_cache)
# Put
self.high_cache.put(self.key_prefix + "L1L2_IMPLICIT",
b"L1L2_IMPLICIT", 60000)
self.high_cache.put(self.key_prefix + "L1L2_EXPLICIT",
b"L1L2_EXPLICIT",
60000,
l1=True,
l2=True)
self.high_cache.put(self.key_prefix + "L1_ONLY",
b"L1_ONLY",
60000,
l1=True,
l2=False)
self.high_cache.put(self.key_prefix + "L2_ONLY",
b"L2_ONLY",
60000,
l1=False,
l2=True)
self.high_cache.put(self.key_prefix + "NONE",
b"NONE",
60000,
l1=False,
l2=False)
# Check get
self.assertEqual(
self.high_cache.get(self.key_prefix + "L1L2_IMPLICIT"),
b"L1L2_IMPLICIT")
self.assertEqual(
self.high_cache.get(self.key_prefix + "L1L2_IMPLICIT",
l1=True,
l2=False), b"L1L2_IMPLICIT")
self.assertEqual(
self.high_cache.get(self.key_prefix + "L1L2_IMPLICIT",
l1=False,
l2=True), b"L1L2_IMPLICIT")
# Check getex
v, level = self.high_cache.getex(self.key_prefix + "L1L2_IMPLICIT",
l1=True,
l2=False)
self.assertEqual(level, 1)
self.assertEqual(v, b"L1L2_IMPLICIT")
v, level = self.high_cache.getex(self.key_prefix + "L1L2_IMPLICIT",
l1=False,
l2=True)
self.assertEqual(level, 2)
self.assertEqual(v, b"L1L2_IMPLICIT")
v, level = self.high_cache.getex(self.key_prefix + "L1L2_IMPLICIT",
l1=False,
l2=False)
self.assertEqual(level, 0)
self.assertIsNone(v)
self.assertEqual(
self.high_cache.get(self.key_prefix + "L1L2_EXPLICIT"),
b"L1L2_EXPLICIT")
self.assertEqual(
self.high_cache.get(self.key_prefix + "L1L2_EXPLICIT",
l1=True,
l2=False), b"L1L2_EXPLICIT")
self.assertEqual(
self.high_cache.get(self.key_prefix + "L1L2_EXPLICIT",
l1=False,
l2=True), b"L1L2_EXPLICIT")
self.assertEqual(
self.high_cache.get(self.key_prefix + "L1_ONLY", l1=True,
l2=False), b"L1_ONLY")
self.assertIsNone(
self.high_cache.get(self.key_prefix + "L1_ONLY", l1=False,
l2=True))
self.assertEqual(self.high_cache.get(self.key_prefix + "L1_ONLY"),
b"L1_ONLY")
self.assertIsNone(
self.high_cache.get(self.key_prefix + "L2_ONLY", l1=True,
l2=False))
self.assertEqual(
self.high_cache.get(self.key_prefix + "L2_ONLY", l1=False,
l2=True), b"L2_ONLY")
self.assertEqual(self.high_cache.get(self.key_prefix + "L2_ONLY"),
b"L2_ONLY")
self.assertIsNone(
self.high_cache.get(self.key_prefix + "NONE", l1=True, l2=False))
self.assertIsNone(
self.high_cache.get(self.key_prefix + "NONE", l1=False, l2=True))
self.assertIsNone(self.high_cache.get(self.key_prefix + "NONE"))
# REMOVE FROM L1
self.high_cache.remove(self.key_prefix + "L1L2_IMPLICIT",
l1=True,
l2=False)
# Target L2 only
self.assertEqual(
self.high_cache.get(self.key_prefix + "L1L2_IMPLICIT",
l1=False,
l2=True), b"L1L2_IMPLICIT")
# Check L1 miss
self.assertIsNone(
self.high_cache.get(self.key_prefix + "L1L2_IMPLICIT",
l1=True,
l2=False))
# Target both and check L1 HIT
v, level = self.high_cache.getex(self.key_prefix + "L1L2_IMPLICIT",
l1=True,
l2=True)
self.assertEqual(v, b"L1L2_IMPLICIT")
self.assertEqual(level, 2)
v, level = self.high_cache.getex(self.key_prefix + "L1L2_IMPLICIT",
l1=True,
l2=True)
self.assertEqual(v, b"L1L2_IMPLICIT")
self.assertEqual(level, 1)
# REMOVE FROM BOTH
self.high_cache.remove(self.key_prefix + "L1L2_EXPLICIT")
self.assertIsNone(
self.high_cache.get(self.key_prefix + "L1L2_EXPLICIT"))
self.assertIsNone(
self.high_cache.get(self.key_prefix + "L1L2_EXPLICIT",
l1=True,
l2=False))
self.assertIsNone(
self.high_cache.get(self.key_prefix + "L1L2_EXPLICIT",
l1=False,
l2=True))
# UNICODE VALUES
self.high_cache.put(
"K1",
SolBase.unicode_to_binary("VAL\u001B\u0BD9\U0001A10D\u1501",
"utf-8"), 60000)
v = self.high_cache.get("K1")
self.assertEqual(
v,
SolBase.unicode_to_binary("VAL\u001B\u0BD9\U0001A10D\u1501",
"utf-8"))
# Sleep
SolBase.sleep(1000)
# Stop
self.high_cache.stop_cache()
self.assertFalse(self.redis_cache._is_started)
self.assertFalse(self.memory_cache._is_started)
self.high_cache = None
self.redis_cache = None
self.memory_cache = None
def test_basic(self):
"""
Test.
"""
# Alloc
self.memory_cache = MemoryCache()
self.redis_cache = RedisCache()
self.high_cache = HighCache(memory_cache=self.memory_cache,
redis_cache=self.redis_cache)
# Put
self.high_cache.put(self.key_prefix + "L1L2_IMPLICIT",
b"L1L2_IMPLICIT", 60000)
self.high_cache.put(self.key_prefix + "L1L2_EXPLICIT",
b"L1L2_EXPLICIT",
60000,
l1=True,
l2=True)
self.high_cache.put(self.key_prefix + "L1_ONLY",
b"L1_ONLY",
60000,
l1=True,
l2=False)
self.high_cache.put(self.key_prefix + "L2_ONLY",
b"L2_ONLY",
60000,
l1=False,
l2=True)
self.high_cache.put(self.key_prefix + "NONE",
b"NONE",
60000,
l1=False,
l2=False)
# Check get
self.assertEqual(
self.high_cache.get(self.key_prefix + "L1L2_IMPLICIT"),
b"L1L2_IMPLICIT")
self.assertEqual(
self.high_cache.get(self.key_prefix + "L1L2_IMPLICIT",
l1=True,
l2=False), b"L1L2_IMPLICIT")
self.assertEqual(
self.high_cache.get(self.key_prefix + "L1L2_IMPLICIT",
l1=False,
l2=True), b"L1L2_IMPLICIT")
# Check getex
v, level = self.high_cache.getex(self.key_prefix + "L1L2_IMPLICIT",
l1=True,
l2=False)
self.assertEqual(level, 1)
self.assertEqual(v, b"L1L2_IMPLICIT")
v, level = self.high_cache.getex(self.key_prefix + "L1L2_IMPLICIT",
l1=False,
l2=True)
self.assertEqual(level, 2)
self.assertEqual(v, b"L1L2_IMPLICIT")
v, level = self.high_cache.getex(self.key_prefix + "L1L2_IMPLICIT",
l1=False,
l2=False)
self.assertEqual(level, 0)
self.assertIsNone(v)
self.assertEqual(
self.high_cache.get(self.key_prefix + "L1L2_EXPLICIT"),
b"L1L2_EXPLICIT")
self.assertEqual(
self.high_cache.get(self.key_prefix + "L1L2_EXPLICIT",
l1=True,
l2=False), b"L1L2_EXPLICIT")
self.assertEqual(
self.high_cache.get(self.key_prefix + "L1L2_EXPLICIT",
l1=False,
l2=True), b"L1L2_EXPLICIT")
self.assertEqual(
self.high_cache.get(self.key_prefix + "L1_ONLY", l1=True,
l2=False), b"L1_ONLY")
self.assertIsNone(
self.high_cache.get(self.key_prefix + "L1_ONLY", l1=False,
l2=True))
self.assertEqual(self.high_cache.get(self.key_prefix + "L1_ONLY"),
b"L1_ONLY")
self.assertIsNone(
self.high_cache.get(self.key_prefix + "L2_ONLY", l1=True,
l2=False))
self.assertEqual(
self.high_cache.get(self.key_prefix + "L2_ONLY", l1=False,
l2=True), b"L2_ONLY")
self.assertEqual(self.high_cache.get(self.key_prefix + "L2_ONLY"),
b"L2_ONLY")
self.assertIsNone(
self.high_cache.get(self.key_prefix + "NONE", l1=True, l2=False))
self.assertIsNone(
self.high_cache.get(self.key_prefix + "NONE", l1=False, l2=True))
self.assertIsNone(self.high_cache.get(self.key_prefix + "NONE"))
# REMOVE FROM L1
self.high_cache.remove(self.key_prefix + "L1L2_IMPLICIT",
l1=True,
l2=False)
self.assertEqual(
self.high_cache.get(self.key_prefix + "L1L2_IMPLICIT"),
b"L1L2_IMPLICIT")
self.assertIsNone(
self.high_cache.get(self.key_prefix + "L1L2_IMPLICIT",
l1=True,
l2=False))
self.assertEqual(
self.high_cache.get(self.key_prefix + "L1L2_IMPLICIT",
l1=False,
l2=True), b"L1L2_IMPLICIT")
# REMOVE FROM L2
self.high_cache.remove(self.key_prefix + "L1L2_IMPLICIT",
l1=False,
l2=True)
self.assertIsNone(
self.high_cache.get(self.key_prefix + "L1L2_IMPLICIT"))
self.assertIsNone(
self.high_cache.get(self.key_prefix + "L1L2_IMPLICIT",
l1=True,
l2=False))
self.assertIsNone(
self.high_cache.get(self.key_prefix + "L1L2_IMPLICIT",
l1=False,
l2=True))
# REMOVE FROM BOTH
self.high_cache.remove(self.key_prefix + "L1L2_EXPLICIT")
self.assertIsNone(
self.high_cache.get(self.key_prefix + "L1L2_EXPLICIT"))
self.assertIsNone(
self.high_cache.get(self.key_prefix + "L1L2_EXPLICIT",
l1=True,
l2=False))
self.assertIsNone(
self.high_cache.get(self.key_prefix + "L1L2_EXPLICIT",
l1=False,
l2=True))
# Stop
self.high_cache.stop_cache()
self.assertFalse(self.redis_cache._is_started)
self.assertFalse(self.memory_cache._is_started)
self.high_cache = None
self.redis_cache = None
self.memory_cache = None
def _go_greenlet(self,
greenlet_count,
put_count,
get_count,
bench_item_count,
watchdog_interval_ms=60000,
max_item=128000,
max_bytes=32 * 1024 * 1024,
max_single_item_bytes=1 * 1024 * 1024,
purge_min_bytes=8 * 1024 * 1024,
purge_min_count=1000):
"""
Doc
:param greenlet_count: greenlet_count
:param put_count: put_count
:param get_count: get_count
:param bench_item_count: bench_item_count
:param watchdog_interval_ms: watchdog_interval_ms
:param max_item: max_item
:param max_bytes: max_bytes
:param max_single_item_bytes: max_single_item_bytes
:param purge_min_bytes: purge_min_bytes
:param purge_min_count: purge_min_count
"""
g_event = None
g_array = None
try:
# Settings
g_count = greenlet_count
g_ms = 10000
# Continue callback loop
self.callback_return = True
# Go
self.mem_cache = MemoryCache(
watchdog_interval_ms=watchdog_interval_ms,
max_item=max_item,
max_bytes=max_bytes,
max_single_item_bytes=max_single_item_bytes,
purge_min_bytes=purge_min_bytes,
purge_min_count=purge_min_count)
# Item count
self.bench_item_count = bench_item_count
self.bench_put_weight = put_count
self.bench_get_weight = get_count
self.bench_ttl_min_ms = 1000
self.bench_ttl_max_ms = int(g_ms / 2)
# Go
self.run_event = Event()
self.exception_raised = 0
self.open_count = 0
self.thread_running = AtomicIntSafe()
self.thread_running_ok = AtomicIntSafe()
# Item per greenlet
item_per_greenlet = self.bench_item_count / g_count
# Signal
self.gorun_event = Event()
# Alloc greenlet
g_array = list()
g_event = list()
for _ in range(0, g_count):
greenlet = Greenlet()
g_array.append(greenlet)
g_event.append(Event())
# Run them
cur_idx = 0
for idx in range(0, len(g_array)):
greenlet = g_array[idx]
event = g_event[idx]
greenlet.spawn(self._run_cache_bench, event, cur_idx,
cur_idx + item_per_greenlet)
cur_idx += item_per_greenlet
SolBase.sleep(0)
# Signal
self.gorun_event.set()
# Wait a bit
dt = SolBase.mscurrent()
while SolBase.msdiff(dt) < g_ms:
SolBase.sleep(500)
# Stat
ms = SolBase.msdiff(dt)
sec = float(ms / 1000.0)
total_put = Meters.aig("mcs.cache_put")
per_sec_put = round(float(total_put) / sec, 2)
total_get = Meters.aig("mcs.cache_get_hit") + Meters.aig(
"mcs.cache_get_miss")
per_sec_get = round(float(total_get) / sec, 2)
logger.info(
"Running..., count=%s, run=%s, ok=%s, put/sec=%s get/sec=%s, cache=%s",
self.open_count, self.thread_running.get(),
self.thread_running_ok.get(), per_sec_put, per_sec_get,
self.mem_cache)
self.assertEqual(self.exception_raised, 0)
# Over, signal
logger.info("Signaling, count=%s", self.open_count)
self.run_event.set()
# Wait
for g in g_event:
g.wait(30.0)
self.assertTrue(g.isSet())
g_event = None
g_array = None
# Log
Meters.write_to_logger()
finally:
self.run_event.set()
if g_event:
for g in g_event:
g.set()
if g_array:
for g in g_array:
g.kill()
if self.mem_cache:
max_count = 0
total_size = 0
i = 0
for (k, v) in self.mem_cache._hash_key.items():
i += 1
total_size += len(k) + len(v[1])
if i < max_count:
logger.info("%s => %s", k, v)
self.assertEqual(total_size,
self.mem_cache._current_data_bytes.get())
self.mem_cache.stop_cache()
self.mem_cache = None
def test_basic(self):
"""
Test.
"""
# Alloc
self.mem_cache = MemoryCache()
# Get : must return nothing
o = self.mem_cache.get("not_found")
self.assertIsNone(o)
# Put
self.mem_cache.put("keyA", b"valA", 60000)
o = self.mem_cache.get("keyA")
self.assertEqual(o, b"valA")
o = self.mem_cache.get_raw("keyA")
self.assertIsNotNone(o)
self.assertIsInstance(o, tuple)
self.assertEqual(o[1], b"valA")
self.assertLessEqual(o[0] - SolBase.mscurrent(), 60000)
logger.info("TTL approx=%s", o[0] - SolBase.mscurrent())
# Put with lower TTL : TTL MUST BE UPDATED
self.mem_cache.put("keyA", b"valA", 30000)
o = self.mem_cache.get("keyA")
self.assertEqual(o, b"valA")
o = self.mem_cache.get_raw("keyA")
self.assertIsNotNone(o)
self.assertIsInstance(o, tuple)
self.assertEqual(o[1], b"valA")
self.assertLessEqual(o[0] - SolBase.mscurrent(), 30000)
logger.info("TTL approx=%s", o[0] - SolBase.mscurrent())
# Str put : must now be ok
self.mem_cache.put("toto_str", "str_val", 1000)
self.assertEqual(self.mem_cache.get("toto_str"), "str_val")
# Non bytes,str injection (int) : must fail
# noinspection PyBroadException,PyPep8
try:
# noinspection PyTypeChecker
self.mem_cache.put("toto", 12, 1000)
self.fail("Must fail")
except:
pass
# Non bytes injection : must fail
# noinspection PyBroadException,PyPep8
try:
# noinspection PyTypeChecker
self.mem_cache.put("toto", u"unicode_buffer", 1000)
self.fail("Must fail")
except:
pass
# This MUST fail
# noinspection PyBroadException
try:
# noinspection PyTypeChecker
self.mem_cache.put(999, b"value", 60000)
self.fail("Put a key as non bytes,str MUST fail")
except Exception:
pass
# This MUST fail
# noinspection PyBroadException
try:
# noinspection PyTypeChecker
self.mem_cache.remove(999)
self.fail("Remove a key as non bytes,str MUST fail")
except Exception:
pass
# Put/Remove
self.mem_cache.put("todel", b"value", 60000)
self.assertEqual(self.mem_cache.get("todel"), b"value")
self.mem_cache.remove("todel")
self.assertEqual(self.mem_cache.get("todel"), None)
# Put
self.mem_cache.put("KEY \u001B\u0BD9\U0001A10D\u1501\xc3", b"zzz",
60000)
self.assertEqual(
self.mem_cache.get("KEY \u001B\u0BD9\U0001A10D\u1501\xc3"), b"zzz")
# Stop
self.mem_cache.stop_cache()
self.mem_cache = None