def test_size_limit_on_data_size_first(self):
"""
Test.
"""
# Alloc
self.mem_cache = MemoryCache(
max_bytes=5 * 10 * 2,
max_single_item_bytes=6 * 2,
purge_min_bytes=5 * 5 * 2,
purge_min_count=2,
max_item=sys.maxsize,
)
# Put 10 items
for i in range(10, 20):
self.mem_cache.put("key" + str(i),
SolBase.unicode_to_binary("val%s" % i, "utf-8"),
60000)
logger.info("Cache=%s", self.mem_cache)
# Must have all of them
for i in range(10, 20):
self.assertEqual(self.mem_cache.get("key" + str(i)),
SolBase.unicode_to_binary("val%s" % i, "utf-8"))
self.assertEqual(len(self.mem_cache._hash_key), 10)
self.assertEqual(self.mem_cache._current_data_bytes.get(), 5 * 10 * 2)
# Then add a new one : we will over size the cache
self.mem_cache.put("key" + str(99),
SolBase.unicode_to_binary("val99", "utf-8"), 60000)
# We must have evicted AT least :
# 5*5 + 5 bytes => 5 items + the item added => 6 items
# 2 items minimum
# So 6 items : 10 to 15 must be evicted
logger.info("Hash = %s", self.mem_cache._hash_key)
for i in range(10, 16):
self.assertIsNone(self.mem_cache.get("key" + str(i)))
for i in range(16, 20):
self.assertEqual(self.mem_cache.get("key" + str(i)),
SolBase.unicode_to_binary("val%s" % i, "utf-8"))
self.assertEqual(self.mem_cache.get("key" + str(99)),
SolBase.unicode_to_binary("val99", "utf-8"))
self.assertEqual(len(self.mem_cache._hash_key), 5)
self.assertEqual(len(self.mem_cache._hash_context), 5)
self.assertEqual(self.mem_cache._current_data_bytes.get(), 5 * 5 * 2)
# Try add a big one this time : must not be done (over limit)
self.mem_cache.put("BIGDATA", b"aaaaaaaaaaaaaaaaaaaa", 60000)
self.assertIsNone(self.mem_cache.get("BIGDATA"))
# Stop
self.mem_cache.stop_cache()
self.mem_cache = None
def test_size_limit_cache_clear(self):
"""
Test.
"""
# Alloc
self.mem_cache = MemoryCache(
max_bytes=5 * 10 * 2,
max_single_item_bytes=6 * 2,
purge_min_bytes=1 * 5,
purge_min_count=100,
max_item=sys.maxsize,
)
# Put 10 items
for i in range(10, 20):
self.mem_cache.put("key" + str(i),
SolBase.unicode_to_binary("val%s" % i, "utf-8"),
60000)
logger.info("Cache=%s", self.mem_cache)
# Must have all of them
for i in range(10, 20):
self.assertEqual(self.mem_cache.get("key" + str(i)),
SolBase.unicode_to_binary("val%s" % i, "utf-8"))
self.assertEqual(len(self.mem_cache._hash_key), 10)
self.assertEqual(self.mem_cache._current_data_bytes.get(),
5 * len(self.mem_cache._hash_key) * 2)
# Then add a new one : we will over size the cache
self.mem_cache.put("key" + str(99),
SolBase.unicode_to_binary("val99", "utf-8"), 60000)
# We must have evicted AT least :
# 1*5 + 5 bytes => 1 items + the item added => 2 items
# 100 items minimum
# So : All items must be kicked, will remain only the new one
logger.info("Hash = %s", self.mem_cache._hash_key)
self.assertEqual(self.mem_cache.get("key" + str(99)),
SolBase.unicode_to_binary("val99", "utf-8"))
self.assertEqual(len(self.mem_cache._hash_key), 1)
self.assertEqual(len(self.mem_cache._hash_context), 1)
self.assertEqual(self.mem_cache._current_data_bytes.get(),
5 * len(self.mem_cache._hash_key) * 2)
# Stop
self.mem_cache.stop_cache()
self.mem_cache = None
def send_unicode_to_socket(self, unicode_to_send, encoding="utf-8", append_lf=True):
"""
Send text to socket, asynch.
:param unicode_to_send: The text to send (str)
:type unicode_to_send: str
:param encoding: The encoding to use.
:type encoding: str
:param append_lf: If true, append an \n
:type append_lf: bool
:return: True is send has been scheduled, false otherwise.
:rtype: bool
"""
# Check
if not isinstance(unicode_to_send, string_types):
logger.error("unicode_to_send not an string_types, class=%s, str=%s, self=%s", SolBase.get_classname(unicode_to_send), repr(unicode_to_send), self)
return False
# Go
unicode_temp = unicode_to_send
# LF if required
if append_lf:
unicode_temp += u"\n"
# Convert to binary localBuffer
bin_buf = SolBase.unicode_to_binary(unicode_temp, encoding)
# Send binary
return self.send_binary_to_socket(bin_buf)
def test_size_limit_max_item_count(self):
"""
Test.
"""
# Alloc
self.mem_cache = MemoryCache(max_bytes=sys.maxsize,
max_single_item_bytes=6 * 2,
purge_min_bytes=1 * 5,
purge_min_count=0,
max_item=10)
# Put 10 items
for i in range(10, 20):
self.mem_cache.put("key" + str(i),
SolBase.unicode_to_binary("val%s" % i, "utf-8"),
60000)
logger.info("Cache=%s", self.mem_cache)
# Must have all of them
for i in range(10, 20):
self.assertEqual(self.mem_cache.get("key" + str(i)),
SolBase.unicode_to_binary("val%s" % i, "utf-8"))
self.assertEqual(len(self.mem_cache._hash_key), 10)
self.assertEqual(self.mem_cache._current_data_bytes.get(),
5 * len(self.mem_cache._hash_key) * 2)
# Then add a new one : we will over size the cache
self.mem_cache.put("key" + str(99),
SolBase.unicode_to_binary("val99", "utf-8"), 60000)
# We must have evicted only first added
logger.info("Hash = %s", self.mem_cache._hash_key)
self.assertIsNone(self.mem_cache.get("key" + str(10)))
for i in range(11, 20):
self.assertEqual(self.mem_cache.get("key" + str(i)),
SolBase.unicode_to_binary("val%s" % i, "utf-8"))
self.assertEqual(self.mem_cache.get("key" + str(99)),
SolBase.unicode_to_binary("val99", "utf-8"))
self.assertEqual(self.mem_cache._current_data_bytes.get(),
5 * len(self.mem_cache._hash_key) * 2)
# Stop
self.mem_cache.stop_cache()
self.mem_cache = None
def test_encoder(self):
"""
Test
"""
buf = "BUF\u001B\u0BD9\U0001A10D\u1501FUB"
bin_buf = SolBase.unicode_to_binary(buf, "utf-8")
self.assertEqual(buf, SolBase.binary_to_unicode(bin_buf, "utf-8"))
def _on_invalid(self, start_response):
"""
On request callback
:param start_response: start_response
:type start_response: instancemethod
:return: list
:rtype: list
"""
# Debug
status = "400 Bad Request"
body = status
headers = [('Content-Type', 'text/txt')]
start_response(status, headers)
return [SolBase.unicode_to_binary(body, "utf-8")]
def _run_cache_bench(self, event, idx_min, idx_max):
"""
Run
:param idx_min: Index min
:param idx_max: Index max
"""
idx_max -= 1
# Wait
self.gorun_event.wait()
# Go
cur_count = 0
logger.debug("Entering now, idx_min=%s, idx_max=%s", idx_min, idx_max)
self.thread_running.increment()
self.thread_running_ok.increment()
try:
while not self.run_event.isSet():
cur_count += 1
try:
cur_item = random.randint(idx_min, idx_max)
s_cur_item = "%s" % cur_item
b_cur_item = SolBase.unicode_to_binary(s_cur_item, "utf-8")
cur_ttl = random.randint(self.bench_ttl_min_ms,
self.bench_ttl_max_ms)
for _ in range(0, self.bench_put_weight):
self.redis_cache.put(s_cur_item, b_cur_item, cur_ttl)
SolBase.sleep(0)
for _ in range(0, self.bench_get_weight):
v = self.redis_cache.get(s_cur_item)
if v:
self.assertEqual(v, b_cur_item)
SolBase.sleep(0)
except Exception as e:
self.exception_raised += 1
logger.warning("Ex=%s", SolBase.extostr(e))
self.thread_running_ok.increment(-1)
return
finally:
pass
finally:
self.assertGreater(cur_count, 0)
logger.debug("Exiting")
event.set()
self.thread_running.increment(-1)
def _decode(cls, buf):
"""
Decode buffer
:param buf: bytes
:type buf: bytes
:return tuple (data, ttl_ms, ms_added)
:rtype tuple
"""
try:
d = ujson.loads(SolBase.binary_to_unicode(buf, "utf-8"))
return SolBase.unicode_to_binary(
d["data"], "utf-8"), d["ttl_ms"], d["ms_added"]
except Exception as e:
logger.warning("Unable to decode, buf=%s, ex=%s", buf,
SolBase.extostr(e))
return None, None, None
def on_request(self, environ, start_response):
"""
On request callback
:param environ: environ
:type environ: dict
:param start_response: start_response
:type start_response: instancemethod
:return: list
:rtype: list
"""
try:
logger.info("Request start now")
# Log
for k, v in environ.items():
logger.debug("Env: %s=%s", k, v)
# Switch
pi = environ["PATH_INFO"]
logger.debug("pi=%s", pi)
# Sometimes PATH_INFO come with full uri (urllib3) (?!)
# http://127.0.0.1:7900/unittest
if pi.endswith("/unittest"):
logger.debug("call _on_unit_test")
return self._on_unit_test(environ, start_response)
else:
logger.debug("call _on_invalid")
return self._on_invalid(start_response)
except Exception as e:
logger.warning("Ex=%s", SolBase.extostr(e))
status = "500 Internal Server Error"
body = status
headers = [('Content-Type', 'text/plain')]
start_response(status, headers)
return [SolBase.unicode_to_binary(body, "utf-8")]
finally:
logger.debug("exit")
self._lifecycle_log_status()
def _encode(cls, val, ttl_ms, ms_added=None):
"""
Encode
:param val: bytes
:type val: bytes
:param ttl_ms: int
:type ttl_ms: int
:param ms_added: int, float, None
:type ms_added: int, float, None
:return bytes
:rtype bytes
"""
if not ms_added:
ms_added = SolBase.mscurrent()
d = {
"ms_added": int(ms_added),
"ttl_ms": ttl_ms,
"data": val,
}
s = ujson.dumps(d, reject_bytes=False)
return SolBase.unicode_to_binary(s, "utf-8")
def _on_unit_test(self, environ, start_response):
"""
On request callback
:param environ: environ
:type environ: dict
:param start_response: start_response
:type start_response: instancemethod
:return: list
:rtype: list
"""
# Param
logger.debug("_get_param_from_qs")
from_qs = self._get_param_from_qs(environ)
logger.debug("_get_param_from_post_data")
from_post = self._get_param_from_post_data(environ)
logger.debug("_get_method")
from_method = self._get_method(environ)
# Debug
logger.debug("reply set")
status = "200 OK"
if from_method == "HEAD":
# No body
body = ""
headers = [('Content-Type', 'text/txt')]
start_response(status, headers)
logger.debug("reply send")
else:
body = "OK" + "\n"
body += "from_qs=" + str(from_qs) + " -EOL\n"
body += "from_post=" + str(from_post) + " -EOL\n"
body += "from_method=" + from_method + "\n"
headers = [('Content-Type', 'text/txt')]
start_response(status, headers)
logger.debug("reply send")
return [SolBase.unicode_to_binary(body, "utf-8")]
def test_basic(self):
"""
Test.
"""
# Alloc
self.memory_cache = MemoryCache()
self.redis_cache = RedisCache()
self.high_cache = HighCacheEx(memory_cache=self.memory_cache,
redis_cache=self.redis_cache)
# Put
self.high_cache.put(self.key_prefix + "L1L2_IMPLICIT",
b"L1L2_IMPLICIT", 60000)
self.high_cache.put(self.key_prefix + "L1L2_EXPLICIT",
b"L1L2_EXPLICIT",
60000,
l1=True,
l2=True)
self.high_cache.put(self.key_prefix + "L1_ONLY",
b"L1_ONLY",
60000,
l1=True,
l2=False)
self.high_cache.put(self.key_prefix + "L2_ONLY",
b"L2_ONLY",
60000,
l1=False,
l2=True)
self.high_cache.put(self.key_prefix + "NONE",
b"NONE",
60000,
l1=False,
l2=False)
# Check get
self.assertEqual(
self.high_cache.get(self.key_prefix + "L1L2_IMPLICIT"),
b"L1L2_IMPLICIT")
self.assertEqual(
self.high_cache.get(self.key_prefix + "L1L2_IMPLICIT",
l1=True,
l2=False), b"L1L2_IMPLICIT")
self.assertEqual(
self.high_cache.get(self.key_prefix + "L1L2_IMPLICIT",
l1=False,
l2=True), b"L1L2_IMPLICIT")
# Check getex
v, level = self.high_cache.getex(self.key_prefix + "L1L2_IMPLICIT",
l1=True,
l2=False)
self.assertEqual(level, 1)
self.assertEqual(v, b"L1L2_IMPLICIT")
v, level = self.high_cache.getex(self.key_prefix + "L1L2_IMPLICIT",
l1=False,
l2=True)
self.assertEqual(level, 2)
self.assertEqual(v, b"L1L2_IMPLICIT")
v, level = self.high_cache.getex(self.key_prefix + "L1L2_IMPLICIT",
l1=False,
l2=False)
self.assertEqual(level, 0)
self.assertIsNone(v)
self.assertEqual(
self.high_cache.get(self.key_prefix + "L1L2_EXPLICIT"),
b"L1L2_EXPLICIT")
self.assertEqual(
self.high_cache.get(self.key_prefix + "L1L2_EXPLICIT",
l1=True,
l2=False), b"L1L2_EXPLICIT")
self.assertEqual(
self.high_cache.get(self.key_prefix + "L1L2_EXPLICIT",
l1=False,
l2=True), b"L1L2_EXPLICIT")
self.assertEqual(
self.high_cache.get(self.key_prefix + "L1_ONLY", l1=True,
l2=False), b"L1_ONLY")
self.assertIsNone(
self.high_cache.get(self.key_prefix + "L1_ONLY", l1=False,
l2=True))
self.assertEqual(self.high_cache.get(self.key_prefix + "L1_ONLY"),
b"L1_ONLY")
self.assertIsNone(
self.high_cache.get(self.key_prefix + "L2_ONLY", l1=True,
l2=False))
self.assertEqual(
self.high_cache.get(self.key_prefix + "L2_ONLY", l1=False,
l2=True), b"L2_ONLY")
self.assertEqual(self.high_cache.get(self.key_prefix + "L2_ONLY"),
b"L2_ONLY")
self.assertIsNone(
self.high_cache.get(self.key_prefix + "NONE", l1=True, l2=False))
self.assertIsNone(
self.high_cache.get(self.key_prefix + "NONE", l1=False, l2=True))
self.assertIsNone(self.high_cache.get(self.key_prefix + "NONE"))
# REMOVE FROM L1
self.high_cache.remove(self.key_prefix + "L1L2_IMPLICIT",
l1=True,
l2=False)
# Target L2 only
self.assertEqual(
self.high_cache.get(self.key_prefix + "L1L2_IMPLICIT",
l1=False,
l2=True), b"L1L2_IMPLICIT")
# Check L1 miss
self.assertIsNone(
self.high_cache.get(self.key_prefix + "L1L2_IMPLICIT",
l1=True,
l2=False))
# Target both and check L1 HIT
v, level = self.high_cache.getex(self.key_prefix + "L1L2_IMPLICIT",
l1=True,
l2=True)
self.assertEqual(v, b"L1L2_IMPLICIT")
self.assertEqual(level, 2)
v, level = self.high_cache.getex(self.key_prefix + "L1L2_IMPLICIT",
l1=True,
l2=True)
self.assertEqual(v, b"L1L2_IMPLICIT")
self.assertEqual(level, 1)
# REMOVE FROM BOTH
self.high_cache.remove(self.key_prefix + "L1L2_EXPLICIT")
self.assertIsNone(
self.high_cache.get(self.key_prefix + "L1L2_EXPLICIT"))
self.assertIsNone(
self.high_cache.get(self.key_prefix + "L1L2_EXPLICIT",
l1=True,
l2=False))
self.assertIsNone(
self.high_cache.get(self.key_prefix + "L1L2_EXPLICIT",
l1=False,
l2=True))
# UNICODE VALUES
self.high_cache.put(
"K1",
SolBase.unicode_to_binary("VAL\u001B\u0BD9\U0001A10D\u1501",
"utf-8"), 60000)
v = self.high_cache.get("K1")
self.assertEqual(
v,
SolBase.unicode_to_binary("VAL\u001B\u0BD9\U0001A10D\u1501",
"utf-8"))
# Sleep
SolBase.sleep(1000)
# Stop
self.high_cache.stop_cache()
self.assertFalse(self.redis_cache._is_started)
self.assertFalse(self.memory_cache._is_started)
self.high_cache = None
self.redis_cache = None
self.memory_cache = None
