def test_size_limit_on_data_size_first(self): """ Test. """ # Alloc self.mem_cache = MemoryCache( max_bytes=5 * 10 * 2, max_single_item_bytes=6 * 2, purge_min_bytes=5 * 5 * 2, purge_min_count=2, max_item=sys.maxsize, ) # Put 10 items for i in range(10, 20): self.mem_cache.put("key" + str(i), SolBase.unicode_to_binary("val%s" % i, "utf-8"), 60000) logger.info("Cache=%s", self.mem_cache) # Must have all of them for i in range(10, 20): self.assertEqual(self.mem_cache.get("key" + str(i)), SolBase.unicode_to_binary("val%s" % i, "utf-8")) self.assertEqual(len(self.mem_cache._hash_key), 10) self.assertEqual(self.mem_cache._current_data_bytes.get(), 5 * 10 * 2) # Then add a new one : we will over size the cache self.mem_cache.put("key" + str(99), SolBase.unicode_to_binary("val99", "utf-8"), 60000) # We must have evicted AT least : # 5*5 + 5 bytes => 5 items + the item added => 6 items # 2 items minimum # So 6 items : 10 to 15 must be evicted logger.info("Hash = %s", self.mem_cache._hash_key) for i in range(10, 16): self.assertIsNone(self.mem_cache.get("key" + str(i))) for i in range(16, 20): self.assertEqual(self.mem_cache.get("key" + str(i)), SolBase.unicode_to_binary("val%s" % i, "utf-8")) self.assertEqual(self.mem_cache.get("key" + str(99)), SolBase.unicode_to_binary("val99", "utf-8")) self.assertEqual(len(self.mem_cache._hash_key), 5) self.assertEqual(len(self.mem_cache._hash_context), 5) self.assertEqual(self.mem_cache._current_data_bytes.get(), 5 * 5 * 2) # Try add a big one this time : must not be done (over limit) self.mem_cache.put("BIGDATA", b"aaaaaaaaaaaaaaaaaaaa", 60000) self.assertIsNone(self.mem_cache.get("BIGDATA")) # Stop self.mem_cache.stop_cache() self.mem_cache = None
def test_size_limit_cache_clear(self): """ Test. """ # Alloc self.mem_cache = MemoryCache( max_bytes=5 * 10 * 2, max_single_item_bytes=6 * 2, purge_min_bytes=1 * 5, purge_min_count=100, max_item=sys.maxsize, ) # Put 10 items for i in range(10, 20): self.mem_cache.put("key" + str(i), SolBase.unicode_to_binary("val%s" % i, "utf-8"), 60000) logger.info("Cache=%s", self.mem_cache) # Must have all of them for i in range(10, 20): self.assertEqual(self.mem_cache.get("key" + str(i)), SolBase.unicode_to_binary("val%s" % i, "utf-8")) self.assertEqual(len(self.mem_cache._hash_key), 10) self.assertEqual(self.mem_cache._current_data_bytes.get(), 5 * len(self.mem_cache._hash_key) * 2) # Then add a new one : we will over size the cache self.mem_cache.put("key" + str(99), SolBase.unicode_to_binary("val99", "utf-8"), 60000) # We must have evicted AT least : # 1*5 + 5 bytes => 1 items + the item added => 2 items # 100 items minimum # So : All items must be kicked, will remain only the new one logger.info("Hash = %s", self.mem_cache._hash_key) self.assertEqual(self.mem_cache.get("key" + str(99)), SolBase.unicode_to_binary("val99", "utf-8")) self.assertEqual(len(self.mem_cache._hash_key), 1) self.assertEqual(len(self.mem_cache._hash_context), 1) self.assertEqual(self.mem_cache._current_data_bytes.get(), 5 * len(self.mem_cache._hash_key) * 2) # Stop self.mem_cache.stop_cache() self.mem_cache = None
def send_unicode_to_socket(self, unicode_to_send, encoding="utf-8", append_lf=True): """ Send text to socket, asynch. :param unicode_to_send: The text to send (str) :type unicode_to_send: str :param encoding: The encoding to use. :type encoding: str :param append_lf: If true, append an \n :type append_lf: bool :return: True is send has been scheduled, false otherwise. :rtype: bool """ # Check if not isinstance(unicode_to_send, string_types): logger.error("unicode_to_send not an string_types, class=%s, str=%s, self=%s", SolBase.get_classname(unicode_to_send), repr(unicode_to_send), self) return False # Go unicode_temp = unicode_to_send # LF if required if append_lf: unicode_temp += u"\n" # Convert to binary localBuffer bin_buf = SolBase.unicode_to_binary(unicode_temp, encoding) # Send binary return self.send_binary_to_socket(bin_buf)
def test_size_limit_max_item_count(self): """ Test. """ # Alloc self.mem_cache = MemoryCache(max_bytes=sys.maxsize, max_single_item_bytes=6 * 2, purge_min_bytes=1 * 5, purge_min_count=0, max_item=10) # Put 10 items for i in range(10, 20): self.mem_cache.put("key" + str(i), SolBase.unicode_to_binary("val%s" % i, "utf-8"), 60000) logger.info("Cache=%s", self.mem_cache) # Must have all of them for i in range(10, 20): self.assertEqual(self.mem_cache.get("key" + str(i)), SolBase.unicode_to_binary("val%s" % i, "utf-8")) self.assertEqual(len(self.mem_cache._hash_key), 10) self.assertEqual(self.mem_cache._current_data_bytes.get(), 5 * len(self.mem_cache._hash_key) * 2) # Then add a new one : we will over size the cache self.mem_cache.put("key" + str(99), SolBase.unicode_to_binary("val99", "utf-8"), 60000) # We must have evicted only first added logger.info("Hash = %s", self.mem_cache._hash_key) self.assertIsNone(self.mem_cache.get("key" + str(10))) for i in range(11, 20): self.assertEqual(self.mem_cache.get("key" + str(i)), SolBase.unicode_to_binary("val%s" % i, "utf-8")) self.assertEqual(self.mem_cache.get("key" + str(99)), SolBase.unicode_to_binary("val99", "utf-8")) self.assertEqual(self.mem_cache._current_data_bytes.get(), 5 * len(self.mem_cache._hash_key) * 2) # Stop self.mem_cache.stop_cache() self.mem_cache = None
def test_encoder(self): """ Test """ buf = "BUF\u001B\u0BD9\U0001A10D\u1501FUB" bin_buf = SolBase.unicode_to_binary(buf, "utf-8") self.assertEqual(buf, SolBase.binary_to_unicode(bin_buf, "utf-8"))
def _on_invalid(self, start_response): """ On request callback :param start_response: start_response :type start_response: instancemethod :return: list :rtype: list """ # Debug status = "400 Bad Request" body = status headers = [('Content-Type', 'text/txt')] start_response(status, headers) return [SolBase.unicode_to_binary(body, "utf-8")]
def _run_cache_bench(self, event, idx_min, idx_max): """ Run :param idx_min: Index min :param idx_max: Index max """ idx_max -= 1 # Wait self.gorun_event.wait() # Go cur_count = 0 logger.debug("Entering now, idx_min=%s, idx_max=%s", idx_min, idx_max) self.thread_running.increment() self.thread_running_ok.increment() try: while not self.run_event.isSet(): cur_count += 1 try: cur_item = random.randint(idx_min, idx_max) s_cur_item = "%s" % cur_item b_cur_item = SolBase.unicode_to_binary(s_cur_item, "utf-8") cur_ttl = random.randint(self.bench_ttl_min_ms, self.bench_ttl_max_ms) for _ in range(0, self.bench_put_weight): self.redis_cache.put(s_cur_item, b_cur_item, cur_ttl) SolBase.sleep(0) for _ in range(0, self.bench_get_weight): v = self.redis_cache.get(s_cur_item) if v: self.assertEqual(v, b_cur_item) SolBase.sleep(0) except Exception as e: self.exception_raised += 1 logger.warning("Ex=%s", SolBase.extostr(e)) self.thread_running_ok.increment(-1) return finally: pass finally: self.assertGreater(cur_count, 0) logger.debug("Exiting") event.set() self.thread_running.increment(-1)
def _decode(cls, buf): """ Decode buffer :param buf: bytes :type buf: bytes :return tuple (data, ttl_ms, ms_added) :rtype tuple """ try: d = ujson.loads(SolBase.binary_to_unicode(buf, "utf-8")) return SolBase.unicode_to_binary( d["data"], "utf-8"), d["ttl_ms"], d["ms_added"] except Exception as e: logger.warning("Unable to decode, buf=%s, ex=%s", buf, SolBase.extostr(e)) return None, None, None
def on_request(self, environ, start_response): """ On request callback :param environ: environ :type environ: dict :param start_response: start_response :type start_response: instancemethod :return: list :rtype: list """ try: logger.info("Request start now") # Log for k, v in environ.items(): logger.debug("Env: %s=%s", k, v) # Switch pi = environ["PATH_INFO"] logger.debug("pi=%s", pi) # Sometimes PATH_INFO come with full uri (urllib3) (?!) # http://127.0.0.1:7900/unittest if pi.endswith("/unittest"): logger.debug("call _on_unit_test") return self._on_unit_test(environ, start_response) else: logger.debug("call _on_invalid") return self._on_invalid(start_response) except Exception as e: logger.warning("Ex=%s", SolBase.extostr(e)) status = "500 Internal Server Error" body = status headers = [('Content-Type', 'text/plain')] start_response(status, headers) return [SolBase.unicode_to_binary(body, "utf-8")] finally: logger.debug("exit") self._lifecycle_log_status()
def _encode(cls, val, ttl_ms, ms_added=None): """ Encode :param val: bytes :type val: bytes :param ttl_ms: int :type ttl_ms: int :param ms_added: int, float, None :type ms_added: int, float, None :return bytes :rtype bytes """ if not ms_added: ms_added = SolBase.mscurrent() d = { "ms_added": int(ms_added), "ttl_ms": ttl_ms, "data": val, } s = ujson.dumps(d, reject_bytes=False) return SolBase.unicode_to_binary(s, "utf-8")
def _on_unit_test(self, environ, start_response): """ On request callback :param environ: environ :type environ: dict :param start_response: start_response :type start_response: instancemethod :return: list :rtype: list """ # Param logger.debug("_get_param_from_qs") from_qs = self._get_param_from_qs(environ) logger.debug("_get_param_from_post_data") from_post = self._get_param_from_post_data(environ) logger.debug("_get_method") from_method = self._get_method(environ) # Debug logger.debug("reply set") status = "200 OK" if from_method == "HEAD": # No body body = "" headers = [('Content-Type', 'text/txt')] start_response(status, headers) logger.debug("reply send") else: body = "OK" + "\n" body += "from_qs=" + str(from_qs) + " -EOL\n" body += "from_post=" + str(from_post) + " -EOL\n" body += "from_method=" + from_method + "\n" headers = [('Content-Type', 'text/txt')] start_response(status, headers) logger.debug("reply send") return [SolBase.unicode_to_binary(body, "utf-8")]
def test_basic(self): """ Test. """ # Alloc self.memory_cache = MemoryCache() self.redis_cache = RedisCache() self.high_cache = HighCacheEx(memory_cache=self.memory_cache, redis_cache=self.redis_cache) # Put self.high_cache.put(self.key_prefix + "L1L2_IMPLICIT", b"L1L2_IMPLICIT", 60000) self.high_cache.put(self.key_prefix + "L1L2_EXPLICIT", b"L1L2_EXPLICIT", 60000, l1=True, l2=True) self.high_cache.put(self.key_prefix + "L1_ONLY", b"L1_ONLY", 60000, l1=True, l2=False) self.high_cache.put(self.key_prefix + "L2_ONLY", b"L2_ONLY", 60000, l1=False, l2=True) self.high_cache.put(self.key_prefix + "NONE", b"NONE", 60000, l1=False, l2=False) # Check get self.assertEqual( self.high_cache.get(self.key_prefix + "L1L2_IMPLICIT"), b"L1L2_IMPLICIT") self.assertEqual( self.high_cache.get(self.key_prefix + "L1L2_IMPLICIT", l1=True, l2=False), b"L1L2_IMPLICIT") self.assertEqual( self.high_cache.get(self.key_prefix + "L1L2_IMPLICIT", l1=False, l2=True), b"L1L2_IMPLICIT") # Check getex v, level = self.high_cache.getex(self.key_prefix + "L1L2_IMPLICIT", l1=True, l2=False) self.assertEqual(level, 1) self.assertEqual(v, b"L1L2_IMPLICIT") v, level = self.high_cache.getex(self.key_prefix + "L1L2_IMPLICIT", l1=False, l2=True) self.assertEqual(level, 2) self.assertEqual(v, b"L1L2_IMPLICIT") v, level = self.high_cache.getex(self.key_prefix + "L1L2_IMPLICIT", l1=False, l2=False) self.assertEqual(level, 0) self.assertIsNone(v) self.assertEqual( self.high_cache.get(self.key_prefix + "L1L2_EXPLICIT"), b"L1L2_EXPLICIT") self.assertEqual( self.high_cache.get(self.key_prefix + "L1L2_EXPLICIT", l1=True, l2=False), b"L1L2_EXPLICIT") self.assertEqual( self.high_cache.get(self.key_prefix + "L1L2_EXPLICIT", l1=False, l2=True), b"L1L2_EXPLICIT") self.assertEqual( self.high_cache.get(self.key_prefix + "L1_ONLY", l1=True, l2=False), b"L1_ONLY") self.assertIsNone( self.high_cache.get(self.key_prefix + "L1_ONLY", l1=False, l2=True)) self.assertEqual(self.high_cache.get(self.key_prefix + "L1_ONLY"), b"L1_ONLY") self.assertIsNone( self.high_cache.get(self.key_prefix + "L2_ONLY", l1=True, l2=False)) self.assertEqual( self.high_cache.get(self.key_prefix + "L2_ONLY", l1=False, l2=True), b"L2_ONLY") self.assertEqual(self.high_cache.get(self.key_prefix + "L2_ONLY"), b"L2_ONLY") self.assertIsNone( self.high_cache.get(self.key_prefix + "NONE", l1=True, l2=False)) self.assertIsNone( self.high_cache.get(self.key_prefix + "NONE", l1=False, l2=True)) self.assertIsNone(self.high_cache.get(self.key_prefix + "NONE")) # REMOVE FROM L1 self.high_cache.remove(self.key_prefix + "L1L2_IMPLICIT", l1=True, l2=False) # Target L2 only self.assertEqual( self.high_cache.get(self.key_prefix + "L1L2_IMPLICIT", l1=False, l2=True), b"L1L2_IMPLICIT") # Check L1 miss self.assertIsNone( self.high_cache.get(self.key_prefix + "L1L2_IMPLICIT", l1=True, l2=False)) # Target both and check L1 HIT v, level = self.high_cache.getex(self.key_prefix + "L1L2_IMPLICIT", l1=True, l2=True) self.assertEqual(v, b"L1L2_IMPLICIT") self.assertEqual(level, 2) v, level = self.high_cache.getex(self.key_prefix + "L1L2_IMPLICIT", l1=True, l2=True) self.assertEqual(v, b"L1L2_IMPLICIT") self.assertEqual(level, 1) # REMOVE FROM BOTH self.high_cache.remove(self.key_prefix + "L1L2_EXPLICIT") self.assertIsNone( self.high_cache.get(self.key_prefix + "L1L2_EXPLICIT")) self.assertIsNone( self.high_cache.get(self.key_prefix + "L1L2_EXPLICIT", l1=True, l2=False)) self.assertIsNone( self.high_cache.get(self.key_prefix + "L1L2_EXPLICIT", l1=False, l2=True)) # UNICODE VALUES self.high_cache.put( "K1", SolBase.unicode_to_binary("VAL\u001B\u0BD9\U0001A10D\u1501", "utf-8"), 60000) v = self.high_cache.get("K1") self.assertEqual( v, SolBase.unicode_to_binary("VAL\u001B\u0BD9\U0001A10D\u1501", "utf-8")) # Sleep SolBase.sleep(1000) # Stop self.high_cache.stop_cache() self.assertFalse(self.redis_cache._is_started) self.assertFalse(self.memory_cache._is_started) self.high_cache = None self.redis_cache = None self.memory_cache = None