def _evict_all_expired_keys(self): """ Evict all expired keys :return The evicted key count :rtype: int """ # We use the _hash_key directly since we don't care order here tu_to_evict = list() cur_ms = SolBase.mscurrent() for tu in self._hash_key.items(): # v is a tuple (ms, object) if self._is_expired(tu[1][0], cur_ms): tu_to_evict.append(tu) # Now evict for (k, v) in tu_to_evict: self._safe_unhash(k) self._notify_eviction(k, v[1]) return len(tu_to_evict)
def remove(self, key): """ Remove a key from cache. :param key: Any key :type key: str """ ms_start = SolBase.mscurrent() try: if not isinstance(key, (bytes, str)): raise Exception("Key must be (bytes, str)") # Use write redis self._write_redis.delete(key) except Exception as e: logger.warning("Exception, ex=%s", SolBase.extostr(e)) Meters.aii(self.meters_prefix + "rcs.cache_ex") finally: Meters.dtci(self.meters_prefix + "rcs.cache_dtc_write", SolBase.msdiff(ms_start))
def _get_std_err(self): """ Get :return: list :rtype: list """ try: sys.stderr.flush() except ValueError: pass ms_start = SolBase.mscurrent() while True: ar = self._file_to_list(self.daemon_std_err) if len(ar) > 0: return ar elif SolBase.msdiff(ms_start) > self.std_err_timeout_ms: return list() else: SolBase.sleep(10)
def __init__(self): """ Constructor """ # Daemon control self._locker = Lock() self._is_running = False self._server_greenlet = None # Zip on self._zip_enabled = True # Start event self._start_event = Event() # Server self._wsgi_server = None # Lifecycle stuff (from daemon) self._lifecycle_locker = Lock() self._lifecycle_interval_ms = 30000 self._lifecycle_last_log_ms = SolBase.mscurrent()
def _lifecycle_log_status(self): """ Run """ try: with self._lifecycle_locker: # Check ms_diff = SolBase.msdiff(self._lifecycle_last_log_ms) if ms_diff < self._lifecycle_interval_ms: return # Log now self._lifecycle_last_log_ms = SolBase.mscurrent() # noinspection PyProtectedMember lifecyclelogger.info( "self=%s", # Id id(self), ) except Exception as e: logger.warning("Exception, ex=%s", SolBase.extostr(e))
def _get_std_out_file(self, file_name): """ Get :param file_name: str :type file_name: str :return: list :rtype: list """ try: sys.stdout.flush() except ValueError: pass ms_start = SolBase.mscurrent() while True: ar = self._file_to_list(file_name) if len(ar) > 0: return ar elif SolBase.msdiff(ms_start) > self.stdout_timeout_ms: return list() else: SolBase.sleep(10)
def _encode(cls, val, ttl_ms, ms_added=None): """ Encode :param val: bytes :type val: bytes :param ttl_ms: int :type ttl_ms: int :param ms_added: int, float, None :type ms_added: int, float, None :return bytes :rtype bytes """ if not ms_added: ms_added = SolBase.mscurrent() d = { "ms_added": int(ms_added), "ttl_ms": ttl_ms, "data": val, } s = ujson.dumps(d, reject_bytes=False) return SolBase.unicode_to_binary(s, "utf-8")
def go_http(self, http_request): """ Perform an http request :param http_request: HttpRequest :type http_request: HttpRequest :return HttpResponse :rtype HttpResponse """ ms = SolBase.mscurrent() http_response = HttpResponse() general_timeout_sec = float(http_request.general_timeout_ms) / 1000.0 try: # Assign request http_response.http_request = http_request # Fire gevent.with_timeout( general_timeout_sec, self._go_http_internal, http_request, http_response) SolBase.sleep(0) except Timeout: # Failed http_response.exception = Exception("Timeout while processing, general_timeout_sec={0}".format(general_timeout_sec)) except Exception as e: # Failed http_response.exception = e finally: # Switch SolBase.sleep(0) # Assign ms http_response.elapsed_ms = SolBase.msdiff(ms) # Return return http_response
def _go_greenlet(self, greenlet_count, put_count, get_count, bench_item_count): """ Doc :param greenlet_count: greenlet_count :param put_count: put_count :param get_count: get_count :param bench_item_count: bench_item_count """ g_event = None g_array = None try: # Settings g_count = greenlet_count g_ms = 10000 # Continue callback loop self.callback_return = True # Go self.redis_cache = RedisCache() # Item count self.bench_item_count = bench_item_count self.bench_put_weight = put_count self.bench_get_weight = get_count self.bench_ttl_min_ms = 1000 self.bench_ttl_max_ms = int(g_ms / 2) # Go self.run_event = Event() self.exception_raised = 0 self.open_count = 0 self.thread_running = AtomicIntSafe() self.thread_running_ok = AtomicIntSafe() # Item per greenlet item_per_greenlet = self.bench_item_count / g_count # Signal self.gorun_event = Event() # Alloc greenlet g_array = list() g_event = list() for _ in range(0, g_count): greenlet = Greenlet() g_array.append(greenlet) g_event.append(Event()) # Run them cur_idx = 0 for idx in range(0, len(g_array)): greenlet = g_array[idx] event = g_event[idx] greenlet.spawn(self._run_cache_bench, event, cur_idx, cur_idx + item_per_greenlet) cur_idx += item_per_greenlet SolBase.sleep(0) # Signal self.gorun_event.set() # Wait a bit dt = SolBase.mscurrent() while SolBase.msdiff(dt) < g_ms: SolBase.sleep(500) # Stat ms = SolBase.msdiff(dt) sec = float(ms / 1000.0) total_put = Meters.aig("rcs.cache_put") per_sec_put = round(float(total_put) / sec, 2) total_get = Meters.aig("rcs.cache_get_hit") + Meters.aig( "rcs.cache_get_miss") per_sec_get = round(float(total_get) / sec, 2) logger.info( "Running..., count=%s, run=%s, ok=%s, put/sec=%s get/sec=%s, cache=%s", self.open_count, self.thread_running.get(), self.thread_running_ok.get(), per_sec_put, per_sec_get, self.redis_cache) self.assertEqual(self.exception_raised, 0) # Over, signal logger.info("Signaling, count=%s", self.open_count) self.run_event.set() # Wait for g in g_event: g.wait(30.0) self.assertTrue(g.isSet()) g_event = None g_array = None # Log Meters.write_to_logger() finally: self.run_event.set() if g_event: for g in g_event: g.set() if g_array: for g in g_array: g.kill() if self.redis_cache: self.redis_cache.stop_cache() self.redis_cache = None
def _go_gevent(self, http_request, http_response): """ Perform an http request :param http_request: HttpRequest :type http_request: HttpRequest :param http_response: HttpResponse :type http_response: HttpResponse """ # Implementation http_response.http_implementation = HttpClient.HTTP_IMPL_GEVENT # Uri url = URL(http_request.uri) SolBase.sleep(0) # Patch for path attribute error try: _ = url.path except AttributeError: url.path = "/" # Get instance logger.debug("Get pool") http = self.gevent_from_pool(url, http_request) logger.debug("Get pool done, pool=%s", http) SolBase.sleep(0) # Fire ms_start = SolBase.mscurrent() logger.debug("Http now") if not http_request.method: # ---------------- # Auto-detect # ---------------- if http_request.post_data: # Post response = http.post(url.request_uri, body=http_request.post_data, headers=http_request.headers) else: # Get response = http.get(url.request_uri, headers=http_request.headers) else: # ---------------- # Use input # ---------------- if http_request.method == "GET": response = http.get(url.request_uri, headers=http_request.headers) elif http_request.method == "DELETE": response = http.delete(url.request_uri, body=http_request.post_data, headers=http_request.headers) elif http_request.method == "HEAD": response = http.head(url.request_uri, headers=http_request.headers) elif http_request.method == "PUT": response = http.put(url.request_uri, body=http_request.post_data, headers=http_request.headers) elif http_request.method == "POST": response = http.post(url.request_uri, body=http_request.post_data, headers=http_request.headers) elif http_request.method == "PATCH": raise Exception("Unsupported gevent method={0}".format(http_request.method)) elif http_request.method == "OPTIONS": raise Exception("Unsupported gevent method={0}".format(http_request.method)) elif http_request.method == "TRACE": raise Exception("Unsupported gevent method={0}".format(http_request.method)) else: raise Exception("Invalid gevent method={0}".format(http_request.method)) logger.debug("Http done, ms=%s", SolBase.msdiff(ms_start)) SolBase.sleep(0) # Check if not response: raise Exception("No response from http") # Process it http_response.status_code = response.status_code # Read ms_start = SolBase.mscurrent() logger.debug("Read now") http_response.buffer = response.read() SolBase.sleep(0) logger.debug("Read done, ms=%s", SolBase.msdiff(ms_start)) if response.content_length: http_response.content_length = response.content_length else: if http_response.buffer: http_response.content_length = len(http_response.buffer) else: http_response.content_length = 0 # noinspection PyProtectedMember for k, v in response._headers_index.items(): HttpClient._add_header(http_response.headers, k, v) response.should_close() # Over SolBase.sleep(0)
def test_start_status_reload_stop_logfile(self): """ Test """ try: # Start self._reset_std_capture() main_helper_file = self.current_dir + "CustomDaemon.py" main_helper_file = abspath(main_helper_file) self.assertTrue(FileUtility.is_file_exist(main_helper_file)) # Params ar = list() ar.append(sys.executable) ar.append(main_helper_file) ar.append("-pidfile={0}".format(self.daemon_pid_file)) ar.append("-stderr={0}".format(self.daemon_std_err)) ar.append("-stdout=/dev/null") ar.append("-logfile={0}".format(self.daemon_std_out)) ar.append("start") # ========================= # START # ========================= # Launch logger.info("Start : %s", " ".join(ar)) p = subprocess.Popen(args=ar) logger.info("Started") SolBase.sleep(0) self._wait_process(p) # Try wait for stdout ms_start = SolBase.mscurrent() while SolBase.msdiff(ms_start) < self.stdout_timeout_ms: if "n".join(self._get_std_out()).find( " INFO | CustomDaemon@_on_start") >= 0: break else: SolBase.sleep(10) # Get std (caution, we are async since forked) logger.info("stdOut ### START") for s in self._get_std_out(): logger.info("stdOut => %s", s) logger.info("stdOut ### END") logger.info("stdErr ### START") for s in self._get_std_err(): logger.info("stdErr => %s", s) logger.info("stdErr ### END") # Check self.assertTrue(len(self._get_std_err()) == 0) self.assertTrue(len(self._get_std_out()) > 0) self.assertTrue("n".join(self._get_std_out()).find(" ERROR ") < 0) self.assertTrue("n".join(self._get_std_out()).find( " INFO | CustomDaemon@_on_start") >= 0) self.assertTrue("n".join(self._get_std_out()).find(" WARN ") < 0) # ========================= # STATUS # ========================= for _ in range(0, 10): # Args ar = list() ar.append(sys.executable) ar.append(main_helper_file) ar.append("-pidfile={0}".format(self.daemon_pid_file)) ar.append("status") # Launch p = subprocess.Popen(args=ar) self._wait_process(p) # ========================= # RELOAD # ========================= for _ in range(0, 10): # Args ar = list() ar.append(sys.executable) ar.append(main_helper_file) ar.append("-pidfile={0}".format(self.daemon_pid_file)) ar.append("reload") # Launch p = subprocess.Popen(args=ar) self._wait_process(p) # ========================= # STOP # ========================= # Args ar = list() ar.append(sys.executable) ar.append(main_helper_file) ar.append("-pidfile={0}".format(self.daemon_pid_file)) ar.append("stop") # Launch p = subprocess.Popen(args=ar) self._wait_process(p) # ========================= # OVER, CHECK LOGS # ========================= # Try wait for stdout ms_start = SolBase.mscurrent() while SolBase.msdiff(ms_start) < self.stdout_timeout_ms: if "n".join(self._get_std_out()).find(" INFO | CustomDaemon@_on_stop") >= 0 \ and "n".join(self._get_std_out()).find(" INFO | CustomDaemon@_on_status") >= 0: break else: SolBase.sleep(10) # Get std (caution, we are async since forked) logger.info("stdOut ### START") for s in self._get_std_out(): logger.info("stdOut => %s", s) logger.info("stdOut ### END") logger.info("stdErr ### START") for s in self._get_std_err(): logger.info("stdErr => %s", s) logger.info("stdErr ### END") # Check self.assertTrue(len(self._get_std_err()) == 0) self.assertTrue(len(self._get_std_out()) > 0) self.assertTrue("n".join(self._get_std_out()).find(" ERROR ") < 0) self.assertTrue("n".join(self._get_std_out()).find( " INFO | CustomDaemon@_on_start") >= 0) self.assertTrue("n".join(self._get_std_out()).find( " INFO | CustomDaemon@_on_stop") >= 0) self.assertTrue("n".join(self._get_std_out()).find( " INFO | CustomDaemon@_on_status") >= 0) self.assertTrue("n".join(self._get_std_out()).find(" WARN ") < 0) # ========================= # OVER, CHECK ACTION FILE # ========================= buf = FileUtility.file_to_textbuffer( CustomDaemon.DAEMON_LAST_ACTION_FILE, "ascii") self.assertTrue(buf.find("is_running=False") >= 0) self.assertTrue(buf.find("start_count=1") >= 0) self.assertTrue(buf.find("stop_count=1") >= 0) self.assertTrue(buf.find("status_count=10") >= 0) self.assertTrue(buf.find("reload_count=10") >= 0) self.assertTrue(buf.find("last_action=stop") >= 0) finally: logger.debug("Exiting test, idx=%s", self.run_idx)
def _go_greenlet(self, greenlet_count, pool_max, sql="SELECT user, host FROM mysql.user LIMIT 1;", check_exception=True): """ Doc :param greenlet_count: greenlet_count :type greenlet_count: int :param pool_max: Pool max size :type pool_max: int :param sql: str :type sql: str :param check_exception: bool :type check_exception: bool """ MysqlApi.reset_pools() Meters.reset() g_event = None g_array = None try: # Settings g_count = greenlet_count g_ms = 5000 # Go self.pool_max = pool_max self.run_event = Event() self.exception_raised = 0 self.pool_sql = sql self.thread_running = AtomicIntSafe() self.thread_running_ok = AtomicIntSafe() # Signal self.gorun_event = Event() # Alloc greenlet g_array = list() g_event = list() for _ in range(0, g_count): greenlet = Greenlet() g_array.append(greenlet) g_event.append(Event()) # Run them for idx in range(0, len(g_array)): greenlet = g_array[idx] event = g_event[idx] greenlet.spawn(self._run_mysql_bench, event) SolBase.sleep(0) # Signal self.gorun_event.set() # Wait a bit dt = SolBase.mscurrent() while SolBase.msdiff(dt) < g_ms: SolBase.sleep(1000) # Stat ms = SolBase.msdiff(dt) sec = float(ms / 1000.0) total_acquire = Meters.aig( "k.db_pool.base.call.connection_acquire") per_sec_acquire = round(float(total_acquire) / sec, 2) total_release = Meters.aig( "k.db_pool.base.call.connection_release") per_sec_release = round(float(total_release) / sec, 2) logger.info("Running..., run=%s, ok=%s, ps.ack/rel=%s/%s", self.thread_running.get(), self.thread_running_ok.get(), per_sec_acquire, per_sec_release) if check_exception: self.assertEqual(self.exception_raised, 0) # Over, signal logger.info("Signaling") self.run_event.set() # Wait for g in g_event: g.wait(30.0) self.assertTrue(g.isSet()) g_event = None g_array = None # Check it self.assertEquals( Meters.aig("k.db_pool.base.call.connection_acquire"), Meters.aig("k.db_pool.base.call.connection_release")) if check_exception: self.assertEquals( Meters.aig("k.db_pool.base.call.connection_acquire"), Meters.aig("k.db_pool.mysql.call._connection_ping")) self.assertLessEqual( Meters.aig("k.db_pool.mysql.call._get_connection"), pool_max) self.assertLessEqual( Meters.aig("k.db_pool.mysql.call._connection_create"), pool_max) self.assertLessEqual(Meters.aig("k.db_pool.hash.cur"), 1) self.assertLessEqual(Meters.aig("k.db_pool.base.cur_size"), pool_max) self.assertLessEqual(Meters.aig("k.db_pool.base.max_size"), pool_max) self.assertEquals(Meters.aig("k.db_pool.mysql.call.__init"), 1) finally: self.run_event.set() if g_event: for g in g_event: g.set() if g_array: for g in g_array: g.kill()
def _go_greenlet(self, greenlet_count, put_count, get_count, bench_item_count, watchdog_interval_ms=60000, max_item=128000, max_bytes=32 * 1024 * 1024, max_single_item_bytes=1 * 1024 * 1024, purge_min_bytes=8 * 1024 * 1024, purge_min_count=1000): """ Doc :param greenlet_count: greenlet_count :param put_count: put_count :param get_count: get_count :param bench_item_count: bench_item_count :param watchdog_interval_ms: watchdog_interval_ms :param max_item: max_item :param max_bytes: max_bytes :param max_single_item_bytes: max_single_item_bytes :param purge_min_bytes: purge_min_bytes :param purge_min_count: purge_min_count """ g_event = None g_array = None try: # Settings g_count = greenlet_count g_ms = 10000 # Continue callback loop self.callback_return = True # Go self.mem_cache = MemoryCache( watchdog_interval_ms=watchdog_interval_ms, max_item=max_item, max_bytes=max_bytes, max_single_item_bytes=max_single_item_bytes, purge_min_bytes=purge_min_bytes, purge_min_count=purge_min_count) # Item count self.bench_item_count = bench_item_count self.bench_put_weight = put_count self.bench_get_weight = get_count self.bench_ttl_min_ms = 1000 self.bench_ttl_max_ms = int(g_ms / 2) # Go self.run_event = Event() self.exception_raised = 0 self.open_count = 0 self.thread_running = AtomicIntSafe() self.thread_running_ok = AtomicIntSafe() # Item per greenlet item_per_greenlet = self.bench_item_count / g_count # Signal self.gorun_event = Event() # Alloc greenlet g_array = list() g_event = list() for _ in range(0, g_count): greenlet = Greenlet() g_array.append(greenlet) g_event.append(Event()) # Run them cur_idx = 0 for idx in range(0, len(g_array)): greenlet = g_array[idx] event = g_event[idx] greenlet.spawn(self._run_cache_bench, event, cur_idx, cur_idx + item_per_greenlet) cur_idx += item_per_greenlet SolBase.sleep(0) # Signal self.gorun_event.set() # Wait a bit dt = SolBase.mscurrent() while SolBase.msdiff(dt) < g_ms: SolBase.sleep(500) # Stat ms = SolBase.msdiff(dt) sec = float(ms / 1000.0) total_put = Meters.aig("mcs.cache_put") per_sec_put = round(float(total_put) / sec, 2) total_get = Meters.aig("mcs.cache_get_hit") + Meters.aig( "mcs.cache_get_miss") per_sec_get = round(float(total_get) / sec, 2) logger.info( "Running..., count=%s, run=%s, ok=%s, put/sec=%s get/sec=%s, cache=%s", self.open_count, self.thread_running.get(), self.thread_running_ok.get(), per_sec_put, per_sec_get, self.mem_cache) self.assertEqual(self.exception_raised, 0) # Over, signal logger.info("Signaling, count=%s", self.open_count) self.run_event.set() # Wait for g in g_event: g.wait(30.0) self.assertTrue(g.isSet()) g_event = None g_array = None # Log Meters.write_to_logger() finally: self.run_event.set() if g_event: for g in g_event: g.set() if g_array: for g in g_array: g.kill() if self.mem_cache: max_count = 0 total_size = 0 i = 0 for (k, v) in self.mem_cache._hash_key.items(): i += 1 total_size += len(k) + len(v[1]) if i < max_count: logger.info("%s => %s", k, v) self.assertEqual(total_size, self.mem_cache._current_data_bytes.get()) self.mem_cache.stop_cache() self.mem_cache = None
def test_basic(self): """ Test. """ # Alloc self.mem_cache = MemoryCache() # Get : must return nothing o = self.mem_cache.get("not_found") self.assertIsNone(o) # Put self.mem_cache.put("keyA", b"valA", 60000) o = self.mem_cache.get("keyA") self.assertEqual(o, b"valA") o = self.mem_cache.get_raw("keyA") self.assertIsNotNone(o) self.assertIsInstance(o, tuple) self.assertEqual(o[1], b"valA") self.assertLessEqual(o[0] - SolBase.mscurrent(), 60000) logger.info("TTL approx=%s", o[0] - SolBase.mscurrent()) # Put with lower TTL : TTL MUST BE UPDATED self.mem_cache.put("keyA", b"valA", 30000) o = self.mem_cache.get("keyA") self.assertEqual(o, b"valA") o = self.mem_cache.get_raw("keyA") self.assertIsNotNone(o) self.assertIsInstance(o, tuple) self.assertEqual(o[1], b"valA") self.assertLessEqual(o[0] - SolBase.mscurrent(), 30000) logger.info("TTL approx=%s", o[0] - SolBase.mscurrent()) # Str put : must now be ok self.mem_cache.put("toto_str", "str_val", 1000) self.assertEqual(self.mem_cache.get("toto_str"), "str_val") # Non bytes,str injection (int) : must fail # noinspection PyBroadException,PyPep8 try: # noinspection PyTypeChecker self.mem_cache.put("toto", 12, 1000) self.fail("Must fail") except: pass # Non bytes injection : must fail # noinspection PyBroadException,PyPep8 try: # noinspection PyTypeChecker self.mem_cache.put("toto", u"unicode_buffer", 1000) self.fail("Must fail") except: pass # This MUST fail # noinspection PyBroadException try: # noinspection PyTypeChecker self.mem_cache.put(999, b"value", 60000) self.fail("Put a key as non bytes,str MUST fail") except Exception: pass # This MUST fail # noinspection PyBroadException try: # noinspection PyTypeChecker self.mem_cache.remove(999) self.fail("Remove a key as non bytes,str MUST fail") except Exception: pass # Put/Remove self.mem_cache.put("todel", b"value", 60000) self.assertEqual(self.mem_cache.get("todel"), b"value") self.mem_cache.remove("todel") self.assertEqual(self.mem_cache.get("todel"), None) # Put self.mem_cache.put("KEY \u001B\u0BD9\U0001A10D\u1501\xc3", b"zzz", 60000) self.assertEqual( self.mem_cache.get("KEY \u001B\u0BD9\U0001A10D\u1501\xc3"), b"zzz") # Stop self.mem_cache.stop_cache() self.mem_cache = None