def _evict_all_expired_keys(self):
"""
Evict all expired keys
:return The evicted key count
:rtype: int
"""
# We use the _hash_key directly since we don't care order here
tu_to_evict = list()
cur_ms = SolBase.mscurrent()
for tu in self._hash_key.items():
# v is a tuple (ms, object)
if self._is_expired(tu[1][0], cur_ms):
tu_to_evict.append(tu)
# Now evict
for (k, v) in tu_to_evict:
self._safe_unhash(k)
self._notify_eviction(k, v[1])
return len(tu_to_evict)
def remove(self, key):
"""
Remove a key from cache.
:param key: Any key
:type key: str
"""
ms_start = SolBase.mscurrent()
try:
if not isinstance(key, (bytes, str)):
raise Exception("Key must be (bytes, str)")
# Use write redis
self._write_redis.delete(key)
except Exception as e:
logger.warning("Exception, ex=%s", SolBase.extostr(e))
Meters.aii(self.meters_prefix + "rcs.cache_ex")
finally:
Meters.dtci(self.meters_prefix + "rcs.cache_dtc_write",
SolBase.msdiff(ms_start))
def _get_std_err(self):
"""
Get
:return: list
:rtype: list
"""
try:
sys.stderr.flush()
except ValueError:
pass
ms_start = SolBase.mscurrent()
while True:
ar = self._file_to_list(self.daemon_std_err)
if len(ar) > 0:
return ar
elif SolBase.msdiff(ms_start) > self.std_err_timeout_ms:
return list()
else:
SolBase.sleep(10)
def __init__(self):
"""
Constructor
"""
# Daemon control
self._locker = Lock()
self._is_running = False
self._server_greenlet = None
# Zip on
self._zip_enabled = True
# Start event
self._start_event = Event()
# Server
self._wsgi_server = None
# Lifecycle stuff (from daemon)
self._lifecycle_locker = Lock()
self._lifecycle_interval_ms = 30000
self._lifecycle_last_log_ms = SolBase.mscurrent()
def _lifecycle_log_status(self):
"""
Run
"""
try:
with self._lifecycle_locker:
# Check
ms_diff = SolBase.msdiff(self._lifecycle_last_log_ms)
if ms_diff < self._lifecycle_interval_ms:
return
# Log now
self._lifecycle_last_log_ms = SolBase.mscurrent()
# noinspection PyProtectedMember
lifecyclelogger.info(
"self=%s",
# Id
id(self),
)
except Exception as e:
logger.warning("Exception, ex=%s", SolBase.extostr(e))
def _get_std_out_file(self, file_name):
"""
Get
:param file_name: str
:type file_name: str
:return: list
:rtype: list
"""
try:
sys.stdout.flush()
except ValueError:
pass
ms_start = SolBase.mscurrent()
while True:
ar = self._file_to_list(file_name)
if len(ar) > 0:
return ar
elif SolBase.msdiff(ms_start) > self.stdout_timeout_ms:
return list()
else:
SolBase.sleep(10)
def _encode(cls, val, ttl_ms, ms_added=None):
"""
Encode
:param val: bytes
:type val: bytes
:param ttl_ms: int
:type ttl_ms: int
:param ms_added: int, float, None
:type ms_added: int, float, None
:return bytes
:rtype bytes
"""
if not ms_added:
ms_added = SolBase.mscurrent()
d = {
"ms_added": int(ms_added),
"ttl_ms": ttl_ms,
"data": val,
}
s = ujson.dumps(d, reject_bytes=False)
return SolBase.unicode_to_binary(s, "utf-8")
def go_http(self, http_request):
"""
Perform an http request
:param http_request: HttpRequest
:type http_request: HttpRequest
:return HttpResponse
:rtype HttpResponse
"""
ms = SolBase.mscurrent()
http_response = HttpResponse()
general_timeout_sec = float(http_request.general_timeout_ms) / 1000.0
try:
# Assign request
http_response.http_request = http_request
# Fire
gevent.with_timeout(
general_timeout_sec,
self._go_http_internal,
http_request, http_response)
SolBase.sleep(0)
except Timeout:
# Failed
http_response.exception = Exception("Timeout while processing, general_timeout_sec={0}".format(general_timeout_sec))
except Exception as e:
# Failed
http_response.exception = e
finally:
# Switch
SolBase.sleep(0)
# Assign ms
http_response.elapsed_ms = SolBase.msdiff(ms)
# Return
return http_response
def _go_greenlet(self, greenlet_count, put_count, get_count,
bench_item_count):
"""
Doc
:param greenlet_count: greenlet_count
:param put_count: put_count
:param get_count: get_count
:param bench_item_count: bench_item_count
"""
g_event = None
g_array = None
try:
# Settings
g_count = greenlet_count
g_ms = 10000
# Continue callback loop
self.callback_return = True
# Go
self.redis_cache = RedisCache()
# Item count
self.bench_item_count = bench_item_count
self.bench_put_weight = put_count
self.bench_get_weight = get_count
self.bench_ttl_min_ms = 1000
self.bench_ttl_max_ms = int(g_ms / 2)
# Go
self.run_event = Event()
self.exception_raised = 0
self.open_count = 0
self.thread_running = AtomicIntSafe()
self.thread_running_ok = AtomicIntSafe()
# Item per greenlet
item_per_greenlet = self.bench_item_count / g_count
# Signal
self.gorun_event = Event()
# Alloc greenlet
g_array = list()
g_event = list()
for _ in range(0, g_count):
greenlet = Greenlet()
g_array.append(greenlet)
g_event.append(Event())
# Run them
cur_idx = 0
for idx in range(0, len(g_array)):
greenlet = g_array[idx]
event = g_event[idx]
greenlet.spawn(self._run_cache_bench, event, cur_idx,
cur_idx + item_per_greenlet)
cur_idx += item_per_greenlet
SolBase.sleep(0)
# Signal
self.gorun_event.set()
# Wait a bit
dt = SolBase.mscurrent()
while SolBase.msdiff(dt) < g_ms:
SolBase.sleep(500)
# Stat
ms = SolBase.msdiff(dt)
sec = float(ms / 1000.0)
total_put = Meters.aig("rcs.cache_put")
per_sec_put = round(float(total_put) / sec, 2)
total_get = Meters.aig("rcs.cache_get_hit") + Meters.aig(
"rcs.cache_get_miss")
per_sec_get = round(float(total_get) / sec, 2)
logger.info(
"Running..., count=%s, run=%s, ok=%s, put/sec=%s get/sec=%s, cache=%s",
self.open_count, self.thread_running.get(),
self.thread_running_ok.get(), per_sec_put, per_sec_get,
self.redis_cache)
self.assertEqual(self.exception_raised, 0)
# Over, signal
logger.info("Signaling, count=%s", self.open_count)
self.run_event.set()
# Wait
for g in g_event:
g.wait(30.0)
self.assertTrue(g.isSet())
g_event = None
g_array = None
# Log
Meters.write_to_logger()
finally:
self.run_event.set()
if g_event:
for g in g_event:
g.set()
if g_array:
for g in g_array:
g.kill()
if self.redis_cache:
self.redis_cache.stop_cache()
self.redis_cache = None
def _go_gevent(self, http_request, http_response):
"""
Perform an http request
:param http_request: HttpRequest
:type http_request: HttpRequest
:param http_response: HttpResponse
:type http_response: HttpResponse
"""
# Implementation
http_response.http_implementation = HttpClient.HTTP_IMPL_GEVENT
# Uri
url = URL(http_request.uri)
SolBase.sleep(0)
# Patch for path attribute error
try:
_ = url.path
except AttributeError:
url.path = "/"
# Get instance
logger.debug("Get pool")
http = self.gevent_from_pool(url, http_request)
logger.debug("Get pool done, pool=%s", http)
SolBase.sleep(0)
# Fire
ms_start = SolBase.mscurrent()
logger.debug("Http now")
if not http_request.method:
# ----------------
# Auto-detect
# ----------------
if http_request.post_data:
# Post
response = http.post(url.request_uri,
body=http_request.post_data,
headers=http_request.headers)
else:
# Get
response = http.get(url.request_uri,
headers=http_request.headers)
else:
# ----------------
# Use input
# ----------------
if http_request.method == "GET":
response = http.get(url.request_uri,
headers=http_request.headers)
elif http_request.method == "DELETE":
response = http.delete(url.request_uri,
body=http_request.post_data,
headers=http_request.headers)
elif http_request.method == "HEAD":
response = http.head(url.request_uri,
headers=http_request.headers)
elif http_request.method == "PUT":
response = http.put(url.request_uri,
body=http_request.post_data,
headers=http_request.headers)
elif http_request.method == "POST":
response = http.post(url.request_uri,
body=http_request.post_data,
headers=http_request.headers)
elif http_request.method == "PATCH":
raise Exception("Unsupported gevent method={0}".format(http_request.method))
elif http_request.method == "OPTIONS":
raise Exception("Unsupported gevent method={0}".format(http_request.method))
elif http_request.method == "TRACE":
raise Exception("Unsupported gevent method={0}".format(http_request.method))
else:
raise Exception("Invalid gevent method={0}".format(http_request.method))
logger.debug("Http done, ms=%s", SolBase.msdiff(ms_start))
SolBase.sleep(0)
# Check
if not response:
raise Exception("No response from http")
# Process it
http_response.status_code = response.status_code
# Read
ms_start = SolBase.mscurrent()
logger.debug("Read now")
http_response.buffer = response.read()
SolBase.sleep(0)
logger.debug("Read done, ms=%s", SolBase.msdiff(ms_start))
if response.content_length:
http_response.content_length = response.content_length
else:
if http_response.buffer:
http_response.content_length = len(http_response.buffer)
else:
http_response.content_length = 0
# noinspection PyProtectedMember
for k, v in response._headers_index.items():
HttpClient._add_header(http_response.headers, k, v)
response.should_close()
# Over
SolBase.sleep(0)
def test_start_status_reload_stop_logfile(self):
"""
Test
"""
try:
# Start
self._reset_std_capture()
main_helper_file = self.current_dir + "CustomDaemon.py"
main_helper_file = abspath(main_helper_file)
self.assertTrue(FileUtility.is_file_exist(main_helper_file))
# Params
ar = list()
ar.append(sys.executable)
ar.append(main_helper_file)
ar.append("-pidfile={0}".format(self.daemon_pid_file))
ar.append("-stderr={0}".format(self.daemon_std_err))
ar.append("-stdout=/dev/null")
ar.append("-logfile={0}".format(self.daemon_std_out))
ar.append("start")
# =========================
# START
# =========================
# Launch
logger.info("Start : %s", " ".join(ar))
p = subprocess.Popen(args=ar)
logger.info("Started")
SolBase.sleep(0)
self._wait_process(p)
# Try wait for stdout
ms_start = SolBase.mscurrent()
while SolBase.msdiff(ms_start) < self.stdout_timeout_ms:
if "n".join(self._get_std_out()).find(
" INFO | CustomDaemon@_on_start") >= 0:
break
else:
SolBase.sleep(10)
# Get std (caution, we are async since forked)
logger.info("stdOut ### START")
for s in self._get_std_out():
logger.info("stdOut => %s", s)
logger.info("stdOut ### END")
logger.info("stdErr ### START")
for s in self._get_std_err():
logger.info("stdErr => %s", s)
logger.info("stdErr ### END")
# Check
self.assertTrue(len(self._get_std_err()) == 0)
self.assertTrue(len(self._get_std_out()) > 0)
self.assertTrue("n".join(self._get_std_out()).find(" ERROR ") < 0)
self.assertTrue("n".join(self._get_std_out()).find(
" INFO | CustomDaemon@_on_start") >= 0)
self.assertTrue("n".join(self._get_std_out()).find(" WARN ") < 0)
# =========================
# STATUS
# =========================
for _ in range(0, 10):
# Args
ar = list()
ar.append(sys.executable)
ar.append(main_helper_file)
ar.append("-pidfile={0}".format(self.daemon_pid_file))
ar.append("status")
# Launch
p = subprocess.Popen(args=ar)
self._wait_process(p)
# =========================
# RELOAD
# =========================
for _ in range(0, 10):
# Args
ar = list()
ar.append(sys.executable)
ar.append(main_helper_file)
ar.append("-pidfile={0}".format(self.daemon_pid_file))
ar.append("reload")
# Launch
p = subprocess.Popen(args=ar)
self._wait_process(p)
# =========================
# STOP
# =========================
# Args
ar = list()
ar.append(sys.executable)
ar.append(main_helper_file)
ar.append("-pidfile={0}".format(self.daemon_pid_file))
ar.append("stop")
# Launch
p = subprocess.Popen(args=ar)
self._wait_process(p)
# =========================
# OVER, CHECK LOGS
# =========================
# Try wait for stdout
ms_start = SolBase.mscurrent()
while SolBase.msdiff(ms_start) < self.stdout_timeout_ms:
if "n".join(self._get_std_out()).find(" INFO | CustomDaemon@_on_stop") >= 0 \
and "n".join(self._get_std_out()).find(" INFO | CustomDaemon@_on_status") >= 0:
break
else:
SolBase.sleep(10)
# Get std (caution, we are async since forked)
logger.info("stdOut ### START")
for s in self._get_std_out():
logger.info("stdOut => %s", s)
logger.info("stdOut ### END")
logger.info("stdErr ### START")
for s in self._get_std_err():
logger.info("stdErr => %s", s)
logger.info("stdErr ### END")
# Check
self.assertTrue(len(self._get_std_err()) == 0)
self.assertTrue(len(self._get_std_out()) > 0)
self.assertTrue("n".join(self._get_std_out()).find(" ERROR ") < 0)
self.assertTrue("n".join(self._get_std_out()).find(
" INFO | CustomDaemon@_on_start") >= 0)
self.assertTrue("n".join(self._get_std_out()).find(
" INFO | CustomDaemon@_on_stop") >= 0)
self.assertTrue("n".join(self._get_std_out()).find(
" INFO | CustomDaemon@_on_status") >= 0)
self.assertTrue("n".join(self._get_std_out()).find(" WARN ") < 0)
# =========================
# OVER, CHECK ACTION FILE
# =========================
buf = FileUtility.file_to_textbuffer(
CustomDaemon.DAEMON_LAST_ACTION_FILE, "ascii")
self.assertTrue(buf.find("is_running=False") >= 0)
self.assertTrue(buf.find("start_count=1") >= 0)
self.assertTrue(buf.find("stop_count=1") >= 0)
self.assertTrue(buf.find("status_count=10") >= 0)
self.assertTrue(buf.find("reload_count=10") >= 0)
self.assertTrue(buf.find("last_action=stop") >= 0)
finally:
logger.debug("Exiting test, idx=%s", self.run_idx)
def _go_greenlet(self,
greenlet_count,
pool_max,
sql="SELECT user, host FROM mysql.user LIMIT 1;",
check_exception=True):
"""
Doc
:param greenlet_count: greenlet_count
:type greenlet_count: int
:param pool_max: Pool max size
:type pool_max: int
:param sql: str
:type sql: str
:param check_exception: bool
:type check_exception: bool
"""
MysqlApi.reset_pools()
Meters.reset()
g_event = None
g_array = None
try:
# Settings
g_count = greenlet_count
g_ms = 5000
# Go
self.pool_max = pool_max
self.run_event = Event()
self.exception_raised = 0
self.pool_sql = sql
self.thread_running = AtomicIntSafe()
self.thread_running_ok = AtomicIntSafe()
# Signal
self.gorun_event = Event()
# Alloc greenlet
g_array = list()
g_event = list()
for _ in range(0, g_count):
greenlet = Greenlet()
g_array.append(greenlet)
g_event.append(Event())
# Run them
for idx in range(0, len(g_array)):
greenlet = g_array[idx]
event = g_event[idx]
greenlet.spawn(self._run_mysql_bench, event)
SolBase.sleep(0)
# Signal
self.gorun_event.set()
# Wait a bit
dt = SolBase.mscurrent()
while SolBase.msdiff(dt) < g_ms:
SolBase.sleep(1000)
# Stat
ms = SolBase.msdiff(dt)
sec = float(ms / 1000.0)
total_acquire = Meters.aig(
"k.db_pool.base.call.connection_acquire")
per_sec_acquire = round(float(total_acquire) / sec, 2)
total_release = Meters.aig(
"k.db_pool.base.call.connection_release")
per_sec_release = round(float(total_release) / sec, 2)
logger.info("Running..., run=%s, ok=%s, ps.ack/rel=%s/%s",
self.thread_running.get(),
self.thread_running_ok.get(), per_sec_acquire,
per_sec_release)
if check_exception:
self.assertEqual(self.exception_raised, 0)
# Over, signal
logger.info("Signaling")
self.run_event.set()
# Wait
for g in g_event:
g.wait(30.0)
self.assertTrue(g.isSet())
g_event = None
g_array = None
# Check it
self.assertEquals(
Meters.aig("k.db_pool.base.call.connection_acquire"),
Meters.aig("k.db_pool.base.call.connection_release"))
if check_exception:
self.assertEquals(
Meters.aig("k.db_pool.base.call.connection_acquire"),
Meters.aig("k.db_pool.mysql.call._connection_ping"))
self.assertLessEqual(
Meters.aig("k.db_pool.mysql.call._get_connection"), pool_max)
self.assertLessEqual(
Meters.aig("k.db_pool.mysql.call._connection_create"),
pool_max)
self.assertLessEqual(Meters.aig("k.db_pool.hash.cur"), 1)
self.assertLessEqual(Meters.aig("k.db_pool.base.cur_size"),
pool_max)
self.assertLessEqual(Meters.aig("k.db_pool.base.max_size"),
pool_max)
self.assertEquals(Meters.aig("k.db_pool.mysql.call.__init"), 1)
finally:
self.run_event.set()
if g_event:
for g in g_event:
g.set()
if g_array:
for g in g_array:
g.kill()
def _go_greenlet(self,
greenlet_count,
put_count,
get_count,
bench_item_count,
watchdog_interval_ms=60000,
max_item=128000,
max_bytes=32 * 1024 * 1024,
max_single_item_bytes=1 * 1024 * 1024,
purge_min_bytes=8 * 1024 * 1024,
purge_min_count=1000):
"""
Doc
:param greenlet_count: greenlet_count
:param put_count: put_count
:param get_count: get_count
:param bench_item_count: bench_item_count
:param watchdog_interval_ms: watchdog_interval_ms
:param max_item: max_item
:param max_bytes: max_bytes
:param max_single_item_bytes: max_single_item_bytes
:param purge_min_bytes: purge_min_bytes
:param purge_min_count: purge_min_count
"""
g_event = None
g_array = None
try:
# Settings
g_count = greenlet_count
g_ms = 10000
# Continue callback loop
self.callback_return = True
# Go
self.mem_cache = MemoryCache(
watchdog_interval_ms=watchdog_interval_ms,
max_item=max_item,
max_bytes=max_bytes,
max_single_item_bytes=max_single_item_bytes,
purge_min_bytes=purge_min_bytes,
purge_min_count=purge_min_count)
# Item count
self.bench_item_count = bench_item_count
self.bench_put_weight = put_count
self.bench_get_weight = get_count
self.bench_ttl_min_ms = 1000
self.bench_ttl_max_ms = int(g_ms / 2)
# Go
self.run_event = Event()
self.exception_raised = 0
self.open_count = 0
self.thread_running = AtomicIntSafe()
self.thread_running_ok = AtomicIntSafe()
# Item per greenlet
item_per_greenlet = self.bench_item_count / g_count
# Signal
self.gorun_event = Event()
# Alloc greenlet
g_array = list()
g_event = list()
for _ in range(0, g_count):
greenlet = Greenlet()
g_array.append(greenlet)
g_event.append(Event())
# Run them
cur_idx = 0
for idx in range(0, len(g_array)):
greenlet = g_array[idx]
event = g_event[idx]
greenlet.spawn(self._run_cache_bench, event, cur_idx,
cur_idx + item_per_greenlet)
cur_idx += item_per_greenlet
SolBase.sleep(0)
# Signal
self.gorun_event.set()
# Wait a bit
dt = SolBase.mscurrent()
while SolBase.msdiff(dt) < g_ms:
SolBase.sleep(500)
# Stat
ms = SolBase.msdiff(dt)
sec = float(ms / 1000.0)
total_put = Meters.aig("mcs.cache_put")
per_sec_put = round(float(total_put) / sec, 2)
total_get = Meters.aig("mcs.cache_get_hit") + Meters.aig(
"mcs.cache_get_miss")
per_sec_get = round(float(total_get) / sec, 2)
logger.info(
"Running..., count=%s, run=%s, ok=%s, put/sec=%s get/sec=%s, cache=%s",
self.open_count, self.thread_running.get(),
self.thread_running_ok.get(), per_sec_put, per_sec_get,
self.mem_cache)
self.assertEqual(self.exception_raised, 0)
# Over, signal
logger.info("Signaling, count=%s", self.open_count)
self.run_event.set()
# Wait
for g in g_event:
g.wait(30.0)
self.assertTrue(g.isSet())
g_event = None
g_array = None
# Log
Meters.write_to_logger()
finally:
self.run_event.set()
if g_event:
for g in g_event:
g.set()
if g_array:
for g in g_array:
g.kill()
if self.mem_cache:
max_count = 0
total_size = 0
i = 0
for (k, v) in self.mem_cache._hash_key.items():
i += 1
total_size += len(k) + len(v[1])
if i < max_count:
logger.info("%s => %s", k, v)
self.assertEqual(total_size,
self.mem_cache._current_data_bytes.get())
self.mem_cache.stop_cache()
self.mem_cache = None
def test_basic(self):
"""
Test.
"""
# Alloc
self.mem_cache = MemoryCache()
# Get : must return nothing
o = self.mem_cache.get("not_found")
self.assertIsNone(o)
# Put
self.mem_cache.put("keyA", b"valA", 60000)
o = self.mem_cache.get("keyA")
self.assertEqual(o, b"valA")
o = self.mem_cache.get_raw("keyA")
self.assertIsNotNone(o)
self.assertIsInstance(o, tuple)
self.assertEqual(o[1], b"valA")
self.assertLessEqual(o[0] - SolBase.mscurrent(), 60000)
logger.info("TTL approx=%s", o[0] - SolBase.mscurrent())
# Put with lower TTL : TTL MUST BE UPDATED
self.mem_cache.put("keyA", b"valA", 30000)
o = self.mem_cache.get("keyA")
self.assertEqual(o, b"valA")
o = self.mem_cache.get_raw("keyA")
self.assertIsNotNone(o)
self.assertIsInstance(o, tuple)
self.assertEqual(o[1], b"valA")
self.assertLessEqual(o[0] - SolBase.mscurrent(), 30000)
logger.info("TTL approx=%s", o[0] - SolBase.mscurrent())
# Str put : must now be ok
self.mem_cache.put("toto_str", "str_val", 1000)
self.assertEqual(self.mem_cache.get("toto_str"), "str_val")
# Non bytes,str injection (int) : must fail
# noinspection PyBroadException,PyPep8
try:
# noinspection PyTypeChecker
self.mem_cache.put("toto", 12, 1000)
self.fail("Must fail")
except:
pass
# Non bytes injection : must fail
# noinspection PyBroadException,PyPep8
try:
# noinspection PyTypeChecker
self.mem_cache.put("toto", u"unicode_buffer", 1000)
self.fail("Must fail")
except:
pass
# This MUST fail
# noinspection PyBroadException
try:
# noinspection PyTypeChecker
self.mem_cache.put(999, b"value", 60000)
self.fail("Put a key as non bytes,str MUST fail")
except Exception:
pass
# This MUST fail
# noinspection PyBroadException
try:
# noinspection PyTypeChecker
self.mem_cache.remove(999)
self.fail("Remove a key as non bytes,str MUST fail")
except Exception:
pass
# Put/Remove
self.mem_cache.put("todel", b"value", 60000)
self.assertEqual(self.mem_cache.get("todel"), b"value")
self.mem_cache.remove("todel")
self.assertEqual(self.mem_cache.get("todel"), None)
# Put
self.mem_cache.put("KEY \u001B\u0BD9\U0001A10D\u1501\xc3", b"zzz",
60000)
self.assertEqual(
self.mem_cache.get("KEY \u001B\u0BD9\U0001A10D\u1501\xc3"), b"zzz")
# Stop
self.mem_cache.stop_cache()
self.mem_cache = None
