def MEMCACHED_CLIENT():
global MEMCACHED__CLIENT__
global MEMCACHED_SERVER
global MEMCACHED_VERSION
global flush_types
if not MEMCACHED_SERVER:
try:
MEMCACHED_SERVER = eval(request.env['ir.config_parameter'].get_param('website_memcached.memcached_db') or '("localhost",11211)')
except:
MEMCACHED_SERVER = ("localhost",11211)
if not MEMCACHED__CLIENT__:
try:
#~ if type(servers) == list:
#~ MEMCACHED__CLIENT__ = HashClient(servers, serializer=serialize_pickle, deserializer=deserialize_pickle)
#~ else:
MEMCACHED__CLIENT__ = PooledClient(MEMCACHED_SERVER, serializer=serialize_pickle, deserializer=deserialize_pickle,no_delay=MEMCACHE_NODELAY,connect_timeout=MEMCACHE_CONNECT_TIMEOUT,timeout=MEMCACHE_TIMEOUT)
MEMCACHED_VERSION = MEMCACHED__CLIENT__.version()
## Retreive all flush_types per database
# https://www.tutorialspoint.com/memcached/memcached_stats_items.htm
# echo "stats items"|nc localhost 11211|grep number
# STAT items:11:number 3
# STAT items:12:number 3
# STAT items:13:number 1
# STAT items:14:number 19
# STAT items:15:number 1212
items = MEMCACHED__CLIENT__.stats('items')
slab_limit = {k.split(':')[1]:v for k,v in MEMCACHED__CLIENT__.stats('items').items() if k.split(':')[2] == 'number' } # slab -> limit
# echo "stats cachedump 15 1212 "|nc localhost 11211 # slab limit
# ITEM 4092067750 [2231 b; 1561018218 s]
# ITEM 3699334878 [1974 b; 1560964179 s]
# ITEM 2768968127 [2071 b; 1560968016 s]
# ITEM 2482188247 [2126 b; 1561020033 s]
# ITEM 2293401784 [2086 b; 1560972986 s]
key_lists = [MEMCACHED__CLIENT__.stats('cachedump',slab,str(limit)) for slab,limit in slab_limit.items()] # List of lists
keys = [key for sublist in key_lists for key in sublist.keys()] # [4092067750, 3699334878 ...] flattended list
for key in keys:
page = MEMCACHED__CLIENT__.get(key)
# echo "get 4092067750 "|nc localhost 11211 -> dict with data
if page and page.get('db'):
if not flush_types.get(page['db'], None):
flush_types[page['db']] = set()
flush_types[page['db']].add(page.get('flush_type'))
except Exception as e:
err = sys.exc_info()
error = ''.join(traceback.format_exception(err[0], err[1], err[2]))
_logger.info('Cannot instantiate MEMCACHED CLIENT\n%s' % error)
raise MemcacheServerError(e)
except TypeError as e:
_logger.info('Type error MEMCACHED CLIENT %s.' % e)
raise MemcacheServerError(e)
return MEMCACHED__CLIENT__
class PymemcacheCacheBackend(NoCacheBackend):
def __init__(self, config):
cache_settings = config['cache_backend_settings']
host = cache_settings.get('host')
port = cache_settings.get('port')
if not host or not port:
raise ValueError(
('Must provide `config.cache_backend_settings.[host|port]`'
' with `PymemcacheCacheBackend!'))
self.host = host
self.port = port
self._make_client()
def _make_client(self):
self.client = PooledClient(
(self.host, self.port),
serde=serde.pickle_serde,
)
def get(self, key):
try:
return self.client.get(key)
except Exception as e:
print(f'FAILED CACHE GET: {e.__class__.__name__}({e.args})')
def set(self, key, value):
try:
return self.client.set(key, value)
except Exception as e:
print(f'FAILED CACHE SET: {e.__class__.__name__}({e.args})')
def delete(self, key):
try:
return self.client.delete(key)
except Exception as e:
print(f'FAILED CACHE DELETE: {e.__class__.__name__}({e.args})')
def MEMCACHED_CLIENT():
global MEMCACHED__CLIENT__
global MEMCACHED_SERVER
global MEMCACHED_VERSION
global flush_types
if not MEMCACHED_SERVER:
try:
MEMCACHED_SERVER = eval(request.env['ir.config_parameter'].get_param('website_memcached.memcached_db') or '("localhost",11211)')
except:
MEMCACHED_SERVER = ("localhost",11211)
if not MEMCACHED__CLIENT__:
try:
#~ if type(servers) == list:
#~ MEMCACHED__CLIENT__ = HashClient(servers, serializer=serialize_pickle, deserializer=deserialize_pickle)
#~ else:
MEMCACHED__CLIENT__ = PooledClient(MEMCACHED_SERVER, serializer=serialize_pickle, deserializer=deserialize_pickle,no_delay=MEMCACHE_NODELAY,connect_timeout=MEMCACHE_CONNECT_TIMEOUT,timeout=MEMCACHE_TIMEOUT)
MEMCACHED_VERSION = MEMCACHED__CLIENT__.version()
items = MEMCACHED__CLIENT__.stats('items')
slab_limit = {k.split(':')[1]:v for k,v in MEMCACHED__CLIENT__.stats('items').items() if k.split(':')[2] == 'number' }
key_lists = [MEMCACHED__CLIENT__.stats('cachedump',slab,str(limit)) for slab,limit in slab_limit.items()]
keys = [key for sublist in key_lists for key in sublist.keys()]
for key in keys:
page = MEMCACHED__CLIENT__.get(key)
if page and page.get('db'):
if not flush_types.get(page['db'], None):
flush_types[page['db']] = set()
flush_types[page['db']].add(page.get('flush_type'))
#~ server: tuple(hostname, port)
#~ serializer: optional function, see notes in the class docs.
#~ deserializer: optional function, see notes in the class docs.
#~ connect_timeout: optional float, seconds to wait for a connection to
#~ the memcached server. Defaults to "forever" (uses the underlying
#~ default socket timeout, which can be very long).
#~ timeout: optional float, seconds to wait for send or recv calls on
#~ the socket connected to memcached. Defaults to "forever" (uses the
#~ underlying default socket timeout, which can be very long).
#~ no_delay: optional bool, set the TCP_NODELAY flag, which may help
#~ with performance in some cases. Defaults to False.
#~ ignore_exc: optional bool, True to cause the "get", "gets",
#~ "get_many" and "gets_many" calls to treat any errors as cache
#~ misses. Defaults to False.
#~ socket_module: socket module to use, e.g. gevent.socket. Defaults to
#~ the standard library's socket module.
#~ key_prefix: Prefix of key. You can use this as namespace. Defaults
#~ to b''.
#~ default_noreply: bool, the default value for 'noreply' as passed to
#~ store commands (except from cas, incr, and decr, which default to
#~ False).
#~ allow_unicode_keys: bool, support unicode (utf8) keys
#http://pymemcache.readthedocs.io/en/latest/getting_started.html
except Exception as e:
err = sys.exc_info()
error = ''.join(traceback.format_exception(err[0], err[1], err[2]))
_logger.info('Cannot instantiate MEMCACHED CLIENT\n%s' % error)
raise MemcacheServerError(e)
except TypeError as e:
_logger.info('Type error MEMCACHED CLIENT %s.' % e)
raise MemcacheServerError(e)
return MEMCACHED__CLIENT__
class MemcachedCache(Cache):
def serialize(self, key, value):
print key, type(value)
if type(value) == str: # only str, not unicode or extend classes
#print 1, len(value)
return value, 1
if isinstance(value, np.ndarray): # any ndarray
#return zlib.compress(value.dumps()), 2
#print 2, key, len(value.dumps())
return value.dumps(), 2
# other types
#print 3, len(json.dumps(value))
try:
print json.dumps(value)
except Exception as e:
#print 4, e
raise
return json.dumps(value), 3
def deserialize(self, key, value, flags):
if flags == 1: # str
return value
if flags == 2: # ndarray
#return np.loads(zlib.decompress(value))
return np.loads(value)
if flags == 3: # other
return json.loads(value)
raise TypeError("Unknown flags for value: %d" % flags)
def __init__(self, server=('localhost', 11211), key_prefix='',
del_on_server=False, raise_on_key=False, raise_on_none=True):
self._client = PooledClient(server, key_prefix=key_prefix,
serializer=self.serialize, deserializer=self.deserialize)
self._keys = set()
self._del_on_server = del_on_server
self._raise_on_key = raise_on_key
self._raise_on_none = raise_on_none
def __getitem__(self, key):
if self._raise_on_key and key not in self._keys:
raise KeyError
value = self._client.get(key)
if self._raise_on_none and value is None:
raise KeyError
return value
def __setitem__(self, key, value):
self._client.set(key, value)
self._keys.add(key)
def __delitem__(self, key):
if self._del_on_server:
self._client.delete(key)
self._keys.discard(key)
def __len__(self):
return len(self._keys)
def __iter__(self):
return self._keys.__iter__()
def keys(self):
return list(self._keys)
def clear(self):
for key in self._keys:
del self[key]
def __del__(self):
self.clear()
self._client.close()
