def MEMCACHED_CLIENT(): global MEMCACHED__CLIENT__ global MEMCACHED_SERVER global MEMCACHED_VERSION global flush_types if not MEMCACHED_SERVER: try: MEMCACHED_SERVER = eval(request.env['ir.config_parameter'].get_param('website_memcached.memcached_db') or '("localhost",11211)') except: MEMCACHED_SERVER = ("localhost",11211) if not MEMCACHED__CLIENT__: try: #~ if type(servers) == list: #~ MEMCACHED__CLIENT__ = HashClient(servers, serializer=serialize_pickle, deserializer=deserialize_pickle) #~ else: MEMCACHED__CLIENT__ = PooledClient(MEMCACHED_SERVER, serializer=serialize_pickle, deserializer=deserialize_pickle,no_delay=MEMCACHE_NODELAY,connect_timeout=MEMCACHE_CONNECT_TIMEOUT,timeout=MEMCACHE_TIMEOUT) MEMCACHED_VERSION = MEMCACHED__CLIENT__.version() ## Retreive all flush_types per database # https://www.tutorialspoint.com/memcached/memcached_stats_items.htm # echo "stats items"|nc localhost 11211|grep number # STAT items:11:number 3 # STAT items:12:number 3 # STAT items:13:number 1 # STAT items:14:number 19 # STAT items:15:number 1212 items = MEMCACHED__CLIENT__.stats('items') slab_limit = {k.split(':')[1]:v for k,v in MEMCACHED__CLIENT__.stats('items').items() if k.split(':')[2] == 'number' } # slab -> limit # echo "stats cachedump 15 1212 "|nc localhost 11211 # slab limit # ITEM 4092067750 [2231 b; 1561018218 s] # ITEM 3699334878 [1974 b; 1560964179 s] # ITEM 2768968127 [2071 b; 1560968016 s] # ITEM 2482188247 [2126 b; 1561020033 s] # ITEM 2293401784 [2086 b; 1560972986 s] key_lists = [MEMCACHED__CLIENT__.stats('cachedump',slab,str(limit)) for slab,limit in slab_limit.items()] # List of lists keys = [key for sublist in key_lists for key in sublist.keys()] # [4092067750, 3699334878 ...] flattended list for key in keys: page = MEMCACHED__CLIENT__.get(key) # echo "get 4092067750 "|nc localhost 11211 -> dict with data if page and page.get('db'): if not flush_types.get(page['db'], None): flush_types[page['db']] = set() flush_types[page['db']].add(page.get('flush_type')) except Exception as e: err = sys.exc_info() error = ''.join(traceback.format_exception(err[0], err[1], err[2])) _logger.info('Cannot instantiate MEMCACHED CLIENT\n%s' % error) raise MemcacheServerError(e) except TypeError as e: _logger.info('Type error MEMCACHED CLIENT %s.' % e) raise MemcacheServerError(e) return MEMCACHED__CLIENT__
class PymemcacheCacheBackend(NoCacheBackend): def __init__(self, config): cache_settings = config['cache_backend_settings'] host = cache_settings.get('host') port = cache_settings.get('port') if not host or not port: raise ValueError( ('Must provide `config.cache_backend_settings.[host|port]`' ' with `PymemcacheCacheBackend!')) self.host = host self.port = port self._make_client() def _make_client(self): self.client = PooledClient( (self.host, self.port), serde=serde.pickle_serde, ) def get(self, key): try: return self.client.get(key) except Exception as e: print(f'FAILED CACHE GET: {e.__class__.__name__}({e.args})') def set(self, key, value): try: return self.client.set(key, value) except Exception as e: print(f'FAILED CACHE SET: {e.__class__.__name__}({e.args})') def delete(self, key): try: return self.client.delete(key) except Exception as e: print(f'FAILED CACHE DELETE: {e.__class__.__name__}({e.args})')
def MEMCACHED_CLIENT(): global MEMCACHED__CLIENT__ global MEMCACHED_SERVER global MEMCACHED_VERSION global flush_types if not MEMCACHED_SERVER: try: MEMCACHED_SERVER = eval(request.env['ir.config_parameter'].get_param('website_memcached.memcached_db') or '("localhost",11211)') except: MEMCACHED_SERVER = ("localhost",11211) if not MEMCACHED__CLIENT__: try: #~ if type(servers) == list: #~ MEMCACHED__CLIENT__ = HashClient(servers, serializer=serialize_pickle, deserializer=deserialize_pickle) #~ else: MEMCACHED__CLIENT__ = PooledClient(MEMCACHED_SERVER, serializer=serialize_pickle, deserializer=deserialize_pickle,no_delay=MEMCACHE_NODELAY,connect_timeout=MEMCACHE_CONNECT_TIMEOUT,timeout=MEMCACHE_TIMEOUT) MEMCACHED_VERSION = MEMCACHED__CLIENT__.version() items = MEMCACHED__CLIENT__.stats('items') slab_limit = {k.split(':')[1]:v for k,v in MEMCACHED__CLIENT__.stats('items').items() if k.split(':')[2] == 'number' } key_lists = [MEMCACHED__CLIENT__.stats('cachedump',slab,str(limit)) for slab,limit in slab_limit.items()] keys = [key for sublist in key_lists for key in sublist.keys()] for key in keys: page = MEMCACHED__CLIENT__.get(key) if page and page.get('db'): if not flush_types.get(page['db'], None): flush_types[page['db']] = set() flush_types[page['db']].add(page.get('flush_type')) #~ server: tuple(hostname, port) #~ serializer: optional function, see notes in the class docs. #~ deserializer: optional function, see notes in the class docs. #~ connect_timeout: optional float, seconds to wait for a connection to #~ the memcached server. Defaults to "forever" (uses the underlying #~ default socket timeout, which can be very long). #~ timeout: optional float, seconds to wait for send or recv calls on #~ the socket connected to memcached. Defaults to "forever" (uses the #~ underlying default socket timeout, which can be very long). #~ no_delay: optional bool, set the TCP_NODELAY flag, which may help #~ with performance in some cases. Defaults to False. #~ ignore_exc: optional bool, True to cause the "get", "gets", #~ "get_many" and "gets_many" calls to treat any errors as cache #~ misses. Defaults to False. #~ socket_module: socket module to use, e.g. gevent.socket. Defaults to #~ the standard library's socket module. #~ key_prefix: Prefix of key. You can use this as namespace. Defaults #~ to b''. #~ default_noreply: bool, the default value for 'noreply' as passed to #~ store commands (except from cas, incr, and decr, which default to #~ False). #~ allow_unicode_keys: bool, support unicode (utf8) keys #http://pymemcache.readthedocs.io/en/latest/getting_started.html except Exception as e: err = sys.exc_info() error = ''.join(traceback.format_exception(err[0], err[1], err[2])) _logger.info('Cannot instantiate MEMCACHED CLIENT\n%s' % error) raise MemcacheServerError(e) except TypeError as e: _logger.info('Type error MEMCACHED CLIENT %s.' % e) raise MemcacheServerError(e) return MEMCACHED__CLIENT__
class MemcachedCache(Cache): def serialize(self, key, value): print key, type(value) if type(value) == str: # only str, not unicode or extend classes #print 1, len(value) return value, 1 if isinstance(value, np.ndarray): # any ndarray #return zlib.compress(value.dumps()), 2 #print 2, key, len(value.dumps()) return value.dumps(), 2 # other types #print 3, len(json.dumps(value)) try: print json.dumps(value) except Exception as e: #print 4, e raise return json.dumps(value), 3 def deserialize(self, key, value, flags): if flags == 1: # str return value if flags == 2: # ndarray #return np.loads(zlib.decompress(value)) return np.loads(value) if flags == 3: # other return json.loads(value) raise TypeError("Unknown flags for value: %d" % flags) def __init__(self, server=('localhost', 11211), key_prefix='', del_on_server=False, raise_on_key=False, raise_on_none=True): self._client = PooledClient(server, key_prefix=key_prefix, serializer=self.serialize, deserializer=self.deserialize) self._keys = set() self._del_on_server = del_on_server self._raise_on_key = raise_on_key self._raise_on_none = raise_on_none def __getitem__(self, key): if self._raise_on_key and key not in self._keys: raise KeyError value = self._client.get(key) if self._raise_on_none and value is None: raise KeyError return value def __setitem__(self, key, value): self._client.set(key, value) self._keys.add(key) def __delitem__(self, key): if self._del_on_server: self._client.delete(key) self._keys.discard(key) def __len__(self): return len(self._keys) def __iter__(self): return self._keys.__iter__() def keys(self): return list(self._keys) def clear(self): for key in self._keys: del self[key] def __del__(self): self.clear() self._client.close()