def __init__(self): """Connect to the memcache server(s).""" self.pool = None try: import pylibmc #TODO: ALL INPUT IS EVIL #regex check these variables mc = pylibmc.Client( servers=[os.environ.get(self.CACHE_SERVER_VAR)], username=os.environ.get(self.CACHE_USER_VAR), password=os.environ.get(self.CACHE_PASS_VAR), binary=True) self.pool = pylibmc.ThreadMappedPool(mc) except ImportError: print >> sys.stderr, "ERROR: Could not import module 'pylibmc' - memcache is disabled. Please install the module and try again." self.pool = None except AttributeError: print >> sys.stderr, "ERROR: Could not connect to the memcache server '%s' as user '%s'. memcache is disabled.\ Please check that the server is running, check your memcache environment variables, and try again." \ % (os.environ.get(self.CACHE_SERVER_VAR), os.environ.get(self.CACHE_PASS_VAR)) self.pool = None except TypeError, e: # thrown by pylibmc e.g. if the wrong password was supplied print >> sys.stderr, "ERROR: Could not connect to memcache server: '%s'. memcache is disabled." % ( str(e)) self.pool = None
def test_simple(self): a_str = "a" p = pylibmc.ThreadMappedPool(self.mc) with p.reserve() as smc: self.assertTrue(smc) self.assertTrue(smc.set(a_str, 1)) self.assertEqual(smc[a_str], 1)
def test_simple(self): a_str = "a" p = pylibmc.ThreadMappedPool(self.mc) with p.reserve() as smc: ok_(smc) ok_(smc.set(a_str, 1)) eq_(smc[a_str], 1)
def __init__(self, *args, **kwargs): self.pool = kwargs.pop('pool', None) or ThreadPoolExecutor(1) self.mc = pylibmc.Client(*args, **kwargs) self.mc_pool = pylibmc.ThreadMappedPool(self.mc) self.loop = asyncio.get_event_loop()
def get_cs(addrs, key, binary=None, behaviors=None, ping_test=None): cs = None cache_key = (os.getpid(), key) error_key = (os.getpid(), cache_key[1], tuple(addrs)) trace = None error = False if ping_test is None: ping_test = True if not behaviors: behaviors = {"tcp_nodelay": True, "ketama": True} if binary is None: binary = True try: cs = _MC_SERVERS['cache'][cache_key] except KeyError: cs = None if not cs: try: _MC_SERVERS['error'][error_key] except KeyError: pass else: # retry failed memcache server in ten minutes if time.time() < (_MC_SERVERS['error'][error_key] + (10 * 60)): ping_test = False error = True if not error and not cs: try: mcs = pylibmc.Client(addrs, binary=binary, behaviors=behaviors) # threadsafe pools cs = _MC_SERVERS['cache'][cache_key] = pylibmc.ThreadMappedPool( mcs) trace, error = None, False except (pylibmc.WriteError, ): error = 'Local memcached server is not reachable (w)' except (Exception, ): trace = traceback.format_exc() error = 'Local memcached server is not reachable' if cs is not None and ping_test and not error: try: cache_set(cs, 'mc_states_ping', 'ping') trace, error = None, False except (pylibmc.WriteError, ): error = 'Local memcached server is not usable (w)' except (Exception, ): trace = traceback.format_exc() error = 'Local memcached server is not usable' if error: _MC_SERVERS['error'][error_key] = time.time() try: del _MC_SERVERS['cache'][cache_key] except KeyError: pass cs = None else: _MC_SERVERS['cache'][cache_key] = cs return cs, error, trace
def __init__(self): self.pool = pylibmc.ThreadMappedPool( pylibmc.Client(servers=[ '{}:{}'.format(settings.MEMCACHED_HOST, settings.MEMCACHED_PORT) ], behaviors={ "tcp_keepalive": True, "tcp_nodelay": True }), )
def __init__(self, nntscid, cachetime): self.memcache = pylibmc.Client( ["127.0.0.1"], behaviors={ "tcp_nodelay": True, "no_block": True, }) self.mcpool = pylibmc.ThreadMappedPool(self.memcache) self.nntscid = nntscid self.cachetime = int(cachetime) self.collections = {}
def threaded_load(self, key): if not self.pool: self.pool = pylibmc.ThreadMappedPool(self.conn) key = "{}_{}".format(self.site, key) result = False with self.pool.reserve() as mc: try: result = mc.get(key) except pylibmc.ConnectionError: self.connect() result = False self.pool.relinquish() return result
def __init__(self, ampy): self.event_timeseries = {} self.site_counts = {} self.common_events = {} self.rare_events = {} self.allevents = set() self.ampy = ampy self.binsize = 30 * 60 self.memcache = pylibmc.Client( ["127.0.0.1"], behaviors={ "tcp_nodelay": True, "no_block": True, }) self.mcpool = pylibmc.ThreadMappedPool(self.memcache)
def threaded_save(self, key, value): if not self.pool: self.pool = pylibmc.ThreadMappedPool(self.conn) key = self.site + "-" + key with self.pool.reserve() as mc: for i in range(MAX_RETRIES): try: mc.set(str(key), str(value)) break except Exception: log_traceback(f"Cache save failed ({key})") time.sleep(0.3) self.connect() else: critical_error( "Memcache save failed. This should never happen.") self.pool.relinquish() return True
def __init__(self, *arg, **kw): super(PyLibMCNamespaceManager, self).__init__(*arg, **kw) memcache_module = kw.get('memcache_module', 'auto') _memcache_module = _client_libs[memcache_module] protocol = kw.get('protocol', 'text') username = kw.get('username', None) password = kw.get('password', None) url = kw.get('url') behaviors = parse_memcached_behaviors(kw) self.mc = MemcachedNamespaceManager.clients.get( (memcache_module, url), _memcache_module.Client, servers=url.split(';'), behaviors=behaviors, binary=(protocol == 'binary'), username=username, password=password) self.pool = pylibmc.ThreadMappedPool(self.mc)
def __init__(self, blocksize): """ Init function for the AmpyCache class. Parameters: blocksize -- the number of binned measurements to include in a data block. """ self.blocksize = blocksize self.memcache = pylibmc.Client(["127.0.0.1"], behaviors={ "tcp_nodelay": True, "no_block": True, }) self.mcpool = pylibmc.ThreadMappedPool(self.memcache) self.streamview_cachetime = 60 * 60 * 24 self.viewgroups_cachetime = 60 * 60 * 6
def threaded_save(self, key, value): if not self.pool: self.pool = pylibmc.ThreadMappedPool(self.conn) key = self.site + "-" + key with self.pool.reserve() as mc: for i in range(10): try: mc.set(str(key), str(value)) break except Exception: log_traceback("Cache save failed ({})".format(key)) time.sleep(.3) self.connect() else: critical_error( "Memcache save failed. This should never happen. Check MC server" ) sys.exit(-1) self.pool.relinquish() return True
def threaded_delete(self, key): if not self.pool: self.pool = pylibmc.ThreadMappedPool(self.conn) key = "{}_{}".format(self.site, key) with self.pool.reserve() as mc: for i in range(10): try: mc.delete(key) break except: log_traceback("Cache delete failed ({})".format(key)) time.sleep(.3) self.connect() else: critical_error( "Memcache delete failed. This should never happen. Check MC server" ) sys.exit(-1) self.pool.relinquish() return True
def __init__(self, server, params): super(CacheClass, self).__init__(params) mc = pylibmc.Client(server.split(';')) mc.behaviors = {"tcp_nodelay": True} self._pool = pylibmc.ThreadMappedPool(mc)
def __init__(self, hosts=['localhost'], *args, **kw): client = pylibmc.Client(hosts) self.pool = pylibmc.ThreadMappedPool(client) BaseBackend.__init__(self, *args, **kw)
return self._mc.set(key, value, time) # FIXME: deprecated 以下为兼容代码 def mclient(servers): return Client(servers) ################################################### # cache define # ################################################### # master func_cache = mclient(memcache_settings["func_cache"]) pool_model_cache = pylibmc.ThreadMappedPool(func_cache) class PyPoolMemcache(object): @classmethod def incr(cls, key, pool=pool_model_cache): with pool.reserve() as _mc: result = _mc.incr(key) return result @classmethod def decr(cls, key, pool=pool_model_cache): with pool.reserve() as _mc: result = _mc.decr(key) return result
def __init__(self, servers): ObjectCache.__init__(self) self.mc = pylibmc.Client(servers) self.mc_pool = pylibmc.ThreadMappedPool(self.mc)
def __init__(self, client, adapter, *, prefix="anom"): self.client_pool = pylibmc.ThreadMappedPool(client) self.adapter = adapter self.prefix = prefix self.query = self.adapter.query
def __init__(self, *arg, **kw): super(PyLibMCNamespaceManager, self).__init__(*arg, **kw) self.pool = pylibmc.ThreadMappedPool(self.mc)
import hashlib import logging import os import re import sys from contextlib import closing from threading import local import requests import pylibmc import remoulade logger = logging.getLogger("example") memcache_client = pylibmc.Client(["localhost"], binary=True) memcache_pool = pylibmc.ThreadMappedPool(memcache_client) anchor_re = re.compile(rb'<a href="([^"]+)">') state = local() if os.getenv("REDIS") == "1": from remoulade.brokers.redis import RedisBroker broker = RedisBroker() remoulade.set_broker(broker) def get_session(): session = getattr(state, "session", None) if session is None: session = state.session = requests.Session() return session
def __init__(self, server, params): super(CacheClass, self).__init__(params) mc = pylibmc.Client(server.split(';')) mc.behaviors = PYLIBMC_BEHAVIORS self._pool = pylibmc.ThreadMappedPool(mc) self._logger = CacheLogger()
def __init__(self, client, adapter, *, namespace="anom"): self.client_pool = pylibmc.ThreadMappedPool(client) self.adapter = adapter self.namespace = namespace self.query = self.adapter.query