def __init__(self):
"""Connect to the memcache server(s)."""
self.pool = None
try:
import pylibmc
#TODO: ALL INPUT IS EVIL
#regex check these variables
mc = pylibmc.Client(
servers=[os.environ.get(self.CACHE_SERVER_VAR)],
username=os.environ.get(self.CACHE_USER_VAR),
password=os.environ.get(self.CACHE_PASS_VAR),
binary=True)
self.pool = pylibmc.ThreadMappedPool(mc)
except ImportError:
print >> sys.stderr, "ERROR: Could not import module 'pylibmc' - memcache is disabled. Please install the module and try again."
self.pool = None
except AttributeError:
print >> sys.stderr, "ERROR: Could not connect to the memcache server '%s' as user '%s'. memcache is disabled.\
Please check that the server is running, check your memcache environment variables, and try again." \
% (os.environ.get(self.CACHE_SERVER_VAR), os.environ.get(self.CACHE_PASS_VAR))
self.pool = None
except TypeError, e:
# thrown by pylibmc e.g. if the wrong password was supplied
print >> sys.stderr, "ERROR: Could not connect to memcache server: '%s'. memcache is disabled." % (
str(e))
self.pool = None
def test_simple(self):
a_str = "a"
p = pylibmc.ThreadMappedPool(self.mc)
with p.reserve() as smc:
self.assertTrue(smc)
self.assertTrue(smc.set(a_str, 1))
self.assertEqual(smc[a_str], 1)
def test_simple(self):
a_str = "a"
p = pylibmc.ThreadMappedPool(self.mc)
with p.reserve() as smc:
ok_(smc)
ok_(smc.set(a_str, 1))
eq_(smc[a_str], 1)
def __init__(self, *args, **kwargs):
self.pool = kwargs.pop('pool', None) or ThreadPoolExecutor(1)
self.mc = pylibmc.Client(*args, **kwargs)
self.mc_pool = pylibmc.ThreadMappedPool(self.mc)
self.loop = asyncio.get_event_loop()
def get_cs(addrs, key, binary=None, behaviors=None, ping_test=None):
cs = None
cache_key = (os.getpid(), key)
error_key = (os.getpid(), cache_key[1], tuple(addrs))
trace = None
error = False
if ping_test is None:
ping_test = True
if not behaviors:
behaviors = {"tcp_nodelay": True, "ketama": True}
if binary is None:
binary = True
try:
cs = _MC_SERVERS['cache'][cache_key]
except KeyError:
cs = None
if not cs:
try:
_MC_SERVERS['error'][error_key]
except KeyError:
pass
else:
# retry failed memcache server in ten minutes
if time.time() < (_MC_SERVERS['error'][error_key] + (10 * 60)):
ping_test = False
error = True
if not error and not cs:
try:
mcs = pylibmc.Client(addrs, binary=binary, behaviors=behaviors)
# threadsafe pools
cs = _MC_SERVERS['cache'][cache_key] = pylibmc.ThreadMappedPool(
mcs)
trace, error = None, False
except (pylibmc.WriteError, ):
error = 'Local memcached server is not reachable (w)'
except (Exception, ):
trace = traceback.format_exc()
error = 'Local memcached server is not reachable'
if cs is not None and ping_test and not error:
try:
cache_set(cs, 'mc_states_ping', 'ping')
trace, error = None, False
except (pylibmc.WriteError, ):
error = 'Local memcached server is not usable (w)'
except (Exception, ):
trace = traceback.format_exc()
error = 'Local memcached server is not usable'
if error:
_MC_SERVERS['error'][error_key] = time.time()
try:
del _MC_SERVERS['cache'][cache_key]
except KeyError:
pass
cs = None
else:
_MC_SERVERS['cache'][cache_key] = cs
return cs, error, trace
def __init__(self):
self.pool = pylibmc.ThreadMappedPool(
pylibmc.Client(servers=[
'{}:{}'.format(settings.MEMCACHED_HOST,
settings.MEMCACHED_PORT)
],
behaviors={
"tcp_keepalive": True,
"tcp_nodelay": True
}), )
def __init__(self, nntscid, cachetime):
self.memcache = pylibmc.Client(
["127.0.0.1"],
behaviors={
"tcp_nodelay": True,
"no_block": True,
})
self.mcpool = pylibmc.ThreadMappedPool(self.memcache)
self.nntscid = nntscid
self.cachetime = int(cachetime)
self.collections = {}
def threaded_load(self, key):
if not self.pool:
self.pool = pylibmc.ThreadMappedPool(self.conn)
key = "{}_{}".format(self.site, key)
result = False
with self.pool.reserve() as mc:
try:
result = mc.get(key)
except pylibmc.ConnectionError:
self.connect()
result = False
self.pool.relinquish()
return result
def __init__(self, ampy):
self.event_timeseries = {}
self.site_counts = {}
self.common_events = {}
self.rare_events = {}
self.allevents = set()
self.ampy = ampy
self.binsize = 30 * 60
self.memcache = pylibmc.Client(
["127.0.0.1"],
behaviors={
"tcp_nodelay": True,
"no_block": True,
})
self.mcpool = pylibmc.ThreadMappedPool(self.memcache)
def threaded_save(self, key, value):
if not self.pool:
self.pool = pylibmc.ThreadMappedPool(self.conn)
key = self.site + "-" + key
with self.pool.reserve() as mc:
for i in range(MAX_RETRIES):
try:
mc.set(str(key), str(value))
break
except Exception:
log_traceback(f"Cache save failed ({key})")
time.sleep(0.3)
self.connect()
else:
critical_error(
"Memcache save failed. This should never happen.")
self.pool.relinquish()
return True
def __init__(self, *arg, **kw):
super(PyLibMCNamespaceManager, self).__init__(*arg, **kw)
memcache_module = kw.get('memcache_module', 'auto')
_memcache_module = _client_libs[memcache_module]
protocol = kw.get('protocol', 'text')
username = kw.get('username', None)
password = kw.get('password', None)
url = kw.get('url')
behaviors = parse_memcached_behaviors(kw)
self.mc = MemcachedNamespaceManager.clients.get(
(memcache_module, url),
_memcache_module.Client,
servers=url.split(';'), behaviors=behaviors,
binary=(protocol == 'binary'), username=username,
password=password)
self.pool = pylibmc.ThreadMappedPool(self.mc)
def __init__(self, blocksize):
"""
Init function for the AmpyCache class.
Parameters:
blocksize -- the number of binned measurements to include in a
data block.
"""
self.blocksize = blocksize
self.memcache = pylibmc.Client(["127.0.0.1"],
behaviors={
"tcp_nodelay": True,
"no_block": True,
})
self.mcpool = pylibmc.ThreadMappedPool(self.memcache)
self.streamview_cachetime = 60 * 60 * 24
self.viewgroups_cachetime = 60 * 60 * 6
def threaded_save(self, key, value):
if not self.pool:
self.pool = pylibmc.ThreadMappedPool(self.conn)
key = self.site + "-" + key
with self.pool.reserve() as mc:
for i in range(10):
try:
mc.set(str(key), str(value))
break
except Exception:
log_traceback("Cache save failed ({})".format(key))
time.sleep(.3)
self.connect()
else:
critical_error(
"Memcache save failed. This should never happen. Check MC server"
)
sys.exit(-1)
self.pool.relinquish()
return True
def threaded_delete(self, key):
if not self.pool:
self.pool = pylibmc.ThreadMappedPool(self.conn)
key = "{}_{}".format(self.site, key)
with self.pool.reserve() as mc:
for i in range(10):
try:
mc.delete(key)
break
except:
log_traceback("Cache delete failed ({})".format(key))
time.sleep(.3)
self.connect()
else:
critical_error(
"Memcache delete failed. This should never happen. Check MC server"
)
sys.exit(-1)
self.pool.relinquish()
return True
def __init__(self, server, params):
super(CacheClass, self).__init__(params)
mc = pylibmc.Client(server.split(';'))
mc.behaviors = {"tcp_nodelay": True}
self._pool = pylibmc.ThreadMappedPool(mc)
def __init__(self, hosts=['localhost'], *args, **kw):
client = pylibmc.Client(hosts)
self.pool = pylibmc.ThreadMappedPool(client)
BaseBackend.__init__(self, *args, **kw)
return self._mc.set(key, value, time)
# FIXME: deprecated 以下为兼容代码
def mclient(servers):
return Client(servers)
###################################################
# cache define #
###################################################
# master
func_cache = mclient(memcache_settings["func_cache"])
pool_model_cache = pylibmc.ThreadMappedPool(func_cache)
class PyPoolMemcache(object):
@classmethod
def incr(cls, key, pool=pool_model_cache):
with pool.reserve() as _mc:
result = _mc.incr(key)
return result
@classmethod
def decr(cls, key, pool=pool_model_cache):
with pool.reserve() as _mc:
result = _mc.decr(key)
return result
def __init__(self, servers):
ObjectCache.__init__(self)
self.mc = pylibmc.Client(servers)
self.mc_pool = pylibmc.ThreadMappedPool(self.mc)
def __init__(self, client, adapter, *, prefix="anom"):
self.client_pool = pylibmc.ThreadMappedPool(client)
self.adapter = adapter
self.prefix = prefix
self.query = self.adapter.query
def __init__(self, *arg, **kw):
super(PyLibMCNamespaceManager, self).__init__(*arg, **kw)
self.pool = pylibmc.ThreadMappedPool(self.mc)
import hashlib
import logging
import os
import re
import sys
from contextlib import closing
from threading import local
import requests
import pylibmc
import remoulade
logger = logging.getLogger("example")
memcache_client = pylibmc.Client(["localhost"], binary=True)
memcache_pool = pylibmc.ThreadMappedPool(memcache_client)
anchor_re = re.compile(rb'<a href="([^"]+)">')
state = local()
if os.getenv("REDIS") == "1":
from remoulade.brokers.redis import RedisBroker
broker = RedisBroker()
remoulade.set_broker(broker)
def get_session():
session = getattr(state, "session", None)
if session is None:
session = state.session = requests.Session()
return session
def __init__(self, server, params):
super(CacheClass, self).__init__(params)
mc = pylibmc.Client(server.split(';'))
mc.behaviors = PYLIBMC_BEHAVIORS
self._pool = pylibmc.ThreadMappedPool(mc)
self._logger = CacheLogger()
def __init__(self, client, adapter, *, namespace="anom"):
self.client_pool = pylibmc.ThreadMappedPool(client)
self.adapter = adapter
self.namespace = namespace
self.query = self.adapter.query
