def mc_check(config):
host = config.get("host", "localhost")
port = int(config.get("port", 11211))
try:
mc = Client((host, port))
mc.stats()
except:
return False
else:
return True
class MemcachedWrapper(object):
'''Class to allow readonly access to underlying memcached connection'''
def __init__(self, counter, host, port=11211, socket_connect_timeout=1):
if not host:
raise ConfigurationError('Memcached wrapper improperly configured. Valid memcached host is required!')
self.__con = Client((host, port))
self._counter = counter('')
def __del__(self):
self.__con.quit()
def get(self, key):
return self.__con.get(key)
def json(self, key):
return pyjson.loads(self.get(key))
def stats(self, extra_keys=[]):
data = self.__con.stats()
ret = {}
for key in data:
if key in COUNTER_KEYS:
ret['{}_per_sec'.format(key.replace('total_', ''))] = \
round(self._counter.key(key).per_second(data.get(key, 0)), 2)
elif key in VALUE_KEYS:
ret[key] = data[key]
elif key in extra_keys:
ret[key] = data[key]
return ret
class MemcachedWrapper(object):
'''Class to allow readonly access to underlying memcached connection'''
def __init__(self, counter, host, port=11211, socket_connect_timeout=1):
if not host:
raise ConfigurationError(
'Memcached wrapper improperly configured. Valid memcached host is required!'
)
self.__con = Client((host, port))
self._counter = counter('')
def __del__(self):
self.__con.quit()
def get(self, key):
return self.__con.get(key)
def json(self, key):
return pyjson.loads(self.get(key))
def stats(self, extra_keys=[]):
data = self.__con.stats()
ret = {}
for key in data:
if key in COUNTER_KEYS:
ret['{}_per_sec'.format(key.replace('total_', ''))] = \
round(self._counter.key(key).per_second(data.get(key, 0)), 2)
elif key in VALUE_KEYS:
ret[key] = data[key]
elif key in extra_keys:
ret[key] = data[key]
return ret
def check(self):
local_port = self.get_port()
from pymemcache.client.base import Client
client = Client(("localhost", local_port))
try:
server_stats = client.stats()
return server_stats[b"accepting_conns"] == 1
except (ConnectionRefusedError, KeyError):
# not ready yet
return False
def memcached_load_nosql(objects, numbers):
from pymemcache.client.base import Client
import hashlib
mc = Client(('localhost', 11211), serializer=mem_serializer, deserializer=mem_deserializer)
test_name = 'memcached nosql '
for i, o in enumerate(objects):
mc.set(str(hashlib.sha1(o["name"]).hexdigest()), o["coords"])
if (i + 1) in numbers:
print test_name + "Objects: %s, Usage: %s" % (i + 1, mc.stats()["bytes"] * 1.0 / (1024 * 1024))
print "finsihed"
def memcached_load_sql(df):
from pymemcache.client.base import Client
#import zlib
mc = Client(('localhost', 11211))#, serializer=mem_serializer, deserializer=mem_deserializer)
#from bson.binary import Binary
#df['text'] = df['text'].map(zlib.compress)#.map(Binary)
test_name = 'mongo sql '
d = df.to_dict(orient='records')
for i, row in enumerate(df.values):
mc.set(str(i), str(d[i]))
print test_name + "Usage: %s" % (mc.stats()["bytes"] * 1.0 / (1024 * 1024))
def check(self, config):
data = {}
client = Client((config["host"], config["port"]),
connect_timeout=config["timeout"],
timeout=config["timeout"])
memcached_stats = client.stats()
# 遍历 memcache_stats 字典，做 key 的 split 转换处理，经过 split 后，分为 3 种情况进行处理：
# 如果 split 后的数组元素长度为 1， 那么增加 server 前缀；
# 如果 split 后的数组元素长度为 2， 那么什么都不需要做，直接split即可;
# 如果 split 后的数组元素长度大于 2， 那么只需要 split 1 个字符即可；
for key in memcached_stats.keys():
split_len = len(key.split("_"))
if split_len == 1:
corrected_key = "server." + key
elif split_len == 2:
corrected_key = '.'.join(key.split("_"))
else:
corrected_key = '.'.join(key.split("_", 1))
data[corrected_key] = self._convert_to_float(memcached_stats[key])
# 数据处理
total_cmd_get = self._convert_to_float(memcached_stats["cmd_get"])
total_get_hits = self._convert_to_float(memcached_stats["get_hits"])
if self.last_cmd_get is not None:
cmd_get = total_cmd_get - self.last_cmd_get
get_hits = total_get_hits - self.last_get_hits
if cmd_get <= 0:
data["hit.ratio"] = 0
else:
data["hit.ratio"] = self._decimal_convert_to_percent(get_hits /
cmd_get)
self.last_cmd_get = total_cmd_get
self.last_get_hits = total_get_hits
return data
def generate_packet(hostname):
packet = []
try:
client = Client((serviceip, serviceport))
memstatsdict = client.stats()
for memstats in memstatsdict:
#print(memstats)
#print(memstatsdict[memstats])
packet.append(
ZabbixMetric(hostname,
"memcached_stats[%s]" % memstats.decode(),
memstatsdict[memstats]))
packet.append(
ZabbixMetric(hostname, "memcached_stats[%s]" % 'status', 1))
except Exception as e:
print(e)
packet.append(
ZabbixMetric(hostname, "memcached_stats[%s]" % 'status', 0))
#print(packet)
return packet
def getData(self):
status_content = {}
try:
host = self.taskConf.get("host")
port = self.taskConf.get("port")
mc = Client((host, port))
status_content = mc.stats()
except Exception:
pybixlib.error(self.logHead + traceback.format_exc())
self.errorInfoDone(traceback.format_exc())
status_content = {}
finally:
self.setData({
'agentType': self.agentType,
'uuid': self.uuid,
'code': self.code,
'time': self.getCurTime(),
'data': status_content,
'error_info': self.error_info
})
self.intStatus()
from pymemcache.client.base import Client
client = Client(('localhost', 11211))
set_result = client.set('foo', 'bar')
print "Setting foo: ", set_result
value = client.get('foo')
print "Retrieved value of foo: ", value
print "Max memory: ", client.stats()['limit_maxbytes']
class MemcachedAdapter(BaseAdapter):
"""
Exposes a cache store using Memcached.
Exposes `pymemcache`'s exceptions.
"""
def __init__(self, host='localhost', port=11211, **kwargs):
super().__init__()
self.store = Client((host, port), **kwargs)
def set(self, key, value, ttl):
if ttl == -1:
ttl = 0
return self.store.set(key, value, expire=ttl)
def batch_set(self, keys, values, ttls):
# There's two reasons to recode pymemcache.set_multi():
# - It returns a list of keys that failed to be inserted, and the base expects a boolean
# - It only allows a unique ttl for all keys
commands = []
ttls = [0 if ttl == -1 else ttl for ttl in ttls]
for key, value, ttl in zip(keys, values, ttls):
ttl = self.store._check_integer(ttl, 'expire') # pylint: disable=protected-access
key = self.store.check_key(key)
value, flags = self.store.serde.serialize(key, value)
command = b'set ' + key
command += b' ' + str(flags).encode(self.store.encoding)
command += b' ' + ttl
command += b' ' + str(len(value)).encode(
self.store.encoding) + b'\r\n'
command += value.encode(self.store.encoding) + b'\r\n'
commands.append(command)
results = self.store._misc_cmd(commands, 'set', False) # pylint: disable=protected-access
for line in results:
if line == b'NOT_STORED':
return False
return True
def get(self, key):
value = self.store.get(key)
return value
def batch_get(self, keys):
key_to_value = self.store.get_multi(keys)
values = [
key_to_value[key] if key in key_to_value else None for key in keys
]
return values
def delete(self, key):
return self.store.delete(key, noreply=False)
def batch_delete(self, keys):
# Here as well, pymemcache.delete_multi() always returns True
commands = []
for key in keys:
key = self.store.check_key(key)
command = b'delete ' + key + b'\r\n'
commands.append(command)
results = self.store._misc_cmd(commands, 'delete', False) # pylint: disable=protected-access
for line in results:
print(f"\"{line}\"")
if line == b'NOT_FOUND':
return False
return True
def exists(self, key):
# Can't just cast to bool since we can store falsey values
return self.store.get(key) is not None
def flush(self):
return self.store.flush_all(noreply=False)
def ping(self):
return bool(self.store.stats())
@property
def connection_exceptions(self):
return (MemcacheUnexpectedCloseError, MemcacheServerError,
MemcacheUnknownError)
class Cache:
"""Simple cache backend wrapper using Pinterest's pymemcache;
Attributes:
client (Client): The client object for a single memcached server;
"""
def __init__(self):
self.client = Client((HOST, POST), serde=JsonSerde())
def __repr__(self):
return "<Cache {}>".format(self.client.server)
def get_stats(self, *args):
"""Runs the memcached `stats` command;
Args:
*arg (list): extra string arguments to the “stats” command;
Returns:
A dictionary of the returned stats;
"""
return self.client.stats(args)
def set(self, key, value):
"""Cache setter;
Args:
key (str): unique identifier of pair;
value (str): value for the key;
"""
self.client.set(key, value)
def set_multiple(self, value):
"""A convenience function for setting multiple values;
Args:
key (str): unique identifier of pair;
value (str): value for the key;
"""
self.client.set_multi(value)
def get(self, key):
"""Cache getter;
Args:
key (str): unique identifier of pair;
Returns:
The value for the key if was found;
Raises:
Exception if the key wasn’t found;
"""
return self.client.get(key)
def add(self, key, value):
"""Runs memcached “add” command;
Store data, only if it does not already exist;
Args:
key (str): unique identifier of pair;
value (str): value for the key;
Returns:
return True if value was stored, False if it was not;
"""
return self.client.add(key, value)
def replace(self, key, value):
"""Runs memcached “replace” command;
Store data, but only if the data already exists;
Args:
key (str): unique identifier of pair;
value (str): value for the key;
Returns:
return True if value was stored, False if it was not;
"""
return self.client.replace(key, value)
def append(self, key, value):
"""Runs memcached append command;
Args:
key (str): unique identifier of pair;
value (str): value for the key;
Returns:
return True;
"""
return self.client.append(key, value)
def prepend(self, key, value):
"""Runs memcached “prepend” command;
Same as append, but adding new data before existing data;
Args:
key (str): unique identifier of pair;
value (str): value for the key;
Returns:
return True;
"""
return self.client.prepend(key, value)
def delete(self, key):
"""Runs memcached delete command;"""
return self.client.delete(key)
def delete_many(self, keys):
"""A convenience function to delete multiple keys;"""
return self.client.delete_many(keys)
def get_memcached_stats(host, port):
"""Connect to memcache server for stats."""
conn = Client((host, port))
return conn.stats()
from pymemcache.client.base import Client
client = Client(('localhost', 11211))
client.get("user")
import pickle
pickle.loads(client.get("complex"))
client.stats()['curr_items']
import pprint
pp = pprint.PrettyPrinter(indent=4)
pp.pprint(client.stats("cachedump","1","0"))
def retrieve(key):
result = client.get_multi(['%s-%s' % (key, i) for i in xrange(12)])
serialized = ''
for i in range(12):
serialized=serialized+result[key+'-'+str(i)]
return serialize
from pymemcache.client.base import Client
import pprint
client = Client(('localhost', 11211))
pp = pprint.PrettyPrinter(indent=4)
pp.pprint(client.stats("items"))
pp.pprint(client.stats("cachedump", "4", "0"))
import time
from pymemcache.client.base import Client
client = Client(('localhost', 11211))
print("[info] sending commands to memcached")
while True:
client.stats()
for i in range(5):
client.get('a')
time.sleep(1)
