class MemcachedWrapper():
MEMCACHED_CONNECT_TIMEOUT = 5 # 5 seconds
MEMCACHED_TIMEOUT = 5 # 5 seconds
def __init__(self, memcached_location):
memcached_location_portions = memcached_location.split(":")
if len(memcached_location_portions) != 2:
raise ValueError(
f"Found incorrectly formatted parameter memcached location: {memcached_location}"
)
memcached_host = memcached_location_portions[0]
memcached_port = int(memcached_location_portions[1])
self.memcached_client = Client(
server=(memcached_host, memcached_port),
serializer=serde.python_memcache_serializer,
deserializer=serde.python_memcache_deserializer,
connect_timeout=MemcachedWrapper.MEMCACHED_CONNECT_TIMEOUT,
timeout=MemcachedWrapper.MEMCACHED_TIMEOUT)
def get(self, key):
return self.memcached_client.get(key)
def set(self, key, value, timeout=None):
self.memcached_client.set(key, value, timeout)
def delete(self, key):
self.memcached_client.delete(key)
def memcclient(self, host):
c = Client(host)
lock = c.get('lock')
while lock:
sleep(.25)
lock = c.get('lock')
c.set('lock', 'true')
try:
yield c
finally:
c.delete('lock')
class Cache():
def __init__(self, hostname=cache_server_host, port=cache_server_port):
self.server = Client((hostname, port))
def set(self, key, value, expiry=0):
self.server.set(str(key), value, expiry)
def get(self, key):
if self.server.get(str(key)):
value = self.server.get(str(key)).decode("utf-8")
return int(value)
def delete(self, key):
self.server.delete(str(key))
def main():
client = Client(('localhost', 11211))
client.set('some_key', 'some_value')
client.set('some_key1', 'some_value')
client.set('some_key2', 'some_value')
client.set('some_key3', 'some_value')
client.set('some_key4', 'some_value')
# client.set('some_key1', { "a": 3, "b": 1234, "c": 1234567 })
# result = client.get('some_key')
# client.set('some_key2', { "a": 3, "b": 1234, "c": 1234567 })
# result = client.get('some_key')
# client.set('some_key3', { "a": 3, "b": 1234, "c": 1234567 })
# result = client.get('some_key')
# print (result)
result = client.get_multi(['some_key2', 'some_key3'])
print(result)
print(client.delete('some_key'))
print(client.get('some_key'))
client.set('new', 'unknown value')
client.set('test1', 'tesing unknow reasonable text')
client.set('test2', 'although i know it will be hash value')
client.set('test3', 'show me the value')
client.set('test4', 'whatever it takes')
client.set('test5', 'something at works')
return True
class Memcached(object):
"""Memcached based caching.
This is extremely lightweight in terms of memory usage, but it
appears to be slightly slower than local database caching, and a
lot slower than local memory based caching. This should be used if
the cache is to be shared between multiple users.
"""
LRU = LeastRecentlyUsed = 0
def __init__(self, url='127.0.0.1', mode=LRU, ttl=None):
"""Create a new engine.
Parameters:
url (str): Memcached server to connect to.
mode (int): How to purge the old keys.
This does not affect anything as memcached is LRU only.
The option is there to match the other engines.
ttl (int): Time the cache is valid for.
Set to None or 0 for infinite.
"""
try:
from pymemcache.client.base import Client
self.client = Client(url, timeout=3, connect_timeout=3)
except ImportError:
from memcache import Client
self.client = Client([url], socket_timeout=3)
self.ttl = ttl
# Don't allow cross version caches
# Pickle may have incompatibilities between versions
self._key_prefix = '{}.{}.'.format(sys.version_info.major,
sys.version_info.minor)
def get(self, key):
"""Get the value belonging to a key.
An error will be raised if the cache is expired or doesn't
exist.
"""
value = self.client.get(self._key_prefix + key)
if value is None:
raise exceptions.CacheNotFound(key)
return _decode(value)
def put(self, key, value, ttl=None):
"""Write a new value to the cache.
This will overwrite any old cache with the same key.
"""
if ttl is None:
ttl = self.ttl
self.client.set(self._key_prefix + key,
_encode(value),
ttl or 0,
noreply=True)
def delete(self, key):
"""Delete an item of cache if it exists."""
return self.client.delete(self._key_prefix + key, noreply=False)
def delMemcache(self, configkey, key, **kwargs):
try:
clients = self.getMemcacheConfig(configkey)
for client_conf in clients:
client = Client(client_conf)
result = client.get(key=key)
if result is not None:
result = client.delete(key=key)
logging.info("删除memcache的key={0}，操作结果={1}".format(key, result))
except Exception as e:
logging.error(e)
class clientRuler:
def json_serializer(self, key, value):
if type(value) == str:
return value, 1
return json.dumps(value), 2
def json_deserializer(self, key, value, flags):
if flags == 1:
return value
if flags == 2:
return json.loads(value)
raise Exception(key)
def __init__(self, log_):
try:
self.log = log_
self.client = Client(('localhost', 11211),
serializer=self.json_serializer,
deserializer=self.json_deserializer)
except Exception as e:
log_.error('{0}'.format(e.message))
def setKey(self, set_key, set_value):
try:
self.client.delete(set_key)
self.client.set(set_key, set_value, 900)
except Exception as e:
self.log.error('{0}'.format(e.message))
def getKey(self, get_key):
try:
result = self.client.get(get_key)
return result
except Exception as e:
self.log.error('{0}'.format(e.message))
def deleteKey(self, delete_key):
try:
self.client.delete(delete_key)
except Exception as e:
self.log.error('{0}'.format(e.message))
class Memcached(object):
DELAY = 0.5
DEBUG = False
def __init__(self, hostname, port, **params):
self.mc = Client((hostname, port))
def handle(self, topic, message):
"""
"""
if 'cmd' not in message:
raise Exception("Bad message: no command")
cmd = message['cmd']
if not hasattr(self, cmd):
raise Exception("Unknown command: " + cmd)
tryit = True
while tryit:
tryit = False
try:
getattr(self, cmd)(message)
except MemcacheUnexpectedCloseError:
# Server dropped dead - we'll retry
tryit = True
except IOError:
# Something network-related - retry
tryit = True
if tryit:
time.sleep(self.DELAY)
def set(self, message):
text = message['val'].encode('utf-8')
if message.get('sbt', None):
purge_time = time.time() + message.get('uto', 0)
text = text.replace('$UNIXTIME$', '%.6f' % purge_time)
if self.DEBUG:
print("Set {0}-{1}-{2}".format(message['key'].encode('utf-8'),
text, int(message['ttl'])))
self.mc.set(message['key'].encode('utf-8'), text, int(message['ttl']))
def delete(self, message):
self.mc.delete(message['key'])
class Memcached(object):
DELAY = 0.5
DEBUG = False
def __init__(self, hostname, port, **params):
self.mc = Client((hostname, port))
def handle(self, topic, message):
"""
"""
if 'cmd' not in message:
raise Exception("Bad message: no command")
cmd = message['cmd']
if not hasattr(self, cmd):
raise Exception("Unknown command: " + cmd)
tryit = True
while tryit:
tryit = False
try:
getattr(self, cmd)(message)
except MemcacheUnexpectedCloseError:
# Server dropped dead - we'll retry
tryit = True
except IOError:
# Something network-related - retry
tryit = True
if tryit:
time.sleep(self.DELAY)
def set(self, message):
text = message['val'].encode('utf-8')
if message.get('sbt', None):
purge_time = time.time() + message.get('uto', 0)
text = text.replace('$UNIXTIME$', '%.6f' % purge_time)
if self.DEBUG:
print("Set {0}-{1}-{2}".format(message['key'].encode('utf-8'), text, int(message['ttl'])))
self.mc.set(message['key'].encode('utf-8'), text, int(message['ttl']))
def delete(self, message):
self.mc.delete(message['key'])
class Memcache(object):
def __init__(self):
from pymemcache.client.base import Client
self.client = Client(("127.0.0.1", 11211))
def setmem(self, key, value):
return self.client.set(str(key), value)
def getmem(self, key):
return self.client.get(key)
def delmem(self, key):
return self.client.delete(key)
class Memcache(object):
'''
Memcache缓存
setmem:写入和更新缓存,传递key和value
getmem：读取缓存,传递key
delmem:删除缓存,根据key删除
'''
def __init__(self):
from pymemcache.client.base import Client
self.client = Client(("127.0.0.1",11211))
def setmem(self,key,value):
return self.client.set(str(key),value)
def getmem(self,key):
return self.client.get(key)
def delmem(self,key):
return self.client.delete(key)
class StandardServerConfigTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
print "PLEASE MAKE SURE THE SERVER IS NOT ALREADY RUNNING!"
print "Note: this test run will start and stop the server for each test case, this may take a minute or two...\n"
def setUp(self):
self.startServer()
self.client = Client(('localhost', 11211))
def tearDown(self):
self.killServer()
def startServer(self):
FNULL = open(os.devnull, 'w')
path = os.path.dirname(os.path.realpath(__file__))
proc = subprocess.Popen(["java", "-jar", path+"/../bin/mcsvr.jar"],
stdout=FNULL, stderr=subprocess.STDOUT)
self.server_pid = proc.pid
time.sleep(3)
def killServer(self):
time.sleep(1)
os.kill(int(self.server_pid), signal.SIGTERM)
def testGetSingle(self):
self.client.set('first_key', 'first_value', 0, False)
res = self.client.get('first_key')
self.assertEqual(res, "first_value")
def testGetMany(self):
self.client.set('first_key', 'first_value', 0, False)
self.client.set('second_key', 'second_value', 0, True)
res = self.client.get_many(['first_key', 'second_key'])
self.assertEqual(res["first_key"], "first_value")
self.assertEqual(res["second_key"], "second_value")
def testGetExpired(self):
self.client.set('first_key', 'first_value', 0, False)
self.client.set('first_key', 'first_value', 2, False)
self.client.get_many(['first_key', 'second_key'])
time.sleep(3)
self.assertEqual(None, self.client.get('first_key'))
def testGetDeleted(self):
self.client.set('first_key', 'first_value', 0, False)
res = self.client.get('first_key')
self.assertEqual('first_value', res)
self.client.delete('first_key')
res = self.client.get('first_key')
self.assertEqual(None, res)
def testUpdateExpired(self):
self.client.set('first_key', 'first_value', 2, False)
time.sleep(3)
self.client.set('first_key', 'second_value', 0, False)
res = self.client.get('first_key')
self.assertEqual('second_value', res)
def testCaseUpdateOk(self):
self.client.set('first_key', 'first_value', 0, False)
res = self.client.gets('first_key')
res = self.client.cas('first_key', 'second_value', res[1], 0, False)
self.assertTrue(res)
res = self.client.get('first_key')
self.assertEqual('second_value', res)
def testCaseUpdateInvalidUniq(self):
self.client.set('first_key', 'first_value', 0, False)
res = self.client.gets('first_key')
casUniq = res[1]
self.client.set('first_key', 'second_value', 0, False)
res = self.client.cas('first_key', 'third_value', casUniq, 0, False)
self.assertFalse(res)
res = self.client.get('first_key')
self.assertEqual('second_value', res)
def testCaseUpdateNotOkOnExpired(self):
self.client.set('first_key', 'first_value', 2, False)
res = self.client.gets('first_key')
time.sleep(3)
res = self.client.cas('first_key', 'second_value', res[1], 0, False)
self.assertEqual(None, res)
def testCaseUpdateNotOkOnMissing(self):
self.client.set('first_key', 'first_value', 0, False)
res = self.client.gets('first_key')
casUniq = res[1]
self.client.delete('first_key')
res = self.client.cas('first_key', 'second_value', casUniq, 0, False)
self.assertEqual(None, res)
class MemcachedAdapter(BaseAdapter):
"""
Exposes a cache store using Memcached.
Exposes `pymemcache`'s exceptions.
"""
def __init__(self, host='localhost', port=11211, **kwargs):
super().__init__()
self.store = Client((host, port), **kwargs)
def set(self, key, value, ttl):
if ttl == -1:
ttl = 0
return self.store.set(key, value, expire=ttl)
def batch_set(self, keys, values, ttls):
# There's two reasons to recode pymemcache.set_multi():
# - It returns a list of keys that failed to be inserted, and the base expects a boolean
# - It only allows a unique ttl for all keys
commands = []
ttls = [0 if ttl == -1 else ttl for ttl in ttls]
for key, value, ttl in zip(keys, values, ttls):
ttl = self.store._check_integer(ttl, 'expire') # pylint: disable=protected-access
key = self.store.check_key(key)
value, flags = self.store.serde.serialize(key, value)
command = b'set ' + key
command += b' ' + str(flags).encode(self.store.encoding)
command += b' ' + ttl
command += b' ' + str(len(value)).encode(
self.store.encoding) + b'\r\n'
command += value.encode(self.store.encoding) + b'\r\n'
commands.append(command)
results = self.store._misc_cmd(commands, 'set', False) # pylint: disable=protected-access
for line in results:
if line == b'NOT_STORED':
return False
return True
def get(self, key):
value = self.store.get(key)
return value
def batch_get(self, keys):
key_to_value = self.store.get_multi(keys)
values = [
key_to_value[key] if key in key_to_value else None for key in keys
]
return values
def delete(self, key):
return self.store.delete(key, noreply=False)
def batch_delete(self, keys):
# Here as well, pymemcache.delete_multi() always returns True
commands = []
for key in keys:
key = self.store.check_key(key)
command = b'delete ' + key + b'\r\n'
commands.append(command)
results = self.store._misc_cmd(commands, 'delete', False) # pylint: disable=protected-access
for line in results:
print(f"\"{line}\"")
if line == b'NOT_FOUND':
return False
return True
def exists(self, key):
# Can't just cast to bool since we can store falsey values
return self.store.get(key) is not None
def flush(self):
return self.store.flush_all(noreply=False)
def ping(self):
return bool(self.store.stats())
@property
def connection_exceptions(self):
return (MemcacheUnexpectedCloseError, MemcacheServerError,
MemcacheUnknownError)
class Cache:
"""Simple cache backend wrapper using Pinterest's pymemcache;
Attributes:
client (Client): The client object for a single memcached server;
"""
def __init__(self):
self.client = Client((HOST, POST), serde=JsonSerde())
def __repr__(self):
return "<Cache {}>".format(self.client.server)
def get_stats(self, *args):
"""Runs the memcached `stats` command;
Args:
*arg (list): extra string arguments to the “stats” command;
Returns:
A dictionary of the returned stats;
"""
return self.client.stats(args)
def set(self, key, value):
"""Cache setter;
Args:
key (str): unique identifier of pair;
value (str): value for the key;
"""
self.client.set(key, value)
def set_multiple(self, value):
"""A convenience function for setting multiple values;
Args:
key (str): unique identifier of pair;
value (str): value for the key;
"""
self.client.set_multi(value)
def get(self, key):
"""Cache getter;
Args:
key (str): unique identifier of pair;
Returns:
The value for the key if was found;
Raises:
Exception if the key wasn’t found;
"""
return self.client.get(key)
def add(self, key, value):
"""Runs memcached “add” command;
Store data, only if it does not already exist;
Args:
key (str): unique identifier of pair;
value (str): value for the key;
Returns:
return True if value was stored, False if it was not;
"""
return self.client.add(key, value)
def replace(self, key, value):
"""Runs memcached “replace” command;
Store data, but only if the data already exists;
Args:
key (str): unique identifier of pair;
value (str): value for the key;
Returns:
return True if value was stored, False if it was not;
"""
return self.client.replace(key, value)
def append(self, key, value):
"""Runs memcached append command;
Args:
key (str): unique identifier of pair;
value (str): value for the key;
Returns:
return True;
"""
return self.client.append(key, value)
def prepend(self, key, value):
"""Runs memcached “prepend” command;
Same as append, but adding new data before existing data;
Args:
key (str): unique identifier of pair;
value (str): value for the key;
Returns:
return True;
"""
return self.client.prepend(key, value)
def delete(self, key):
"""Runs memcached delete command;"""
return self.client.delete(key)
def delete_many(self, keys):
"""A convenience function to delete multiple keys;"""
return self.client.delete_many(keys)
def res():
client = Client(('localhost', 11211))
text = client.get('info')
client.delete('info')
client.close()
return text
class MemcachedMonitor:
'''
Abstracts some of the logic of setting up a parameter fitting problem.
Provides information via MC for monitoring.
'''
def __init__(self, name, mc_host, mc_port, run=None, run_id=None):
self.name = name
self.run = run
self.run_id = run_id
self.mc_host = mc_host
self.mc_port = mc_port
self.mc_client = Client((self.mc_host, self.mc_port))
def getName(self):
return self.name
def __enter__(self):
from sabaody.diagnostics import test_memcached
test_memcached(self.mc_host, self.mc_port)
self.setupMonitoringVariables()
self.reset_best_score()
return self
def domainAppend(self, s):
return '.'.join((self.getDomain(), s))
def __exit__(self, exception_type, exception_val, trace):
self.update('finished', self.run, 'status')
self.update(str(time()), self.run, 'endTime')
def setupMonitoringVariables(self):
if self.run is None:
# self.run = int(self.mc_client.get(self.domainAppend('run')) or 0)
# self.run += 1
self.run = 0
self.update(self.run, 'run')
if self.run_id is None:
self.run_id = str(uuid4())
self.update(self.run_id, 'runId')
print('monitor at {}'.format(self.getNameQualifier()()))
self.update(str(time()), 'startTime')
self.update('active', 'status')
# print('Starting run {} of {} with id {}...'.format(self.run, self.getName(), self.run_id))
def getNameQualifier(self):
from toolz import partial
return partial(getQualifiedName, self.getName(), str(self.run_id))
def __getstate__(self):
return {
'name': self.name,
'run': self.run,
'run_id': self.run_id,
'mc_host': self.mc_host,
'mc_port': self.mc_port
}
def __setstate__(self, state):
self.name = state['name']
self.run = state['run']
self.run_id = state['run_id']
self.mc_host = state['mc_host']
self.mc_port = state['mc_port']
self.mc_client = Client((self.mc_host, self.mc_port))
def update(self, value, *key):
# print('update {}'.format(self.getNameQualifier()(*list(str(k) for k in key))))
self.mc_client.set(self.getNameQualifier()(*list(str(k) for k in key)),
str(value), 604800)
def best_score_candidate(self, best_f, best_x):
from json import dumps, loads
current_best_f = self.mc_client.get(self.getNameQualifier()('global',
'best_f'))
if current_best_f is not None:
current_best_f = array(loads(current_best_f))
if current_best_f is None or mean(best_f) < mean(current_best_f):
self.update(dumps(best_f.tolist()), 'global', 'best_f')
self.update(dumps(best_x.tolist()), 'global', 'best_x')
def get_best_x(self):
best_x = self.mc_client.get(self.getNameQualifier()('global',
'best_x'))
if best_x is not None:
from json import loads
best_x = array(loads(best_x))
return best_x
def get_best_f(self):
best_f = self.mc_client.get(self.getNameQualifier()('global',
'best_f'))
if best_f is not None:
from json import loads
best_f = array(loads(best_f))
return best_f
def reset_best_score(self):
self.mc_client.delete(self.getNameQualifier()('global', 'best_x'))
self.mc_client.delete(self.getNameQualifier()('global', 'best_f'))
def test_delete(self):
client = Client(('localhost', 11212))
client.set('some_key', 'some_value')
client.delete('some_key')
result = client.get('some_key')
self.assertEqual(result, None)
client = Client((ip, port),
connect_timeout=TIMEOUT_SEC,
timeout=TIMEOUT_SEC)
for i in range(ITERATION_NUM):
key, value = f"some_key-{i}", f"some_value-{i}"
# cmd "set", create/update key:value
assert client.set(key, value)
# cmd "get", get value with key
result = client.get(key).decode("utf-8")
assert result == value
# cmd "delete", delete key
assert client.delete(key)
# cmd "add", create key:value
assert client.set(key, value)
assert client.add(key, value)
# cmd "append", append value
res = client.append(key, "_suffix")
value += "_suffix"
assert client.get(key).decode("utf-8") == value
# cmd "preppend", preppend value
res = client.prepend(key, "prefix_")
value = "prefix_" + value
assert client.get(key).decode("utf-8") == value
