class MemcachedWrapper(): MEMCACHED_CONNECT_TIMEOUT = 5 # 5 seconds MEMCACHED_TIMEOUT = 5 # 5 seconds def __init__(self, memcached_location): memcached_location_portions = memcached_location.split(":") if len(memcached_location_portions) != 2: raise ValueError( f"Found incorrectly formatted parameter memcached location: {memcached_location}" ) memcached_host = memcached_location_portions[0] memcached_port = int(memcached_location_portions[1]) self.memcached_client = Client( server=(memcached_host, memcached_port), serializer=serde.python_memcache_serializer, deserializer=serde.python_memcache_deserializer, connect_timeout=MemcachedWrapper.MEMCACHED_CONNECT_TIMEOUT, timeout=MemcachedWrapper.MEMCACHED_TIMEOUT) def get(self, key): return self.memcached_client.get(key) def set(self, key, value, timeout=None): self.memcached_client.set(key, value, timeout) def delete(self, key): self.memcached_client.delete(key)
def memcclient(self, host): c = Client(host) lock = c.get('lock') while lock: sleep(.25) lock = c.get('lock') c.set('lock', 'true') try: yield c finally: c.delete('lock')
class Cache(): def __init__(self, hostname=cache_server_host, port=cache_server_port): self.server = Client((hostname, port)) def set(self, key, value, expiry=0): self.server.set(str(key), value, expiry) def get(self, key): if self.server.get(str(key)): value = self.server.get(str(key)).decode("utf-8") return int(value) def delete(self, key): self.server.delete(str(key))
def main(): client = Client(('localhost', 11211)) client.set('some_key', 'some_value') client.set('some_key1', 'some_value') client.set('some_key2', 'some_value') client.set('some_key3', 'some_value') client.set('some_key4', 'some_value') # client.set('some_key1', { "a": 3, "b": 1234, "c": 1234567 }) # result = client.get('some_key') # client.set('some_key2', { "a": 3, "b": 1234, "c": 1234567 }) # result = client.get('some_key') # client.set('some_key3', { "a": 3, "b": 1234, "c": 1234567 }) # result = client.get('some_key') # print (result) result = client.get_multi(['some_key2', 'some_key3']) print(result) print(client.delete('some_key')) print(client.get('some_key')) client.set('new', 'unknown value') client.set('test1', 'tesing unknow reasonable text') client.set('test2', 'although i know it will be hash value') client.set('test3', 'show me the value') client.set('test4', 'whatever it takes') client.set('test5', 'something at works') return True
class Memcached(object): """Memcached based caching. This is extremely lightweight in terms of memory usage, but it appears to be slightly slower than local database caching, and a lot slower than local memory based caching. This should be used if the cache is to be shared between multiple users. """ LRU = LeastRecentlyUsed = 0 def __init__(self, url='127.0.0.1', mode=LRU, ttl=None): """Create a new engine. Parameters: url (str): Memcached server to connect to. mode (int): How to purge the old keys. This does not affect anything as memcached is LRU only. The option is there to match the other engines. ttl (int): Time the cache is valid for. Set to None or 0 for infinite. """ try: from pymemcache.client.base import Client self.client = Client(url, timeout=3, connect_timeout=3) except ImportError: from memcache import Client self.client = Client([url], socket_timeout=3) self.ttl = ttl # Don't allow cross version caches # Pickle may have incompatibilities between versions self._key_prefix = '{}.{}.'.format(sys.version_info.major, sys.version_info.minor) def get(self, key): """Get the value belonging to a key. An error will be raised if the cache is expired or doesn't exist. """ value = self.client.get(self._key_prefix + key) if value is None: raise exceptions.CacheNotFound(key) return _decode(value) def put(self, key, value, ttl=None): """Write a new value to the cache. This will overwrite any old cache with the same key. """ if ttl is None: ttl = self.ttl self.client.set(self._key_prefix + key, _encode(value), ttl or 0, noreply=True) def delete(self, key): """Delete an item of cache if it exists.""" return self.client.delete(self._key_prefix + key, noreply=False)
def delMemcache(self, configkey, key, **kwargs): try: clients = self.getMemcacheConfig(configkey) for client_conf in clients: client = Client(client_conf) result = client.get(key=key) if result is not None: result = client.delete(key=key) logging.info("删除memcache的key={0},操作结果={1}".format(key, result)) except Exception as e: logging.error(e)
class clientRuler: def json_serializer(self, key, value): if type(value) == str: return value, 1 return json.dumps(value), 2 def json_deserializer(self, key, value, flags): if flags == 1: return value if flags == 2: return json.loads(value) raise Exception(key) def __init__(self, log_): try: self.log = log_ self.client = Client(('localhost', 11211), serializer=self.json_serializer, deserializer=self.json_deserializer) except Exception as e: log_.error('{0}'.format(e.message)) def setKey(self, set_key, set_value): try: self.client.delete(set_key) self.client.set(set_key, set_value, 900) except Exception as e: self.log.error('{0}'.format(e.message)) def getKey(self, get_key): try: result = self.client.get(get_key) return result except Exception as e: self.log.error('{0}'.format(e.message)) def deleteKey(self, delete_key): try: self.client.delete(delete_key) except Exception as e: self.log.error('{0}'.format(e.message))
class Memcached(object): DELAY = 0.5 DEBUG = False def __init__(self, hostname, port, **params): self.mc = Client((hostname, port)) def handle(self, topic, message): """ """ if 'cmd' not in message: raise Exception("Bad message: no command") cmd = message['cmd'] if not hasattr(self, cmd): raise Exception("Unknown command: " + cmd) tryit = True while tryit: tryit = False try: getattr(self, cmd)(message) except MemcacheUnexpectedCloseError: # Server dropped dead - we'll retry tryit = True except IOError: # Something network-related - retry tryit = True if tryit: time.sleep(self.DELAY) def set(self, message): text = message['val'].encode('utf-8') if message.get('sbt', None): purge_time = time.time() + message.get('uto', 0) text = text.replace('$UNIXTIME$', '%.6f' % purge_time) if self.DEBUG: print("Set {0}-{1}-{2}".format(message['key'].encode('utf-8'), text, int(message['ttl']))) self.mc.set(message['key'].encode('utf-8'), text, int(message['ttl'])) def delete(self, message): self.mc.delete(message['key'])
class Memcached(object): DELAY = 0.5 DEBUG = False def __init__(self, hostname, port, **params): self.mc = Client((hostname, port)) def handle(self, topic, message): """ """ if 'cmd' not in message: raise Exception("Bad message: no command") cmd = message['cmd'] if not hasattr(self, cmd): raise Exception("Unknown command: " + cmd) tryit = True while tryit: tryit = False try: getattr(self, cmd)(message) except MemcacheUnexpectedCloseError: # Server dropped dead - we'll retry tryit = True except IOError: # Something network-related - retry tryit = True if tryit: time.sleep(self.DELAY) def set(self, message): text = message['val'].encode('utf-8') if message.get('sbt', None): purge_time = time.time() + message.get('uto', 0) text = text.replace('$UNIXTIME$', '%.6f' % purge_time) if self.DEBUG: print("Set {0}-{1}-{2}".format(message['key'].encode('utf-8'), text, int(message['ttl']))) self.mc.set(message['key'].encode('utf-8'), text, int(message['ttl'])) def delete(self, message): self.mc.delete(message['key'])
class Memcache(object): def __init__(self): from pymemcache.client.base import Client self.client = Client(("127.0.0.1", 11211)) def setmem(self, key, value): return self.client.set(str(key), value) def getmem(self, key): return self.client.get(key) def delmem(self, key): return self.client.delete(key)
class Memcache(object): ''' Memcache缓存 setmem:写入和更新缓存,传递key和value getmem:读取缓存,传递key delmem:删除缓存,根据key删除 ''' def __init__(self): from pymemcache.client.base import Client self.client = Client(("127.0.0.1",11211)) def setmem(self,key,value): return self.client.set(str(key),value) def getmem(self,key): return self.client.get(key) def delmem(self,key): return self.client.delete(key)
class StandardServerConfigTests(unittest.TestCase): @classmethod def setUpClass(cls): print "PLEASE MAKE SURE THE SERVER IS NOT ALREADY RUNNING!" print "Note: this test run will start and stop the server for each test case, this may take a minute or two...\n" def setUp(self): self.startServer() self.client = Client(('localhost', 11211)) def tearDown(self): self.killServer() def startServer(self): FNULL = open(os.devnull, 'w') path = os.path.dirname(os.path.realpath(__file__)) proc = subprocess.Popen(["java", "-jar", path+"/../bin/mcsvr.jar"], stdout=FNULL, stderr=subprocess.STDOUT) self.server_pid = proc.pid time.sleep(3) def killServer(self): time.sleep(1) os.kill(int(self.server_pid), signal.SIGTERM) def testGetSingle(self): self.client.set('first_key', 'first_value', 0, False) res = self.client.get('first_key') self.assertEqual(res, "first_value") def testGetMany(self): self.client.set('first_key', 'first_value', 0, False) self.client.set('second_key', 'second_value', 0, True) res = self.client.get_many(['first_key', 'second_key']) self.assertEqual(res["first_key"], "first_value") self.assertEqual(res["second_key"], "second_value") def testGetExpired(self): self.client.set('first_key', 'first_value', 0, False) self.client.set('first_key', 'first_value', 2, False) self.client.get_many(['first_key', 'second_key']) time.sleep(3) self.assertEqual(None, self.client.get('first_key')) def testGetDeleted(self): self.client.set('first_key', 'first_value', 0, False) res = self.client.get('first_key') self.assertEqual('first_value', res) self.client.delete('first_key') res = self.client.get('first_key') self.assertEqual(None, res) def testUpdateExpired(self): self.client.set('first_key', 'first_value', 2, False) time.sleep(3) self.client.set('first_key', 'second_value', 0, False) res = self.client.get('first_key') self.assertEqual('second_value', res) def testCaseUpdateOk(self): self.client.set('first_key', 'first_value', 0, False) res = self.client.gets('first_key') res = self.client.cas('first_key', 'second_value', res[1], 0, False) self.assertTrue(res) res = self.client.get('first_key') self.assertEqual('second_value', res) def testCaseUpdateInvalidUniq(self): self.client.set('first_key', 'first_value', 0, False) res = self.client.gets('first_key') casUniq = res[1] self.client.set('first_key', 'second_value', 0, False) res = self.client.cas('first_key', 'third_value', casUniq, 0, False) self.assertFalse(res) res = self.client.get('first_key') self.assertEqual('second_value', res) def testCaseUpdateNotOkOnExpired(self): self.client.set('first_key', 'first_value', 2, False) res = self.client.gets('first_key') time.sleep(3) res = self.client.cas('first_key', 'second_value', res[1], 0, False) self.assertEqual(None, res) def testCaseUpdateNotOkOnMissing(self): self.client.set('first_key', 'first_value', 0, False) res = self.client.gets('first_key') casUniq = res[1] self.client.delete('first_key') res = self.client.cas('first_key', 'second_value', casUniq, 0, False) self.assertEqual(None, res)
class MemcachedAdapter(BaseAdapter): """ Exposes a cache store using Memcached. Exposes `pymemcache`'s exceptions. """ def __init__(self, host='localhost', port=11211, **kwargs): super().__init__() self.store = Client((host, port), **kwargs) def set(self, key, value, ttl): if ttl == -1: ttl = 0 return self.store.set(key, value, expire=ttl) def batch_set(self, keys, values, ttls): # There's two reasons to recode pymemcache.set_multi(): # - It returns a list of keys that failed to be inserted, and the base expects a boolean # - It only allows a unique ttl for all keys commands = [] ttls = [0 if ttl == -1 else ttl for ttl in ttls] for key, value, ttl in zip(keys, values, ttls): ttl = self.store._check_integer(ttl, 'expire') # pylint: disable=protected-access key = self.store.check_key(key) value, flags = self.store.serde.serialize(key, value) command = b'set ' + key command += b' ' + str(flags).encode(self.store.encoding) command += b' ' + ttl command += b' ' + str(len(value)).encode( self.store.encoding) + b'\r\n' command += value.encode(self.store.encoding) + b'\r\n' commands.append(command) results = self.store._misc_cmd(commands, 'set', False) # pylint: disable=protected-access for line in results: if line == b'NOT_STORED': return False return True def get(self, key): value = self.store.get(key) return value def batch_get(self, keys): key_to_value = self.store.get_multi(keys) values = [ key_to_value[key] if key in key_to_value else None for key in keys ] return values def delete(self, key): return self.store.delete(key, noreply=False) def batch_delete(self, keys): # Here as well, pymemcache.delete_multi() always returns True commands = [] for key in keys: key = self.store.check_key(key) command = b'delete ' + key + b'\r\n' commands.append(command) results = self.store._misc_cmd(commands, 'delete', False) # pylint: disable=protected-access for line in results: print(f"\"{line}\"") if line == b'NOT_FOUND': return False return True def exists(self, key): # Can't just cast to bool since we can store falsey values return self.store.get(key) is not None def flush(self): return self.store.flush_all(noreply=False) def ping(self): return bool(self.store.stats()) @property def connection_exceptions(self): return (MemcacheUnexpectedCloseError, MemcacheServerError, MemcacheUnknownError)
class Cache: """Simple cache backend wrapper using Pinterest's pymemcache; Attributes: client (Client): The client object for a single memcached server; """ def __init__(self): self.client = Client((HOST, POST), serde=JsonSerde()) def __repr__(self): return "<Cache {}>".format(self.client.server) def get_stats(self, *args): """Runs the memcached `stats` command; Args: *arg (list): extra string arguments to the “stats” command; Returns: A dictionary of the returned stats; """ return self.client.stats(args) def set(self, key, value): """Cache setter; Args: key (str): unique identifier of pair; value (str): value for the key; """ self.client.set(key, value) def set_multiple(self, value): """A convenience function for setting multiple values; Args: key (str): unique identifier of pair; value (str): value for the key; """ self.client.set_multi(value) def get(self, key): """Cache getter; Args: key (str): unique identifier of pair; Returns: The value for the key if was found; Raises: Exception if the key wasn’t found; """ return self.client.get(key) def add(self, key, value): """Runs memcached “add” command; Store data, only if it does not already exist; Args: key (str): unique identifier of pair; value (str): value for the key; Returns: return True if value was stored, False if it was not; """ return self.client.add(key, value) def replace(self, key, value): """Runs memcached “replace” command; Store data, but only if the data already exists; Args: key (str): unique identifier of pair; value (str): value for the key; Returns: return True if value was stored, False if it was not; """ return self.client.replace(key, value) def append(self, key, value): """Runs memcached append command; Args: key (str): unique identifier of pair; value (str): value for the key; Returns: return True; """ return self.client.append(key, value) def prepend(self, key, value): """Runs memcached “prepend” command; Same as append, but adding new data before existing data; Args: key (str): unique identifier of pair; value (str): value for the key; Returns: return True; """ return self.client.prepend(key, value) def delete(self, key): """Runs memcached delete command;""" return self.client.delete(key) def delete_many(self, keys): """A convenience function to delete multiple keys;""" return self.client.delete_many(keys)
def res(): client = Client(('localhost', 11211)) text = client.get('info') client.delete('info') client.close() return text
class MemcachedMonitor: ''' Abstracts some of the logic of setting up a parameter fitting problem. Provides information via MC for monitoring. ''' def __init__(self, name, mc_host, mc_port, run=None, run_id=None): self.name = name self.run = run self.run_id = run_id self.mc_host = mc_host self.mc_port = mc_port self.mc_client = Client((self.mc_host, self.mc_port)) def getName(self): return self.name def __enter__(self): from sabaody.diagnostics import test_memcached test_memcached(self.mc_host, self.mc_port) self.setupMonitoringVariables() self.reset_best_score() return self def domainAppend(self, s): return '.'.join((self.getDomain(), s)) def __exit__(self, exception_type, exception_val, trace): self.update('finished', self.run, 'status') self.update(str(time()), self.run, 'endTime') def setupMonitoringVariables(self): if self.run is None: # self.run = int(self.mc_client.get(self.domainAppend('run')) or 0) # self.run += 1 self.run = 0 self.update(self.run, 'run') if self.run_id is None: self.run_id = str(uuid4()) self.update(self.run_id, 'runId') print('monitor at {}'.format(self.getNameQualifier()())) self.update(str(time()), 'startTime') self.update('active', 'status') # print('Starting run {} of {} with id {}...'.format(self.run, self.getName(), self.run_id)) def getNameQualifier(self): from toolz import partial return partial(getQualifiedName, self.getName(), str(self.run_id)) def __getstate__(self): return { 'name': self.name, 'run': self.run, 'run_id': self.run_id, 'mc_host': self.mc_host, 'mc_port': self.mc_port } def __setstate__(self, state): self.name = state['name'] self.run = state['run'] self.run_id = state['run_id'] self.mc_host = state['mc_host'] self.mc_port = state['mc_port'] self.mc_client = Client((self.mc_host, self.mc_port)) def update(self, value, *key): # print('update {}'.format(self.getNameQualifier()(*list(str(k) for k in key)))) self.mc_client.set(self.getNameQualifier()(*list(str(k) for k in key)), str(value), 604800) def best_score_candidate(self, best_f, best_x): from json import dumps, loads current_best_f = self.mc_client.get(self.getNameQualifier()('global', 'best_f')) if current_best_f is not None: current_best_f = array(loads(current_best_f)) if current_best_f is None or mean(best_f) < mean(current_best_f): self.update(dumps(best_f.tolist()), 'global', 'best_f') self.update(dumps(best_x.tolist()), 'global', 'best_x') def get_best_x(self): best_x = self.mc_client.get(self.getNameQualifier()('global', 'best_x')) if best_x is not None: from json import loads best_x = array(loads(best_x)) return best_x def get_best_f(self): best_f = self.mc_client.get(self.getNameQualifier()('global', 'best_f')) if best_f is not None: from json import loads best_f = array(loads(best_f)) return best_f def reset_best_score(self): self.mc_client.delete(self.getNameQualifier()('global', 'best_x')) self.mc_client.delete(self.getNameQualifier()('global', 'best_f'))
def test_delete(self): client = Client(('localhost', 11212)) client.set('some_key', 'some_value') client.delete('some_key') result = client.get('some_key') self.assertEqual(result, None)
client = Client((ip, port), connect_timeout=TIMEOUT_SEC, timeout=TIMEOUT_SEC) for i in range(ITERATION_NUM): key, value = f"some_key-{i}", f"some_value-{i}" # cmd "set", create/update key:value assert client.set(key, value) # cmd "get", get value with key result = client.get(key).decode("utf-8") assert result == value # cmd "delete", delete key assert client.delete(key) # cmd "add", create key:value assert client.set(key, value) assert client.add(key, value) # cmd "append", append value res = client.append(key, "_suffix") value += "_suffix" assert client.get(key).decode("utf-8") == value # cmd "preppend", preppend value res = client.prepend(key, "prefix_") value = "prefix_" + value assert client.get(key).decode("utf-8") == value