class MemcacheHashCli(object): def __init__(self, hosts): self.client = HashClient(hosts) def set(self, key, value, expire): try: return self.client.set(key, value, expire) except Exception as e: return False def get(self, key): try: return self.client.get(key, default=None) except Exception as e: return None def mset(self, values, expire): try: return self.client.set_many(values, expire) except Exception as e: return False def mget(self, keys): try: return self.client.get_many(keys) except Exception as e: return None
def test_no_servers_left_with_set_many(self): from pymemcache.client.hash import HashClient client = HashClient( [], use_pooling=True, ignore_exc=True, timeout=1, connect_timeout=1 ) result = client.set_many({'foo': 'bar'}) assert result == ['foo']
class MemcachedCache(CacheBase): """ Memcached-backed cache implementation. Compatible with AWS ElastiCache when using their memcached interface. """ def __init__( self, servers: Optional[Tuple[str, int]] = None, connect_timeout=None, read_timeout=None, serde=None, testing=False, ignore_exc=False, ): client_kwargs = dict( connect_timeout=connect_timeout, timeout=read_timeout, serde=serde or JsonSerializerDeserializer(), ignore_exc=ignore_exc, ) if testing: self.client = MockMemcacheClient( server=None, **client_kwargs, ) else: self.client = HashClient( servers=servers, **client_kwargs, ) def get(self, key: str): """ Return the value for a key, or None if not found """ return self.client.get(key) def add(self, key: str, value, ttl=None): """ Set the value for a key, but only that key hasn't been set. """ if ttl is None: # pymemcache interprets 0 as no expiration ttl = 0 # NB: If input is malformed, this will not raise errors. # set `noreply` to False for further debugging return self.client.add(key, value, expire=ttl) def set(self, key: str, value, ttl=None): """ Set the value for a key, but overwriting existing values """ if ttl is None: # pymemcache interprets 0 as no expiration ttl = 0 # NB: If input is malformed, this will not raise errors. # set `noreply` to False for further debugging return self.client.set(key, value, expire=ttl) def set_many(self, values, ttl=None): """ Set the many key-value pairs at a time, overwriting existing values """ if ttl is None: # pymemcache interprets 0 as no expiration ttl = 0 return self.client.set_many(values, expire=ttl)
class CouchbaseMemcacheMirror(object): def __init__(self, couchbase_uri, memcached_hosts, primary=PRIMARY_COUCHBASE): """ :param couchbase_uri: Connection string for Couchbase :param memcached_hosts: List of Memcached nodes :param primary: Determines which datastore is authoritative. This affects how get operations are performed and which datastore is used for CAS operations. PRIMARY_COUCHBASE: Couchbase is authoritative PRIMARY_MEMCACHED: Memcached is authoritative By default, Couchbase is the primary store :return: """ self.cb = CbBucket(couchbase_uri) self.mc = McClient(memcached_hosts) self._primary = primary @property def primary(self): return self._primary def _cb_get(self, key): try: return self.cb.get(key).value except NotFoundError: return None def get(self, key, try_alternate=True): """ Gets a document :param key: The key to retrieve :param try_alternate: Whether to try the secondary data source if the item is not found in the primary. :return: The value as a Python object """ if self._primary == PRIMARY_COUCHBASE: order = [self._cb_get, self.mc.get] else: order = [self.mc.get, self._cb_get] for meth in order: ret = meth(key) if ret or not try_alternate: return ret return None def _cb_mget(self, keys): """ Internal method to execute a Couchbase multi-get :param keys: The keys to retrieve :return: A tuple of {found_key:found_value, ...}, [missing_key1,...] """ try: ok_rvs = self.cb.get_multi(keys) bad_rvs = {} except NotFoundError as e: ok_rvs, bad_rvs = e.split_results() ok_dict = {k: (v.value, v.cas) for k, v in ok_rvs} return ok_dict, bad_rvs.keys() def get_multi(self, keys, try_alternate=True): """ Gets multiple items from the server :param keys: The keys to fetch as an iterable :param try_alternate: Whether to fetch missing items from alternate store :return: A dictionary of key:value. Only contains keys which exist and have values """ if self._primary == PRIMARY_COUCHBASE: ok, err = self._cb_get(keys) if err and try_alternate: ok.update(self.mc.get_many(err)) return ok else: ok = self.mc.get_many(keys) if len(ok) < len(keys) and try_alternate: keys_err = set(keys) - set(ok) ok.update(self._cb_mget(list(keys_err))[0]) return ok def gets(self, key): """ Get an item with its CAS. The item will always be fetched from the primary data store. :param key: the key to get :return: the value of the key, or None if no such value """ if self._primary == PRIMARY_COUCHBASE: try: rv = self.cb.get(key) return key, rv.cas except NotFoundError: return None, None else: return self.mc.gets(key) def gets_multi(self, keys): if self._primary == PRIMARY_COUCHBASE: try: rvs = self.cb.get_multi(keys) except NotFoundError as e: rvs, _ = e.split_results() return {k: (v.value, v.cas) for k, v in rvs} else: # TODO: I'm not sure if this is implemented in HasClient :( return self.mc.gets_many(keys) def delete(self, key): st = Status() try: self.cb.remove(key) except NotFoundError as e: st.cb_error = e st.mc_status = self.mc.delete(key) return st def delete_multi(self, keys): st = Status() try: self.cb.remove_multi(keys) except NotFoundError as e: st.cb_error = e st.mc_status = self.mc.delete_many(keys) def _do_incrdecr(self, key, value, is_incr): cb_value = value if is_incr else -value mc_meth = self.mc.incr if is_incr else self.mc.decr st = Status() try: self.cb.counter(key, delta=cb_value) except NotFoundError as e: st.cb_error = e st.mc_status = mc_meth(key, value) def incr(self, key, value): return self._do_incrdecr(key, value, True) def decr(self, key, value): return self._do_incrdecr(key, value, False) def touch(self, key, expire=0): st = Status() try: self.cb.touch(key, ttl=expire) except NotFoundError as e: st.cb_error = st st.mc_status = self.mc.touch(key) def set(self, key, value, expire=0): """ Write first to Couchbase, and then to Memcached :param key: Key to use :param value: Value to use :param expire: If set, the item will expire in the given amount of time :return: Status object if successful (will always be success). on failure an exception is raised """ self.cb.upsert(key, value, ttl=expire) self.mc.set(key, value, expire=expire) return Status() def set_multi(self, values, expire=0): """ Set multiple items. :param values: A dictionary of key, value indicating values to store :param expire: If present, expiration time for all the items :return: """ self.cb.upsert_multi(values, ttl=expire) self.mc.set_many(values, expire=expire) return Status() def replace(self, key, value, expire=0): """ Replace existing items :param key: key to replace :param value: new value :param expire: expiration for item :return: Status object. Will be OK """ status = Status() try: self.cb.replace(key, value, ttl=expire) except NotFoundError as e: status.cb_error = e status.mc_status = self.mc.replace(key, value, expire=expire) return status def add(self, key, value, expire=0): status = Status() try: self.cb.insert(key, value, ttl=expire) except KeyExistsError as e: status.cb_error = e status.mc_status = self.mc.add(key, value, expire=expire) return status def _append_prepend(self, key, value, is_append): cb_meth = self.cb.append if is_append else self.cb.prepend mc_meth = self.mc.append if is_append else self.mc.prepend st = Status() try: cb_meth(key, value, format=FMT_UTF8) except (NotStoredError, NotFoundError) as e: st.cb_error = e st.mc_status = mc_meth(key, value) def append(self, key, value): return self._append_prepend(key, value, True) def prepend(self, key, value): return self._append_prepend(key, value, False) def cas(self, key, value, cas, expire=0): if self._primary == PRIMARY_COUCHBASE: try: self.cb.replace(key, value, cas=cas, ttl=expire) self.mc.set(key, value, ttl=expire) return True except KeyExistsError: return False except NotFoundError: return None else: return self.mc.cas(key, value, cas)