def __init__( self, servers: Optional[Tuple[str, int]] = None, connect_timeout=None, read_timeout=None, serde=None, testing=False, ignore_exc=False, ): client_kwargs = dict( connect_timeout=connect_timeout, timeout=read_timeout, serde=serde or JsonSerializerDeserializer(), ignore_exc=ignore_exc, ) if testing: self.client = MockMemcacheClient( server=None, **client_kwargs, ) else: self.client = HashClient( servers=servers, **client_kwargs, )
def __init__(self, servers="127.0.0.1:11211", **kwargs): """Constructor""" Cache.__init__(self, **kwargs) if isinstance(servers, string_types): servers = [s.strip() for s in servers.split(",")] self.cache = HashClient(servers, use_pooling=True) self.timeout = int(kwargs.get("timeout", 0))
class MemcacheHashCli(object): def __init__(self, hosts): self.client = HashClient(hosts) def set(self, key, value, expire): try: return self.client.set(key, value, expire) except Exception as e: return False def get(self, key): try: return self.client.get(key, default=None) except Exception as e: return None def mset(self, values, expire): try: return self.client.set_many(values, expire) except Exception as e: return False def mget(self, keys): try: return self.client.get_many(keys) except Exception as e: return None
def remove_node(self): if self.internal_scale: print('Cannot scale more than one at a time') try: self.internal_scale = True response = self.elasticache.describe_cache_clusters( CacheClusterId=self.cluster_id, ShowCacheNodeInfo=True) count = response['CacheClusters'][0]['NumCacheNodes'] nodes = response['CacheClusters'][0]['CacheNodes'] node = nodes[count - 1] id_to_remove = node['CacheNodeId'] endpoint = (node['Endpoint']['Address'], node['Endpoint']['Port']) print('Removing node: ' + str(endpoint) + ' with id ' + str(id_to_remove)) new_servers = self.servers.copy() new_servers.remove(endpoint) print('Remaining servers: ' + str(new_servers)) self.hash_client = HashClient(new_servers, serializer=self.json_serializer, deserializer=self.json_deserializer, use_pooling=True) self.remap_keys(self.servers) # remove the node response = self.elasticache.modify_cache_cluster( CacheClusterId=self.cluster_id, NumCacheNodes=count - 1, CacheNodeIdsToRemove=[id_to_remove], ApplyImmediately=True) print(response) except Exception as e: print(str(e))
def test_custom_client(self): class MyClient(Client): pass client = HashClient([]) client.client_class = MyClient client.add_server(("host", 11211)) assert isinstance(client.clients["host:11211"], MyClient)
def test_setup_client_without_pooling(self): with mock.patch('pymemcache.client.hash.Client') as internal_client: client = HashClient([], timeout=999, key_prefix='foo_bar_baz') client.add_server('127.0.0.1', '11211') assert internal_client.call_args[0][0] == ('127.0.0.1', '11211') kwargs = internal_client.call_args[1] assert kwargs['timeout'] == 999 assert kwargs['key_prefix'] == 'foo_bar_baz'
class MemCacheHelper(object): def __init__(self): self.client = HashClient([(MEMCACHED_ENDPOINT, MEMCACHED_PORT)]) def set(self, key, value): self.client.set(key, value) def get(self, key): return self.client.get(key)
def make_unix_client(self, sockets, *mock_socket_values, **kwargs): client = HashClient([], **kwargs) for socket_, vals in zip(sockets, mock_socket_values): c = self.make_client_pool(socket_, vals, **kwargs) client.clients[socket_] = c client.hasher.add_node(socket_) return client
class Memcache(object): """ This class is an interface to a memcache storage backend it implements needed methods to store and fetch for an url shortener. """ from pymemcache.client.hash import HashClient def __init__(self, host, key_expiration, username=None, password=None): """ Instanciate a memcache storage backend object @params: host: the server to connect to key_expiration: key expiration in seconds username: not use password: not use @returns: """ self.client = HashClient(servers='{} 11211'.format(host), connect_timeout=True, timeout=True, use_pooling=True, max_pool_size=100) self.key_expiration = key_expiration def set_value(self, key, value): """ Set a new record in the datavase for a short code @params: key: the short code to insert value: the long url corresponding @returns: True or False if succeeded or not """ return self.client.set(key, value.encode('utf-8'), self.key_expiration) def get_value(self, key): """ Get a long url from the shorten form @params: key: the short form to lookup @returns: a long url if found, None otherwise """ res = None res = self.client.get(key) if res is not None: return res.decode('utf-8') return None def get_stat(self, key): pass def get_all(self): pass
def test_unavailable_servers_zero_retry_raise_exception(self): from pymemcache.client.hash import HashClient client = HashClient( [('example.com', 11211)], use_pooling=True, ignore_exc=False, retry_attempts=0, timeout=1, connect_timeout=1 ) with pytest.raises(socket.error): client.get('foo')
class MetricCache: def __init__(self, server_list): # server_list = list of (host,port) self.client = HashClient(server_list) def increment(self, *, metric_name, value): if not self.client.get(metric_name): self.client.set(metric_name, 0) return self.client.incr(metric_name, value)
def test_no_servers_left_with_get_many(self): from pymemcache.client.hash import HashClient client = HashClient( [], use_pooling=True, ignore_exc=True, timeout=1, connect_timeout=1 ) result = client.get_many(['foo', 'bar']) assert result == {'foo': False, 'bar': False}
def test_no_servers_left_with_commands(self): from pymemcache.client.hash import HashClient client = HashClient( [], use_pooling=True, ignore_exc=True, timeout=1, connect_timeout=1 ) result = client.get('foo') assert result is False
def test_no_servers_left(self): from pymemcache.client.hash import HashClient client = HashClient( [], use_pooling=True, ignore_exc=True, timeout=1, connect_timeout=1 ) hashed_client = client._get_client('foo') assert hashed_client is None
def test_setup_client_without_pooling(self): client_class = "pymemcache.client.hash.HashClient.client_class" with mock.patch(client_class) as internal_client: client = HashClient([], timeout=999, key_prefix="foo_bar_baz") client.add_server(("127.0.0.1", "11211")) assert internal_client.call_args[0][0] == ("127.0.0.1", "11211") kwargs = internal_client.call_args[1] assert kwargs["timeout"] == 999 assert kwargs["key_prefix"] == "foo_bar_baz"
def __init__(self, cluster_id): self.elasticache = boto3.client('elasticache', region_name='us-east-1') self.cluster_id = cluster_id self.servers = self.new_server_list() self.internal_scale = False self.hash_client = HashClient(self.servers, serializer=self.json_serializer, deserializer=self.json_deserializer, use_pooling=True) thread = TimerThread(self.check_cluster) thread.start()
def test_no_servers_left_with_set_many(self): from pymemcache.client.hash import HashClient client = HashClient([], use_pooling=True, ignore_exc=True, timeout=1, connect_timeout=1) result = client.set_many({"foo": "bar"}) assert result == ["foo"]
def test_custom_client_with_pooling(self): class MyClient(Client): pass client = HashClient([], use_pooling=True) client.client_class = MyClient client.add_server(("host", 11211)) assert isinstance(client.clients["host:11211"], PooledClient) pool = client.clients["host:11211"].client_pool with pool.get_and_release(destroy_on_fail=True) as c: assert isinstance(c, MyClient)
def test_no_servers_left_raise_exception(self): from pymemcache.client.hash import HashClient client = HashClient( [], use_pooling=True, ignore_exc=False, timeout=1, connect_timeout=1 ) with pytest.raises(MemcacheError) as e: client._get_client('foo') assert str(e.value) == 'All servers seem to be down right now'
def make_client(self, *mock_socket_values, **kwargs): current_port = 11012 client = HashClient([], **kwargs) ip = '127.0.0.1' for vals in mock_socket_values: s = '%s:%s' % (ip, current_port) c = self.make_client_pool((ip, current_port), vals, **kwargs) client.clients[s] = c client.hasher.add_node(s) current_port += 1 return client
def get_game_info(group_id): """ Get the game info for a group. :param group_id: ID for the group :type group_id: String :return: String representation of group information """ nodes = elasticache_auto_discovery.discover('hackgt5mem.gy46cz.cfg.use1.cache.amazonaws.com:11211') nodes = map(lambda x: (x[1], int(x[2])), nodes) memcache_client = HashClient(nodes) return memcache_client.get(str(group_id)).decode("utf-8")
def test_no_servers_left_return_positional_default(self): from pymemcache.client.hash import HashClient client = HashClient([], use_pooling=True, ignore_exc=True, timeout=1, connect_timeout=1) # Ensure compatibility with clients that pass the default as a # positional argument result = client.get("foo", "default") assert result == "default"
def test_no_servers_left_with_commands_return_default_value(self): from pymemcache.client.hash import HashClient client = HashClient([], use_pooling=True, ignore_exc=True, timeout=1, connect_timeout=1) result = client.get("foo") assert result is None result = client.get("foo", default="default") assert result == "default" result = client.set("foo", "bar") assert result is False
def _start(self): #self.client = pymemcache_client.PooledClient( from pymemcache.client.hash import HashClient self.client = HashClient(self.host, serializer=self._msgpack_serializer, deserializer=self._msgpack_deserializer, timeout=self.timeout, connect_timeout=self.timeout, max_pool_size=self.max_pool_size) # Run heartbeat here because pymemcache use a lazy connection # method and only connect once you do an operation. self.heartbeat() self._group_members = collections.defaultdict(set) self._executor.start()
class MemcacheMap(Map): def open(self): self._db = HashClient(('127.0.0.1', 11211)) return self def _put(self, key: Key, value: bytes) -> Key: self._db.set(key_bytes(key), value) return key def _get(self, uuid: UUID, time: int) -> Optional[bytes]: return self._db.get(key_bytes(key)) def close(self): del (self._db) return self
def _get_memcache_client(endpoint, is_local): client = None if is_local: address = endpoint.split(':') node = (address[0], int(address[1])) client = Client(node, serializer=ECClient._json_serializer, deserializer=ECClient._json_deserializer, timeout=10000, connect_timeout=10000) else: save_log_info('Trying to get nodes') nodes = elasticache_auto_discovery.discover(endpoint, 10) nodes = list(map(lambda x: (x[1], int(x[2])), nodes)) # save_log_info(nodes) # address = endpoint.split(':') # nodes = [(address[0], int(address[1]))] save_log_info('Nodes found: {}'.format(nodes)) client = HashClient(nodes, serializer=ECClient._json_serializer, deserializer=ECClient._json_deserializer, timeout=10000, connect_timeout=10000) save_log_info('Return ECClient: {}'.format(client)) return client
def getClient(config): if type(config) == list: from pymemcache.client.hash import HashClient return HashClient(config) else: from pymemcache.client.base import Client return Client(config[0])
def __init__(self, host, key_expiration, username=None, password=None): """ Instanciate a memcache storage backend object @params: host: the server to connect to key_expiration: key expiration in seconds username: not use password: not use @returns: """ self.client = HashClient(servers='{} 11211'.format(host), connect_timeout=True, timeout=True, use_pooling=True, max_pool_size=100) self.key_expiration = key_expiration
def make_client(self, *mock_socket_values, **kwargs): current_port = 11012 client = HashClient([], **kwargs) ip = '127.0.0.1' for vals in mock_socket_values: s = '%s:%s' % (ip, current_port) c = self.make_client_pool( (ip, current_port), vals ) client.clients[s] = c client.hasher.add_node(s) current_port += 1 return client
def hash_client_test(): '''一致性Hash客户端,通过一致性HASH算法,减少因为某个节点宕机而导致整个集群缓存失效的问题,但是由于set key的时候需要计算hash,所以效率会有一定的影响''' hash_client = HashClient([('192.168.24.138', 11211), ('192.168.24.139', 11211), ('192.168.24.140', 11211)]) hash_key_prefix = 'hash_test_key' hash_value_prefix = 'hash_test_value' #在第一次执行完kv_check方法后,停掉上面3台的任意一台memcache服务,把kv_check的client.set相关代码注释掉,再次执行kv_check方法时,发现有1/3左右的kv丢失 kv_check(hash_client, hash_key_prefix, hash_value_prefix)
def make_client(self, mock_socket_values, **kwargs): from pymemcache.client.hash import HashClient tracer = DummyTracer() Pin.override(pymemcache, tracer=tracer) self.client = HashClient([(TEST_HOST, TEST_PORT)], **kwargs) for _c in self.client.clients.values(): _c.sock = MockSocket(list(mock_socket_values)) return self.client
def get_pymemcache_client(servers, timeout=None, **kwargs): from pymemcache.client.hash import HashClient from pymemcache.serde import ( python_memcache_serializer, python_memcache_deserializer) kwargs.setdefault('serializer', python_memcache_serializer) kwargs.setdefault('deserializer', python_memcache_deserializer) if timeout: kwargs['timeout'] = timeout split = [s.split(':') for s in servers] _servers = [(host, int(port)) for host, port in split] client = HashClient(_servers, **kwargs) try: client.TooBig = ConnectionResetError except NameError: import socket client.TooBig = socket.error return client
def __init__(self, couchbase_uri, memcached_hosts, primary=PRIMARY_COUCHBASE): """ :param couchbase_uri: Connection string for Couchbase :param memcached_hosts: List of Memcached nodes :param primary: Determines which datastore is authoritative. This affects how get operations are performed and which datastore is used for CAS operations. PRIMARY_COUCHBASE: Couchbase is authoritative PRIMARY_MEMCACHED: Memcached is authoritative By default, Couchbase is the primary store :return: """ self.cb = CbBucket(couchbase_uri) self.mc = McClient(memcached_hosts) self._primary = primary
def _start(self): #self.client = pymemcache_client.PooledClient( from pymemcache.client.hash import HashClient self.client = HashClient( self.host, serializer=self._msgpack_serializer, deserializer=self._msgpack_deserializer, timeout=self.timeout, connect_timeout=self.timeout, max_pool_size=self.max_pool_size) # Run heartbeat here because pymemcache use a lazy connection # method and only connect once you do an operation. self.heartbeat() self._group_members = collections.defaultdict(set) self._executor.start()
def setupMemcacheClient(self): """ Sets up the memcache client. """ if os.path.exists(self.APPSCALE_MEMCACHE_FILE): memcache_file = open(self.APPSCALE_MEMCACHE_FILE, "r") all_ips = memcache_file.read().split("\n") memcache_file.close() else: all_ips = ['localhost'] memcaches = [(ip, self.MEMCACHE_PORT) for ip in all_ips if ip] memcaches.sort() self._memcache = HashClient( memcaches, serializer=serializer, deserializer=deserializer, connect_timeout=5, timeout=1, use_pooling=True) # The GAE API expects return values for all mutate operations. for client in six.itervalues(self._memcache.clients): client.default_noreply = False
class CouchbaseMemcacheMirror(object): def __init__(self, couchbase_uri, memcached_hosts, primary=PRIMARY_COUCHBASE): """ :param couchbase_uri: Connection string for Couchbase :param memcached_hosts: List of Memcached nodes :param primary: Determines which datastore is authoritative. This affects how get operations are performed and which datastore is used for CAS operations. PRIMARY_COUCHBASE: Couchbase is authoritative PRIMARY_MEMCACHED: Memcached is authoritative By default, Couchbase is the primary store :return: """ self.cb = CbBucket(couchbase_uri) self.mc = McClient(memcached_hosts) self._primary = primary @property def primary(self): return self._primary def _cb_get(self, key): try: return self.cb.get(key).value except NotFoundError: return None def get(self, key, try_alternate=True): """ Gets a document :param key: The key to retrieve :param try_alternate: Whether to try the secondary data source if the item is not found in the primary. :return: The value as a Python object """ if self._primary == PRIMARY_COUCHBASE: order = [self._cb_get, self.mc.get] else: order = [self.mc.get, self._cb_get] for meth in order: ret = meth(key) if ret or not try_alternate: return ret return None def _cb_mget(self, keys): """ Internal method to execute a Couchbase multi-get :param keys: The keys to retrieve :return: A tuple of {found_key:found_value, ...}, [missing_key1,...] """ try: ok_rvs = self.cb.get_multi(keys) bad_rvs = {} except NotFoundError as e: ok_rvs, bad_rvs = e.split_results() ok_dict = {k: (v.value, v.cas) for k, v in ok_rvs} return ok_dict, bad_rvs.keys() def get_multi(self, keys, try_alternate=True): """ Gets multiple items from the server :param keys: The keys to fetch as an iterable :param try_alternate: Whether to fetch missing items from alternate store :return: A dictionary of key:value. Only contains keys which exist and have values """ if self._primary == PRIMARY_COUCHBASE: ok, err = self._cb_get(keys) if err and try_alternate: ok.update(self.mc.get_many(err)) return ok else: ok = self.mc.get_many(keys) if len(ok) < len(keys) and try_alternate: keys_err = set(keys) - set(ok) ok.update(self._cb_mget(list(keys_err))[0]) return ok def gets(self, key): """ Get an item with its CAS. The item will always be fetched from the primary data store. :param key: the key to get :return: the value of the key, or None if no such value """ if self._primary == PRIMARY_COUCHBASE: try: rv = self.cb.get(key) return key, rv.cas except NotFoundError: return None, None else: return self.mc.gets(key) def gets_multi(self, keys): if self._primary == PRIMARY_COUCHBASE: try: rvs = self.cb.get_multi(keys) except NotFoundError as e: rvs, _ = e.split_results() return {k: (v.value, v.cas) for k, v in rvs} else: # TODO: I'm not sure if this is implemented in HasClient :( return self.mc.gets_many(keys) def delete(self, key): st = Status() try: self.cb.remove(key) except NotFoundError as e: st.cb_error = e st.mc_status = self.mc.delete(key) return st def delete_multi(self, keys): st = Status() try: self.cb.remove_multi(keys) except NotFoundError as e: st.cb_error = e st.mc_status = self.mc.delete_many(keys) def _do_incrdecr(self, key, value, is_incr): cb_value = value if is_incr else -value mc_meth = self.mc.incr if is_incr else self.mc.decr st = Status() try: self.cb.counter(key, delta=cb_value) except NotFoundError as e: st.cb_error = e st.mc_status = mc_meth(key, value) def incr(self, key, value): return self._do_incrdecr(key, value, True) def decr(self, key, value): return self._do_incrdecr(key, value, False) def touch(self, key, expire=0): st = Status() try: self.cb.touch(key, ttl=expire) except NotFoundError as e: st.cb_error = st st.mc_status = self.mc.touch(key) def set(self, key, value, expire=0): """ Write first to Couchbase, and then to Memcached :param key: Key to use :param value: Value to use :param expire: If set, the item will expire in the given amount of time :return: Status object if successful (will always be success). on failure an exception is raised """ self.cb.upsert(key, value, ttl=expire) self.mc.set(key, value, expire=expire) return Status() def set_multi(self, values, expire=0): """ Set multiple items. :param values: A dictionary of key, value indicating values to store :param expire: If present, expiration time for all the items :return: """ self.cb.upsert_multi(values, ttl=expire) self.mc.set_many(values, expire=expire) return Status() def replace(self, key, value, expire=0): """ Replace existing items :param key: key to replace :param value: new value :param expire: expiration for item :return: Status object. Will be OK """ status = Status() try: self.cb.replace(key, value, ttl=expire) except NotFoundError as e: status.cb_error = e status.mc_status = self.mc.replace(key, value, expire=expire) return status def add(self, key, value, expire=0): status = Status() try: self.cb.insert(key, value, ttl=expire) except KeyExistsError as e: status.cb_error = e status.mc_status = self.mc.add(key, value, expire=expire) return status def _append_prepend(self, key, value, is_append): cb_meth = self.cb.append if is_append else self.cb.prepend mc_meth = self.mc.append if is_append else self.mc.prepend st = Status() try: cb_meth(key, value, format=FMT_UTF8) except (NotStoredError, NotFoundError) as e: st.cb_error = e st.mc_status = mc_meth(key, value) def append(self, key, value): return self._append_prepend(key, value, True) def prepend(self, key, value): return self._append_prepend(key, value, False) def cas(self, key, value, cas, expire=0): if self._primary == PRIMARY_COUCHBASE: try: self.cb.replace(key, value, cas=cas, ttl=expire) self.mc.set(key, value, ttl=expire) return True except KeyExistsError: return False except NotFoundError: return None else: return self.mc.cas(key, value, cas)
class MemcachedDriver(coordination._RunWatchersMixin, coordination.CoordinationDriver): """A `memcached`_ based driver. This driver users `memcached`_ concepts to provide the coordination driver semantics and required API(s). It **is** fully functional and implements all of the coordination driver API(s). It stores data into memcache using expiries and `msgpack`_ encoded values. General recommendations/usage considerations: - Memcache (without different backend technology) is a **cache** enough said. .. _memcached: http://memcached.org/ .. _msgpack: http://msgpack.org/ """ CHARACTERISTICS = ( coordination.Characteristics.DISTRIBUTED_ACROSS_THREADS, coordination.Characteristics.DISTRIBUTED_ACROSS_PROCESSES, coordination.Characteristics.DISTRIBUTED_ACROSS_HOSTS, coordination.Characteristics.CAUSAL, ) """ Tuple of :py:class:`~tooz.coordination.Characteristics` introspectable enum member(s) that can be used to interogate how this driver works. """ #: Key prefix attached to groups (used in name-spacing keys) GROUP_PREFIX = b'_TOOZ_GROUP_' #: Key prefix attached to leaders of groups (used in name-spacing keys) GROUP_LEADER_PREFIX = b'_TOOZ_GROUP_LEADER_' #: Key prefix attached to members of groups (used in name-spacing keys) MEMBER_PREFIX = b'_TOOZ_MEMBER_' #: Key where all groups 'known' are stored. GROUP_LIST_KEY = b'_TOOZ_GROUP_LIST' #: Default socket/lock/member/leader timeout used when none is provided. DEFAULT_TIMEOUT = 30 #: String used to keep a key/member alive (until it next expires). STILL_ALIVE = b"It's alive!" def __init__(self, member_id, parsed_url, options): super(MemcachedDriver, self).__init__() options = utils.collapse(options) self._options = options self._member_id = member_id self._joined_groups = set() self._executor = utils.ProxyExecutor.build("Memcached", options) # self.host = (parsed_url.hostname or "localhost", # parsed_url.port or 11211) self.host = [] for one_url in parsed_url: tmp = (one_url.hostname or "localhost", one_url.port or 11211) self.host.append(tmp) default_timeout = options.get('timeout', self.DEFAULT_TIMEOUT) self.timeout = int(default_timeout) self.membership_timeout = int(options.get( 'membership_timeout', default_timeout)) self.lock_timeout = int(options.get( 'lock_timeout', default_timeout)) self.leader_timeout = int(options.get( 'leader_timeout', default_timeout)) max_pool_size = options.get('max_pool_size', None) if max_pool_size is not None: self.max_pool_size = int(max_pool_size) else: self.max_pool_size = None self._acquired_locks = [] @staticmethod def _msgpack_serializer(key, value): if isinstance(value, six.binary_type): return value, 1 return utils.dumps(value), 2 @staticmethod def _msgpack_deserializer(key, value, flags): if flags == 1: return value if flags == 2: return utils.loads(value) raise coordination.SerializationError("Unknown serialization" " format '%s'" % flags) @_translate_failures def _start(self): #self.client = pymemcache_client.PooledClient( from pymemcache.client.hash import HashClient self.client = HashClient( self.host, serializer=self._msgpack_serializer, deserializer=self._msgpack_deserializer, timeout=self.timeout, connect_timeout=self.timeout, max_pool_size=self.max_pool_size) # Run heartbeat here because pymemcache use a lazy connection # method and only connect once you do an operation. self.heartbeat() self._group_members = collections.defaultdict(set) self._executor.start() @_translate_failures def _stop(self): for lock in list(self._acquired_locks): lock.release() self.client.delete(self._encode_member_id(self._member_id)) for g in list(self._joined_groups): try: self.leave_group(g).get() except (coordination.MemberNotJoined, coordination.GroupNotCreated): # Guess we got booted out/never existed in the first place... pass except coordination.ToozError: LOG.warning("Unable to leave group '%s'", g, exc_info=True) self._executor.stop() # self.client.close() def _encode_group_id(self, group_id): return self.GROUP_PREFIX + group_id def _encode_member_id(self, member_id): return self.MEMBER_PREFIX + member_id def _encode_group_leader(self, group_id): return self.GROUP_LEADER_PREFIX + group_id @_retry.retry() def _add_group_to_group_list(self, group_id): """Add group to the group list. :param group_id: The group id """ group_list, cas = self.client.gets(self.GROUP_LIST_KEY) if cas: group_list = set(group_list) group_list.add(group_id) if not self.client.cas(self.GROUP_LIST_KEY, list(group_list), cas): # Someone updated the group list before us, try again! raise _retry.Retry else: if not self.client.add(self.GROUP_LIST_KEY, [group_id], noreply=False): # Someone updated the group list before us, try again! raise _retry.Retry @_retry.retry() def _remove_from_group_list(self, group_id): """Remove group from the group list. :param group_id: The group id """ group_list, cas = self.client.gets(self.GROUP_LIST_KEY) group_list = set(group_list) group_list.remove(group_id) if not self.client.cas(self.GROUP_LIST_KEY, list(group_list), cas): # Someone updated the group list before us, try again! raise _retry.Retry def create_group(self, group_id): encoded_group = self._encode_group_id(group_id) @_translate_failures def _create_group(): if not self.client.add(encoded_group, {}, noreply=False): raise coordination.GroupAlreadyExist(group_id) self._add_group_to_group_list(group_id) return MemcachedFutureResult(self._executor.submit(_create_group)) def get_groups(self): @_translate_failures def _get_groups(): return self.client.get(self.GROUP_LIST_KEY) or [] return MemcachedFutureResult(self._executor.submit(_get_groups)) def join_group(self, group_id, capabilities=b""): encoded_group = self._encode_group_id(group_id) @_retry.retry() @_translate_failures def _join_group(): group_members, cas = self.client.gets(encoded_group) if group_members is None: raise coordination.GroupNotCreated(group_id) if self._member_id in group_members: raise coordination.MemberAlreadyExist(group_id, self._member_id) group_members[self._member_id] = { b"capabilities": capabilities, } if not self.client.cas(encoded_group, group_members, cas): # It changed, let's try again raise _retry.Retry self._joined_groups.add(group_id) return MemcachedFutureResult(self._executor.submit(_join_group)) def leave_group(self, group_id): encoded_group = self._encode_group_id(group_id) @_retry.retry() @_translate_failures def _leave_group(): group_members, cas = self.client.gets(encoded_group) if group_members is None: raise coordination.GroupNotCreated(group_id) if self._member_id not in group_members: raise coordination.MemberNotJoined(group_id, self._member_id) del group_members[self._member_id] if not self.client.cas(encoded_group, group_members, cas): # It changed, let's try again raise _retry.Retry self._joined_groups.discard(group_id) return MemcachedFutureResult(self._executor.submit(_leave_group)) def _destroy_group(self, group_id): self.client.delete(self._encode_group_id(group_id)) def delete_group(self, group_id): encoded_group = self._encode_group_id(group_id) @_retry.retry() @_translate_failures def _delete_group(): group_members, cas = self.client.gets(encoded_group) if group_members is None: raise coordination.GroupNotCreated(group_id) if group_members != {}: raise coordination.GroupNotEmpty(group_id) # Delete is not atomic, so we first set the group to # using CAS, and then we delete it, to avoid race conditions. if not self.client.cas(encoded_group, None, cas): raise _retry.Retry self.client.delete(encoded_group) self._remove_from_group_list(group_id) return MemcachedFutureResult(self._executor.submit(_delete_group)) @_retry.retry() @_translate_failures def _get_members(self, group_id): encoded_group = self._encode_group_id(group_id) group_members, cas = self.client.gets(encoded_group) if group_members is None: raise coordination.GroupNotCreated(group_id) actual_group_members = {} for m, v in six.iteritems(group_members): # Never kick self from the group, we know we're alive if (m == self._member_id or self.client.get(self._encode_member_id(m))): actual_group_members[m] = v if group_members != actual_group_members: # There are some dead members, update the group if not self.client.cas(encoded_group, actual_group_members, cas): # It changed, let's try again raise _retry.Retry return actual_group_members def get_members(self, group_id): def _get_members(): return self._get_members(group_id).keys() return MemcachedFutureResult(self._executor.submit(_get_members)) def get_member_capabilities(self, group_id, member_id): def _get_member_capabilities(): group_members = self._get_members(group_id) if member_id not in group_members: raise coordination.MemberNotJoined(group_id, member_id) return group_members[member_id][b'capabilities'] return MemcachedFutureResult( self._executor.submit(_get_member_capabilities)) def update_capabilities(self, group_id, capabilities): encoded_group = self._encode_group_id(group_id) @_retry.retry() @_translate_failures def _update_capabilities(): group_members, cas = self.client.gets(encoded_group) if group_members is None: raise coordination.GroupNotCreated(group_id) if self._member_id not in group_members: raise coordination.MemberNotJoined(group_id, self._member_id) group_members[self._member_id][b'capabilities'] = capabilities if not self.client.cas(encoded_group, group_members, cas): # It changed, try again raise _retry.Retry return MemcachedFutureResult( self._executor.submit(_update_capabilities)) def get_leader(self, group_id): def _get_leader(): return self._get_leader_lock(group_id).get_owner() return MemcachedFutureResult(self._executor.submit(_get_leader)) @_translate_failures def heartbeat(self): self.client.set(self._encode_member_id(self._member_id), self.STILL_ALIVE, expire=self.membership_timeout) # Reset the acquired locks for lock in self._acquired_locks: lock.heartbeat() return min(self.membership_timeout, self.leader_timeout, self.lock_timeout) @_translate_failures def _init_watch_group(self, group_id): members = self.client.get(self._encode_group_id(group_id)) if members is None: raise coordination.GroupNotCreated(group_id) # Initialize with the current group member list if group_id not in self._group_members: self._group_members[group_id] = set(members.keys()) def watch_join_group(self, group_id, callback): self._init_watch_group(group_id) return super(MemcachedDriver, self).watch_join_group( group_id, callback) def unwatch_join_group(self, group_id, callback): return super(MemcachedDriver, self).unwatch_join_group( group_id, callback) def watch_leave_group(self, group_id, callback): self._init_watch_group(group_id) return super(MemcachedDriver, self).watch_leave_group( group_id, callback) def unwatch_leave_group(self, group_id, callback): return super(MemcachedDriver, self).unwatch_leave_group( group_id, callback) def watch_elected_as_leader(self, group_id, callback): return super(MemcachedDriver, self).watch_elected_as_leader( group_id, callback) def unwatch_elected_as_leader(self, group_id, callback): return super(MemcachedDriver, self).unwatch_elected_as_leader( group_id, callback) def get_lock(self, name): return MemcachedLock(self, name, self.lock_timeout) def _get_leader_lock(self, group_id): return MemcachedLock(self, self._encode_group_leader(group_id), self.leader_timeout) @_translate_failures def run_elect_coordinator(self): for group_id, hooks in six.iteritems(self._hooks_elected_leader): # Try to grab the lock, if that fails, that means someone has it # already. leader_lock = self._get_leader_lock(group_id) if leader_lock.acquire(blocking=False): # We got the lock hooks.run(coordination.LeaderElected( group_id, self._member_id)) def run_watchers(self, timeout=None): result = super(MemcachedDriver, self).run_watchers(timeout=timeout) self.run_elect_coordinator() return result
class MemcacheService(apiproxy_stub.APIProxyStub): """Python only memcache service. This service keeps all data in any external servers running memcached. """ # The memcached default port. MEMCACHE_PORT = 11211 # An AppScale file which has a list of IPs running memcached. APPSCALE_MEMCACHE_FILE = "/etc/appscale/memcache_ips" def __init__(self, project_id, service_name='memcache'): """Initializer. Args: service_name: Service name expected for all calls. """ super(MemcacheService, self).__init__(service_name) self._memcache = None self.setupMemcacheClient() self._methods = {MemcacheSetRequest.SET: self._memcache.set, MemcacheSetRequest.ADD: self._memcache.add, MemcacheSetRequest.REPLACE: self._memcache.replace, MemcacheSetRequest.CAS: self._memcache.cas} self._project_id = project_id def setupMemcacheClient(self): """ Sets up the memcache client. """ if os.path.exists(self.APPSCALE_MEMCACHE_FILE): memcache_file = open(self.APPSCALE_MEMCACHE_FILE, "r") all_ips = memcache_file.read().split("\n") memcache_file.close() else: all_ips = ['localhost'] memcaches = [(ip, self.MEMCACHE_PORT) for ip in all_ips if ip] memcaches.sort() self._memcache = HashClient( memcaches, serializer=serializer, deserializer=deserializer, connect_timeout=5, timeout=1, use_pooling=True) # The GAE API expects return values for all mutate operations. for client in six.itervalues(self._memcache.clients): client.default_noreply = False def _Dynamic_Get(self, request, response): """Implementation of gets for memcache. Args: request: A MemcacheGetRequest protocol buffer. response: A MemcacheGetResponse protocol buffer. """ # Remove duplicate keys. original_keys = { encode_key(self._project_id, request.name_space(), key): key for key in request.key_list()} try: backend_response = self._memcache.get_many( original_keys.keys(), gets=request.for_cas()) except MemcacheClientError as error: raise apiproxy_errors.ApplicationError(INVALID_VALUE, str(error)) except TRANSIENT_ERRORS as error: raise apiproxy_errors.ApplicationError( UNSPECIFIED_ERROR, 'Transient memcache error: {}'.format(error)) for encoded_key, value_tuple in six.iteritems(backend_response): item = response.add_item() item.set_key(original_keys[encoded_key]) if request.for_cas(): item.set_cas_id(int(value_tuple[1])) value_tuple = value_tuple[0] item.set_value(value_tuple[0]) item.set_flags(value_tuple[1]) def _Dynamic_Set(self, request, response): """Implementation of sets for memcache. Args: request: A MemcacheSetRequest. response: A MemcacheSetResponse. """ namespace = request.name_space() if any(item.set_policy() not in self._methods for item in request.item_list()): raise apiproxy_errors.ApplicationError( INVALID_VALUE, 'Unsupported set_policy') if not all(item.has_cas_id() for item in request.item_list() if item.set_policy() == MemcacheSetRequest.CAS): raise apiproxy_errors.ApplicationError( INVALID_VALUE, 'All CAS items must have a cas_id') for item in request.item_list(): try: encoded_key = encode_key(self._project_id, namespace, item.key()) except apiproxy_errors.ApplicationError: response.add_set_status(MemcacheSetResponse.ERROR) continue args = {'key': encoded_key, 'value': (item.value(), item.flags()), 'expire': int(item.expiration_time())} is_cas = item.set_policy() == MemcacheSetRequest.CAS if is_cas: args['cas'] = six.binary_type(item.cas_id()) try: backend_response = self._methods[item.set_policy()](**args) except (TRANSIENT_ERRORS + (MemcacheClientError,)): response.add_set_status(MemcacheSetResponse.ERROR) continue if backend_response: response.add_set_status(MemcacheSetResponse.STORED) continue if is_cas and backend_response is False: response.add_set_status(MemcacheSetResponse.EXISTS) continue response.add_set_status(MemcacheSetResponse.NOT_STORED) def _Dynamic_Delete(self, request, response): """Implementation of delete in memcache. Args: request: A MemcacheDeleteRequest protocol buffer. response: A MemcacheDeleteResponse protocol buffer. """ for item in request.item_list(): encoded_key = encode_key(self._project_id, request.name_space(), item.key()) try: key_existed = self._memcache.delete(encoded_key) except MemcacheClientError as error: raise apiproxy_errors.ApplicationError(INVALID_VALUE, str(error)) except TRANSIENT_ERRORS as error: raise apiproxy_errors.ApplicationError( UNSPECIFIED_ERROR, 'Transient memcache error: {}'.format(error)) response.add_delete_status(MemcacheDeleteResponse.DELETED if key_existed else MemcacheDeleteResponse.NOT_FOUND) def _Increment(self, namespace, request): """Internal function for incrementing from a MemcacheIncrementRequest. Args: namespace: A string containing the namespace for the request, if any. Pass an empty string if there is no namespace. request: A MemcacheIncrementRequest instance. Returns: An integer indicating the new value. Raises: ApplicationError if unable to perform the mutation. """ encoded_key = encode_key(self._project_id, namespace, request.key()) method = self._memcache.incr if request.direction() == MemcacheIncrementRequest.DECREMENT: method = self._memcache.decr try: response = method(encoded_key, request.delta()) except MemcacheClientError as error: raise apiproxy_errors.ApplicationError(INVALID_VALUE, str(error)) except TRANSIENT_ERRORS as error: raise apiproxy_errors.ApplicationError( UNSPECIFIED_ERROR, 'Transient memcache error: {}'.format(error)) if response is None and not request.has_initial_value(): raise apiproxy_errors.ApplicationError( UNSPECIFIED_ERROR, 'Key does not exist') if response is not None: return response # If the key was not present and an initial value was provided, perform # the mutation client-side and set the key if it still doesn't exist. flags = 0 if request.has_initial_flags(): flags = request.initial_flags() if request.direction() == MemcacheIncrementRequest.INCREMENT: updated_val = request.initial_value() + request.delta() else: updated_val = request.initial_value() - request.delta() updated_val = max(updated_val, 0) % (MAX_INCR + 1) try: response = self._memcache.add( encoded_key, (six.binary_type(updated_val), flags)) except (TRANSIENT_ERRORS + (MemcacheClientError,)): raise apiproxy_errors.ApplicationError( UNSPECIFIED_ERROR, 'Unable to set initial value') if response is False: raise apiproxy_errors.ApplicationError( UNSPECIFIED_ERROR, 'Unable to set initial value') return updated_val def _Dynamic_Increment(self, request, response): """Implementation of increment for memcache. Args: request: A MemcacheIncrementRequest protocol buffer. response: A MemcacheIncrementResponse protocol buffer. """ new_value = self._Increment(request.name_space(), request) response.set_new_value(new_value) def _Dynamic_BatchIncrement(self, request, response): """Implementation of batch increment for memcache. Args: request: A MemcacheBatchIncrementRequest protocol buffer. response: A MemcacheBatchIncrementResponse protocol buffer. """ for request_item in request.item_list(): item = response.add_item() try: new_value = self._Increment(request.name_space(), request_item) except apiproxy_errors.ApplicationError as error: if error.application_error == INVALID_VALUE: item.set_increment_status(MemcacheIncrementResponse.NOT_CHANGED) else: item.set_increment_status(MemcacheIncrementResponse.ERROR) continue item.set_increment_status(MemcacheIncrementResponse.OK) item.set_new_value(new_value) def _Dynamic_FlushAll(self, request, response): """Implementation of MemcacheService::FlushAll(). Args: request: A MemcacheFlushRequest. response: A MemcacheFlushResponse. """ # TODO: Prevent a project from clearing another project's namespace. self._memcache.flush_all() def _Dynamic_Stats(self, request, response): """Implementation of MemcacheService::Stats(). Args: request: A MemcacheStatsRequest. response: A MemcacheStatsResponse. """ # TODO: Gather stats for a project rather than the deployment. hits = 0 misses = 0 byte_hits = 0 items = 0 byte_count = 0 oldest_item_age = 0 for server in six.itervalues(self._memcache.clients): server_stats = server.stats() hits += server_stats.get('get_hits', 0) misses += server_stats.get('get_misses', 0) byte_hits += server_stats.get('bytes_read', 0) items += server_stats.get('curr_items', 0) byte_count += server_stats.get('bytes', 0) # Using the "age" field may not be correct here. The GAE docs claim this # should specify "how long in seconds since the oldest item in the cache # was accessed" rather than when it was created. item_stats = server.stats('items') oldest_server_item = max(age for key, age in six.iteritems(item_stats) if key.endswith(':age')) oldest_item_age = max(oldest_item_age, oldest_server_item) stats = response.mutable_stats() stats.set_hits(hits) stats.set_misses(misses) stats.set_byte_hits(byte_hits) stats.set_items(items) stats.set_bytes(byte_count) stats.set_oldest_item_age(oldest_item_age)
def _get_client(self, key): client = HashClient._get_client(self, key) client.default_noreply = False return client