def __init__(
self,
servers: Optional[Tuple[str, int]] = None,
connect_timeout=None,
read_timeout=None,
serde=None,
testing=False,
ignore_exc=False,
):
client_kwargs = dict(
connect_timeout=connect_timeout,
timeout=read_timeout,
serde=serde or JsonSerializerDeserializer(),
ignore_exc=ignore_exc,
)
if testing:
self.client = MockMemcacheClient(
server=None,
**client_kwargs,
)
else:
self.client = HashClient(
servers=servers,
**client_kwargs,
)
def getClient(config):
if type(config) == list:
from pymemcache.client.hash import HashClient
return HashClient(config)
else:
from pymemcache.client.base import Client
return Client(config[0])
def __init__(self, servers="127.0.0.1:11211", **kwargs):
"""Constructor"""
Cache.__init__(self, **kwargs)
if isinstance(servers, string_types):
servers = [s.strip() for s in servers.split(",")]
self.cache = HashClient(servers, use_pooling=True)
self.timeout = int(kwargs.get("timeout", 0))
def remove_node(self):
if self.internal_scale:
print('Cannot scale more than one at a time')
try:
self.internal_scale = True
response = self.elasticache.describe_cache_clusters(
CacheClusterId=self.cluster_id, ShowCacheNodeInfo=True)
count = response['CacheClusters'][0]['NumCacheNodes']
nodes = response['CacheClusters'][0]['CacheNodes']
node = nodes[count - 1]
id_to_remove = node['CacheNodeId']
endpoint = (node['Endpoint']['Address'], node['Endpoint']['Port'])
print('Removing node: ' + str(endpoint) + ' with id ' +
str(id_to_remove))
new_servers = self.servers.copy()
new_servers.remove(endpoint)
print('Remaining servers: ' + str(new_servers))
self.hash_client = HashClient(new_servers,
serializer=self.json_serializer,
deserializer=self.json_deserializer,
use_pooling=True)
self.remap_keys(self.servers)
# remove the node
response = self.elasticache.modify_cache_cluster(
CacheClusterId=self.cluster_id,
NumCacheNodes=count - 1,
CacheNodeIdsToRemove=[id_to_remove],
ApplyImmediately=True)
print(response)
except Exception as e:
print(str(e))
def main_hashclient_test():
client = HashClient([('192.168.204.128', 11211)
]) #('192.168.204.128', 11212),
client.set('heshanshan', 'some value')
result = client.get('heshanshan')
logging.debug(client.get('heshanshan'))
def _get_memcache_client(endpoint, is_local):
client = None
if is_local:
address = endpoint.split(':')
node = (address[0], int(address[1]))
client = Client(node,
serializer=ECClient._json_serializer,
deserializer=ECClient._json_deserializer,
timeout=10000,
connect_timeout=10000)
else:
save_log_info('Trying to get nodes')
nodes = elasticache_auto_discovery.discover(endpoint, 10)
nodes = list(map(lambda x: (x[1], int(x[2])), nodes))
# save_log_info(nodes)
# address = endpoint.split(':')
# nodes = [(address[0], int(address[1]))]
save_log_info('Nodes found: {}'.format(nodes))
client = HashClient(nodes,
serializer=ECClient._json_serializer,
deserializer=ECClient._json_deserializer,
timeout=10000,
connect_timeout=10000)
save_log_info('Return ECClient: {}'.format(client))
return client
def test_custom_client(self):
class MyClient(Client):
pass
client = HashClient([])
client.client_class = MyClient
client.add_server(("host", 11211))
assert isinstance(client.clients["host:11211"], MyClient)
def make_client(self, mock_socket_values, **kwargs):
from pymemcache.client.hash import HashClient
tracer = DummyTracer()
Pin.override(pymemcache, tracer=tracer)
self.client = HashClient([(TEST_HOST, TEST_PORT)], **kwargs)
for _c in self.client.clients.values():
_c.sock = MockSocket(list(mock_socket_values))
return self.client
def make_unix_client(self, sockets, *mock_socket_values, **kwargs):
client = HashClient([], **kwargs)
for socket_, vals in zip(sockets, mock_socket_values):
c = self.make_client_pool(socket_, vals, **kwargs)
client.clients[socket_] = c
client.hasher.add_node(socket_)
return client
def hash_client_test():
'''一致性Hash客户端，通过一致性HASH算法，减少因为某个节点宕机而导致整个集群缓存失效的问题,但是由于set key的时候需要计算hash,所以效率会有一定的影响'''
hash_client = HashClient([('192.168.24.138', 11211),
('192.168.24.139', 11211),
('192.168.24.140', 11211)])
hash_key_prefix = 'hash_test_key'
hash_value_prefix = 'hash_test_value'
#在第一次执行完kv_check方法后，停掉上面3台的任意一台memcache服务，把kv_check的client.set相关代码注释掉，再次执行kv_check方法时，发现有1/3左右的kv丢失
kv_check(hash_client, hash_key_prefix, hash_value_prefix)
def test_setup_client_without_pooling(self):
with mock.patch('pymemcache.client.hash.Client') as internal_client:
client = HashClient([], timeout=999, key_prefix='foo_bar_baz')
client.add_server('127.0.0.1', '11211')
assert internal_client.call_args[0][0] == ('127.0.0.1', '11211')
kwargs = internal_client.call_args[1]
assert kwargs['timeout'] == 999
assert kwargs['key_prefix'] == 'foo_bar_baz'
def check_cluster(self):
print('Checking cluster')
new_servers = self.new_server_list()
try:
new_server_set = set(new_servers)
cur_server_set = set(self.servers)
servers_changed = False
if (new_server_set - cur_server_set) or (cur_server_set -
new_server_set):
print('Found a node difference')
# self.dump_keys('all_keys.txt')
servers_changed = True
if (new_server_set - cur_server_set):
print('Server added')
self.internal_scale = False
self.hash_client = HashClient(
new_servers,
serializer=self.json_serializer,
deserializer=self.json_deserializer,
use_pooling=True)
self.remap_keys(self.servers)
# server removed not with our code
if (cur_server_set - new_server_set):
print('Server removed')
if self.internal_scale:
self.internal_scale = False
else:
print('Removing server')
self.hash_client = HashClient(
new_servers,
serializer=self.json_serializer,
deserializer=self.json_deserializer,
use_pooling=True)
if servers_changed:
self.servers = new_servers
# self.dump_keys('all_keys_2.txt')
self.internal_scale = False
except Exception as e:
print(str(e))
def test_setup_client_without_pooling(self):
client_class = "pymemcache.client.hash.HashClient.client_class"
with mock.patch(client_class) as internal_client:
client = HashClient([], timeout=999, key_prefix="foo_bar_baz")
client.add_server(("127.0.0.1", "11211"))
assert internal_client.call_args[0][0] == ("127.0.0.1", "11211")
kwargs = internal_client.call_args[1]
assert kwargs["timeout"] == 999
assert kwargs["key_prefix"] == "foo_bar_baz"
def get_elastic_cache_client():
global memcache_client
if memcache_client:
return memcache_client
elasticache_config_endpoint = config.config["elastic_cache_url"] + ":11211"
nodes = elasticache_auto_discovery.discover(elasticache_config_endpoint)
nodes = map(lambda x: (x[1], int(x[2])), nodes)
memcache_client = HashClient(nodes)
return memcache_client
def sendtocache(link):
elasticache_config_endpoint = "giftbot.h0k94j4.cfg.use1.cache.amazonaws.com:11211"
nodes = elasticache_auto_discovery.discover(elasticache_config_endpoint)
nodes = map(lambda x: (x[1], int(x[2])), nodes)
memcache_client = HashClient(nodes)
print nodes
memcache_client.set('ImageLink', link[5])
print link[5]
memcache_client.set('ImageName', link[4])
print link[4]
def test_no_servers_left_with_get_many(self):
from pymemcache.client.hash import HashClient
client = HashClient(
[], use_pooling=True,
ignore_exc=True,
timeout=1, connect_timeout=1
)
result = client.get_many(['foo', 'bar'])
assert result == {'foo': False, 'bar': False}
def test_no_servers_left_with_commands(self):
from pymemcache.client.hash import HashClient
client = HashClient(
[], use_pooling=True,
ignore_exc=True,
timeout=1, connect_timeout=1
)
result = client.get('foo')
assert result is False
def test_unavailable_servers_zero_retry_raise_exception(self):
from pymemcache.client.hash import HashClient
client = HashClient(
[('example.com', 11211)], use_pooling=True,
ignore_exc=False,
retry_attempts=0, timeout=1, connect_timeout=1
)
with pytest.raises(socket.error):
client.get('foo')
def test_no_servers_left(self):
from pymemcache.client.hash import HashClient
client = HashClient(
[], use_pooling=True,
ignore_exc=True,
timeout=1, connect_timeout=1
)
hashed_client = client._get_client('foo')
assert hashed_client is None
def test_server_encoding_client(self):
"""
test passed encoding from hash client to clients
"""
encoding = 'utf8'
from pymemcache.client.hash import HashClient
hash_client = HashClient([('example.com', 11211)], encoding=encoding)
for client in hash_client.clients.values():
assert client.encoding == encoding
def __init__(self, cluster_id):
self.elasticache = boto3.client('elasticache', region_name='us-east-1')
self.cluster_id = cluster_id
self.servers = self.new_server_list()
self.internal_scale = False
self.hash_client = HashClient(self.servers,
serializer=self.json_serializer,
deserializer=self.json_deserializer,
use_pooling=True)
thread = TimerThread(self.check_cluster)
thread.start()
def test_no_servers_left_with_set_many(self):
from pymemcache.client.hash import HashClient
client = HashClient([],
use_pooling=True,
ignore_exc=True,
timeout=1,
connect_timeout=1)
result = client.set_many({"foo": "bar"})
assert result == ["foo"]
def test_no_servers_left_raise_exception(self):
from pymemcache.client.hash import HashClient
client = HashClient(
[], use_pooling=True,
ignore_exc=False,
timeout=1, connect_timeout=1
)
with pytest.raises(MemcacheError) as e:
client._get_client('foo')
assert str(e.value) == 'All servers seem to be down right now'
def test_custom_client_with_pooling(self):
class MyClient(Client):
pass
client = HashClient([], use_pooling=True)
client.client_class = MyClient
client.add_server(("host", 11211))
assert isinstance(client.clients["host:11211"], PooledClient)
pool = client.clients["host:11211"].client_pool
with pool.get_and_release(destroy_on_fail=True) as c:
assert isinstance(c, MyClient)
def test_no_servers_left_return_positional_default(self):
from pymemcache.client.hash import HashClient
client = HashClient([],
use_pooling=True,
ignore_exc=True,
timeout=1,
connect_timeout=1)
# Ensure compatibility with clients that pass the default as a
# positional argument
result = client.get("foo", "default")
assert result == "default"
def make_client(self, *mock_socket_values, **kwargs):
current_port = 11012
client = HashClient([], **kwargs)
ip = '127.0.0.1'
for vals in mock_socket_values:
s = '%s:%s' % (ip, current_port)
c = self.make_client_pool((ip, current_port), vals, **kwargs)
client.clients[s] = c
client.hasher.add_node(s)
current_port += 1
return client
def test_server_encoding_pooled(self):
"""
test passed encoding from hash client to pooled clients
"""
encoding = "utf8"
from pymemcache.client.hash import HashClient
hash_client = HashClient([("example.com", 11211)],
use_pooling=True,
encoding=encoding)
for client in hash_client.clients.values():
assert client.encoding == encoding
def get_game_info(group_id):
"""
Get the game info for a group.
:param group_id: ID for the group
:type group_id: String
:return: String representation of group information
"""
nodes = elasticache_auto_discovery.discover('hackgt5mem.gy46cz.cfg.use1.cache.amazonaws.com:11211')
nodes = map(lambda x: (x[1], int(x[2])), nodes)
memcache_client = HashClient(nodes)
return memcache_client.get(str(group_id)).decode("utf-8")
def get_default_client(server_config = None):
if server_config and isinstance(server_config, list):
return HashClient(
server_config,
serializer = serializer,
deserializer = deserializer
)
return Client(
server_config or ('127.0.0.1', 11211),
serializer = serializer,
deserializer = deserializer
)
def make_client(self, *mock_socket_values, **kwargs):
current_port = TEST_PORT
from pymemcache.client.hash import HashClient
self.client = HashClient([], **kwargs)
ip = TEST_HOST
for vals in mock_socket_values:
s = '{}:{}'.format(ip, current_port)
c = self.make_client_pool((ip, current_port), vals, **kwargs)
self.client.clients[s] = c
self.client.hasher.add_node(s)
current_port += 1
return self.client
