class KetamaMemcacheClient(memcache.Client):
""" A memcache subclass. This implementation allows you to add a new host at run
time. It uses consistent hash library (libketama). All server are initilized with
weight 1, that should uniformly distribute the keys, you can assign different weights
that is more reflective of your memcached setup. It takes
By default all servers have a weight of 1.
{ '192.168.0.101:11212': 1, '192.168.0.102:11212': 2, '192.168.0.103:11212': 1 }
would generate a 25/50/25 distribution of the keys.
@see https://github.com/yummybian/consistent-hash
"""
def __init__(self, *args, **kwargs):
super(KetamaMemcacheClient, self).__init__(*args, **kwargs)
if self.servers:
# ConsistentHash expects dictionary of servers and associated weights
s = {
str(server).split(":", 1)[1]: server.weight
for server in self.servers
}
self.consistent_hash = ConsistentHash(s)
def _get_server(self, key):
""" Returns the most likely server to hold the key
"""
server_ip_address = self.consistent_hash.get_node(key)
# find the server object
if server_ip_address:
server = next(
(x for x in self.servers if server_ip_address in str(x)), None)
if server and server.connect():
return server, key
return (None, None)
def add_server(self, server):
""" Adds a host at runtime to client
"""
server_tmp = server
# Create a new host entry
server = memcache._Host(server,
self.debug,
dead_retry=self.dead_retry,
socket_timeout=self.socket_timeout,
flush_on_reconnect=self.flush_on_reconnect)
# Add this to our server choices
self.servers.append(server)
self.buckets.append(server)
# Adds this node to the circle
self.consistent_hash.add_nodes([server_tmp])
class KetamaMemcacheClient(memcache.Client):
""" A memcache subclass. This implementation allows you to add a new host at run
time. It uses consistent hash library (libketama). All server are initilized with
weight 1, that should uniformly distribute the keys, you can assign different weights
that is more reflective of your memcached setup. It takes
By default all servers have a weight of 1.
{ '192.168.0.101:11212': 1, '192.168.0.102:11212': 2, '192.168.0.103:11212': 1 }
would generate a 25/50/25 distribution of the keys.
@see https://github.com/yummybian/consistent-hash
"""
def __init__(self, *args, **kwargs):
super(KetamaMemcacheClient, self).__init__(*args, **kwargs)
if self.servers:
# ConsistentHash expects dictionary of servers and associated weights
s = {str(server).split(":", 1)[1]: server.weight for server in self.servers}
self.consistent_hash = ConsistentHash(s)
def _get_server(self, key):
""" Returns the most likely server to hold the key
"""
server_ip_address = self.consistent_hash.get_node(key)
# find the server object
if server_ip_address:
server = next((x for x in self.servers if server_ip_address in str(x)), None)
if server and server.connect():
return server, key
return (None, None)
def add_server(self, server):
""" Adds a host at runtime to client
"""
server_tmp = server
# Create a new host entry
server = memcache._Host(
server, self.debug, dead_retry=self.dead_retry,
socket_timeout=self.socket_timeout,
flush_on_reconnect=self.flush_on_reconnect
)
# Add this to our server choices
self.servers.append(server)
self.buckets.append(server)
# Adds this node to the circle
self.consistent_hash.add_nodes([server_tmp])
class YamClient(object):
clientFromString = staticmethod(endpoints.clientFromString)
def __init__(self, reactor, hosts, retryDelay=2, **kw):
self.reactor = reactor
self._allHosts = hosts
self._consistentHash = ConsistentHash([])
self._connectionDeferreds = set()
self._protocols = {}
self._retryDelay = retryDelay
self._protocolKwargs = kw
self.disconnecting = False
def connect(self):
self.disconnecting = False
deferreds = []
for host in self._allHosts:
deferreds.append(self._connectHost(host))
dl = defer.DeferredList(deferreds)
dl.addCallback(lambda ign: self)
return dl
def _connectHost(self, host):
endpoint = self.clientFromString(self.reactor, host)
d = endpoint.connect(
MemCacheClientFactory(self.reactor, **self._protocolKwargs))
self._connectionDeferreds.add(d)
d.addCallback(self._gotProtocol, host, d)
d.addErrback(self._connectionFailed, host, d)
return d
def _gotProtocol(self, protocol, host, deferred):
self._connectionDeferreds.discard(deferred)
self._protocols[host] = protocol
self._consistentHash.add_nodes([host])
protocol.deferred.addErrback(self._lostProtocol, host)
def _connectionFailed(self, reason, host, deferred):
self._connectionDeferreds.discard(deferred)
if self.disconnecting:
return
log.err(reason, 'connection to %r failed' % (host,), system='txyam')
self.reactor.callLater(self._retryDelay, self._connectHost, host)
def _lostProtocol(self, reason, host):
if not self.disconnecting:
log.err(reason, 'connection to %r lost' % (host,), system='txyam')
del self._protocols[host]
self._consistentHash.del_nodes([host])
if self.disconnecting:
return
if reason.check(ConnectionAborted):
self._connectHost(host)
else:
self.reactor.callLater(self._retryDelay, self._connectHost, host)
@property
def _allConnections(self):
return itervalues(self._protocols)
def disconnect(self):
self.disconnecting = True
log.msg('disconnecting from all clients', system='txyam')
for d in list(self._connectionDeferreds):
d.cancel()
for proto in self._allConnections:
proto.transport.loseConnection()
def flushAll(self):
return defer.gatherResults(
[proto.flushAll() for proto in self._allConnections])
def stats(self, arg=None):
ds = {}
for host, proto in iteritems(self._protocols):
ds[host] = proto.stats(arg)
return deferredDict(ds)
def version(self):
ds = {}
for host, proto in iteritems(self._protocols):
ds[host] = proto.version()
return deferredDict(ds)
def getClient(self, key):
return self._protocols.get(self._consistentHash.get_node(key))
def getMultiple(self, keys, withIdentifier=False):
clients = defaultdict(list)
for key in keys:
clients[self.getClient(key)].append(key)
dl = defer.DeferredList(
[c.getMultiple(ks, withIdentifier) for c, ks in iteritems(clients)
if c is not None],
consumeErrors=True)
dl.addCallback(self._consolidateMultiple)
return dl
def setMultiple(self, items, flags=0, expireTime=0):
ds = {}
for key, value in iteritems(items):
ds[key] = self.set(key, value, flags, expireTime)
return deferredDict(ds)
def deleteMultiple(self, keys):
ds = {}
for key in keys:
ds[key] = self.delete(key)
return deferredDict(ds)
def _consolidateMultiple(self, results):
ret = {}
for succeeded, result in results:
if succeeded:
ret.update(result)
return ret
set = _wrap('set')
get = _wrap('get')
increment = _wrap('increment')
decrement = _wrap('decrement')
replace = _wrap('replace')
add = _wrap('add')
checkAndSet = _wrap('checkAndSet')
append = _wrap('append')
prepend = _wrap('prepend')
delete = _wrap('delete')
class TestConsistentHash:
init_nodes = {'192.168.0.101:11212': 1,
'192.168.0.102:11212': 1,
'192.168.0.103:11212': 1,
'192.168.0.104:11212': 1}
obj_nums = 10000
@classmethod
def setup_class(cls):
cls.objs = cls.gen_random_objs()
print('Initial nodes {nodes}'.format(nodes=cls.init_nodes))
@classmethod
def teardown_class(cls):
pass
def setUp(self):
self.hit_nums = {}
def tearDown(self):
pass
def test___init__(self):
self.con_hash = ConsistentHash(self.init_nodes)
# Get nodes from hashing ring
for obj in self.objs:
node = self.con_hash.get_node(obj)
self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
distribution = self.show_nodes_balance()
self.validate_distribution(distribution, {
'192.168.0.101:11212': (23, 27),
'192.168.0.102:11212': (23, 27),
'192.168.0.103:11212': (23, 27),
'192.168.0.104:11212': (23, 27)
})
def test_empty__init__(self):
self.con_hash = ConsistentHash()
for obj in self.objs:
node = self.con_hash.get_node(obj)
if node is not None:
raise Exception("Should have received an exception \
when hashing using an empty LUT")
self.con_hash.add_nodes(self.init_nodes)
for obj in self.objs:
node = self.con_hash.get_node(obj)
self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
distribution = self.show_nodes_balance()
self.validate_distribution(distribution, {
'192.168.0.101:11212': (23, 27),
'192.168.0.102:11212': (23, 27),
'192.168.0.103:11212': (23, 27),
'192.168.0.104:11212': (23, 27)
})
def test_add_nodes(self):
self.con_hash = ConsistentHash(self.init_nodes)
# Add nodes to hashing ring
add_nodes = {'192.168.0.105:11212': 1}
self.con_hash.add_nodes(add_nodes)
# Get nodes from hashing ring
for obj in self.objs:
node = self.con_hash.get_node(obj)
self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
distribution = self.show_nodes_balance()
self.validate_distribution(distribution, {
'192.168.0.105:11212': (17, 23),
'192.168.0.102:11212': (17, 23),
'192.168.0.104:11212': (17, 23),
'192.168.0.101:11212': (17, 23),
'192.168.0.103:11212': (17, 23)
})
print('->The {nodes} added!!!'.format(nodes=add_nodes))
def test_del_nodes(self):
self.con_hash = ConsistentHash(self.init_nodes)
# del_nodes = self.nodes[0:2]
del_nodes = ['192.168.0.102:11212', '192.168.0.104:11212']
# Delete the nodes from hashing ring
self.con_hash.del_nodes(del_nodes)
# Get nodes from hashing ring after deleting
for obj in self.objs:
node = self.con_hash.get_node(obj)
self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
distribution = self.show_nodes_balance()
self.validate_distribution(distribution, {
'192.168.0.101:11212': (48, 52),
'192.168.0.103:11212': (48, 52)
})
print('->The {nodes} deleted!!!'.format(nodes=del_nodes))
# -------------Help functions-------------
def show_nodes_balance(self):
distribution = {}
print('-' * 67)
print('Nodes count:{nNodes} Objects count:{nObjs}'.format(
nNodes=self.con_hash.get_nodes_cnt(),
nObjs=len(self.objs)
))
print('-' * 27 + 'Nodes balance' + '-' * 27)
for node in self.con_hash.get_all_nodes():
substitutions = {
'nNodes': node,
'nObjs': self.hit_nums[node],
'percentage': self.get_percent(self.hit_nums[node],
self.obj_nums)
}
print('Nodes:{nNodes} \
- Objects count:{nObjs} \
- percent:{percentage}%'.format(**substitutions))
distribution[node] = substitutions['percentage']
return distribution
def validate_distribution(self, actual, expected):
if expected.keys() != actual.keys():
raise Exception("Expected nodes does not match actual nodes")
for i in expected.keys():
actual_value = actual[i]
min_value = expected[i][0]
max_value = expected[i][1]
if actual_value < min_value or actual_value > max_value:
print(min_value, actual_value, max_value)
raise Exception("Value outside of expected range")
print("Validated ranges")
def get_percent(self, num, sum):
return int(float(num) / sum * 100)
@classmethod
def gen_random_objs(cls, num=10000, len=10):
objs = []
for i in range(num):
objs.append(''.join([random.choice(chars) for i in range(len)]))
return objs
class TestConsistentHash:
init_nodes = {
'192.168.0.101:11212': 1,
'192.168.0.102:11212': 1,
'192.168.0.103:11212': 1,
'192.168.0.104:11212': 1
}
obj_nums = 10000
@classmethod
def setup_class(cls):
cls.objs = cls.gen_random_objs()
print('Initial nodes {nodes}'.format(nodes=cls.init_nodes))
@classmethod
def teardown_class(cls):
pass
def setUp(self):
self.hit_nums = {}
def tearDown(self):
pass
def test___init__(self):
self.con_hash = ConsistentHash(self.init_nodes)
# Get nodes from hashing ring
for obj in self.objs:
node = self.con_hash.get_node(obj)
self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
distribution = self.show_nodes_balance()
self.validate_distribution(
distribution, {
'192.168.0.101:11212': (23, 27),
'192.168.0.102:11212': (23, 27),
'192.168.0.103:11212': (23, 27),
'192.168.0.104:11212': (23, 27)
})
def test_empty__init__(self):
self.con_hash = ConsistentHash()
for obj in self.objs:
node = self.con_hash.get_node(obj)
if node is not None:
raise Exception("Should have received an exception \
when hashing using an empty LUT")
self.con_hash.add_nodes(self.init_nodes)
for obj in self.objs:
node = self.con_hash.get_node(obj)
self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
distribution = self.show_nodes_balance()
self.validate_distribution(
distribution, {
'192.168.0.101:11212': (23, 27),
'192.168.0.102:11212': (23, 27),
'192.168.0.103:11212': (23, 27),
'192.168.0.104:11212': (23, 27)
})
def test_add_nodes(self):
self.con_hash = ConsistentHash(self.init_nodes)
# Add nodes to hashing ring
add_nodes = {'192.168.0.105:11212': 1}
self.con_hash.add_nodes(add_nodes)
# Get nodes from hashing ring
for obj in self.objs:
node = self.con_hash.get_node(obj)
self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
distribution = self.show_nodes_balance()
self.validate_distribution(
distribution, {
'192.168.0.105:11212': (17, 23),
'192.168.0.102:11212': (17, 23),
'192.168.0.104:11212': (17, 23),
'192.168.0.101:11212': (17, 23),
'192.168.0.103:11212': (17, 23)
})
print('->The {nodes} added!!!'.format(nodes=add_nodes))
def test_del_nodes(self):
self.con_hash = ConsistentHash(self.init_nodes)
# del_nodes = self.nodes[0:2]
del_nodes = ['192.168.0.102:11212', '192.168.0.104:11212']
# Delete the nodes from hashing ring
self.con_hash.del_nodes(del_nodes)
# Get nodes from hashing ring after deleting
for obj in self.objs:
node = self.con_hash.get_node(obj)
self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
distribution = self.show_nodes_balance()
self.validate_distribution(distribution, {
'192.168.0.101:11212': (48, 52),
'192.168.0.103:11212': (48, 52)
})
print('->The {nodes} deleted!!!'.format(nodes=del_nodes))
# -------------Help functions-------------
def show_nodes_balance(self):
distribution = {}
print('-' * 67)
print('Nodes count:{nNodes} Objects count:{nObjs}'.format(
nNodes=self.con_hash.get_nodes_cnt(), nObjs=len(self.objs)))
print('-' * 27 + 'Nodes balance' + '-' * 27)
for node in self.con_hash.get_all_nodes():
substitutions = {
'nNodes': node,
'nObjs': self.hit_nums[node],
'percentage': self.get_percent(self.hit_nums[node],
self.obj_nums)
}
print('Nodes:{nNodes} \
- Objects count:{nObjs} \
- percent:{percentage}%'.format(**substitutions))
distribution[node] = substitutions['percentage']
return distribution
def validate_distribution(self, actual, expected):
if expected.keys() != actual.keys():
raise Exception("Expected nodes does not match actual nodes")
for i in expected.keys():
actual_value = actual[i]
min_value = expected[i][0]
max_value = expected[i][1]
if actual_value < min_value or actual_value > max_value:
print(min_value, actual_value, max_value)
raise Exception("Value outside of expected range")
print("Validated ranges")
def get_percent(self, num, sum):
return int(float(num) / sum * 100)
@classmethod
def gen_random_objs(cls, num=10000, len=10):
objs = []
for i in range(num):
objs.append(''.join([random.choice(chars) for i in range(len)]))
return objs
class TestConsistentHash(unittest.TestCase):
init_nodes = {
'192.168.0.101:11212': 1,
'192.168.0.102:11212': 1,
'192.168.0.103:11212': 1,
'192.168.0.104:11212': 1,
}
obj_nums = 10000
@classmethod
def setup_class(cls):
cls.objects = cls.generate_random_objects()
print('Initial nodes {nodes}'.format(nodes=cls.init_nodes))
@classmethod
def teardown_class(cls):
pass
def setUp(self):
self.hit_nums = {}
def tearDown(self):
pass
def test___init__(self):
self.con_hash = ConsistentHash(self.init_nodes)
# Get nodes from hashing ring
for obj in self.objects:
node = self.con_hash.get_node(obj)
self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
distribution = self.show_nodes_balance()
self.validate_distribution(
distribution, {
'192.168.0.101:11212': (23, 27),
'192.168.0.102:11212': (23, 27),
'192.168.0.103:11212': (23, 27),
'192.168.0.104:11212': (23, 27)
})
def test_empty__init__(self):
self.con_hash = ConsistentHash()
for obj in self.objects:
node = self.con_hash.get_node(obj)
if node is not None:
raise AssertionError(
'Should have received an exception when hashing using an empty LUT'
)
self.con_hash.add_nodes(self.init_nodes)
for obj in self.objects:
node = self.con_hash.get_node(obj)
self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
distribution = self.show_nodes_balance()
self.validate_distribution(
distribution, {
'192.168.0.101:11212': (23, 27),
'192.168.0.102:11212': (23, 27),
'192.168.0.103:11212': (23, 27),
'192.168.0.104:11212': (23, 27),
})
def test_add_nodes(self):
self.con_hash = ConsistentHash(self.init_nodes)
# Add nodes to hashing ring
add_nodes = {'192.168.0.105:11212': 1}
self.con_hash.add_nodes(add_nodes)
# Get nodes from hashing ring
for obj in self.objects:
node = self.con_hash.get_node(obj)
self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
distribution = self.show_nodes_balance()
self.validate_distribution(
distribution, {
'192.168.0.105:11212': (17, 23),
'192.168.0.102:11212': (17, 23),
'192.168.0.104:11212': (17, 23),
'192.168.0.101:11212': (17, 23),
'192.168.0.103:11212': (17, 23),
})
print('->The {nodes} added!!!'.format(nodes=add_nodes))
def test_add_nodes_unicode(self):
self.con_hash = ConsistentHash({
u'192.168.0.101:11212': 1,
u'192.168.0.102:11212': 1,
u'192.168.0.103:11212': 1,
u'192.168.0.104:11212': 1,
})
# Add nodes to hashing ring
add_nodes = u'192.168.0.105:11212'
self.con_hash.add_nodes(add_nodes)
# Get nodes from hashing ring
for obj in self.objects:
node = self.con_hash.get_node(obj)
self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
distribution = self.show_nodes_balance()
self.validate_distribution(
distribution, {
'192.168.0.105:11212': (17, 23),
'192.168.0.102:11212': (17, 23),
'192.168.0.104:11212': (17, 23),
'192.168.0.101:11212': (17, 23),
'192.168.0.103:11212': (17, 23),
})
print('->The {nodes} added!!!'.format(nodes=add_nodes))
def test_add_nodes_tuple(self):
self.con_hash = ConsistentHash(self.init_nodes)
# Add nodes to hashing ring
add_nodes = ('192.168.0.105:11212', '192.168.0.106:11212')
self.con_hash.add_nodes(add_nodes)
# Get nodes from hashing ring
for obj in self.objects:
node = self.con_hash.get_node(obj)
self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
distribution = self.show_nodes_balance()
self.validate_distribution(
distribution, {
'192.168.0.106:11212': (15, 17),
'192.168.0.105:11212': (15, 17),
'192.168.0.102:11212': (15, 17),
'192.168.0.104:11212': (15, 17),
'192.168.0.101:11212': (15, 17),
'192.168.0.103:11212': (15, 17),
})
print('->The {nodes} added!!!'.format(nodes=add_nodes))
def test_del_nodes(self):
self.con_hash = ConsistentHash(self.init_nodes)
# del_nodes = self.nodes[0:2]
del_nodes = ['192.168.0.102:11212', '192.168.0.104:11212']
# Delete the nodes from hashing ring
self.con_hash.del_nodes(del_nodes)
# Get nodes from hashing ring after deleting
for obj in self.objects:
node = self.con_hash.get_node(obj)
self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
distribution = self.show_nodes_balance()
self.validate_distribution(distribution, {
'192.168.0.101:11212': (48, 52),
'192.168.0.103:11212': (48, 52)
})
print('->The {nodes} deleted!!!'.format(nodes=del_nodes))
def test_del_nodes_tuple(self):
self.con_hash = ConsistentHash(self.init_nodes)
# del_nodes = self.nodes[0:2]
del_nodes = ('192.168.0.102:11212', '192.168.0.104:11212')
# Delete the nodes from hashing ring
self.con_hash.del_nodes(del_nodes)
# Get nodes from hashing ring after deleting
for obj in self.objects:
node = self.con_hash.get_node(obj)
self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
distribution = self.show_nodes_balance()
self.validate_distribution(distribution, {
'192.168.0.101:11212': (48, 52),
'192.168.0.103:11212': (48, 52)
})
print('->The {nodes} deleted!!!'.format(nodes=del_nodes))
# -------------Help functions-------------
def show_nodes_balance(self):
distribution = {}
print('-' * 67)
print('Nodes count:{nNodes} Objects count:{nObjects}'.format(
nNodes=self.con_hash.get_nodes_cnt(), nObjects=len(self.objects)))
print('-' * 27 + 'Nodes balance' + '-' * 27)
for node in self.con_hash.get_all_nodes():
substitutions = {
'nNodes': node,
'nObjects': self.hit_nums[node],
'percentage': self.get_percent(self.hit_nums[node],
self.obj_nums)
}
print('Nodes: {nNodes} \
- Objects count: {nObjects} \
- percent:{percentage}%'.format(**substitutions))
distribution[node] = substitutions['percentage']
return distribution
@staticmethod
def validate_distribution(actual, expected):
if expected.keys() != actual.keys():
raise AssertionError('Expected nodes does not match actual nodes')
for i in expected.keys():
actual_value = actual[i]
min_value = expected[i][0]
max_value = expected[i][1]
if actual_value < min_value or actual_value > max_value:
print(min_value, actual_value, max_value)
raise AssertionError(
'Value {actual} outside of expected range ({expected1},{expected2})'
.format(
expected1=min_value,
expected2=max_value,
actual=actual_value,
))
print('Validated ranges')
@staticmethod
def get_percent(numerator, denominator):
return int(float(numerator) / denominator * 100)
@staticmethod
def generate_random_objects(num=10000, length=10):
objects = []
for i in range(num):
objects.append(''.join(
[random.choice(chars) for _ in range(length)]))
return objects
def test_sample_hash_output(self):
ConsistentHash.interleave_count = 40
# Test backward compatibility with version 1.0
samples = {
'35132097': 'B',
'25291004': 'D',
'48182416': 'F',
'45818378': 'H',
'52733021': 'A',
'94027025': 'I',
'18116713': 'F',
'75531098': 'J',
'99011825': 'F',
'99371754': 'A',
'19630740': 'D',
'87823770': 'G',
'32160063': 'A',
'28054420': 'E',
'75904283': 'H',
'08458048': 'E',
'51583844': 'I',
'16226754': 'B',
'95450503': 'E',
'47557476': 'C',
'38808589': 'A',
}
hash_ring = ConsistentHash(
objects=['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J'])
for node, output in samples.items():
result = hash_ring.get_node(node)
if result != output:
raise AssertionError(
'Expected node does not match actual node. Expected: {}. Got: {}'
.format(
output,
result,
))