class YamClient(object):
clientFromString = staticmethod(endpoints.clientFromString)
def __init__(self, reactor, hosts, retryDelay=2, **kw):
self.reactor = reactor
self._allHosts = hosts
self._consistentHash = ConsistentHash([])
self._connectionDeferreds = set()
self._protocols = {}
self._retryDelay = retryDelay
self._protocolKwargs = kw
self.disconnecting = False
def connect(self):
self.disconnecting = False
deferreds = []
for host in self._allHosts:
deferreds.append(self._connectHost(host))
dl = defer.DeferredList(deferreds)
dl.addCallback(lambda ign: self)
return dl
def _connectHost(self, host):
endpoint = self.clientFromString(self.reactor, host)
d = endpoint.connect(
MemCacheClientFactory(self.reactor, **self._protocolKwargs))
self._connectionDeferreds.add(d)
d.addCallback(self._gotProtocol, host, d)
d.addErrback(self._connectionFailed, host, d)
return d
def _gotProtocol(self, protocol, host, deferred):
self._connectionDeferreds.discard(deferred)
self._protocols[host] = protocol
self._consistentHash.add_nodes([host])
protocol.deferred.addErrback(self._lostProtocol, host)
def _connectionFailed(self, reason, host, deferred):
self._connectionDeferreds.discard(deferred)
if self.disconnecting:
return
log.err(reason, 'connection to %r failed' % (host,), system='txyam')
self.reactor.callLater(self._retryDelay, self._connectHost, host)
def _lostProtocol(self, reason, host):
if not self.disconnecting:
log.err(reason, 'connection to %r lost' % (host,), system='txyam')
del self._protocols[host]
self._consistentHash.del_nodes([host])
if self.disconnecting:
return
if reason.check(ConnectionAborted):
self._connectHost(host)
else:
self.reactor.callLater(self._retryDelay, self._connectHost, host)
@property
def _allConnections(self):
return itervalues(self._protocols)
def disconnect(self):
self.disconnecting = True
log.msg('disconnecting from all clients', system='txyam')
for d in list(self._connectionDeferreds):
d.cancel()
for proto in self._allConnections:
proto.transport.loseConnection()
def flushAll(self):
return defer.gatherResults(
[proto.flushAll() for proto in self._allConnections])
def stats(self, arg=None):
ds = {}
for host, proto in iteritems(self._protocols):
ds[host] = proto.stats(arg)
return deferredDict(ds)
def version(self):
ds = {}
for host, proto in iteritems(self._protocols):
ds[host] = proto.version()
return deferredDict(ds)
def getClient(self, key):
return self._protocols.get(self._consistentHash.get_node(key))
def getMultiple(self, keys, withIdentifier=False):
clients = defaultdict(list)
for key in keys:
clients[self.getClient(key)].append(key)
dl = defer.DeferredList(
[c.getMultiple(ks, withIdentifier) for c, ks in iteritems(clients)
if c is not None],
consumeErrors=True)
dl.addCallback(self._consolidateMultiple)
return dl
def setMultiple(self, items, flags=0, expireTime=0):
ds = {}
for key, value in iteritems(items):
ds[key] = self.set(key, value, flags, expireTime)
return deferredDict(ds)
def deleteMultiple(self, keys):
ds = {}
for key in keys:
ds[key] = self.delete(key)
return deferredDict(ds)
def _consolidateMultiple(self, results):
ret = {}
for succeeded, result in results:
if succeeded:
ret.update(result)
return ret
set = _wrap('set')
get = _wrap('get')
increment = _wrap('increment')
decrement = _wrap('decrement')
replace = _wrap('replace')
add = _wrap('add')
checkAndSet = _wrap('checkAndSet')
append = _wrap('append')
prepend = _wrap('prepend')
delete = _wrap('delete')
class TestConsistentHash:
init_nodes = {
'192.168.0.101:11212': 1,
'192.168.0.102:11212': 1,
'192.168.0.103:11212': 1,
'192.168.0.104:11212': 1
}
obj_nums = 10000
@classmethod
def setup_class(cls):
cls.objs = cls.gen_random_objs()
print('Initial nodes {nodes}'.format(nodes=cls.init_nodes))
@classmethod
def teardown_class(cls):
pass
def setUp(self):
self.hit_nums = {}
def tearDown(self):
pass
def test___init__(self):
self.con_hash = ConsistentHash(self.init_nodes)
# Get nodes from hashing ring
for obj in self.objs:
node = self.con_hash.get_node(obj)
self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
distribution = self.show_nodes_balance()
self.validate_distribution(
distribution, {
'192.168.0.101:11212': (23, 27),
'192.168.0.102:11212': (23, 27),
'192.168.0.103:11212': (23, 27),
'192.168.0.104:11212': (23, 27)
})
def test_empty__init__(self):
self.con_hash = ConsistentHash()
for obj in self.objs:
node = self.con_hash.get_node(obj)
if node is not None:
raise Exception("Should have received an exception \
when hashing using an empty LUT")
self.con_hash.add_nodes(self.init_nodes)
for obj in self.objs:
node = self.con_hash.get_node(obj)
self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
distribution = self.show_nodes_balance()
self.validate_distribution(
distribution, {
'192.168.0.101:11212': (23, 27),
'192.168.0.102:11212': (23, 27),
'192.168.0.103:11212': (23, 27),
'192.168.0.104:11212': (23, 27)
})
def test_add_nodes(self):
self.con_hash = ConsistentHash(self.init_nodes)
# Add nodes to hashing ring
add_nodes = {'192.168.0.105:11212': 1}
self.con_hash.add_nodes(add_nodes)
# Get nodes from hashing ring
for obj in self.objs:
node = self.con_hash.get_node(obj)
self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
distribution = self.show_nodes_balance()
self.validate_distribution(
distribution, {
'192.168.0.105:11212': (17, 23),
'192.168.0.102:11212': (17, 23),
'192.168.0.104:11212': (17, 23),
'192.168.0.101:11212': (17, 23),
'192.168.0.103:11212': (17, 23)
})
print('->The {nodes} added!!!'.format(nodes=add_nodes))
def test_del_nodes(self):
self.con_hash = ConsistentHash(self.init_nodes)
# del_nodes = self.nodes[0:2]
del_nodes = ['192.168.0.102:11212', '192.168.0.104:11212']
# Delete the nodes from hashing ring
self.con_hash.del_nodes(del_nodes)
# Get nodes from hashing ring after deleting
for obj in self.objs:
node = self.con_hash.get_node(obj)
self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
distribution = self.show_nodes_balance()
self.validate_distribution(distribution, {
'192.168.0.101:11212': (48, 52),
'192.168.0.103:11212': (48, 52)
})
print('->The {nodes} deleted!!!'.format(nodes=del_nodes))
# -------------Help functions-------------
def show_nodes_balance(self):
distribution = {}
print('-' * 67)
print('Nodes count:{nNodes} Objects count:{nObjs}'.format(
nNodes=self.con_hash.get_nodes_cnt(), nObjs=len(self.objs)))
print('-' * 27 + 'Nodes balance' + '-' * 27)
for node in self.con_hash.get_all_nodes():
substitutions = {
'nNodes': node,
'nObjs': self.hit_nums[node],
'percentage': self.get_percent(self.hit_nums[node],
self.obj_nums)
}
print('Nodes:{nNodes} \
- Objects count:{nObjs} \
- percent:{percentage}%'.format(**substitutions))
distribution[node] = substitutions['percentage']
return distribution
def validate_distribution(self, actual, expected):
if expected.keys() != actual.keys():
raise Exception("Expected nodes does not match actual nodes")
for i in expected.keys():
actual_value = actual[i]
min_value = expected[i][0]
max_value = expected[i][1]
if actual_value < min_value or actual_value > max_value:
print(min_value, actual_value, max_value)
raise Exception("Value outside of expected range")
print("Validated ranges")
def get_percent(self, num, sum):
return int(float(num) / sum * 100)
@classmethod
def gen_random_objs(cls, num=10000, len=10):
objs = []
for i in range(num):
objs.append(''.join([random.choice(chars) for i in range(len)]))
return objs
class TestConsistentHash:
init_nodes = {'192.168.0.101:11212': 1,
'192.168.0.102:11212': 1,
'192.168.0.103:11212': 1,
'192.168.0.104:11212': 1}
obj_nums = 10000
@classmethod
def setup_class(cls):
cls.objs = cls.gen_random_objs()
print('Initial nodes {nodes}'.format(nodes=cls.init_nodes))
@classmethod
def teardown_class(cls):
pass
def setUp(self):
self.hit_nums = {}
def tearDown(self):
pass
def test___init__(self):
self.con_hash = ConsistentHash(self.init_nodes)
# Get nodes from hashing ring
for obj in self.objs:
node = self.con_hash.get_node(obj)
self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
distribution = self.show_nodes_balance()
self.validate_distribution(distribution, {
'192.168.0.101:11212': (23, 27),
'192.168.0.102:11212': (23, 27),
'192.168.0.103:11212': (23, 27),
'192.168.0.104:11212': (23, 27)
})
def test_empty__init__(self):
self.con_hash = ConsistentHash()
for obj in self.objs:
node = self.con_hash.get_node(obj)
if node is not None:
raise Exception("Should have received an exception \
when hashing using an empty LUT")
self.con_hash.add_nodes(self.init_nodes)
for obj in self.objs:
node = self.con_hash.get_node(obj)
self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
distribution = self.show_nodes_balance()
self.validate_distribution(distribution, {
'192.168.0.101:11212': (23, 27),
'192.168.0.102:11212': (23, 27),
'192.168.0.103:11212': (23, 27),
'192.168.0.104:11212': (23, 27)
})
def test_add_nodes(self):
self.con_hash = ConsistentHash(self.init_nodes)
# Add nodes to hashing ring
add_nodes = {'192.168.0.105:11212': 1}
self.con_hash.add_nodes(add_nodes)
# Get nodes from hashing ring
for obj in self.objs:
node = self.con_hash.get_node(obj)
self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
distribution = self.show_nodes_balance()
self.validate_distribution(distribution, {
'192.168.0.105:11212': (17, 23),
'192.168.0.102:11212': (17, 23),
'192.168.0.104:11212': (17, 23),
'192.168.0.101:11212': (17, 23),
'192.168.0.103:11212': (17, 23)
})
print('->The {nodes} added!!!'.format(nodes=add_nodes))
def test_del_nodes(self):
self.con_hash = ConsistentHash(self.init_nodes)
# del_nodes = self.nodes[0:2]
del_nodes = ['192.168.0.102:11212', '192.168.0.104:11212']
# Delete the nodes from hashing ring
self.con_hash.del_nodes(del_nodes)
# Get nodes from hashing ring after deleting
for obj in self.objs:
node = self.con_hash.get_node(obj)
self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
distribution = self.show_nodes_balance()
self.validate_distribution(distribution, {
'192.168.0.101:11212': (48, 52),
'192.168.0.103:11212': (48, 52)
})
print('->The {nodes} deleted!!!'.format(nodes=del_nodes))
# -------------Help functions-------------
def show_nodes_balance(self):
distribution = {}
print('-' * 67)
print('Nodes count:{nNodes} Objects count:{nObjs}'.format(
nNodes=self.con_hash.get_nodes_cnt(),
nObjs=len(self.objs)
))
print('-' * 27 + 'Nodes balance' + '-' * 27)
for node in self.con_hash.get_all_nodes():
substitutions = {
'nNodes': node,
'nObjs': self.hit_nums[node],
'percentage': self.get_percent(self.hit_nums[node],
self.obj_nums)
}
print('Nodes:{nNodes} \
- Objects count:{nObjs} \
- percent:{percentage}%'.format(**substitutions))
distribution[node] = substitutions['percentage']
return distribution
def validate_distribution(self, actual, expected):
if expected.keys() != actual.keys():
raise Exception("Expected nodes does not match actual nodes")
for i in expected.keys():
actual_value = actual[i]
min_value = expected[i][0]
max_value = expected[i][1]
if actual_value < min_value or actual_value > max_value:
print(min_value, actual_value, max_value)
raise Exception("Value outside of expected range")
print("Validated ranges")
def get_percent(self, num, sum):
return int(float(num) / sum * 100)
@classmethod
def gen_random_objs(cls, num=10000, len=10):
objs = []
for i in range(num):
objs.append(''.join([random.choice(chars) for i in range(len)]))
return objs
class TestConsistentHash(unittest.TestCase):
init_nodes = {
'192.168.0.101:11212': 1,
'192.168.0.102:11212': 1,
'192.168.0.103:11212': 1,
'192.168.0.104:11212': 1,
}
obj_nums = 10000
@classmethod
def setup_class(cls):
cls.objects = cls.generate_random_objects()
print('Initial nodes {nodes}'.format(nodes=cls.init_nodes))
@classmethod
def teardown_class(cls):
pass
def setUp(self):
self.hit_nums = {}
def tearDown(self):
pass
def test___init__(self):
self.con_hash = ConsistentHash(self.init_nodes)
# Get nodes from hashing ring
for obj in self.objects:
node = self.con_hash.get_node(obj)
self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
distribution = self.show_nodes_balance()
self.validate_distribution(
distribution, {
'192.168.0.101:11212': (23, 27),
'192.168.0.102:11212': (23, 27),
'192.168.0.103:11212': (23, 27),
'192.168.0.104:11212': (23, 27)
})
def test_empty__init__(self):
self.con_hash = ConsistentHash()
for obj in self.objects:
node = self.con_hash.get_node(obj)
if node is not None:
raise AssertionError(
'Should have received an exception when hashing using an empty LUT'
)
self.con_hash.add_nodes(self.init_nodes)
for obj in self.objects:
node = self.con_hash.get_node(obj)
self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
distribution = self.show_nodes_balance()
self.validate_distribution(
distribution, {
'192.168.0.101:11212': (23, 27),
'192.168.0.102:11212': (23, 27),
'192.168.0.103:11212': (23, 27),
'192.168.0.104:11212': (23, 27),
})
def test_add_nodes(self):
self.con_hash = ConsistentHash(self.init_nodes)
# Add nodes to hashing ring
add_nodes = {'192.168.0.105:11212': 1}
self.con_hash.add_nodes(add_nodes)
# Get nodes from hashing ring
for obj in self.objects:
node = self.con_hash.get_node(obj)
self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
distribution = self.show_nodes_balance()
self.validate_distribution(
distribution, {
'192.168.0.105:11212': (17, 23),
'192.168.0.102:11212': (17, 23),
'192.168.0.104:11212': (17, 23),
'192.168.0.101:11212': (17, 23),
'192.168.0.103:11212': (17, 23),
})
print('->The {nodes} added!!!'.format(nodes=add_nodes))
def test_add_nodes_unicode(self):
self.con_hash = ConsistentHash({
u'192.168.0.101:11212': 1,
u'192.168.0.102:11212': 1,
u'192.168.0.103:11212': 1,
u'192.168.0.104:11212': 1,
})
# Add nodes to hashing ring
add_nodes = u'192.168.0.105:11212'
self.con_hash.add_nodes(add_nodes)
# Get nodes from hashing ring
for obj in self.objects:
node = self.con_hash.get_node(obj)
self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
distribution = self.show_nodes_balance()
self.validate_distribution(
distribution, {
'192.168.0.105:11212': (17, 23),
'192.168.0.102:11212': (17, 23),
'192.168.0.104:11212': (17, 23),
'192.168.0.101:11212': (17, 23),
'192.168.0.103:11212': (17, 23),
})
print('->The {nodes} added!!!'.format(nodes=add_nodes))
def test_add_nodes_tuple(self):
self.con_hash = ConsistentHash(self.init_nodes)
# Add nodes to hashing ring
add_nodes = ('192.168.0.105:11212', '192.168.0.106:11212')
self.con_hash.add_nodes(add_nodes)
# Get nodes from hashing ring
for obj in self.objects:
node = self.con_hash.get_node(obj)
self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
distribution = self.show_nodes_balance()
self.validate_distribution(
distribution, {
'192.168.0.106:11212': (15, 17),
'192.168.0.105:11212': (15, 17),
'192.168.0.102:11212': (15, 17),
'192.168.0.104:11212': (15, 17),
'192.168.0.101:11212': (15, 17),
'192.168.0.103:11212': (15, 17),
})
print('->The {nodes} added!!!'.format(nodes=add_nodes))
def test_del_nodes(self):
self.con_hash = ConsistentHash(self.init_nodes)
# del_nodes = self.nodes[0:2]
del_nodes = ['192.168.0.102:11212', '192.168.0.104:11212']
# Delete the nodes from hashing ring
self.con_hash.del_nodes(del_nodes)
# Get nodes from hashing ring after deleting
for obj in self.objects:
node = self.con_hash.get_node(obj)
self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
distribution = self.show_nodes_balance()
self.validate_distribution(distribution, {
'192.168.0.101:11212': (48, 52),
'192.168.0.103:11212': (48, 52)
})
print('->The {nodes} deleted!!!'.format(nodes=del_nodes))
def test_del_nodes_tuple(self):
self.con_hash = ConsistentHash(self.init_nodes)
# del_nodes = self.nodes[0:2]
del_nodes = ('192.168.0.102:11212', '192.168.0.104:11212')
# Delete the nodes from hashing ring
self.con_hash.del_nodes(del_nodes)
# Get nodes from hashing ring after deleting
for obj in self.objects:
node = self.con_hash.get_node(obj)
self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
distribution = self.show_nodes_balance()
self.validate_distribution(distribution, {
'192.168.0.101:11212': (48, 52),
'192.168.0.103:11212': (48, 52)
})
print('->The {nodes} deleted!!!'.format(nodes=del_nodes))
# -------------Help functions-------------
def show_nodes_balance(self):
distribution = {}
print('-' * 67)
print('Nodes count:{nNodes} Objects count:{nObjects}'.format(
nNodes=self.con_hash.get_nodes_cnt(), nObjects=len(self.objects)))
print('-' * 27 + 'Nodes balance' + '-' * 27)
for node in self.con_hash.get_all_nodes():
substitutions = {
'nNodes': node,
'nObjects': self.hit_nums[node],
'percentage': self.get_percent(self.hit_nums[node],
self.obj_nums)
}
print('Nodes: {nNodes} \
- Objects count: {nObjects} \
- percent:{percentage}%'.format(**substitutions))
distribution[node] = substitutions['percentage']
return distribution
@staticmethod
def validate_distribution(actual, expected):
if expected.keys() != actual.keys():
raise AssertionError('Expected nodes does not match actual nodes')
for i in expected.keys():
actual_value = actual[i]
min_value = expected[i][0]
max_value = expected[i][1]
if actual_value < min_value or actual_value > max_value:
print(min_value, actual_value, max_value)
raise AssertionError(
'Value {actual} outside of expected range ({expected1},{expected2})'
.format(
expected1=min_value,
expected2=max_value,
actual=actual_value,
))
print('Validated ranges')
@staticmethod
def get_percent(numerator, denominator):
return int(float(numerator) / denominator * 100)
@staticmethod
def generate_random_objects(num=10000, length=10):
objects = []
for i in range(num):
objects.append(''.join(
[random.choice(chars) for _ in range(length)]))
return objects
def test_sample_hash_output(self):
ConsistentHash.interleave_count = 40
# Test backward compatibility with version 1.0
samples = {
'35132097': 'B',
'25291004': 'D',
'48182416': 'F',
'45818378': 'H',
'52733021': 'A',
'94027025': 'I',
'18116713': 'F',
'75531098': 'J',
'99011825': 'F',
'99371754': 'A',
'19630740': 'D',
'87823770': 'G',
'32160063': 'A',
'28054420': 'E',
'75904283': 'H',
'08458048': 'E',
'51583844': 'I',
'16226754': 'B',
'95450503': 'E',
'47557476': 'C',
'38808589': 'A',
}
hash_ring = ConsistentHash(
objects=['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J'])
for node, output in samples.items():
result = hash_ring.get_node(node)
if result != output:
raise AssertionError(
'Expected node does not match actual node. Expected: {}. Got: {}'
.format(
output,
result,
))