Exemple #1
0
class YamClient(object):
    clientFromString = staticmethod(endpoints.clientFromString)

    def __init__(self, reactor, hosts, retryDelay=2, **kw):
        self.reactor = reactor
        self._allHosts = hosts
        self._consistentHash = ConsistentHash([])
        self._connectionDeferreds = set()
        self._protocols = {}
        self._retryDelay = retryDelay
        self._protocolKwargs = kw
        self.disconnecting = False

    def connect(self):
        self.disconnecting = False
        deferreds = []
        for host in self._allHosts:
            deferreds.append(self._connectHost(host))

        dl = defer.DeferredList(deferreds)
        dl.addCallback(lambda ign: self)
        return dl

    def _connectHost(self, host):
        endpoint = self.clientFromString(self.reactor, host)
        d = endpoint.connect(
            MemCacheClientFactory(self.reactor, **self._protocolKwargs))
        self._connectionDeferreds.add(d)
        d.addCallback(self._gotProtocol, host, d)
        d.addErrback(self._connectionFailed, host, d)
        return d

    def _gotProtocol(self, protocol, host, deferred):
        self._connectionDeferreds.discard(deferred)
        self._protocols[host] = protocol
        self._consistentHash.add_nodes([host])
        protocol.deferred.addErrback(self._lostProtocol, host)

    def _connectionFailed(self, reason, host, deferred):
        self._connectionDeferreds.discard(deferred)
        if self.disconnecting:
            return
        log.err(reason, 'connection to %r failed' % (host,), system='txyam')
        self.reactor.callLater(self._retryDelay, self._connectHost, host)

    def _lostProtocol(self, reason, host):
        if not self.disconnecting:
            log.err(reason, 'connection to %r lost' % (host,), system='txyam')
        del self._protocols[host]
        self._consistentHash.del_nodes([host])
        if self.disconnecting:
            return
        if reason.check(ConnectionAborted):
            self._connectHost(host)
        else:
            self.reactor.callLater(self._retryDelay, self._connectHost, host)

    @property
    def _allConnections(self):
        return itervalues(self._protocols)

    def disconnect(self):
        self.disconnecting = True
        log.msg('disconnecting from all clients', system='txyam')
        for d in list(self._connectionDeferreds):
            d.cancel()
        for proto in self._allConnections:
            proto.transport.loseConnection()

    def flushAll(self):
        return defer.gatherResults(
            [proto.flushAll() for proto in self._allConnections])

    def stats(self, arg=None):
        ds = {}
        for host, proto in iteritems(self._protocols):
            ds[host] = proto.stats(arg)
        return deferredDict(ds)

    def version(self):
        ds = {}
        for host, proto in iteritems(self._protocols):
            ds[host] = proto.version()
        return deferredDict(ds)

    def getClient(self, key):
        return self._protocols.get(self._consistentHash.get_node(key))

    def getMultiple(self, keys, withIdentifier=False):
        clients = defaultdict(list)
        for key in keys:
            clients[self.getClient(key)].append(key)
        dl = defer.DeferredList(
            [c.getMultiple(ks, withIdentifier) for c, ks in iteritems(clients)
             if c is not None],
            consumeErrors=True)
        dl.addCallback(self._consolidateMultiple)
        return dl

    def setMultiple(self, items, flags=0, expireTime=0):
        ds = {}
        for key, value in iteritems(items):
            ds[key] = self.set(key, value, flags, expireTime)
        return deferredDict(ds)

    def deleteMultiple(self, keys):
        ds = {}
        for key in keys:
            ds[key] = self.delete(key)
        return deferredDict(ds)

    def _consolidateMultiple(self, results):
        ret = {}
        for succeeded, result in results:
            if succeeded:
                ret.update(result)
        return ret

    set = _wrap('set')
    get = _wrap('get')
    increment = _wrap('increment')
    decrement = _wrap('decrement')
    replace = _wrap('replace')
    add = _wrap('add')
    checkAndSet = _wrap('checkAndSet')
    append = _wrap('append')
    prepend = _wrap('prepend')
    delete = _wrap('delete')
Exemple #2
0
class TestConsistentHash:
    init_nodes = {
        '192.168.0.101:11212': 1,
        '192.168.0.102:11212': 1,
        '192.168.0.103:11212': 1,
        '192.168.0.104:11212': 1
    }
    obj_nums = 10000

    @classmethod
    def setup_class(cls):
        cls.objs = cls.gen_random_objs()
        print('Initial nodes {nodes}'.format(nodes=cls.init_nodes))

    @classmethod
    def teardown_class(cls):
        pass

    def setUp(self):
        self.hit_nums = {}

    def tearDown(self):
        pass

    def test___init__(self):
        self.con_hash = ConsistentHash(self.init_nodes)
        # Get nodes from hashing ring
        for obj in self.objs:
            node = self.con_hash.get_node(obj)
            self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
        distribution = self.show_nodes_balance()

        self.validate_distribution(
            distribution, {
                '192.168.0.101:11212': (23, 27),
                '192.168.0.102:11212': (23, 27),
                '192.168.0.103:11212': (23, 27),
                '192.168.0.104:11212': (23, 27)
            })

    def test_empty__init__(self):
        self.con_hash = ConsistentHash()
        for obj in self.objs:
            node = self.con_hash.get_node(obj)

            if node is not None:
                raise Exception("Should have received an exception \
                                 when hashing using an empty LUT")

        self.con_hash.add_nodes(self.init_nodes)

        for obj in self.objs:
            node = self.con_hash.get_node(obj)
            self.hit_nums[node] = self.hit_nums.get(node, 0) + 1

        distribution = self.show_nodes_balance()

        self.validate_distribution(
            distribution, {
                '192.168.0.101:11212': (23, 27),
                '192.168.0.102:11212': (23, 27),
                '192.168.0.103:11212': (23, 27),
                '192.168.0.104:11212': (23, 27)
            })

    def test_add_nodes(self):
        self.con_hash = ConsistentHash(self.init_nodes)
        # Add nodes to hashing ring
        add_nodes = {'192.168.0.105:11212': 1}
        self.con_hash.add_nodes(add_nodes)
        # Get nodes from hashing ring
        for obj in self.objs:
            node = self.con_hash.get_node(obj)
            self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
        distribution = self.show_nodes_balance()

        self.validate_distribution(
            distribution, {
                '192.168.0.105:11212': (17, 23),
                '192.168.0.102:11212': (17, 23),
                '192.168.0.104:11212': (17, 23),
                '192.168.0.101:11212': (17, 23),
                '192.168.0.103:11212': (17, 23)
            })
        print('->The {nodes} added!!!'.format(nodes=add_nodes))

    def test_del_nodes(self):
        self.con_hash = ConsistentHash(self.init_nodes)
        # del_nodes = self.nodes[0:2]
        del_nodes = ['192.168.0.102:11212', '192.168.0.104:11212']
        # Delete the nodes from hashing ring
        self.con_hash.del_nodes(del_nodes)
        # Get nodes from hashing ring after deleting
        for obj in self.objs:
            node = self.con_hash.get_node(obj)
            self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
        distribution = self.show_nodes_balance()

        self.validate_distribution(distribution, {
            '192.168.0.101:11212': (48, 52),
            '192.168.0.103:11212': (48, 52)
        })
        print('->The {nodes} deleted!!!'.format(nodes=del_nodes))

    # -------------Help functions-------------
    def show_nodes_balance(self):
        distribution = {}
        print('-' * 67)
        print('Nodes count:{nNodes} Objects count:{nObjs}'.format(
            nNodes=self.con_hash.get_nodes_cnt(), nObjs=len(self.objs)))
        print('-' * 27 + 'Nodes balance' + '-' * 27)

        for node in self.con_hash.get_all_nodes():
            substitutions = {
                'nNodes': node,
                'nObjs': self.hit_nums[node],
                'percentage': self.get_percent(self.hit_nums[node],
                                               self.obj_nums)
            }

            print('Nodes:{nNodes} \
                   - Objects count:{nObjs} \
                   - percent:{percentage}%'.format(**substitutions))

            distribution[node] = substitutions['percentage']

        return distribution

    def validate_distribution(self, actual, expected):
        if expected.keys() != actual.keys():
            raise Exception("Expected nodes does not match actual nodes")

        for i in expected.keys():
            actual_value = actual[i]
            min_value = expected[i][0]
            max_value = expected[i][1]

            if actual_value < min_value or actual_value > max_value:
                print(min_value, actual_value, max_value)
                raise Exception("Value outside of expected range")

        print("Validated ranges")

    def get_percent(self, num, sum):
        return int(float(num) / sum * 100)

    @classmethod
    def gen_random_objs(cls, num=10000, len=10):
        objs = []
        for i in range(num):
            objs.append(''.join([random.choice(chars) for i in range(len)]))
        return objs
class TestConsistentHash:
    init_nodes = {'192.168.0.101:11212': 1,
                  '192.168.0.102:11212': 1,
                  '192.168.0.103:11212': 1,
                  '192.168.0.104:11212': 1}
    obj_nums = 10000

    @classmethod
    def setup_class(cls):
        cls.objs = cls.gen_random_objs()
        print('Initial nodes {nodes}'.format(nodes=cls.init_nodes))

    @classmethod
    def teardown_class(cls):
        pass

    def setUp(self):
        self.hit_nums = {}

    def tearDown(self):
        pass

    def test___init__(self):
        self.con_hash = ConsistentHash(self.init_nodes)
        # Get nodes from hashing ring
        for obj in self.objs:
            node = self.con_hash.get_node(obj)
            self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
        distribution = self.show_nodes_balance()

        self.validate_distribution(distribution, {
            '192.168.0.101:11212': (23, 27),
            '192.168.0.102:11212': (23, 27),
            '192.168.0.103:11212': (23, 27),
            '192.168.0.104:11212': (23, 27)
        })

    def test_empty__init__(self):
        self.con_hash = ConsistentHash()
        for obj in self.objs:
            node = self.con_hash.get_node(obj)

            if node is not None:
                raise Exception("Should have received an exception \
                                 when hashing using an empty LUT")

        self.con_hash.add_nodes(self.init_nodes)

        for obj in self.objs:
            node = self.con_hash.get_node(obj)
            self.hit_nums[node] = self.hit_nums.get(node, 0) + 1

        distribution = self.show_nodes_balance()

        self.validate_distribution(distribution, {
            '192.168.0.101:11212': (23, 27),
            '192.168.0.102:11212': (23, 27),
            '192.168.0.103:11212': (23, 27),
            '192.168.0.104:11212': (23, 27)
        })

    def test_add_nodes(self):
        self.con_hash = ConsistentHash(self.init_nodes)
        # Add nodes to hashing ring
        add_nodes = {'192.168.0.105:11212': 1}
        self.con_hash.add_nodes(add_nodes)
        # Get nodes from hashing ring
        for obj in self.objs:
            node = self.con_hash.get_node(obj)
            self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
        distribution = self.show_nodes_balance()

        self.validate_distribution(distribution, {
            '192.168.0.105:11212': (17, 23),
            '192.168.0.102:11212': (17, 23),
            '192.168.0.104:11212': (17, 23),
            '192.168.0.101:11212': (17, 23),
            '192.168.0.103:11212': (17, 23)
        })
        print('->The {nodes} added!!!'.format(nodes=add_nodes))

    def test_del_nodes(self):
        self.con_hash = ConsistentHash(self.init_nodes)
        # del_nodes = self.nodes[0:2]
        del_nodes = ['192.168.0.102:11212', '192.168.0.104:11212']
        # Delete the nodes from hashing ring
        self.con_hash.del_nodes(del_nodes)
        # Get nodes from hashing ring after deleting
        for obj in self.objs:
            node = self.con_hash.get_node(obj)
            self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
        distribution = self.show_nodes_balance()

        self.validate_distribution(distribution, {
            '192.168.0.101:11212': (48, 52),
            '192.168.0.103:11212': (48, 52)
        })
        print('->The {nodes} deleted!!!'.format(nodes=del_nodes))

    # -------------Help functions-------------
    def show_nodes_balance(self):
        distribution = {}
        print('-' * 67)
        print('Nodes count:{nNodes} Objects count:{nObjs}'.format(
            nNodes=self.con_hash.get_nodes_cnt(),
            nObjs=len(self.objs)
        ))
        print('-' * 27 + 'Nodes balance' + '-' * 27)

        for node in self.con_hash.get_all_nodes():
            substitutions = {
                'nNodes': node,
                'nObjs': self.hit_nums[node],
                'percentage': self.get_percent(self.hit_nums[node],
                                               self.obj_nums)
            }

            print('Nodes:{nNodes} \
                   - Objects count:{nObjs} \
                   - percent:{percentage}%'.format(**substitutions))

            distribution[node] = substitutions['percentage']

        return distribution

    def validate_distribution(self, actual, expected):
        if expected.keys() != actual.keys():
            raise Exception("Expected nodes does not match actual nodes")

        for i in expected.keys():
            actual_value = actual[i]
            min_value = expected[i][0]
            max_value = expected[i][1]

            if actual_value < min_value or actual_value > max_value:
                print(min_value, actual_value, max_value)
                raise Exception("Value outside of expected range")

        print("Validated ranges")

    def get_percent(self, num, sum):
        return int(float(num) / sum * 100)

    @classmethod
    def gen_random_objs(cls, num=10000, len=10):
        objs = []
        for i in range(num):
            objs.append(''.join([random.choice(chars) for i in range(len)]))
        return objs
Exemple #4
0
class TestConsistentHash(unittest.TestCase):
    init_nodes = {
        '192.168.0.101:11212': 1,
        '192.168.0.102:11212': 1,
        '192.168.0.103:11212': 1,
        '192.168.0.104:11212': 1,
    }
    obj_nums = 10000

    @classmethod
    def setup_class(cls):
        cls.objects = cls.generate_random_objects()
        print('Initial nodes {nodes}'.format(nodes=cls.init_nodes))

    @classmethod
    def teardown_class(cls):
        pass

    def setUp(self):
        self.hit_nums = {}

    def tearDown(self):
        pass

    def test___init__(self):
        self.con_hash = ConsistentHash(self.init_nodes)
        # Get nodes from hashing ring
        for obj in self.objects:
            node = self.con_hash.get_node(obj)
            self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
        distribution = self.show_nodes_balance()

        self.validate_distribution(
            distribution, {
                '192.168.0.101:11212': (23, 27),
                '192.168.0.102:11212': (23, 27),
                '192.168.0.103:11212': (23, 27),
                '192.168.0.104:11212': (23, 27)
            })

    def test_empty__init__(self):
        self.con_hash = ConsistentHash()
        for obj in self.objects:
            node = self.con_hash.get_node(obj)

            if node is not None:
                raise AssertionError(
                    'Should have received an exception when hashing using an empty LUT'
                )

        self.con_hash.add_nodes(self.init_nodes)

        for obj in self.objects:
            node = self.con_hash.get_node(obj)
            self.hit_nums[node] = self.hit_nums.get(node, 0) + 1

        distribution = self.show_nodes_balance()

        self.validate_distribution(
            distribution, {
                '192.168.0.101:11212': (23, 27),
                '192.168.0.102:11212': (23, 27),
                '192.168.0.103:11212': (23, 27),
                '192.168.0.104:11212': (23, 27),
            })

    def test_add_nodes(self):
        self.con_hash = ConsistentHash(self.init_nodes)
        # Add nodes to hashing ring
        add_nodes = {'192.168.0.105:11212': 1}
        self.con_hash.add_nodes(add_nodes)
        # Get nodes from hashing ring
        for obj in self.objects:
            node = self.con_hash.get_node(obj)
            self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
        distribution = self.show_nodes_balance()

        self.validate_distribution(
            distribution, {
                '192.168.0.105:11212': (17, 23),
                '192.168.0.102:11212': (17, 23),
                '192.168.0.104:11212': (17, 23),
                '192.168.0.101:11212': (17, 23),
                '192.168.0.103:11212': (17, 23),
            })
        print('->The {nodes} added!!!'.format(nodes=add_nodes))

    def test_add_nodes_unicode(self):
        self.con_hash = ConsistentHash({
            u'192.168.0.101:11212': 1,
            u'192.168.0.102:11212': 1,
            u'192.168.0.103:11212': 1,
            u'192.168.0.104:11212': 1,
        })
        # Add nodes to hashing ring
        add_nodes = u'192.168.0.105:11212'
        self.con_hash.add_nodes(add_nodes)
        # Get nodes from hashing ring
        for obj in self.objects:
            node = self.con_hash.get_node(obj)
            self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
        distribution = self.show_nodes_balance()

        self.validate_distribution(
            distribution, {
                '192.168.0.105:11212': (17, 23),
                '192.168.0.102:11212': (17, 23),
                '192.168.0.104:11212': (17, 23),
                '192.168.0.101:11212': (17, 23),
                '192.168.0.103:11212': (17, 23),
            })
        print('->The {nodes} added!!!'.format(nodes=add_nodes))

    def test_add_nodes_tuple(self):
        self.con_hash = ConsistentHash(self.init_nodes)
        # Add nodes to hashing ring
        add_nodes = ('192.168.0.105:11212', '192.168.0.106:11212')
        self.con_hash.add_nodes(add_nodes)
        # Get nodes from hashing ring
        for obj in self.objects:
            node = self.con_hash.get_node(obj)
            self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
        distribution = self.show_nodes_balance()

        self.validate_distribution(
            distribution, {
                '192.168.0.106:11212': (15, 17),
                '192.168.0.105:11212': (15, 17),
                '192.168.0.102:11212': (15, 17),
                '192.168.0.104:11212': (15, 17),
                '192.168.0.101:11212': (15, 17),
                '192.168.0.103:11212': (15, 17),
            })
        print('->The {nodes} added!!!'.format(nodes=add_nodes))

    def test_del_nodes(self):
        self.con_hash = ConsistentHash(self.init_nodes)
        # del_nodes = self.nodes[0:2]
        del_nodes = ['192.168.0.102:11212', '192.168.0.104:11212']
        # Delete the nodes from hashing ring
        self.con_hash.del_nodes(del_nodes)
        # Get nodes from hashing ring after deleting
        for obj in self.objects:
            node = self.con_hash.get_node(obj)
            self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
        distribution = self.show_nodes_balance()

        self.validate_distribution(distribution, {
            '192.168.0.101:11212': (48, 52),
            '192.168.0.103:11212': (48, 52)
        })
        print('->The {nodes} deleted!!!'.format(nodes=del_nodes))

    def test_del_nodes_tuple(self):
        self.con_hash = ConsistentHash(self.init_nodes)
        # del_nodes = self.nodes[0:2]
        del_nodes = ('192.168.0.102:11212', '192.168.0.104:11212')
        # Delete the nodes from hashing ring
        self.con_hash.del_nodes(del_nodes)
        # Get nodes from hashing ring after deleting
        for obj in self.objects:
            node = self.con_hash.get_node(obj)
            self.hit_nums[node] = self.hit_nums.get(node, 0) + 1
        distribution = self.show_nodes_balance()

        self.validate_distribution(distribution, {
            '192.168.0.101:11212': (48, 52),
            '192.168.0.103:11212': (48, 52)
        })
        print('->The {nodes} deleted!!!'.format(nodes=del_nodes))

    # -------------Help functions-------------
    def show_nodes_balance(self):
        distribution = {}
        print('-' * 67)
        print('Nodes count:{nNodes} Objects count:{nObjects}'.format(
            nNodes=self.con_hash.get_nodes_cnt(), nObjects=len(self.objects)))
        print('-' * 27 + 'Nodes balance' + '-' * 27)

        for node in self.con_hash.get_all_nodes():
            substitutions = {
                'nNodes': node,
                'nObjects': self.hit_nums[node],
                'percentage': self.get_percent(self.hit_nums[node],
                                               self.obj_nums)
            }

            print('Nodes: {nNodes} \
                   - Objects count: {nObjects} \
                   - percent:{percentage}%'.format(**substitutions))

            distribution[node] = substitutions['percentage']

        return distribution

    @staticmethod
    def validate_distribution(actual, expected):
        if expected.keys() != actual.keys():
            raise AssertionError('Expected nodes does not match actual nodes')

        for i in expected.keys():
            actual_value = actual[i]
            min_value = expected[i][0]
            max_value = expected[i][1]

            if actual_value < min_value or actual_value > max_value:
                print(min_value, actual_value, max_value)
                raise AssertionError(
                    'Value {actual} outside of expected range ({expected1},{expected2})'
                    .format(
                        expected1=min_value,
                        expected2=max_value,
                        actual=actual_value,
                    ))

        print('Validated ranges')

    @staticmethod
    def get_percent(numerator, denominator):
        return int(float(numerator) / denominator * 100)

    @staticmethod
    def generate_random_objects(num=10000, length=10):
        objects = []
        for i in range(num):
            objects.append(''.join(
                [random.choice(chars) for _ in range(length)]))
        return objects

    def test_sample_hash_output(self):
        ConsistentHash.interleave_count = 40
        # Test backward compatibility with version 1.0
        samples = {
            '35132097': 'B',
            '25291004': 'D',
            '48182416': 'F',
            '45818378': 'H',
            '52733021': 'A',
            '94027025': 'I',
            '18116713': 'F',
            '75531098': 'J',
            '99011825': 'F',
            '99371754': 'A',
            '19630740': 'D',
            '87823770': 'G',
            '32160063': 'A',
            '28054420': 'E',
            '75904283': 'H',
            '08458048': 'E',
            '51583844': 'I',
            '16226754': 'B',
            '95450503': 'E',
            '47557476': 'C',
            '38808589': 'A',
        }

        hash_ring = ConsistentHash(
            objects=['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J'])
        for node, output in samples.items():
            result = hash_ring.get_node(node)
            if result != output:
                raise AssertionError(
                    'Expected node does not match actual node. Expected: {}. Got: {}'
                    .format(
                        output,
                        result,
                    ))