Exemple #1
0
    def _setup_hash_and_connections(self, cluster, *args, **kwargs):
        # Create the hash if it doesn't exist yet
        if not hasattr(self, '_hash'):
            strings = [h.identifier for (i, h) in cluster.hosts.items()]
            self._hash = Ketama(strings)

        self._handle_host_retries(cluster, retry_for=kwargs.get('retry_for'))
Exemple #2
0
    def _setup_router(self, args, kwargs, **fkwargs):
        self._db_num_id_map = dict([
            (db_num, host.identifier)
            for db_num, host in self.cluster.hosts.iteritems()
        ])
        self._hash = Ketama(self._db_num_id_map.values())

        return True
Exemple #3
0
class ConsistentHashingRouter(RoundRobinRouter):
    """
    Router that returns host number based on a consistent hashing algorithm.
    The consistent hashing algorithm only works if a key argument is provided.

    If a key is not provided, then all hosts are returned.

    The first argument is assumed to be the ``key`` for routing. Keyword arguments
    are not supported.
    """

    def __init__(self, *args, **kwargs):
        self._db_num_id_map = {}
        super(ConsistentHashingRouter, self).__init__(*args, **kwargs)

    def mark_connection_down(self, db_num):
        db_num = self.ensure_db_num(db_num)
        self._hash.remove_node(self._db_num_id_map[db_num])

        super(ConsistentHashingRouter, self).mark_connection_down(db_num)

    def mark_connection_up(self, db_num):
        db_num = self.ensure_db_num(db_num)
        node_key = self._db_num_id_map[db_num]
        if self._hash.get_node(node_key) is None:
            self._hash.add_node(node_key)

        super(ConsistentHashingRouter, self).mark_connection_up(db_num)

    @routing_params
    def _setup_router(self, args, kwargs, **fkwargs):
        self._db_num_id_map = dict([(db_num, host.identifier) for db_num, host in self.cluster.hosts.iteritems()])
        self._hash = Ketama(self._db_num_id_map.values())

        return True

    @routing_params
    def _pre_routing(self, *args, **kwargs):
        self.check_down_connections()

        return super(ConsistentHashingRouter, self)._pre_routing(*args, **kwargs)

    @routing_params
    def _route(self, attr, args, kwargs, **fkwargs):
        """
        The first argument is assumed to be the ``key`` for routing.
        """

        key = get_key(args, kwargs)

        found = self._hash.get_node(key)

        if not found and len(self._down_connections) > 0:
            raise self.HostListExhausted()

        return [i for i, h in self.cluster.hosts.iteritems()
                if h.identifier == found]
Exemple #4
0
class ConsistentHashingRouter(RoundRobinRouter):
    """
    Router that returns host number based on a consistent hashing algorithm.
    The consistent hashing algorithm only works if a key argument is provided.

    If a key is not provided, then all hosts are returned.

    The first argument is assumed to be the ``key`` for routing. Keyword arguments
    are not supported.
    """

    def __init__(self, *args, **kwargs):
        self._db_num_id_map = {}
        super(ConsistentHashingRouter, self).__init__(*args, **kwargs)

    def mark_connection_down(self, db_num):
        db_num = self.ensure_db_num(db_num)
        self._hash.remove_node(self._db_num_id_map[db_num])

        super(ConsistentHashingRouter, self).mark_connection_down(db_num)

    def mark_connection_up(self, db_num):
        db_num = self.ensure_db_num(db_num)
        self._hash.add_node(self._db_num_id_map[db_num])

        super(ConsistentHashingRouter, self).mark_connection_up(db_num)

    @routing_params
    def _setup_router(self, args, kwargs, **fkwargs):
        self._db_num_id_map = dict([(db_num, host.identifier) for db_num, host in compat.iteritems(self.cluster.hosts)])
        self._hash = Ketama(self._db_num_id_map.values())

        return True

    @routing_params
    def _pre_routing(self, *args, **kwargs):
        self.check_down_connections()

        return super(ConsistentHashingRouter, self)._pre_routing(*args, **kwargs)

    @routing_params
    def _route(self, attr, args, kwargs, **fkwargs):
        """
        The first argument is assumed to be the ``key`` for routing.
        """

        key = get_key(args, kwargs)

        found = self._hash.get_node(key)

        if not found and len(self._down_connections) > 0:
            raise self.HostListExhausted()

        return [i for i, h in compat.iteritems(self.cluster.hosts)
                if h.identifier == found]
Exemple #5
0
class ConsistentHashingRouter(RoundRobinRouter):
    '''
    Router that returns host number based on a consistent hashing algorithm.
    The consistent hashing algorithm only works if a key argument is provided.
    If a key is not provided, then all hosts are returned.
    '''

    def __init__(self, *args, **kwargs):
        self._db_num_id_map = {}
        super(ConsistentHashingRouter, self).__init__(*args, **kwargs)

    def flush_down_connections(self):
        for db_num in self._down_connections:
            self._hash.add_node(self._db_num_id_map[db_num])

        super(ConsistentHashingRouter, self).flush_down_connections()

    def mark_connection_down(self, db_num):
        db_num = self.ensure_db_num(db_num)
        self._hash.remove_node(self._db_num_id_map[db_num])

        super(ConsistentHashingRouter, self).mark_connection_down(db_num)

    def mark_conenction_up(self, db_num):
        db_num = self.ensure_db_num(db_num)
        self._hash.add_node(self._db_num_id_map[db_num])

        super(ConsistentHashingRouter, self).mark_connection_up(db_num)

    def _setup_router(self, cluster, *args, **kwargs):
        self._db_num_id_map = dict([(db_num, host.identifier) for db_num, host in cluster.hosts.iteritems()])
        self._hash = Ketama(self._db_num_id_map.values())

        return True

    def _route(self, cluster, attr, key, *args, **kwargs):
        found = self._hash.get_node(key)

        if not found and len(self._down_connections) > 0:
            raise self.HostListExhausted()

        return [i for i, h in cluster.hosts.iteritems()
                if h.identifier == found]
Exemple #6
0
    def _setup_router(self, args, kwargs, **fkwargs):
        self._db_num_id_map = dict([(db_num, host.identifier) for db_num, host in self.cluster.hosts.iteritems()])
        self._hash = Ketama(self._db_num_id_map.values())

        return True
Exemple #7
0
class ConsistentHashingRouter(BaseRouter):
    '''
    Router that returns host number based on a consistent hashing algorithm.
    The consistent hashing algorithm only works if a key argument is provided.
    If a key is not provided, then all hosts are returned.
    '''

    # Raised if all hosts in the hash have been marked as down
    class HostListExhaused(Exception):
        pass

    # If this router can be retried on if a particular db index it gave out did
    # not work
    retryable = True

    # How many requests to serve in a situation when a host is down before
    # the down hosts are retried
    attempt_reconnect_threshold = 100000

    def __init__(self):
        self._get_db_attempts = 0
        self._down_connections = set()

    # There is one instance of this class that lives inside the Cluster object
    def get_db(self, cluster, func, key=None, *args, **kwargs):
        self._setup_hash_and_connections(cluster, *args, **kwargs)

        if not cluster:
            return []
        elif not key:
            return range(len(cluster))
        else:
            return self._host_indexes_for(key, cluster)

    def flush_down_connections(self):
        for connection in self._down_connections:
            self._hash.add_node(connection.identifier)

        self._down_connections = set()

    def _setup_hash_and_connections(self, cluster, *args, **kwargs):
        # Create the hash if it doesn't exist yet
        if not hasattr(self, '_hash'):
            strings = [h.identifier for (i, h) in cluster.hosts.items()]
            self._hash = Ketama(strings)

        self._handle_host_retries(cluster, retry_for=kwargs.get('retry_for'))

    def _handle_host_retries(self, cluster, retry_for):
        self._get_db_attempts += 1

        if self._get_db_attempts > self.attempt_reconnect_threshold:
            self.flush_down_connections()
            self._get_db_attempts = 0

        if retry_for is not None:
            self._mark_connection_as_down(cluster[retry_for])

    def _mark_connection_as_down(self, connection):
        self._hash.remove_node(connection.identifier)
        self._down_connections.add(connection)

    def _host_indexes_for(self, key, cluster):
        found = self._hash.get_node(key)

        if not found and len(self._down_connections) > 0:
            raise self.HostListExhaused

        return [i for (i, h) in cluster.hosts.items()
                if h.identifier == found]