def _setup_hash_and_connections(self, cluster, *args, **kwargs):
# Create the hash if it doesn't exist yet
if not hasattr(self, '_hash'):
strings = [h.identifier for (i, h) in cluster.hosts.items()]
self._hash = Ketama(strings)
self._handle_host_retries(cluster, retry_for=kwargs.get('retry_for'))
def _setup_router(self, args, kwargs, **fkwargs):
self._db_num_id_map = dict([
(db_num, host.identifier)
for db_num, host in self.cluster.hosts.iteritems()
])
self._hash = Ketama(self._db_num_id_map.values())
return True
class ConsistentHashingRouter(RoundRobinRouter):
"""
Router that returns host number based on a consistent hashing algorithm.
The consistent hashing algorithm only works if a key argument is provided.
If a key is not provided, then all hosts are returned.
The first argument is assumed to be the ``key`` for routing. Keyword arguments
are not supported.
"""
def __init__(self, *args, **kwargs):
self._db_num_id_map = {}
super(ConsistentHashingRouter, self).__init__(*args, **kwargs)
def mark_connection_down(self, db_num):
db_num = self.ensure_db_num(db_num)
self._hash.remove_node(self._db_num_id_map[db_num])
super(ConsistentHashingRouter, self).mark_connection_down(db_num)
def mark_connection_up(self, db_num):
db_num = self.ensure_db_num(db_num)
node_key = self._db_num_id_map[db_num]
if self._hash.get_node(node_key) is None:
self._hash.add_node(node_key)
super(ConsistentHashingRouter, self).mark_connection_up(db_num)
@routing_params
def _setup_router(self, args, kwargs, **fkwargs):
self._db_num_id_map = dict([(db_num, host.identifier) for db_num, host in self.cluster.hosts.iteritems()])
self._hash = Ketama(self._db_num_id_map.values())
return True
@routing_params
def _pre_routing(self, *args, **kwargs):
self.check_down_connections()
return super(ConsistentHashingRouter, self)._pre_routing(*args, **kwargs)
@routing_params
def _route(self, attr, args, kwargs, **fkwargs):
"""
The first argument is assumed to be the ``key`` for routing.
"""
key = get_key(args, kwargs)
found = self._hash.get_node(key)
if not found and len(self._down_connections) > 0:
raise self.HostListExhausted()
return [i for i, h in self.cluster.hosts.iteritems()
if h.identifier == found]
class ConsistentHashingRouter(RoundRobinRouter):
"""
Router that returns host number based on a consistent hashing algorithm.
The consistent hashing algorithm only works if a key argument is provided.
If a key is not provided, then all hosts are returned.
The first argument is assumed to be the ``key`` for routing. Keyword arguments
are not supported.
"""
def __init__(self, *args, **kwargs):
self._db_num_id_map = {}
super(ConsistentHashingRouter, self).__init__(*args, **kwargs)
def mark_connection_down(self, db_num):
db_num = self.ensure_db_num(db_num)
self._hash.remove_node(self._db_num_id_map[db_num])
super(ConsistentHashingRouter, self).mark_connection_down(db_num)
def mark_connection_up(self, db_num):
db_num = self.ensure_db_num(db_num)
self._hash.add_node(self._db_num_id_map[db_num])
super(ConsistentHashingRouter, self).mark_connection_up(db_num)
@routing_params
def _setup_router(self, args, kwargs, **fkwargs):
self._db_num_id_map = dict([(db_num, host.identifier) for db_num, host in compat.iteritems(self.cluster.hosts)])
self._hash = Ketama(self._db_num_id_map.values())
return True
@routing_params
def _pre_routing(self, *args, **kwargs):
self.check_down_connections()
return super(ConsistentHashingRouter, self)._pre_routing(*args, **kwargs)
@routing_params
def _route(self, attr, args, kwargs, **fkwargs):
"""
The first argument is assumed to be the ``key`` for routing.
"""
key = get_key(args, kwargs)
found = self._hash.get_node(key)
if not found and len(self._down_connections) > 0:
raise self.HostListExhausted()
return [i for i, h in compat.iteritems(self.cluster.hosts)
if h.identifier == found]
class ConsistentHashingRouter(RoundRobinRouter):
'''
Router that returns host number based on a consistent hashing algorithm.
The consistent hashing algorithm only works if a key argument is provided.
If a key is not provided, then all hosts are returned.
'''
def __init__(self, *args, **kwargs):
self._db_num_id_map = {}
super(ConsistentHashingRouter, self).__init__(*args, **kwargs)
def flush_down_connections(self):
for db_num in self._down_connections:
self._hash.add_node(self._db_num_id_map[db_num])
super(ConsistentHashingRouter, self).flush_down_connections()
def mark_connection_down(self, db_num):
db_num = self.ensure_db_num(db_num)
self._hash.remove_node(self._db_num_id_map[db_num])
super(ConsistentHashingRouter, self).mark_connection_down(db_num)
def mark_conenction_up(self, db_num):
db_num = self.ensure_db_num(db_num)
self._hash.add_node(self._db_num_id_map[db_num])
super(ConsistentHashingRouter, self).mark_connection_up(db_num)
def _setup_router(self, cluster, *args, **kwargs):
self._db_num_id_map = dict([(db_num, host.identifier) for db_num, host in cluster.hosts.iteritems()])
self._hash = Ketama(self._db_num_id_map.values())
return True
def _route(self, cluster, attr, key, *args, **kwargs):
found = self._hash.get_node(key)
if not found and len(self._down_connections) > 0:
raise self.HostListExhausted()
return [i for i, h in cluster.hosts.iteritems()
if h.identifier == found]
def _setup_router(self, args, kwargs, **fkwargs):
self._db_num_id_map = dict([(db_num, host.identifier) for db_num, host in self.cluster.hosts.iteritems()])
self._hash = Ketama(self._db_num_id_map.values())
return True
class ConsistentHashingRouter(BaseRouter):
'''
Router that returns host number based on a consistent hashing algorithm.
The consistent hashing algorithm only works if a key argument is provided.
If a key is not provided, then all hosts are returned.
'''
# Raised if all hosts in the hash have been marked as down
class HostListExhaused(Exception):
pass
# If this router can be retried on if a particular db index it gave out did
# not work
retryable = True
# How many requests to serve in a situation when a host is down before
# the down hosts are retried
attempt_reconnect_threshold = 100000
def __init__(self):
self._get_db_attempts = 0
self._down_connections = set()
# There is one instance of this class that lives inside the Cluster object
def get_db(self, cluster, func, key=None, *args, **kwargs):
self._setup_hash_and_connections(cluster, *args, **kwargs)
if not cluster:
return []
elif not key:
return range(len(cluster))
else:
return self._host_indexes_for(key, cluster)
def flush_down_connections(self):
for connection in self._down_connections:
self._hash.add_node(connection.identifier)
self._down_connections = set()
def _setup_hash_and_connections(self, cluster, *args, **kwargs):
# Create the hash if it doesn't exist yet
if not hasattr(self, '_hash'):
strings = [h.identifier for (i, h) in cluster.hosts.items()]
self._hash = Ketama(strings)
self._handle_host_retries(cluster, retry_for=kwargs.get('retry_for'))
def _handle_host_retries(self, cluster, retry_for):
self._get_db_attempts += 1
if self._get_db_attempts > self.attempt_reconnect_threshold:
self.flush_down_connections()
self._get_db_attempts = 0
if retry_for is not None:
self._mark_connection_as_down(cluster[retry_for])
def _mark_connection_as_down(self, connection):
self._hash.remove_node(connection.identifier)
self._down_connections.add(connection)
def _host_indexes_for(self, key, cluster):
found = self._hash.get_node(key)
if not found and len(self._down_connections) > 0:
raise self.HostListExhaused
return [i for (i, h) in cluster.hosts.items()
if h.identifier == found]
