def to_byte(value):
"""
Convert value to byte
:param value:
:return:
"""
accepted_types = (bytes, str, int, float)
assert isinstance(value, accepted_types), (
"Given value type cannot be {}, should be one of them {}".format(
type(value), accepted_types))
encoder = Encoder(encoding='utf-8',
encoding_errors='strict',
decode_responses=True)
return encoder.encode(value)
class NodeManager(object):
"""
"""
RedisClusterHashSlots = 16384
def __init__(self,
startup_nodes=None,
reinitialize_steps=None,
skip_full_coverage_check=False,
nodemanager_follow_cluster=False,
host_port_remap=None,
**connection_kwargs):
"""
:skip_full_coverage_check:
Skips the check of cluster-require-full-coverage config, useful for clusters
without the CONFIG command (like aws)
:nodemanager_follow_cluster:
The node manager will during initialization try the last set of nodes that
it was operating on. This will allow the client to drift along side the cluster
if the cluster nodes move around alot.
"""
self.connection_kwargs = connection_kwargs
self.nodes = {}
self.slots = {}
self.startup_nodes = [] if startup_nodes is None else startup_nodes
self.orig_startup_nodes = [node for node in self.startup_nodes]
self.reinitialize_counter = 0
self.reinitialize_steps = reinitialize_steps or 25
self._skip_full_coverage_check = skip_full_coverage_check
self.nodemanager_follow_cluster = nodemanager_follow_cluster
self.encoder = Encoder(
connection_kwargs.get('encoding', 'utf-8'),
connection_kwargs.get('encoding_errors', 'strict'),
connection_kwargs.get('decode_responses', False))
self._validate_host_port_remap(host_port_remap)
self.host_port_remap = host_port_remap
if not self.startup_nodes:
raise RedisClusterException("No startup nodes provided")
def _validate_host_port_remap(self, host_port_remap):
"""
Helper method that validates all entries in the host_port_remap config.
"""
if host_port_remap is None:
# Nothing to validate if config not set
return
if not isinstance(host_port_remap, list):
raise RedisClusterConfigError("host_port_remap must be a list")
for item in host_port_remap:
if not isinstance(item, dict):
raise RedisClusterConfigError(
"items inside host_port_remap list must be of dict type")
# If we have from_host, we must have a to_host option to allow for translation to work
if ('from_host' in item
and 'to_host' not in item) or ('from_host' not in item
and 'to_host' in item):
raise RedisClusterConfigError(
"Both from_host and to_host must be present in remap item if either is defined"
)
if ('from_port' in item
and 'to_port' not in item) or ('from_port' not in item
and 'to_port' in item):
raise RedisClusterConfigError(
"Both from_port and to_port must be present in remap item")
def keyslot(self, key):
"""
Calculate keyslot for a given key.
Tuned for compatibility with python 2.7.x
"""
k = self.encoder.encode(key)
start = k.find(b"{")
if start > -1:
end = k.find(b"}", start + 1)
if end > -1 and end != start + 1:
k = k[start + 1:end]
return crc16(k) % self.RedisClusterHashSlots
def node_from_slot(self, slot):
"""
"""
for node in self.slots[slot]:
if node['server_type'] == 'master':
return node
def all_nodes(self):
"""
"""
for node in self.nodes.values():
yield node
def all_masters(self):
"""
"""
for node in self.nodes.values():
if node["server_type"] == "master":
yield node
def random_startup_node(self):
"""
"""
random.shuffle(self.startup_nodes)
return self.startup_nodes[0]
def random_startup_node_ittr(self):
"""
Generator that will return a random startup nodes. Works as a generator.
"""
while True:
yield random.choice(self.startup_nodes)
def random_node(self):
"""
"""
key = random.choice(list(self.nodes.keys()))
return self.nodes[key]
def get_redis_link(self, host, port, decode_responses=False):
"""
"""
allowed_keys = (
'host',
'port',
'db',
'username',
'password',
'socket_timeout',
'socket_connect_timeout',
'socket_keepalive',
'socket_keepalive_options',
'connection_pool',
'unix_socket_path',
'encoding',
'encoding_errors',
'charset',
'errors',
'decode_responses',
'retry_on_timeout',
'ssl',
'ssl_keyfile',
'ssl_certfile',
'ssl_cert_reqs',
'ssl_ca_certs',
'max_connections',
)
disabled_keys = (
'host',
'port',
'decode_responses',
)
connection_kwargs = {
k: v
for k, v in self.connection_kwargs.items()
if k in set(allowed_keys) - set(disabled_keys)
}
return Redis(host=host,
port=port,
decode_responses=decode_responses,
**connection_kwargs)
def initialize(self):
"""
Init the slots cache by asking all startup nodes what the current cluster configuration is
"""
nodes_cache = {}
tmp_slots = {}
all_slots_covered = False
disagreements = []
startup_nodes_reachable = False
nodes = self.orig_startup_nodes
# With this option the client will attempt to connect to any of the previous set of nodes instead of the original set of nodes
if self.nodemanager_follow_cluster:
nodes = self.startup_nodes
for node in nodes:
try:
r = self.get_redis_link(host=node["host"],
port=node["port"],
decode_responses=True)
cluster_slots = r.execute_command("cluster", "slots")
startup_nodes_reachable = True
except (ConnectionError, TimeoutError):
continue
except ResponseError as e:
# Isn't a cluster connection, so it won't parse these exceptions automatically
message = e.__str__()
if 'CLUSTERDOWN' in message or 'MASTERDOWN' in message:
continue
else:
raise RedisClusterException(
"ERROR sending 'cluster slots' command to redis server: {0}"
.format(node))
except Exception:
raise RedisClusterException(
"ERROR sending 'cluster slots' command to redis server: {0}"
.format(node))
all_slots_covered = True
# If there's only one server in the cluster, its ``host`` is ''
# Fix it to the host in startup_nodes
if (len(cluster_slots) == 1 and len(cluster_slots[0][2][0]) == 0
and len(self.startup_nodes) == 1):
cluster_slots[0][2][0] = self.startup_nodes[0]['host']
# No need to decode response because Redis should handle that for us...
for slot in cluster_slots:
master_node = slot[2]
if master_node[0] == '':
master_node[0] = node['host']
master_node[1] = int(master_node[1])
master_node = self.remap_internal_node_object(master_node)
node, node_name = self.make_node_obj(master_node[0],
master_node[1], 'master')
nodes_cache[node_name] = node
for i in range(int(slot[0]), int(slot[1]) + 1):
if i not in tmp_slots:
tmp_slots[i] = [node]
slave_nodes = [slot[j] for j in range(3, len(slot))]
for slave_node in slave_nodes:
slave_node = self.remap_internal_node_object(
slave_node)
target_slave_node, slave_node_name = self.make_node_obj(
slave_node[0], slave_node[1], 'slave')
nodes_cache[slave_node_name] = target_slave_node
tmp_slots[i].append(target_slave_node)
else:
# Validate that 2 nodes want to use the same slot cache setup
if tmp_slots[i][0]['name'] != node['name']:
disagreements.append(
"{0} vs {1} on slot: {2}".format(
tmp_slots[i][0]['name'], node['name'],
i), )
if len(disagreements) > 5:
raise RedisClusterException(
"startup_nodes could not agree on a valid slots cache. {0}"
.format(", ".join(disagreements)))
self.populate_startup_nodes()
self.refresh_table_asap = False
if self._skip_full_coverage_check:
need_full_slots_coverage = False
else:
need_full_slots_coverage = self.cluster_require_full_coverage(
nodes_cache)
# Validate if all slots are covered or if we should try next startup node
for i in range(0, self.RedisClusterHashSlots):
if i not in tmp_slots and need_full_slots_coverage:
all_slots_covered = False
if all_slots_covered:
# All slots are covered and application can continue to execute
break
if not startup_nodes_reachable:
raise RedisClusterException(
"Redis Cluster cannot be connected. Please provide at least one reachable node."
)
if not all_slots_covered:
raise RedisClusterException(
"All slots are not covered after query all startup_nodes. {0} of {1} covered..."
.format(len(tmp_slots), self.RedisClusterHashSlots))
# Set the tmp variables to the real variables
self.slots = tmp_slots
self.nodes = nodes_cache
self.reinitialize_counter = 0
def remap_internal_node_object(self, node_obj):
if not self.host_port_remap:
# No remapping rule set, return object unmodified
return node_obj
for remap_rule in self.host_port_remap:
if 'from_host' in remap_rule and 'to_host' in remap_rule:
if remap_rule['from_host'] in node_obj[0]:
# print('remapping host', node_obj[0], remap_rule['to_host'])
node_obj[0] = remap_rule['to_host']
## The port value is always an integer
if 'from_port' in remap_rule and 'to_port' in remap_rule:
if remap_rule['from_port'] == node_obj[1]:
# print('remapping port', node_obj[1], remap_rule['to_port'])
node_obj[1] = remap_rule['to_port']
return node_obj
def increment_reinitialize_counter(self, ct=1):
for i in range(min(ct, self.reinitialize_steps)):
self.reinitialize_counter += 1
if self.reinitialize_counter % self.reinitialize_steps == 0:
self.initialize()
def cluster_require_full_coverage(self, nodes_cache):
"""
if exists 'cluster-require-full-coverage no' config on redis servers,
then even all slots are not covered, cluster still will be able to
respond
"""
nodes = nodes_cache or self.nodes
def node_require_full_coverage(node):
try:
r_node = self.get_redis_link(host=node["host"],
port=node["port"],
decode_responses=True)
return "yes" in r_node.config_get(
"cluster-require-full-coverage").values()
except ConnectionError:
return False
except Exception:
raise RedisClusterException(
"ERROR sending 'config get cluster-require-full-coverage' command to redis server: {0}"
.format(node))
# at least one node should have cluster-require-full-coverage yes
return any(node_require_full_coverage(node) for node in nodes.values())
def set_node_name(self, n):
"""
Format the name for the given node object
# TODO: This shold not be constructed this way. It should update the name of the node in the node cache dict
"""
if "name" not in n:
n["name"] = "{0}:{1}".format(n["host"], n["port"])
def make_node_obj(self, host, port, server_type):
"""
Create a node datastructure.
Returns the node datastructure and the node name
"""
node_name = "{0}:{1}".format(host, port)
node = {
'host': host,
'port': port,
'name': node_name,
'server_type': server_type
}
return (node, node_name)
def set_node(self, host, port, server_type=None):
"""
Update data for a node.
"""
node, node_name = self.make_node_obj(host, port, server_type)
self.nodes[node_name] = node
return node
def populate_startup_nodes(self):
"""
Do something with all startup nodes and filters out any duplicates
"""
for item in self.startup_nodes:
self.set_node_name(item)
for n in self.nodes.values():
if n not in self.startup_nodes:
self.startup_nodes.append(n)
# freeze it so we can set() it
uniq = {frozenset(node.items()) for node in self.startup_nodes}
# then thaw it back out into a list of dicts
self.startup_nodes = [dict(node) for node in uniq]
def reset(self):
"""
Drop all node data and start over from startup_nodes
"""
self.initialize()