def _determine_slot(self, *args):
"""
figure out what slot based on command and args
"""
if len(args) <= 1:
raise RedisClusterException(
"No way to dispatch this command to Redis Cluster. Missing key."
)
command = args[0]
if command in ['EVAL', 'EVALSHA']:
numkeys = args[2]
keys = args[3:3 + numkeys]
slots = {self.connection_pool.nodes.keyslot(key) for key in keys}
if len(slots) != 1:
raise RedisClusterException(
"{0} - all keys must map to the same key slot".format(
command))
return slots.pop()
elif command in ('XREAD', 'XREADGROUP'):
try:
idx = args.index('STREAMS') + 1
except ValueError:
raise RedisClusterException(
"{0} arguments do not contain STREAMS operand".format(
command))
key = args[idx]
elif command in ('XGROUP', 'XINFO'):
key = args[2]
else:
key = args[1]
return self.connection_pool.nodes.keyslot(key)
def make_connection(self, node):
"""
Create a new connection
"""
if self.count_all_num_connections(node) >= self.max_connections:
if self.max_connections_per_node:
raise RedisClusterException(
"Too many connection ({0}) for node: {1}".format(
self.count_all_num_connections(node), node['name']))
raise RedisClusterException("Too many connections")
self._created_connections_per_node.setdefault(node['name'], 0)
self._created_connections_per_node[node['name']] += 1
connection = self.connection_class(host=node["host"],
port=node["port"],
**self.connection_kwargs)
# Must store node in the connection to make it eaiser to track
connection.node = node
if self.max_idle_time > self.idle_check_interval > 0:
# do not await the future
asyncio.ensure_future(
self.disconnect_on_idle_time_exceeded(connection))
return connection
def get_connection(self, command_name, *keys, **options):
# Only pubsub command/connection should be allowed here
if command_name != "pubsub":
raise RedisClusterException(
"Only 'pubsub' commands can use get_connection()")
channel = options.pop('channel', None)
if not channel:
return self.get_random_connection()
slot = self.nodes.keyslot(channel)
node = self.get_master_node_by_slot(slot)
self._checkpid()
try:
connection = self._available_connections.get(node["name"],
[]).pop()
except IndexError:
connection = self.make_connection(node)
if node['name'] not in self._in_use_connections:
self._in_use_connections[node['name']] = set()
self._in_use_connections[node['name']].add(connection)
return connection
def get_connection_by_key(self, key):
"""
"""
if not key:
raise RedisClusterException("No way to dispatch this command to Redis Cluster.")
return self.get_connection_by_slot(self.nodes.keyslot(key))
def determine_node(self, *args, **kwargs):
"""
TODO: document
"""
command = args[0]
node_flag = self.nodes_flags.get(command)
if node_flag == NodeFlag.BLOCKED:
return blocked_command(self, command)
elif node_flag == NodeFlag.RANDOM:
return [self.connection_pool.nodes.random_node()]
elif node_flag == NodeFlag.ALL_MASTERS:
return self.connection_pool.nodes.all_masters()
elif node_flag == NodeFlag.ALL_NODES:
return self.connection_pool.nodes.all_nodes()
elif node_flag == NodeFlag.SLOT_ID:
# if node flag of command is SLOT_ID
# `slot_id` should is assumed in kwargs
slot = kwargs.get('slot_id')
if not slot:
raise RedisClusterException(
'slot_id is needed to execute command {}'.format(command))
return [self.connection_pool.nodes.node_from_slot(slot)]
else:
return None
def delete(self, *names):
"""Deletes a key specified by ``names``"""
if len(names) != 1:
raise RedisClusterException(
"deleting multiple keys is not implemented in pipeline command"
)
return self.execute_command('DEL', names[0])
def __init__(self, host=None, port=None, startup_nodes=None, max_connections=32,
max_connections_per_node=False, readonly=False,
reinitialize_steps=None, skip_full_coverage_check=False,
nodemanager_follow_cluster=False, **kwargs):
"""
:startup_nodes:
List of nodes that initial bootstrapping can be done from
:host:
Can be used to point to a startup node
:port:
Can be used to point to a startup node
:max_connections:
Maximum number of connections that should be kept open at one time
:readonly:
enable READONLY mode. You can read possibly stale data from slave.
:skip_full_coverage_check:
Skips the check of cluster-require-full-coverage config, useful for clusters
without the CONFIG command (like aws)
:nodemanager_follow_cluster:
The node manager will during initialization try the last set of nodes that
it was operating on. This will allow the client to drift along side the cluster
if the cluster nodes move around alot.
:**kwargs:
Extra arguments that will be sent into StrictRedis instance when created
(See Official redis-py doc for supported kwargs
[https://github.com/andymccurdy/redis-py/blob/master/redis/client.py])
Some kwargs is not supported and will raise RedisClusterException
- db (Redis do not support database SELECT in cluster mode)
"""
# Tweaks to StrictRedis client arguments when running in cluster mode
if "db" in kwargs:
raise RedisClusterException("Argument 'db' is not possible to use in cluster mode")
if "connection_pool" in kwargs:
pool = kwargs.pop('connection_pool')
else:
startup_nodes = [] if startup_nodes is None else startup_nodes
# Support host/port as argument
if host:
startup_nodes.append({"host": host, "port": port if port else 7000})
pool = ClusterConnectionPool(
startup_nodes=startup_nodes,
max_connections=max_connections,
reinitialize_steps=reinitialize_steps,
max_connections_per_node=max_connections_per_node,
skip_full_coverage_check=skip_full_coverage_check,
nodemanager_follow_cluster=nodemanager_follow_cluster,
readonly=readonly,
**kwargs
)
super(StrictRedisCluster, self).__init__(connection_pool=pool, **kwargs)
self.refresh_table_asap = False
self.nodes_flags = self.__class__.NODES_FLAGS.copy()
self.result_callbacks = self.__class__.RESULT_CALLBACKS.copy()
self.response_callbacks = self.__class__.RESPONSE_CALLBACKS.copy()
def _determine_slot(self, *args):
"""
figure out what slot based on command and args
"""
if len(args) <= 1:
raise RedisClusterException("No way to dispatch this command to Redis Cluster. Missing key.")
command = args[0]
if command in ['EVAL', 'EVALSHA']:
numkeys = args[2]
keys = args[3: 3 + numkeys]
slots = {self.connection_pool.nodes.keyslot(key) for key in keys}
if len(slots) != 1:
raise RedisClusterException("{0} - all keys must map to the same key slot".format(command))
return slots.pop()
key = args[1]
return self.connection_pool.nodes.keyslot(key)
def make_connection(self, node):
"""
Create a new connection
"""
if self.count_all_num_connections(node) >= self.max_connections:
if self.max_connections_per_node:
raise RedisClusterException(
"Too many connection ({0}) for node: {1}".format(
self.count_all_num_connections(node), node['name']))
raise RedisClusterException("Too many connections")
self._created_connections_per_node.setdefault(node['name'], 0)
self._created_connections_per_node[node['name']] += 1
connection = self.connection_class(host=node["host"],
port=node["port"],
**self.connection_kwargs)
# Must store node in the connection to make it eaiser to track
connection.node = node
return connection
async def pipeline(self, transaction=None, shard_hint=None):
"""
Cluster impl:
Pipelines do not work in cluster mode the same way they do in normal mode.
Create a clone of this object so that simulating pipelines will work correctly.
Each command will be called directly when used and when calling execute() will only return the result stack.
"""
await self.connection_pool.initialize()
if shard_hint:
raise RedisClusterException(
"shard_hint is deprecated in cluster mode")
if transaction:
raise RedisClusterException(
"transaction is deprecated in cluster mode")
from aredis.pipeline import StrictClusterPipeline
return StrictClusterPipeline(
connection_pool=self.connection_pool,
startup_nodes=self.connection_pool.nodes.startup_nodes,
result_callbacks=self.result_callbacks,
response_callbacks=self.response_callbacks,
)
async def execute_command(self, *args, **kwargs):
"""
Send a command to a node in the cluster
"""
if not self.connection_pool.initialized:
await self.connection_pool.initialize()
if not args:
raise RedisClusterException("Unable to determine command to use")
command = args[0]
node = self.determine_node(*args, **kwargs)
if node:
return await self._execute_command_on_nodes(node, *args, **kwargs)
# If set externally we must update it before calling any commands
if self.refresh_table_asap:
await self.connection_pool.nodes.initialize()
self.refresh_table_asap = False
redirect_addr = None
asking = False
try_random_node = False
slot = self._determine_slot(*args)
ttl = int(self.RedisClusterRequestTTL)
while ttl > 0:
ttl -= 1
if asking:
node = self.connection_pool.nodes.nodes[redirect_addr]
r = self.connection_pool.get_connection_by_node(node)
elif try_random_node:
r = self.connection_pool.get_random_connection()
try_random_node = False
else:
if self.refresh_table_asap:
# MOVED
node = self.connection_pool.get_master_node_by_slot(slot)
else:
node = self.connection_pool.get_node_by_slot(slot)
r = self.connection_pool.get_connection_by_node(node)
try:
if asking:
await r.send_command('ASKING')
await self.parse_response(r, "ASKING", **kwargs)
asking = False
await r.send_command(*args)
return await self.parse_response(r, command, **kwargs)
except (RedisClusterException, BusyLoadingError):
raise
except (CancelledError, ConnectionError, TimeoutError):
try_random_node = True
if ttl < self.RedisClusterRequestTTL / 2:
await asyncio.sleep(0.1)
except ClusterDownError as e:
self.connection_pool.disconnect()
self.connection_pool.reset()
self.refresh_table_asap = True
raise e
except MovedError as e:
# Reinitialize on ever x number of MovedError.
# This counter will increase faster when the same client object
# is shared between multiple threads. To reduce the frequency you
# can set the variable 'reinitialize_steps' in the constructor.
self.refresh_table_asap = True
await self.connection_pool.nodes.increment_reinitialize_counter()
node = self.connection_pool.nodes.set_node(e.host, e.port, server_type='master')
self.connection_pool.nodes.slots[e.slot_id][0] = node
except TryAgainError as e:
if ttl < self.RedisClusterRequestTTL / 2:
await asyncio.sleep(0.05)
except AskError as e:
redirect_addr, asking = "{0}:{1}".format(e.host, e.port), True
finally:
self.connection_pool.release(r)
raise ClusterError('TTL exhausted.')
async def sort(self, name, start=None, num=None, by=None, get=None, desc=False, alpha=False, store=None, groups=None):
"""Sort and return the list, set or sorted set at ``name``.
:start: and :num:
allow for paging through the sorted data
:by:
allows using an external key to weight and sort the items.
Use an "*" to indicate where in the key the item value is located
:get:
allows for returning items from external keys rather than the
sorted data itself. Use an "*" to indicate where int he key
the item value is located
:desc:
allows for reversing the sort
:alpha:
allows for sorting lexicographically rather than numerically
:store:
allows for storing the result of the sort into the key `store`
ClusterImpl:
A full implementation of the server side sort mechanics because many of the
options work on multiple keys that can exist on multiple servers.
"""
if (start is None and num is not None) or \
(start is not None and num is None):
raise RedisError("RedisError: ``start`` and ``num`` must both be specified")
try:
data_type = b(await self.type(name))
if data_type == b("none"):
return []
elif data_type == b("set"):
data = list(await self.smembers(name))[:]
elif data_type == b("list"):
data = await self.lrange(name, 0, -1)
else:
raise RedisClusterException("Unable to sort data type : {0}".format(data_type))
if by is not None:
# _sort_using_by_arg mutates data so we don't
# need need a return value.
data = await self._sort_using_by_arg(data, by, alpha)
elif not alpha:
data.sort(key=self._strtod_key_func)
else:
data.sort()
if desc:
data = data[::-1]
if not (start is None and num is None):
data = data[start:start + num]
if get:
data = await self._retrive_data_from_sort(data, get)
if store is not None:
if data_type == b("set"):
await self.delete(store)
await self.rpush(store, *data)
elif data_type == b("list"):
await self.delete(store)
await self.rpush(store, *data)
else:
raise RedisClusterException("Unable to store sorted data for data type : {0}".format(data_type))
return len(data)
if groups:
if not get or isinstance(get, str) or len(get) < 2:
raise DataError('when using "groups" the "get" argument '
'must be specified and contain at least '
'two keys')
n = len(get)
return list(zip(*[data[i::n] for i in range(n)]))
else:
return data
except KeyError:
return []
def unwatch(self):
"""
"""
raise RedisClusterException("method unwatch() is not implemented")
def watch(self, *names):
"""
"""
raise RedisClusterException("method watch() is not implemented")
def _execute_transaction(self, *args, **kwargs):
"""
"""
raise RedisClusterException(
"method _execute_transaction() is not implemented")
