async def createCluster(args):
'''
See clusterManagerCommandCreate in redis-cli.c
The main difference is that we first set slots
Then run cluster meet
Then set replicas, as setting replicas requires the master to be known by the
replica, and hence the meet to have happened already
Another difference is master/replica assignment from the input list of ip address
that could be changed easily to match redis-cli behavior.
Tested with redis-server 4.x. In redis-4 time there was still redis-trib and no
redis-cli, so we cannot use redis-cli to initialize the cluster
'''
logging.info(args)
nodes = args.ips.split()
#
# Reset all first
#
for nodeAddress in nodes:
redisUrl = f'redis://{nodeAddress}'
client = RedisClient(redisUrl, args.password, args.user)
dbSize = await client.send('DBSIZE')
if dbSize > 0:
try:
await client.send('FLUSHALL')
except Exception as e:
logging.warning(f'Ignoring error with FLUSHALL: {e}')
await client.send('CLUSTER', 'RESET', 'hard')
#
# Hash slot allocation
#
click.secho(
f'>>> Performing hash slots allocation on {args.shards} nodes...',
bold=True)
slotsPerNode = SLOTS // args.shards
offset = 0
allSlots = []
for i in range(args.shards):
nodeSlots = []
for j in range(slotsPerNode):
nodeSlots.append(offset + j)
offset += slotsPerNode
allSlots.append(nodeSlots)
# reminder
for j in range(SLOTS % args.shards):
allSlots[-1].append(offset + j)
# Print slot assignment
for i, nodeSlots in enumerate(allSlots):
# Master[0] -> Slots 0 - 4095
print('Master[{}] -> Slots {} - {}'.format(i, nodeSlots[0],
nodeSlots[-1]))
# Setting slots
for i in range(args.shards):
nodeAddress = nodes[i]
redisUrl = f'redis://{nodeAddress}'
masterClient = RedisClient(redisUrl, args.password, args.user)
await masterClient.send('CLUSTER', 'ADDSLOTS', *allSlots[i])
# Give each node its own 'epoch'
click.secho('>>> Assign a different config epoch to each node', bold=True)
for i, nodeAddress in enumerate(nodes):
epoch = i + 1
redisUrl = f'redis://{nodeAddress}'
client = RedisClient(redisUrl, args.password, args.user)
try:
await client.send('CLUSTER', 'SET-CONFIG-EPOCH', epoch)
except Exception as e:
logging.warning(f'{redisUrl}: Error with set-config-epoch: {e}')
pass
click.secho('>>> Sending CLUSTER MEET messages to join the cluster',
bold=True)
firstNodeIp, _, firstNodePort = nodes[0].partition(':')
for nodeAddress in nodes[1:]:
redisUrl = f'redis://{nodeAddress}'
client = RedisClient(redisUrl, args.password, args.user)
await client.send('CLUSTER', 'MEET', firstNodeIp, firstNodePort)
# Wait one second
await asyncio.sleep(1)
# Wait for all the nodes in the cluster to agree on node count
# so that they all know about each other, and we can set replicas
# Give us 15 seconds max to do that
start = time.time()
timeout = 15
for nodeAddress in nodes:
redisUrl = f'redis://{nodeAddress}'
nodeCount = await getClusterNodesCount(redisUrl, args.password,
args.user)
if nodeCount != args.shards:
await asyncio.sleep(0.1)
sys.stderr.write('.')
sys.stderr.flush()
if time.time() - start > timeout:
raise ValueError(
'Timed out while waiting for all nodes to see each other')
sys.stderr.write('\n')
#
# [m1, m2, ..., mn, r1, r2, ...]
# [m1, m2, ..., mn, r1, r2, ..., rn, s1, s2, ..., sn]
#
# <----- args.shards -> <--- args.shards --> <--- args.shards ->
#
# r1 = args.shards + 1
# ...
# ri = args.shards + i
#
# s1 = replicas * args.shards + 1
# ...
# si = replicas * args.shards + i
#
# Now we can set the replicas
click.secho('>>> Assigning replicas', bold=True)
for i in range(args.shards):
masterNodeAddress = nodes[i]
redisUrl = f'redis://{masterNodeAddress}'
masterClient = RedisClient(redisUrl, args.password, args.user)
masterNodeId = await masterClient.send('CLUSTER', 'MYID')
for j in range(args.replicas):
replicaIp = nodes[(j + 1) * args.shards + i]
print(f'Setting replica {replicaIp} to {masterNodeAddress}')
url = f'redis://{replicaIp}'
replicaClient = RedisClient(url, args.password, args.user)
await replicaClient.send('CLUSTER', 'REPLICATE', masterNodeId)
def makeClientfromNode(node, redisPassword):
url = f'redis://{node.ip}:{node.port}'
return RedisClient(url, redisPassword)
async def analyzeKeyspace(
redisUrlsStr: str,
redisPassword: str,
redisUser: str,
timeout: int,
count: int = -1,
monitor: bool = False,
):
pattern = '__key*__:*'
redisUrls = redisUrlsStr.split(';')
redisUrl = redisUrls[0]
redisClient = RedisClient(redisUrl, redisPassword, redisUser)
await redisClient.connect()
clients = []
if redisClient.cluster:
nodes = await redisClient.cluster_nodes()
masterNodes = [node for node in nodes if node.role == 'master']
for node in masterNodes:
client = makeClientfromNode(node, redisPassword, redisUser)
clients.append(client)
else:
clients = [RedisClient(url, redisPassword, redisUser) for url in redisUrls]
#
# E for Keyevent events, published with __keyevent@<db>__ prefix.
# A Alias for g$lshztxe, to catch all events
#
keyspaceConfig = 'AE'
async def pubSubCallback(client, obj, message):
'''Need to extract a key and a command from the pubsub payload'''
msg = message[2].decode()
_, _, cmd = msg.partition(':')
cmd = cmd.upper()
key = message[3].decode()
obj.keys[key] += 1
obj.notifications += 1
node = f'{client.host}:{client.port}'
obj.nodes[node] += 1
obj.commands[cmd] += 1
async def monitorCallback(client, obj, message):
'''Need to extract a key and a command from the monitor payload'''
#
# We need to skip the beginning of such lines
# 1586739509.775473 [0 [::1]:54588] "XADD" "58c52262_channel_99" "MAXLEN" ...
#
line = message.decode()
# FIXME / parsing is not robust, if there are spaces in keys
tokens = line.split()
tokens = tokens[3:]
cmd = tokens[0][1:-1] # we need to remove the double quotes from key and cmd
cmd = cmd.upper()
# Some keys are located in odd places
if len(tokens) > 1:
key = tokens[1][1:-1]
if cmd in ('XREAD', 'XREADGROUP'):
try:
idx = tokens.index('"STREAMS"') + 1
except ValueError:
raise ValueError(
"{0} arguments do not contain STREAMS operand".format(cmd)
)
key = tokens[idx]
elif cmd in ('XGROUP', 'XINFO'):
key = tokens[2]
else:
key = tokens[1]
key = key[1:-1]
obj.keys[key] += 1
# We need key and command
obj.notifications += 1
node = f'{client.host}:{client.port}'
obj.nodes[node] += 1
obj.commands[cmd] += 1
tasks = []
keySpace = KeySpace()
# First we need to make sure keyspace notifications are ON
# Do this manually with redis-cli -p 10000 config set notify-keyspace-events KEAt
if not monitor:
confs = []
for client in clients:
conf = await client.send('CONFIG', 'GET', 'notify-keyspace-events')
if conf[1]:
print(f'{client} current keyspace config: {conf[1].decode()}')
confs.append(conf[1].decode())
# Set the new conf
await client.send('CONFIG', 'SET', 'notify-keyspace-events', keyspaceConfig)
try:
for client in clients:
if monitor:
task = asyncio.ensure_future(client.monitor(monitorCallback, keySpace))
else:
task = asyncio.ensure_future(
client.psubscribe(pattern, pubSubCallback, keySpace)
)
tasks.append(task)
if count > 0:
label = f'Capturing {count} events'
with click.progressbar(length=count, label=label) as bar:
progress = 0
while keySpace.notifications < count:
await asyncio.sleep(0.1)
bar.update(keySpace.notifications - progress)
progress = keySpace.notifications
else:
label = f'Capturing events during {timeout} seconds'
with click.progressbar(length=timeout, label=label) as bar:
# Monitor during X seconds
for i in range(timeout):
await asyncio.sleep(1)
bar.update(1)
for task in tasks:
# Cancel the tasks
task.cancel()
await task
finally:
if not monitor:
# Now restore the notification
for client, conf in zip(clients, confs):
# reset the previous conf
print(f'resetting old config {conf}')
await client.send('CONFIG', 'SET', 'notify-keyspace-events', conf)
keySpace.describe()
return keySpace
async def binPackingReshardCoroutine(
redisUrls, redisPassword, weights, timeout, dry=False, nodeId=None
):
redisClient = RedisClient(redisUrls, redisPassword)
nodes = await redisClient.cluster_nodes()
# There will be as many bins as there are master nodes
masterNodes = [node for node in nodes if node.role == 'master']
masterClients = [makeClientfromNode(node, redisPassword) for node in masterNodes]
binCount = len(masterNodes)
# Multiple keys could hash to the same hash-slot (collisions), so
# we need to feed to binpacking a list of [slot: weight], not
# a list of [key: weight]
hashSlotWeights = collections.defaultdict(int)
for key, weight in weights.items():
slot = getHashSlot(key)
hashSlotWeights[slot] += weight
# Run the bin packing algorithm
bins = to_constant_bin_number(hashSlotWeights, binCount)
# A list of list of slots to migrate, for each node
allSlots = []
for b in bins:
# b is a dictionary of [name, weight] / name is the hash slot.
# we want to sort by hash slot to make this process deterministic
binSlots = [slot for slot in b.keys()]
binSlots.sort()
allSlots.append(binSlots)
# We need to know where each slots lives
slotToNodes = await getSlotsToNodesMapping(redisUrls, redisPassword)
totalMigratedSlots = 0
for binSlots, node in zip(allSlots, masterNodes):
print(f'== {node.node_id} / {node.ip}:{node.port} ==')
migratedSlots = 0
if nodeId is not None and node.node_id != nodeId:
continue
for slot in binSlots:
sourceNode = slotToNodes[slot]
logging.debug(f'{slot} owned by {sourceNode.node_id}')
# Migrate each slot
for slot in binSlots:
# recompute the slots to node mapping after each node migration
slotToNodes = await getSlotsToNodesMapping(redisUrls, redisPassword)
sourceNode = slotToNodes[slot]
if sourceNode.node_id != node.node_id:
ret = await migrateSlot(
masterClients, redisPassword, slot, sourceNode, node, dry
)
if not ret:
return False
migratedSlots += 1
print(f'migrated {migratedSlots} slots')
totalMigratedSlots += migratedSlots
#
# This section is key.
# We periodically make sure that all nodes in the cluster agree on their view
# of the cluster, mostly on how slots are allocated
#
# Without this wait, if we try to keep on moving other slots
# the cluster will become broken,
# and commands such as redis-cli --cluste check will report it as inconsistent
#
# note that existing redis cli command do not migrate to multiple nodes at once
# while this script does
#
consistent = await waitForClusterViewToBeConsistent(
redisUrls, redisPassword, timeout
)
if not consistent:
return False
print(f'total migrated slots: {migratedSlots}')
return True
async def redisSubscriber(
client: RedisClient,
stream: str,
position: Optional[str],
messageHandlerClass: RedisSubscriberMessageHandlerClass, # noqa
obj,
):
messageHandler = messageHandlerClass(obj)
logPrefix = f'subscriber[{stream}]: {client}'
streamExists = False
redisHost = client.host
clientId = -1
if client:
# query the stream size
try:
streamExists = await client.send('EXISTS', stream)
clientId = await client.send('CLIENT', 'ID', key=stream)
redisHost = await getHostForKey(client, stream)
except Exception as e:
logging.error(f"{logPrefix} cannot retreive stream metadata: {e}")
client = None
initInfo = {
'success': client is not None,
'redis_node': redisHost,
'redis_client_id': clientId,
'stream_exists': streamExists,
'stream_name': stream,
}
try:
await messageHandler.on_init(initInfo)
except Exception as e:
logging.error(f'{logPrefix} cannot initialize message handler: {e}')
client = None
if client is None:
return messageHandler
# lastId = '0-0'
lastId = '$' if position is None else position
try:
# wait for incoming events.
while True:
results = await client.send('XREAD', 'BLOCK', b'0', b'STREAMS',
stream, lastId)
results = results[stream.encode()]
for result in results:
lastId = result[0].decode()
msg = result[1]
data = msg[b'json']
msgCksum = msg.get(b'sha1')
if msgCksum is not None:
cksum = sha1(data).hexdigest().encode()
if cksum != msgCksum:
err = f'{lastId}: invalid xread msg cksum'
logging.error(err)
continue
payloadSize = len(data)
try:
msg = json.loads(data)
except json.JSONDecodeError:
msgEncoded = base64.b64encode(data).decode()
err = f'{lastId}: malformed json: base64: {msgEncoded} raw: {data}'
logging.error(err)
continue
ret = await messageHandler.handleMsg(msg, lastId, payloadSize)
if not ret:
break
except asyncio.CancelledError:
messageHandler.log('Cancelling redis subscription')
raise
except Exception as e:
messageHandler.log(e)
backtrace = traceback.format_exc()
messageHandler.log(
f'{logPrefix} Generic Exception caught in {backtrace}')
finally:
messageHandler.log('Closing redis subscription')
# When finished, close the connection.
client.close()
return messageHandler
