def rescue_cluster(host, port, subst_host, subst_port):
failed_slots = set(xrange(SLOT_COUNT))
with Talker(host, port) as t:
_ensure_cluster_status_set(t)
for node in _list_masters(t)[0]:
failed_slots -= set(node.assigned_slots)
if len(failed_slots) == 0:
logging.info('No need to rescue cluster at %s:%d', host, port)
return
with Talker(subst_host, subst_port) as s:
_ensure_cluster_status_unset(s)
m = s.talk('cluster', 'meet', host, port)
logging.debug('Ask `cluster meet` Rsp %s', m)
if m.lower() != 'ok':
raise RedisStatusError('Unexpected reply after MEET: %s' % m)
m = s.talk('cluster', 'addslots', *failed_slots)
logging.debug('Ask `cluster addslots` Rsp %s', m)
if m.lower() != 'ok':
raise RedisStatusError('Unexpected reply after ADDSLOTS: %s' %
m)
_poll_check_status(s)
logging.info(
'Instance at %s:%d serves %d slots to rescue the cluster',
subst_host, subst_port, len(failed_slots))
def start_cluster_on_multi(host_port_list, max_slots=SLOT_COUNT):
talkers = []
try:
for host, port in set(host_port_list):
t = Talker(host, port)
talkers.append(t)
_ensure_cluster_status_unset(t)
logging.info('Instance at %s:%d checked', t.host, t.port)
first_talker = talkers[0]
for i, t in enumerate(talkers[1:]):
t.talk('cluster', 'meet', first_talker.host, first_talker.port)
slots_each = SLOT_COUNT / len(talkers)
slots_residue = SLOT_COUNT - slots_each * len(talkers)
first_node_slots = slots_residue + slots_each
_add_slots(first_talker, 0, first_node_slots, max_slots)
logging.info('Add %d slots to %s:%d', slots_residue + slots_each,
first_talker.host, first_talker.port)
for i, t in enumerate(talkers[1:]):
_add_slots(t, i * slots_each + first_node_slots,
(i + 1) * slots_each + first_node_slots, max_slots)
logging.info('Add %d slots to %s:%d', slots_each, t.host, t.port)
for t in talkers:
_poll_check_status(t)
finally:
for t in talkers:
t.close()
def replicate(master_host, master_port, slave_host, slave_port):
master_talker = None
t = Talker(slave_host, slave_port)
try:
master_talker = Talker(master_host, master_port)
_ensure_cluster_status_set(master_talker)
myself = _list_nodes(master_talker)[1]
myid = (myself.node_id if myself.role_in_cluster == 'master'
else myself.master_id)
_join_to_cluster(master_host, master_port, t)
logging.info('Instance at %s:%d has joined %s:%d; now set replica',
slave_host, slave_port, master_host, master_port)
m = t.talk('cluster', 'replicate', myid)
logging.debug('Ask `cluster replicate` Rsp %s', m)
if m.lower() != 'ok':
raise RedisStatusError('Unexpected reply after REPCLIATE: %s' % m)
_check_slave(slave_host, slave_port, master_talker)
logging.info('Instance at %s:%d set as replica to %s',
slave_host, slave_port, myid)
finally:
t.close()
if master_talker is not None:
master_talker.close()
def rescue_cluster(host, port, subst_host, subst_port):
failed_slots = set(xrange(SLOT_COUNT))
t = None
s = None
try:
t = Talker(host, port)
_ensure_cluster_status_set(t)
for node in _list_masters(t)[0]:
failed_slots -= set(node.assigned_slots)
if len(failed_slots) == 0:
logging.info('No need to rescue cluster at %s:%d', host, port)
return
s = Talker(subst_host, subst_port)
_ensure_cluster_status_unset(s)
m = s.talk('cluster', 'meet', host, port)
logging.debug('Ask `cluster meet` Rsp %s', m)
if m.lower() != 'ok':
raise RedisStatusError('Unexpected reply after MEET: %s' % m)
m = s.talk('cluster', 'addslots', *failed_slots)
logging.debug('Ask `cluster addslots` Rsp %s', m)
if m.lower() != 'ok':
raise RedisStatusError('Unexpected reply after ADDSLOTS: %s' % m)
_poll_check_status(s)
logging.info('Instance at %s:%d serves %d slots to rescue the cluster',
subst_host, subst_port, len(failed_slots))
finally:
if t is not None:
t.close()
if s is not None:
s.close()
def start_cluster_on_multi(host_port_list):
talkers = []
try:
for host, port in set(host_port_list):
t = Talker(host, port)
talkers.append(t)
_ensure_cluster_status_unset(t)
logging.info('Instance at %s:%d checked', t.host, t.port)
first_talker = talkers[0]
for i, t in enumerate(talkers[1:]):
t.talk('cluster', 'meet', first_talker.host, first_talker.port)
slots_each = SLOT_COUNT / len(talkers)
slots_residue = SLOT_COUNT - slots_each * len(talkers)
first_node_slots = slots_residue + slots_each
first_talker.talk('cluster', 'addslots', *xrange(first_node_slots))
logging.info('Add %d slots to %s:%d', slots_residue + slots_each,
first_talker.host, first_talker.port)
for i, t in enumerate(talkers[1:]):
t.talk(
'cluster', 'addslots',
*xrange(i * slots_each + first_node_slots,
(i + 1) * slots_each + first_node_slots))
logging.info('Add %d slots to %s:%d', slots_each, t.host, t.port)
for t in talkers:
_poll_check_status(t)
finally:
for t in talkers:
t.close()
def fix_migrating(host, port):
nodes = dict()
mig_srcs = []
mig_dsts = []
t = Talker(host, port)
try:
m = t.talk_raw(CMD_CLUSTER_NODES)
logging.debug('Ask `cluster nodes` Rsp %s', m)
for node_info in m.split('\n'):
if not _valid_node_info(node_info):
continue
node = ClusterNode(*node_info.split(' '))
node.host = node.host or host
nodes[node.node_id] = node
mig_dsts.extend([(node, {
'slot': g[0],
'id': g[1]
}) for g in PAT_MIGRATING_IN.findall(node_info)])
mig_srcs.extend([(node, {
'slot': g[0],
'id': g[1]
}) for g in PAT_MIGRATING_OUT.findall(node_info)])
for n, args in mig_dsts:
node_id = args['id']
if node_id not in nodes:
logging.error(
'Fail to fix %s:%d <- (referenced from %s:%d)'
' - node %s is missing', n.host, n.port, host, port,
node_id)
continue
_migr_one_slot(nodes[node_id], n, int(args['slot']),
nodes.itervalues())
for n, args in mig_srcs:
node_id = args['id']
if node_id not in nodes:
logging.error(
'Fail to fix %s:%d -> (referenced from %s:%d)'
' - node %s is missing', n.host, n.port, host, port,
node_id)
continue
_migr_one_slot(n, nodes[node_id], int(args['slot']),
nodes.itervalues())
finally:
t.close()
for n in nodes.itervalues():
n.close()
def fix_migrating(host, port):
nodes = dict()
mig_srcs = []
mig_dsts = []
t = Talker(host, port)
try:
m = t.talk_raw(CMD_CLUSTER_NODES)
logging.debug("Ask `cluster nodes` Rsp %s", m)
for node_info in m.split("\n"):
if not _valid_node_info(node_info):
continue
node = ClusterNode(*node_info.split(" "))
node.host = node.host or host
nodes[node.node_id] = node
mig_dsts.extend([(node, {"slot": g[0], "id": g[1]}) for g in PAT_MIGRATING_IN.findall(node_info)])
mig_srcs.extend([(node, {"slot": g[0], "id": g[1]}) for g in PAT_MIGRATING_OUT.findall(node_info)])
for n, args in mig_dsts:
node_id = args["id"]
if node_id not in nodes:
logging.error(
"Fail to fix %s:%d <- (referenced from %s:%d)" " - node %s is missing",
n.host,
n.port,
host,
port,
node_id,
)
continue
_migr_one_slot(nodes[node_id], n, int(args["slot"]), nodes.itervalues())
for n, args in mig_srcs:
node_id = args["id"]
if node_id not in nodes:
logging.error(
"Fail to fix %s:%d -> (referenced from %s:%d)" " - node %s is missing",
n.host,
n.port,
host,
port,
node_id,
)
continue
_migr_one_slot(n, nodes[node_id], int(args["slot"]), nodes.itervalues())
finally:
t.close()
for n in nodes.itervalues():
n.close()
def start_cluster(host, port):
t = Talker(host, port)
try:
_ensure_cluster_status_unset(t)
m = t.talk('cluster', 'addslots', *xrange(SLOT_COUNT))
logging.debug('Ask `cluster addslots` Rsp %s', m)
if m.lower() != 'ok':
raise RedisStatusError('Unexpected reply after ADDSLOTS: %s' % m)
_poll_check_status(t)
logging.info('Instance at %s:%d started as a standalone cluster',
host, port)
finally:
t.close()
def start_cluster(host, port):
t = Talker(host, port)
try:
_ensure_cluster_status_unset(t)
m = t.talk('cluster', 'addslots', *xrange(SLOT_COUNT))
logging.debug('Ask `cluster addslots` Rsp %s', m)
if m.lower() != 'ok':
raise RedisStatusError('Unexpected reply after ADDSLOTS: %s' % m)
_poll_check_status(t)
logging.info('Instance at %s:%d started as a standalone cluster',
host, port)
finally:
t.close()
def start_cluster(host, port, max_slots=SLOT_COUNT):
with Talker(host, port) as t:
_ensure_cluster_status_unset(t)
_add_slots(t, 0, SLOT_COUNT, max_slots)
_poll_check_status(t)
logging.info('Instance at %s:%d started as a standalone cluster', host,
port)
def _get_shard_master(shard):
for addr in shard:
host, port = addr.split(":")
with Talker(host, int(port)) as s:
role = s.talk('role')
if role[0] == 'master':
return host, int(port)
raise Exception('No master found in shard: %s' % shard)
def migrate_slots(src_host, src_port, dst_host, dst_port, slots):
if src_host == dst_host and src_port == dst_port:
raise ValueError('Same node')
t = Talker(src_host, src_port)
try:
nodes, myself = _list_masters(t, src_host)
finally:
t.close()
slots = set(slots)
logging.debug('Migrating %s', slots)
if not slots.issubset(set(myself.assigned_slots)):
raise ValueError('Not all slot held by %s:%d' % (src_host, src_port))
for n in nodes:
if n.host == dst_host and n.port == dst_port:
return _migr_slots(myself, n, slots, nodes)
raise ValueError('Two nodes are not in the same cluster')
def shutdown_cluster(host, port):
t = Talker(host, port)
try:
_ensure_cluster_status_set(t)
myself = None
m = t.talk_raw(CMD_CLUSTER_NODES)
logging.debug('Ask `cluster nodes` Rsp %s', m)
nodes_info = filter(None, m.split('\n'))
if len(nodes_info) > 1:
raise RedisStatusError('More than 1 nodes in cluster.')
try:
m = t.talk('cluster', 'reset')
except hiredis.ReplyError, e:
if 'containing keys' in e.message:
raise RedisStatusError('Cluster containing keys')
raise
logging.debug('Ask `cluster delslots` Rsp %s', m)
def shutdown_cluster(host, port):
t = Talker(host, port)
try:
_ensure_cluster_status_set(t)
myself = None
m = t.talk_raw(CMD_CLUSTER_NODES)
logging.debug('Ask `cluster nodes` Rsp %s', m)
nodes_info = filter(None, m.split('\n'))
if len(nodes_info) > 1:
raise RedisStatusError('More than 1 nodes in cluster.')
try:
m = t.talk('cluster', 'reset')
except hiredis.ReplyError, e:
if 'containing keys' in e.message:
raise RedisStatusError('Cluster containing keys')
raise
logging.debug('Ask `cluster delslots` Rsp %s', m)
def fix_migrating(host, port):
nodes = dict()
mig_srcs = []
mig_dsts = []
t = Talker(host, port)
try:
m = t.talk_raw(CMD_CLUSTER_NODES)
logging.debug('Ask `cluster nodes` Rsp %s', m)
for node_info in m.split('\n'):
if not _valid_node_info(node_info):
continue
node = ClusterNode(*node_info.split(' '))
node.host = node.host or host
nodes[node.node_id] = node
search = PAT_MIGRATING_IN.search(node_info)
if search is not None:
mig_dsts.append((node, search.groupdict()))
search = PAT_MIGRATING_OUT.search(node_info)
if search is not None:
mig_srcs.append((node, search.groupdict()))
for n, args in mig_dsts:
node_id = args['id']
if node_id not in nodes:
logging.error('Fail to fix %s:%d <- (referenced from %s:%d)'
' - node %s is missing', n.host, n.port,
host, port, node_id)
continue
_migr_one_slot(nodes[node_id], n, int(args['slot']),
nodes.itervalues())
for n, args in mig_srcs:
node_id = args['id']
if node_id not in nodes:
logging.error('Fail to fix %s:%d -> (referenced from %s:%d)'
' - node %s is missing', n.host, n.port,
host, port, node_id)
continue
_migr_one_slot(n, nodes[node_id], int(args['slot']),
nodes.itervalues())
finally:
t.close()
for n in nodes.itervalues():
n.close()
def start_cluster_on_multi(host_port_list):
talkers = []
try:
for host, port in set(host_port_list):
t = Talker(host, port)
talkers.append(t)
_ensure_cluster_status_unset(t)
logging.info("Instance at %s:%d checked", t.host, t.port)
first_talker = talkers[0]
for i, t in enumerate(talkers[1:]):
t.talk("cluster", "meet", first_talker.host, first_talker.port)
slots_each = SLOT_COUNT / len(talkers)
slots_residue = SLOT_COUNT - slots_each * len(talkers)
first_node_slots = slots_residue + slots_each
first_talker.talk("cluster", "addslots", *xrange(first_node_slots))
logging.info("Add %d slots to %s:%d", slots_residue + slots_each, first_talker.host, first_talker.port)
for i, t in enumerate(talkers[1:]):
t.talk(
"cluster",
"addslots",
*xrange(i * slots_each + first_node_slots, (i + 1) * slots_each + first_node_slots)
)
logging.info("Add %d slots to %s:%d", slots_each, t.host, t.port)
for t in talkers:
_poll_check_status(t)
finally:
for t in talkers:
t.close()
def replicate(master_host, master_port, slave_host, slave_port):
master_talker = None
t = Talker(slave_host, slave_port)
try:
master_talker = Talker(master_host, master_port)
_ensure_cluster_status_set(master_talker)
myself = _list_nodes(master_talker)[1]
myid = (myself.node_id if myself.role_in_cluster == 'master'
else myself.master_id)
_join_to_cluster(master_host, master_port, t)
logging.info('Instance at %s:%d has joined %s:%d; now set replica',
slave_host, slave_port, master_host, master_port)
m = t.talk('cluster', 'replicate', myid)
logging.debug('Ask `cluster replicate` Rsp %s', m)
if m.lower() != 'ok':
raise RedisStatusError('Unexpected reply after REPCLIATE: %s' % m)
_check_slave(slave_host, slave_port, master_talker)
logging.info('Instance at %s:%d set as replica to %s',
slave_host, slave_port, myid)
finally:
t.close()
if master_talker is not None:
master_talker.close()
def quit_cluster(host, port):
nodes = []
myself = None
t = Talker(host, port)
try:
_ensure_cluster_status_set(t)
nodes, myself = _list_nodes(t)
nodes.remove(myself)
if myself.role_in_cluster == 'master':
_check_master_and_migrate_slots(nodes, myself)
logging.info('Migrated for %s / Broadcast a `forget`', myself.node_id)
for node in nodes:
tk = node.talker()
try:
tk.talk('cluster', 'forget', myself.node_id)
except hiredis.ReplyError, e:
if 'Unknown node' not in e.message:
raise
t.talk('cluster', 'reset')
def quit_cluster(host, port):
nodes = []
myself = None
t = Talker(host, port)
try:
_ensure_cluster_status_set(t)
nodes, myself = _list_nodes(t)
nodes.remove(myself)
if myself.role_in_cluster == 'master':
_check_master_and_migrate_slots(nodes, myself)
logging.info('Migrated for %s / Broadcast a `forget`', myself.node_id)
for node in nodes:
tk = node.talker()
try:
tk.talk('cluster', 'forget', myself.node_id)
except hiredis.ReplyError, e:
if 'Unknown node' not in e.message:
raise
t.talk('cluster', 'reset')
def _cluster_slots(shards):
import random
host_port = shards[random.randrange(0, len(shards))][0].split(':')
assert len(host_port) == 2
host, port = host_port
port = int(port)
with Talker(host, port) as s:
cluster_slots = s.talk('cluster', 'slots')
if not cluster_slots:
raise Exception("get cluster slots failed")
return cluster_slots
def join_cluster(cluster_host,
cluster_port,
newin_host,
newin_port,
balancer=None,
balance_plan=base_balance_plan):
with Talker(newin_host, newin_port) as t, \
Talker(cluster_host, cluster_port) as cnode:
_join_to_cluster(cnode, t)
nodes = []
try:
logging.info(
'Instance at %s:%d has joined %s:%d; now balancing slots',
newin_host, newin_port, cluster_host, cluster_port)
nodes = _list_nodes(t, default_host=newin_host)[0]
for src, dst, count in balance_plan(nodes, balancer):
_migr_slots(src, dst, src.assigned_slots[:count], nodes)
finally:
for n in nodes:
n.close()
def quit_cluster(host, port):
nodes = []
myself = None
t = Talker(host, port)
try:
_ensure_cluster_status_set(t)
nodes, myself = _list_nodes(t)
nodes.remove(myself)
if myself.role_in_cluster == 'master':
_check_master_and_migrate_slots(nodes, myself)
logging.info('Migrated for %s / Broadcast a `forget`', myself.node_id)
for node in nodes:
tk = node.talker()
tk.talk('cluster', 'forget', myself.node_id)
t.talk('cluster', 'reset')
finally:
t.close()
if myself is not None:
myself.close()
for n in nodes:
n.close()
def migrate_slots(src_host, src_port, dst_host, dst_port, slots):
if src_host == dst_host and src_port == dst_port:
raise ValueError('Same node')
t = Talker(src_host, src_port)
try:
nodes, myself = _list_masters(t, src_host)
finally:
t.close()
slots = set(slots)
logging.debug('Migrating %s', slots)
if not slots.issubset(set(myself.assigned_slots)):
raise ValueError('Not all slot held by %s:%d' % (src_host, src_port))
for n in nodes:
if n.host == dst_host and n.port == dst_port:
logging.info('Migrating %d slots from %s:%d to %s:%d', len(slots),
src_host, src_port, dst_host, dst_port)
for s in slots:
_migr_one_slot(myself, n, s, nodes)
return
raise ValueError('Two nodes are not in the same cluster')
def migrate_slots(src_host, src_port, dst_host, dst_port, slots):
if src_host == dst_host and src_port == dst_port:
raise ValueError('Same node')
with Talker(src_host, src_port) as t:
nodes, myself = _list_masters(t, src_host)
slots = set(slots)
logging.debug('Migrating %s', slots)
if not slots.issubset(set(myself.assigned_slots)):
raise ValueError('Not all slot held by %s:%d' % (src_host, src_port))
for n in nodes:
if n.host == dst_host and n.port == dst_port:
return _migr_slots(myself, n, slots, nodes)
raise ValueError('Two nodes are not in the same cluster')
def join_cluster(cluster_host, cluster_port, newin_host, newin_port,
balancer=None, balance_plan=base_balance_plan):
nodes = []
t = Talker(newin_host, newin_port)
try:
_join_to_cluster(cluster_host, cluster_port, t)
logging.info('Instance at %s:%d has joined %s:%d; now balancing slots',
newin_host, newin_port, cluster_host, cluster_port)
m = t.talk_raw(CMD_CLUSTER_INFO)
logging.debug('Ask `cluster info` Rsp %s', m)
cluster_state = PAT_CLUSTER_STATE.findall(m)
if cluster_state[0] != 'ok':
raise hiredis.ProtocolError(
'Node %s:%d is already in a cluster' % (t.host, t.port))
slots = int(PAT_CLUSTER_SLOT_ASSIGNED.findall(m)[0])
nodes = _list_nodes(t, default_host=newin_host)[0]
for source, target, count in balance_plan(nodes, balancer):
_migr_slots(source, target, count, nodes)
finally:
t.close()
for n in nodes:
n.close()
def join_cluster(cluster_host, cluster_port, newin_host, newin_port,
balancer=None, balance_plan=base_balance_plan):
nodes = []
t = Talker(newin_host, newin_port)
try:
_join_to_cluster(cluster_host, cluster_port, t)
logging.info('Instance at %s:%d has joined %s:%d; now balancing slots',
newin_host, newin_port, cluster_host, cluster_port)
m = t.talk_raw(CMD_CLUSTER_INFO)
logging.debug('Ask `cluster info` Rsp %s', m)
cluster_state = PAT_CLUSTER_STATE.findall(m)
if cluster_state[0] != 'ok':
raise hiredis.ProtocolError(
'Node %s:%d is already in a cluster' % (t.host, t.port))
slots = int(PAT_CLUSTER_SLOT_ASSIGNED.findall(m)[0])
nodes = _list_nodes(t, default_host=newin_host)[0]
for source, target, count in balance_plan(nodes, balancer):
_migr_slots(source, target, count, nodes)
finally:
t.close()
for n in nodes:
n.close()
def quit_cluster(host, port):
nodes = []
myself = None
t = Talker(host, port)
try:
_ensure_cluster_status_set(t)
nodes, myself = _list_nodes(t)
nodes.remove(myself)
if myself.role_in_cluster == 'master':
_check_master_and_migrate_slots(nodes, myself)
logging.info('Migrated for %s / Broadcast a `forget`', myself.node_id)
for node in nodes:
tk = node.talker()
tk.talk('cluster', 'forget', myself.node_id)
t.talk('cluster', 'reset')
finally:
t.close()
if myself is not None:
myself.close()
for n in nodes:
n.close()
def list_masters(host, port, default_host=None):
t = Talker(host, port)
try:
return _list_masters(t, default_host or host)
finally:
t.close()
def _ensure_cluster_status_set_at(host, port):
t = Talker(host, port)
try:
_ensure_cluster_status_set(t)
finally:
t.close()
def list_masters(host, port, default_host=None):
t = Talker(host, port)
try:
return _list_masters(t, default_host or host)
finally:
t.close()
def join_no_load(cluster_host, cluster_port, newin_host, newin_port):
with Talker(newin_host, newin_port) as t, \
Talker(cluster_host, cluster_port) as c:
_join_to_cluster(c, t)
def list_masters(host, port, default_host=None):
with Talker(host, port) as t:
return _list_masters(t, default_host or host)
def join_no_load(cluster_host, cluster_port, newin_host, newin_port):
t = Talker(newin_host, newin_port)
try:
_join_to_cluster(cluster_host, cluster_port, t)
finally:
t.close()
def join_no_load(cluster_host, cluster_port, newin_host, newin_port):
t = Talker(newin_host, newin_port)
try:
_join_to_cluster(cluster_host, cluster_port, t)
finally:
t.close()
def _ensure_cluster_status_set_at(host, port):
t = Talker(host, port)
try:
_ensure_cluster_status_set(t)
finally:
t.close()
