def add_nodes(self):
for (_, servers), initial_nodes in zip(self.clusters(),
self.initial_nodes):
if initial_nodes < 2: # Single-node cluster
continue
# Adding initial nodes
master = servers[0]
if self.group_number > 1:
groups = self.rest.get_server_groups(master)
else:
groups = {}
for i, host_port in enumerate(servers[1:initial_nodes],
start=1):
uri = groups.get(server_group(servers[:initial_nodes],
self.group_number, i))
self.rest.add_node(master, host_port, self.roles[host_port],
uri)
# Rebalance
master = servers[0]
known_nodes = servers[:initial_nodes]
ejected_nodes = []
self.rest.rebalance(master, known_nodes, ejected_nodes)
self.monitor.monitor_rebalance(master)
def add_nodes(self):
for (_, servers), initial_nodes in zip(self.clusters(),
self.initial_nodes):
if initial_nodes < 2: # Single-node cluster
continue
# Adding initial nodes
master = servers[0]
if self.group_number > 1:
groups = self.rest.get_server_groups(master)
else:
groups = {}
for i, host_port in enumerate(servers[1:initial_nodes], start=1):
uri = groups.get(
server_group(servers[:initial_nodes], self.group_number,
i))
self.rest.add_node(master, host_port, self.roles[host_port],
uri)
# Rebalance
master = servers[0]
known_nodes = servers[:initial_nodes]
ejected_nodes = []
self.rest.rebalance(master, known_nodes, ejected_nodes)
self.monitor.monitor_rebalance(master)
def test_initial_4_out_of_8(self):
servers = range(8)
initial_nodes = 4
group_number = 2
groups = []
for i, host in enumerate(servers[1:initial_nodes], start=1):
g = server_group(servers[:initial_nodes], group_number, i)
groups.append(g)
self.assertEqual(groups, ['Group 1', 'Group 2', 'Group 2'])
def test_initial_4_out_of_8(self):
servers = range(8)
initial_nodes = 4
group_number = 2
groups = []
for i, host in enumerate(servers[1:initial_nodes], start=1):
g = server_group(servers[:initial_nodes], group_number, i)
groups.append(g)
self.assertEqual(groups, ['Group 1', 'Group 2', 'Group 2'])
def test_server_group_6_to_8(self):
servers = range(8)
initial_nodes = 6
nodes_after = 8
group_number = 3
groups = []
for i, host in enumerate(servers[initial_nodes:nodes_after],
start=initial_nodes):
g = server_group(servers[:nodes_after], group_number, i)
groups.append(g)
self.assertEqual(groups, ['Group 3', 'Group 3'])
def test_server_group_3_to_4(self):
servers = range(8)
initial_nodes = 3
nodes_after = 4
group_number = 2
groups = []
for i, host in enumerate(servers[initial_nodes:nodes_after],
start=initial_nodes):
g = server_group(servers[:nodes_after], group_number, i)
groups.append(g)
self.assertEqual(groups, ['Group 2'])
def test_server_group_3_to_4(self):
servers = range(8)
initial_nodes = 3
nodes_after = 4
group_number = 2
groups = []
for i, host in enumerate(servers[initial_nodes:nodes_after],
start=initial_nodes):
g = server_group(servers[:nodes_after], group_number, i)
groups.append(g)
self.assertEqual(groups, ['Group 2'])
def test_server_group_6_to_8(self):
servers = range(8)
initial_nodes = 6
nodes_after = 8
group_number = 3
groups = []
for i, host in enumerate(servers[initial_nodes:nodes_after],
start=initial_nodes):
g = server_group(servers[:nodes_after], group_number, i)
groups.append(g)
self.assertEqual(groups, ['Group 3', 'Group 3'])
def add_nodes(self):
for (_, servers), initial_nodes in zip(self.clusters(),
self.initial_nodes):
if initial_nodes < 2: # Single-node cluster
continue
master = servers[0]
if self.group_number > 1:
groups = self.rest.get_server_groups(master)
else:
groups = {}
for i, host_port in enumerate(servers[1:initial_nodes],
start=1):
uri = groups.get(server_group(servers[:initial_nodes],
self.group_number, i))
self.rest.add_node(master, host_port, self.roles[host_port],
uri)
def rebalance(self):
clusters = self.cluster_spec.yield_clusters()
initial_nodes = self.test_config.cluster.initial_nodes
nodes_after = self.rebalance_settings.nodes_after
group_number = self.test_config.cluster.group_number or 1
self.master = None
for (_, servers), initial_nodes, nodes_after in zip(clusters,
initial_nodes,
nodes_after):
master = servers[0]
groups = group_number > 1 and self.rest.get_server_groups(master) or {}
new_nodes = []
known_nodes = servers[:initial_nodes]
ejected_nodes = []
if nodes_after > initial_nodes: # rebalance-in
new_nodes = enumerate(
servers[initial_nodes:nodes_after],
start=initial_nodes
)
known_nodes = servers[:nodes_after]
elif nodes_after < initial_nodes: # rebalance-out
ejected_nodes = servers[nodes_after:initial_nodes]
else:
continue
for i, host_port in new_nodes:
group = server_group(servers[:nodes_after], group_number, i)
uri = groups.get(group)
self.rest.add_node(master, host_port, uri=uri)
self.rest.rebalance(master, known_nodes, ejected_nodes)
self.master = master
self.enable_xdcr()
start = time.time()
if self.master:
p = multiprocessing.Process(target=self.monitor.monitor_rebalance, args=(self.master,))
p.start()
self.monitor_replication()
self.time_elapsed = int(time.time() - start)
def rebalance(self):
clusters = self.cluster_spec.yield_clusters()
initial_nodes = self.test_config.cluster.initial_nodes
nodes_after = self.rebalance_settings.nodes_after
swap = self.rebalance_settings.swap
failover = self.rebalance_settings.failover
graceful_failover = self.rebalance_settings.graceful_failover
delta_recovery = self.rebalance_settings.delta_recovery
sleep_after_failover = self.rebalance_settings.sleep_after_failover
group_number = self.test_config.cluster.group_number or 1
for (_, servers), initial_nodes, nodes_after in zip(clusters,
initial_nodes,
nodes_after):
master = servers[0]
groups = group_number > 1 and self.rest.get_server_groups(master) or {}
new_nodes = []
known_nodes = servers[:initial_nodes]
ejected_nodes = []
failover_nodes = []
graceful_failover_nodes = []
if nodes_after > initial_nodes: # rebalance-in
new_nodes = enumerate(
servers[initial_nodes:nodes_after],
start=initial_nodes
)
known_nodes = servers[:nodes_after]
elif nodes_after < initial_nodes: # rebalance-out
ejected_nodes = servers[nodes_after:initial_nodes]
elif swap:
new_nodes = enumerate(
servers[initial_nodes:initial_nodes + swap],
start=initial_nodes - swap
)
known_nodes = servers[:initial_nodes + swap]
ejected_nodes = servers[initial_nodes - swap:initial_nodes]
elif failover:
failover_nodes = servers[initial_nodes - failover:initial_nodes]
elif graceful_failover:
graceful_failover_nodes = \
servers[initial_nodes - graceful_failover:initial_nodes]
else:
continue
for i, host_port in new_nodes:
group = server_group(servers[:nodes_after], group_number, i)
uri = groups.get(group)
self.rest.add_node(master, host_port, uri=uri)
for host_port in failover_nodes:
self.rest.fail_over(master, host_port)
self.rest.add_back(master, host_port)
for host_port in graceful_failover_nodes:
self.rest.graceful_fail_over(master, host_port)
self.monitor.monitor_rebalance(master)
self.rest.add_back(master, host_port)
if delta_recovery:
for host_port in failover_nodes + graceful_failover_nodes:
self.rest.set_delta_recovery_type(master, host_port)
if failover or graceful_failover:
logger.info('Sleeping for {} seconds after failover'
.format(sleep_after_failover))
time.sleep(sleep_after_failover)
self.reporter.start()
self.rest.rebalance(master, known_nodes, ejected_nodes)
self.monitor.monitor_rebalance(master)
def rebalance(self):
clusters = self.cluster_spec.yield_clusters()
initial_nodes = self.test_config.cluster.initial_nodes
nodes_after = self.rebalance_settings.nodes_after
swap = self.rebalance_settings.swap
failover = self.rebalance_settings.failover
graceful_failover = self.rebalance_settings.graceful_failover
delta_recovery = self.rebalance_settings.delta_recovery
sleep_after_failover = self.rebalance_settings.sleep_after_failover
group_number = self.test_config.cluster.group_number or 1
for (_, servers), initial_nodes, nodes_after in zip(
clusters, initial_nodes, nodes_after):
master = servers[0]
groups = group_number > 1 and self.rest.get_server_groups(
master) or {}
new_nodes = []
known_nodes = servers[:initial_nodes]
ejected_nodes = []
failover_nodes = []
graceful_failover_nodes = []
if nodes_after > initial_nodes: # rebalance-in
new_nodes = enumerate(servers[initial_nodes:nodes_after],
start=initial_nodes)
known_nodes = servers[:nodes_after]
elif nodes_after < initial_nodes: # rebalance-out
ejected_nodes = servers[nodes_after:initial_nodes]
elif swap:
new_nodes = enumerate(servers[initial_nodes:initial_nodes +
swap],
start=initial_nodes - swap)
known_nodes = servers[:initial_nodes + swap]
ejected_nodes = servers[initial_nodes - swap:initial_nodes]
elif failover:
failover_nodes = servers[initial_nodes -
failover:initial_nodes]
elif graceful_failover:
graceful_failover_nodes = \
servers[initial_nodes - graceful_failover:initial_nodes]
else:
continue
for i, host_port in new_nodes:
host = host_port.split(':')[0]
group = server_group(servers[:nodes_after], group_number, i)
uri = groups.get(group)
self.rest.add_node(master, host, uri)
for host_port in failover_nodes:
self.rest.fail_over(master, host_port)
self.rest.add_back(master, host_port)
if delta_recovery:
self.rest.set_delta_recovery_type(master, host_port)
for host_port in graceful_failover_nodes:
self.rest.graceful_fail_over(master, host_port)
self.monitor.monitor_rebalance(master)
self.rest.add_back(master, host_port)
if graceful_failover:
self.reporter.post_to_sf(
*self.metric_helper.failover_time(self.reporter))
self.reporter.start()
if failover:
time.sleep(sleep_after_failover)
self.reporter.start()
self.rest.rebalance(master, known_nodes, ejected_nodes)
for i, host_port in new_nodes:
host = host_port.split(':')[0]
self.change_watermarks(host)
self.monitor.monitor_rebalance(master)
