def add_nodes(self): for (_, servers), initial_nodes in zip(self.clusters(), self.initial_nodes): if initial_nodes < 2: # Single-node cluster continue # Adding initial nodes master = servers[0] if self.group_number > 1: groups = self.rest.get_server_groups(master) else: groups = {} for i, host_port in enumerate(servers[1:initial_nodes], start=1): uri = groups.get(server_group(servers[:initial_nodes], self.group_number, i)) self.rest.add_node(master, host_port, self.roles[host_port], uri) # Rebalance master = servers[0] known_nodes = servers[:initial_nodes] ejected_nodes = [] self.rest.rebalance(master, known_nodes, ejected_nodes) self.monitor.monitor_rebalance(master)
def add_nodes(self): for (_, servers), initial_nodes in zip(self.clusters(), self.initial_nodes): if initial_nodes < 2: # Single-node cluster continue # Adding initial nodes master = servers[0] if self.group_number > 1: groups = self.rest.get_server_groups(master) else: groups = {} for i, host_port in enumerate(servers[1:initial_nodes], start=1): uri = groups.get( server_group(servers[:initial_nodes], self.group_number, i)) self.rest.add_node(master, host_port, self.roles[host_port], uri) # Rebalance master = servers[0] known_nodes = servers[:initial_nodes] ejected_nodes = [] self.rest.rebalance(master, known_nodes, ejected_nodes) self.monitor.monitor_rebalance(master)
def test_initial_4_out_of_8(self): servers = range(8) initial_nodes = 4 group_number = 2 groups = [] for i, host in enumerate(servers[1:initial_nodes], start=1): g = server_group(servers[:initial_nodes], group_number, i) groups.append(g) self.assertEqual(groups, ['Group 1', 'Group 2', 'Group 2'])
def test_server_group_6_to_8(self): servers = range(8) initial_nodes = 6 nodes_after = 8 group_number = 3 groups = [] for i, host in enumerate(servers[initial_nodes:nodes_after], start=initial_nodes): g = server_group(servers[:nodes_after], group_number, i) groups.append(g) self.assertEqual(groups, ['Group 3', 'Group 3'])
def test_server_group_3_to_4(self): servers = range(8) initial_nodes = 3 nodes_after = 4 group_number = 2 groups = [] for i, host in enumerate(servers[initial_nodes:nodes_after], start=initial_nodes): g = server_group(servers[:nodes_after], group_number, i) groups.append(g) self.assertEqual(groups, ['Group 2'])
def add_nodes(self): for (_, servers), initial_nodes in zip(self.clusters(), self.initial_nodes): if initial_nodes < 2: # Single-node cluster continue master = servers[0] if self.group_number > 1: groups = self.rest.get_server_groups(master) else: groups = {} for i, host_port in enumerate(servers[1:initial_nodes], start=1): uri = groups.get(server_group(servers[:initial_nodes], self.group_number, i)) self.rest.add_node(master, host_port, self.roles[host_port], uri)
def rebalance(self): clusters = self.cluster_spec.yield_clusters() initial_nodes = self.test_config.cluster.initial_nodes nodes_after = self.rebalance_settings.nodes_after group_number = self.test_config.cluster.group_number or 1 self.master = None for (_, servers), initial_nodes, nodes_after in zip(clusters, initial_nodes, nodes_after): master = servers[0] groups = group_number > 1 and self.rest.get_server_groups(master) or {} new_nodes = [] known_nodes = servers[:initial_nodes] ejected_nodes = [] if nodes_after > initial_nodes: # rebalance-in new_nodes = enumerate( servers[initial_nodes:nodes_after], start=initial_nodes ) known_nodes = servers[:nodes_after] elif nodes_after < initial_nodes: # rebalance-out ejected_nodes = servers[nodes_after:initial_nodes] else: continue for i, host_port in new_nodes: group = server_group(servers[:nodes_after], group_number, i) uri = groups.get(group) self.rest.add_node(master, host_port, uri=uri) self.rest.rebalance(master, known_nodes, ejected_nodes) self.master = master self.enable_xdcr() start = time.time() if self.master: p = multiprocessing.Process(target=self.monitor.monitor_rebalance, args=(self.master,)) p.start() self.monitor_replication() self.time_elapsed = int(time.time() - start)
def rebalance(self): clusters = self.cluster_spec.yield_clusters() initial_nodes = self.test_config.cluster.initial_nodes nodes_after = self.rebalance_settings.nodes_after swap = self.rebalance_settings.swap failover = self.rebalance_settings.failover graceful_failover = self.rebalance_settings.graceful_failover delta_recovery = self.rebalance_settings.delta_recovery sleep_after_failover = self.rebalance_settings.sleep_after_failover group_number = self.test_config.cluster.group_number or 1 for (_, servers), initial_nodes, nodes_after in zip(clusters, initial_nodes, nodes_after): master = servers[0] groups = group_number > 1 and self.rest.get_server_groups(master) or {} new_nodes = [] known_nodes = servers[:initial_nodes] ejected_nodes = [] failover_nodes = [] graceful_failover_nodes = [] if nodes_after > initial_nodes: # rebalance-in new_nodes = enumerate( servers[initial_nodes:nodes_after], start=initial_nodes ) known_nodes = servers[:nodes_after] elif nodes_after < initial_nodes: # rebalance-out ejected_nodes = servers[nodes_after:initial_nodes] elif swap: new_nodes = enumerate( servers[initial_nodes:initial_nodes + swap], start=initial_nodes - swap ) known_nodes = servers[:initial_nodes + swap] ejected_nodes = servers[initial_nodes - swap:initial_nodes] elif failover: failover_nodes = servers[initial_nodes - failover:initial_nodes] elif graceful_failover: graceful_failover_nodes = \ servers[initial_nodes - graceful_failover:initial_nodes] else: continue for i, host_port in new_nodes: group = server_group(servers[:nodes_after], group_number, i) uri = groups.get(group) self.rest.add_node(master, host_port, uri=uri) for host_port in failover_nodes: self.rest.fail_over(master, host_port) self.rest.add_back(master, host_port) for host_port in graceful_failover_nodes: self.rest.graceful_fail_over(master, host_port) self.monitor.monitor_rebalance(master) self.rest.add_back(master, host_port) if delta_recovery: for host_port in failover_nodes + graceful_failover_nodes: self.rest.set_delta_recovery_type(master, host_port) if failover or graceful_failover: logger.info('Sleeping for {} seconds after failover' .format(sleep_after_failover)) time.sleep(sleep_after_failover) self.reporter.start() self.rest.rebalance(master, known_nodes, ejected_nodes) self.monitor.monitor_rebalance(master)
def rebalance(self): clusters = self.cluster_spec.yield_clusters() initial_nodes = self.test_config.cluster.initial_nodes nodes_after = self.rebalance_settings.nodes_after swap = self.rebalance_settings.swap failover = self.rebalance_settings.failover graceful_failover = self.rebalance_settings.graceful_failover delta_recovery = self.rebalance_settings.delta_recovery sleep_after_failover = self.rebalance_settings.sleep_after_failover group_number = self.test_config.cluster.group_number or 1 for (_, servers), initial_nodes, nodes_after in zip( clusters, initial_nodes, nodes_after): master = servers[0] groups = group_number > 1 and self.rest.get_server_groups( master) or {} new_nodes = [] known_nodes = servers[:initial_nodes] ejected_nodes = [] failover_nodes = [] graceful_failover_nodes = [] if nodes_after > initial_nodes: # rebalance-in new_nodes = enumerate(servers[initial_nodes:nodes_after], start=initial_nodes) known_nodes = servers[:nodes_after] elif nodes_after < initial_nodes: # rebalance-out ejected_nodes = servers[nodes_after:initial_nodes] elif swap: new_nodes = enumerate(servers[initial_nodes:initial_nodes + swap], start=initial_nodes - swap) known_nodes = servers[:initial_nodes + swap] ejected_nodes = servers[initial_nodes - swap:initial_nodes] elif failover: failover_nodes = servers[initial_nodes - failover:initial_nodes] elif graceful_failover: graceful_failover_nodes = \ servers[initial_nodes - graceful_failover:initial_nodes] else: continue for i, host_port in new_nodes: host = host_port.split(':')[0] group = server_group(servers[:nodes_after], group_number, i) uri = groups.get(group) self.rest.add_node(master, host, uri) for host_port in failover_nodes: self.rest.fail_over(master, host_port) self.rest.add_back(master, host_port) if delta_recovery: self.rest.set_delta_recovery_type(master, host_port) for host_port in graceful_failover_nodes: self.rest.graceful_fail_over(master, host_port) self.monitor.monitor_rebalance(master) self.rest.add_back(master, host_port) if graceful_failover: self.reporter.post_to_sf( *self.metric_helper.failover_time(self.reporter)) self.reporter.start() if failover: time.sleep(sleep_after_failover) self.reporter.start() self.rest.rebalance(master, known_nodes, ejected_nodes) for i, host_port in new_nodes: host = host_port.split(':')[0] self.change_watermarks(host) self.monitor.monitor_rebalance(master)