def test_blinking_node_baseline(self):
# IGNITE-8879
with PiClient(self.ignite, self.get_client_config()):
for iteration in range(0, self.iterations):
log_print("Iteration {}/{}".format(str(iteration + 1),
str(self.iterations)),
color='blue')
self.assert_nodes_alive()
self.ignite.kill_node(2)
self.ignite.wait_for_topology_snapshot(
server_num=len(self.ignite.get_alive_default_nodes()))
self.cu.set_current_topology_as_baseline()
util_sleep(5)
self.ignite.start_node(2)
self.ignite.wait_for_topology_snapshot(
server_num=len(self.ignite.get_alive_default_nodes()))
self.cu.set_current_topology_as_baseline()
self.ignite.jmx.wait_for_finish_rebalance(
self.rebalance_timeout, self.group_names)
tiden_assert_equal(
0, self.ignite.find_exception_in_logs('java.lang.AssertionError'),
"# of AssertionError")
def test_two_blinking_nodes_clean_lfs(self):
with PiClient(self.ignite, self.get_client_config()):
with ExitStack() as stack:
if is_enabled(self.config.get('zookeeper_enabled')) and \
is_enabled(self.config.get('zookeeper_nodes_restart')):
stack.enter_context(ZkNodesRestart(self.zoo, 2))
for iteration in range(0, self.iterations):
log_print("Iteration {}/{}".format(str(iteration + 1),
str(self.iterations)),
color='blue')
self.assert_nodes_alive()
self.ignite.kill_node(2)
self.cleanup_lfs(2)
# self.ignite.start_node(2)
self.start_node(2)
self.ignite.kill_node(3)
util_sleep(10)
# self.ignite.start_node(3)
self.start_node(3)
self.ignite.jmx.wait_for_finish_rebalance(
self.rebalance_timeout, self.group_names)
tiden_assert_equal(
0, self.ignite.find_exception_in_logs('java.lang.AssertionError'),
"# of AssertionError")
def test_blinking_node_with_extra_node_in_blt(self):
# IGNITE-8893
with PiClient(self.ignite, self.get_client_config()):
self.ignite.start_additional_nodes(
self.ignite.add_additional_nodes(self.get_server_config()))
for iteration in range(0, self.iterations):
log_print("Iteration {}/{}".format(str(iteration + 1),
str(self.iterations)),
color='blue')
self.assert_nodes_alive()
self.ignite.kill_node(2)
util_sleep(10)
self.ignite.start_node(2)
self.ignite.jmx.wait_for_finish_rebalance(
self.rebalance_timeout, self.group_names)
util_sleep(60)
tiden_assert_equal(
0, self.ignite.find_exception_in_logs('java.lang.AssertionError'),
"# of AssertionError")
def restart_clients(self, nodes, iterations):
for i in range(1, iterations):
log_print('Going to start client nodes %s' % nodes)
self.ignite.start_additional_nodes(nodes,
client_nodes=True,
skip_topology_check=True)
util_sleep(2)
self.ignite.update_starting_node_attrs()
log_print('Going to kill client nodes %s' % nodes)
self.ignite.kill_nodes(*nodes)
def test_rebalancing_with_ttl_caches(self):
"""
IGN-13549 (IGNITE-11400)
Rebalancing caches with TTL enabled can cause data corruption.
:return:
"""
with PiClient(self.ignite, self.get_client_config()):
checksums = create_distributed_checksum_operation().evaluate()
self.wait_for_running_clients_num(client_num=0, timeout=120)
self.ignite.cu.control_utility(
'--cache idle_verify --dump --skip-zeros')
log_print('Calculating checksums done: %s' % checksums)
self.assert_nodes_alive()
self.ignite.kill_node(2)
PiClientIgniteUtils.load_data_with_putall(
self.ignite,
self.get_client_config(),
start_key=self.preloading_size,
end_key=self.preloading_size + 100000)
util_sleep(5 * 60)
self.ignite.start_node(2)
self.ignite.jmx.wait_for_finish_rebalance(self.rebalance_timeout * 2,
self.group_names)
with PiClient(self.ignite,
self.get_client_config()): # , jvm_options=jvm_options):
checksums = create_distributed_checksum_operation().evaluate()
self.wait_for_running_clients_num(client_num=0, timeout=120)
log_print('Calculating checksums done: %s' % checksums)
tiden_assert_equal(
0, self.ignite.find_exception_in_logs('java.lang.AssertionError'),
"# of AssertionError")
def test_loading_blinking_node_baseline(self):
with PiClient(self.ignite, self.get_client_config()) as piclient:
self.wait_for_running_clients_num(piclient.nodes_num, 90)
with ExitStack() as stack:
stack.enter_context(
TransactionalLoading(
self,
cross_cache_batch=2,
skip_atomic=True,
post_checksum_action=self.idle_verify_action))
if is_enabled(self.config.get('zookeeper_enabled')) and \
is_enabled(self.config.get('zookeeper_nodes_restart')):
stack.enter_context(ZkNodesRestart(self.zoo, 2))
for iteration in range(0, self.iterations):
log_print("Iteration {}/{}".format(str(iteration + 1),
str(self.iterations)),
color='blue')
self.assert_nodes_alive()
self.ignite.kill_node(2)
self.ignite.wait_for_topology_snapshot(
server_num=len(self.ignite.get_alive_default_nodes()))
self.cu.set_current_topology_as_baseline()
util_sleep(5)
self.start_node(2)
self.ignite.wait_for_topology_snapshot(
server_num=len(self.ignite.get_alive_default_nodes()))
self.cu.set_current_topology_as_baseline()
self.ignite.jmx.wait_for_finish_rebalance(
self.rebalance_timeout, self.group_names)
tiden_assert_equal(
0, self.ignite.find_exception_in_logs('java.lang.AssertionError'),
"# of AssertionError")
def test_round_restart(self):
with PiClient(self.ignite, self.get_client_config()):
for node_id in range(1, 5):
util_sleep(10)
log_print(f"Restarting node {node_id}", color='blue')
self.ignite.kill_node(node_id)
self.load_data_with_streamer(
end_key=1000,
value_type=ModelTypes.VALUE_ALL_TYPES_INDEXED.value,
allow_overwrite=True)
util_sleep(10)
self.start_node(node_id)
self.ignite.jmx.wait_for_finish_rebalance(
self.rebalance_timeout, self.group_names)
tiden_assert_equal(
0, self.ignite.find_exception_in_logs('java.lang.AssertionError'),
"# of AssertionError")
def idle_verify_action(self, sum_after):
idle_verify_pass = [
'idle_verify check has finished, no conflicts have been found.'
]
self.cu.control_utility('--cache',
'idle_verify',
all_required=idle_verify_pass)
self.verify_no_assertion_errors()
validate_indexes_pass = ['no issues found.']
try:
self.cu.control_utility('--cache',
'validate_indexes',
all_required=validate_indexes_pass)
except TidenException as e:
log_print('Got exception during index validation: {}'.format(e))
if 'Cluster not idle' in self.cu.latest_utility_output:
util_sleep(5)
self.cu.control_utility('--cache',
'validate_indexes',
all_required=validate_indexes_pass)
else:
raise e
def test_indexes_rebuilded(self):
"""
https://ggsystems.atlassian.net/browse/GG-17428
1. Start cluster.
2. Start transactional loading.
3. Stop one node and remove index.bin files for the caches.
4. Start node and let it finish rebalance.
5. Check indexes are not broken after rebalance.
:return:
"""
self.need_delete_lfs_on_teardown = True
debug = False
with PiClient(self.ignite, self.get_client_config()) as piclient:
self.wait_for_running_clients_num(piclient.nodes_num, 90)
with ExitStack() as stack:
# todo unreachable code
if False:
stack.enter_context(
TransactionalLoading(
self,
cross_cache_batch=2,
skip_atomic=True,
post_checksum_action=self.idle_verify_action))
if is_enabled(self.config.get('zookeeper_enabled')) and \
is_enabled(self.config.get('zookeeper_nodes_restart')):
stack.enter_context(ZkNodesRestart(self.zoo, 2))
for iteration in range(0, self.iterations):
log_print("Iteration {}/{}".format(str(iteration + 1),
str(self.iterations)),
color='blue')
self.assert_nodes_alive()
with TransactionalLoading(self,
cross_cache_batch=2,
skip_atomic=True):
util_sleep(20)
self.ignite.kill_node(2)
if debug:
self.cu.control_utility(
'--cache idle_verify --dump --skip-zeros')
self.remove_index_bin_files(2)
util_sleep(10)
if debug:
self.cu.control_utility(
'--cache idle_verify --dump --skip-zeros')
self.start_node(2)
self.ignite.jmx.wait_for_finish_rebalance(
self.rebalance_timeout, self.group_names)
util_sleep(30)
log_print("Check indexes")
try:
if debug:
self.cu.control_utility(
'--cache idle_verify --dump --skip-zeros')
self.idle_verify_action(None)
except TidenException:
if debug:
self.cu.control_utility(
'--cache idle_verify --dump --skip-zeros')
raise TidenException('validate_index failed')
tiden_assert_equal(
0, self.ignite.find_exception_in_logs('java.lang.AssertionError'),
"# of AssertionError")
