def test_two_blinking_nodes_clean_lfs(self):
with PiClient(self.ignite, self.get_client_config()):
with ExitStack() as stack:
if is_enabled(self.config.get('zookeeper_enabled')) and \
is_enabled(self.config.get('zookeeper_nodes_restart')):
stack.enter_context(ZkNodesRestart(self.zoo, 2))
for iteration in range(0, self.iterations):
log_print("Iteration {}/{}".format(str(iteration + 1),
str(self.iterations)),
color='blue')
self.assert_nodes_alive()
self.ignite.kill_node(2)
self.cleanup_lfs(2)
# self.ignite.start_node(2)
self.start_node(2)
self.ignite.kill_node(3)
util_sleep(10)
# self.ignite.start_node(3)
self.start_node(3)
self.ignite.jmx.wait_for_finish_rebalance(
self.rebalance_timeout, self.group_names)
tiden_assert_equal(
0, self.ignite.find_exception_in_logs('java.lang.AssertionError'),
"# of AssertionError")
def test_loading_blinking_node_baseline(self):
with PiClient(self.ignite, self.get_client_config()) as piclient:
self.wait_for_running_clients_num(piclient.nodes_num, 90)
with ExitStack() as stack:
stack.enter_context(
TransactionalLoading(
self,
cross_cache_batch=2,
skip_atomic=True,
post_checksum_action=self.idle_verify_action))
if is_enabled(self.config.get('zookeeper_enabled')) and \
is_enabled(self.config.get('zookeeper_nodes_restart')):
stack.enter_context(ZkNodesRestart(self.zoo, 2))
for iteration in range(0, self.iterations):
log_print("Iteration {}/{}".format(str(iteration + 1),
str(self.iterations)),
color='blue')
self.assert_nodes_alive()
self.ignite.kill_node(2)
self.ignite.wait_for_topology_snapshot(
server_num=len(self.ignite.get_alive_default_nodes()))
self.cu.set_current_topology_as_baseline()
util_sleep(5)
self.start_node(2)
self.ignite.wait_for_topology_snapshot(
server_num=len(self.ignite.get_alive_default_nodes()))
self.cu.set_current_topology_as_baseline()
self.ignite.jmx.wait_for_finish_rebalance(
self.rebalance_timeout, self.group_names)
tiden_assert_equal(
0, self.ignite.find_exception_in_logs('java.lang.AssertionError'),
"# of AssertionError")
def test_loading_blinking_two_nodes_blt_and_extra_node(self):
with PiClient(self.ignite, self.get_client_config()):
additional_node = self.ignite.add_additional_nodes(
self.get_server_config())[0]
self.ignite.start_additional_nodes(additional_node)
with ExitStack() as stack:
stack.enter_context(
TransactionalLoading(
self,
cross_cache_batch=2,
skip_atomic=True,
post_checksum_action=self.idle_verify_action))
if is_enabled(self.config.get('zookeeper_enabled')) and \
is_enabled(self.config.get('zookeeper_nodes_restart')):
stack.enter_context(ZkNodesRestart(self.zoo, 2))
for iteration in range(0, self.iterations):
log_print("Iteration {}/{}".format(str(iteration + 1),
str(self.iterations)),
color='blue')
self.assert_nodes_alive()
self.ignite.kill_node(2)
self.ignite.kill_node(additional_node)
# self.ignite.start_node(2)
# self.ignite.start_additional_nodes(additional_node)
self.start_node(2)
self.start_node(additional_node)
self.ignite.jmx.wait_for_finish_rebalance(
self.rebalance_timeout, self.group_names)
tiden_assert_equal(
0, self.ignite.find_exception_in_logs('java.lang.AssertionError'),
"# of AssertionError")
def test_cycling_restart_grid_dynamic_caches_no_client(self):
"""
Scenario The Glue
(Assertions should be enabled)
1. Start grid, load some data
2. In the loop:
2.1 define node restart timeout (0.5 - 2.0 seconds)
2.2 Load more data
2.3 Restart each node with defined timeout (DOES NOT LOOK ON TOPOLOGY SNAPSHOT)
2.4 Try to activate, check AssertionErrors
2.5 Try to baseline (If 2 operation failed -> PME, kill all nodes, start new test iteration)
2.6 Try to load data
2.7 Try to calculate checksum
:return:
"""
import random
PiClient.read_timeout = 240
# sleep_for_time = float(random.randrange(1, 15, 1)) / 5
self.set_current_context('in_memory')
self.util_copy_piclient_model_to_libs()
self.ignite.set_activation_timeout(240)
self.ignite.set_snapshot_timeout(240)
self.ignite.set_node_option('*', 'jvm_options', ['-ea'])
self.su.clear_snapshots_list()
self.start_grid(skip_activation=True)
self.start_dynamic_caches_with_node_filter()
last_loaded_key = 1000
PiClientIgniteUtils.load_data_with_streamer(self.ignite,
self.get_client_config(),
end_key=last_loaded_key,
jvm_options=['-ea'])
nodes_before = self.ignite.get_alive_default_nodes()
iterations = 50
last_loaded_key += 1
for i in range(0, iterations):
with ExitStack() as stack:
# load data before start zk restart thread
self.start_dynamic_caches_with_node_filter()
# PiClientIgniteUtils.wait_for_running_clients_num(self.ignite, 0, 120)
PiClientIgniteUtils.load_data_with_streamer(
self.ignite,
self.get_client_config(),
start_key=last_loaded_key,
end_key=last_loaded_key + 500,
jvm_options=['-ea'],
check_clients=True)
last_loaded_key += 500
if self.get_context_variable('zookeeper_enabled') and \
is_enabled(self.config.get('zookeeper_nodes_restart')):
stack.enter_context(ZkNodesRestart(self.zoo, 3))
log_print('Current iteration %s from %s' % (i, iterations),
color='debug')
sleep_for_time = float(
self.the_glue_timeout) if self.the_glue_timeout else round(
random.uniform(0.5, 2.5), 1)
log_print(
"In this run we are going to sleep for {} seconds after each node restart"
.format(sleep_for_time),
color='green')
log_print('Trying to load data into created/existing caches',
color='yellow')
log_print("Round restart")
for node_id in self.ignite.get_alive_default_nodes():
self.ignite.kill_node(node_id)
self.ignite.start_node(node_id, skip_topology_check=True)
sleep(sleep_for_time)
log_print("Wait for topology messages")
for node_id in self.ignite.get_all_default_nodes():
self.ignite.update_started_node_status(node_id)
sleep(15)
last_loaded_key = self.verify_cluster(nodes_before,
last_loaded_key)
def test_indexes_rebuilded(self):
"""
https://ggsystems.atlassian.net/browse/GG-17428
1. Start cluster.
2. Start transactional loading.
3. Stop one node and remove index.bin files for the caches.
4. Start node and let it finish rebalance.
5. Check indexes are not broken after rebalance.
:return:
"""
self.need_delete_lfs_on_teardown = True
debug = False
with PiClient(self.ignite, self.get_client_config()) as piclient:
self.wait_for_running_clients_num(piclient.nodes_num, 90)
with ExitStack() as stack:
# todo unreachable code
if False:
stack.enter_context(
TransactionalLoading(
self,
cross_cache_batch=2,
skip_atomic=True,
post_checksum_action=self.idle_verify_action))
if is_enabled(self.config.get('zookeeper_enabled')) and \
is_enabled(self.config.get('zookeeper_nodes_restart')):
stack.enter_context(ZkNodesRestart(self.zoo, 2))
for iteration in range(0, self.iterations):
log_print("Iteration {}/{}".format(str(iteration + 1),
str(self.iterations)),
color='blue')
self.assert_nodes_alive()
with TransactionalLoading(self,
cross_cache_batch=2,
skip_atomic=True):
util_sleep(20)
self.ignite.kill_node(2)
if debug:
self.cu.control_utility(
'--cache idle_verify --dump --skip-zeros')
self.remove_index_bin_files(2)
util_sleep(10)
if debug:
self.cu.control_utility(
'--cache idle_verify --dump --skip-zeros')
self.start_node(2)
self.ignite.jmx.wait_for_finish_rebalance(
self.rebalance_timeout, self.group_names)
util_sleep(30)
log_print("Check indexes")
try:
if debug:
self.cu.control_utility(
'--cache idle_verify --dump --skip-zeros')
self.idle_verify_action(None)
except TidenException:
if debug:
self.cu.control_utility(
'--cache idle_verify --dump --skip-zeros')
raise TidenException('validate_index failed')
tiden_assert_equal(
0, self.ignite.find_exception_in_logs('java.lang.AssertionError'),
"# of AssertionError")