def setup_testcase(self): if self.lfs_stored: self.restore_lfs('snapshot_util', timeout=1200) self.setup_testcase_without_start_gid() activation_timeout = 60 if self.get_context_variable('sbt_model_enabled'): activation_timeout = 200 self.start_grid(timeout=activation_timeout, activate_on_particular_node=1) if not self.lfs_stored: if self.get_context_variable('dynamic_cache_enabled'): self.start_caches_dynamic( caches_file_name=self.get_context_variable('caches'), batch_size=10000) if self.get_context_variable('sbt_model_enabled'): PiClientIgniteUtils.load_data_with_txput_sbt_model( self.config, self.ignite, self.get_client_config(), only_caches_batch=None, end_key=int(self.max_key * self.load_multiplier)) else: PiClientIgniteUtils.load_data_with_streamer( self.ignite, self.get_client_config(), end_key=int(self.max_key * self.load_multiplier), allow_overwrite=True) log_print(repr(self.ignite), color='debug')
def test_sim(self): version, ignite = self.start_ignite_grid(True) ignite.jmx.start_utility() client_config = Ignite.config_builder.get_config( 'client', config_set_name='base') group_names = PiClientIgniteUtils.collect_cache_group_names( ignite, client_config) PiClientIgniteUtils.load_data_with_streamer(ignite, client_config, end_key=50) server_nodes_num = ignite.get_nodes_num('server') sim_engine = PigeonSimulation(server_nodes_num) for running_iteration in range(1, DEFAULT_ITERATIONS + 1): log_print("Running iteration %s" % running_iteration) ev, node = sim_engine.next_event() log_print("Evaluating event %s on node %s" % (ev, node)) pigeon = self.get_pigeon(ignite, node) pigeon[ev]() ignite.jmx.wait_for_finish_rebalance(120, group_names) self.verify_cluster(ignite) ignite.jmx.kill_utility()
def setup_testcase(self): self.logger.info('TestSetup is called') if self.get_context_variable('zookeeper_enabled'): self.start_zookeeper() self.ignite.set_activation_timeout(240) self.ignite.set_snapshot_timeout(240) self.su.clear_snapshots_list() self.start_grid(activate_on_particular_node=1) PiClientIgniteUtils.load_data_with_streamer(self.ignite, self.get_client_config(), end_key=1000) if self.get_context_variable( 'pitr_enabled') and self.get_context_variable( 'snapshots_enabled'): self.su.wait_no_snapshots_activity_in_cluster() self.group_names = PiClientIgniteUtils.collect_cache_group_names( self.ignite, self.get_client_config()) # if not self.ignite.jmx.is_started(): # self.ignite.jmx.start_utility() # wait for no client on cluster self.ignite.wait_for_topology_snapshot(client_num=0) print_debug(repr(self.ignite))
def setup_shared_storage_test(self): self.success_run = False log_print('create transfer_folder') if self.use_local_shared_directory: self.dr_storage = self.local_shared_dir('transfer_folder_{}'.format(self.config['rt']['test_method']), create=True) log_print('set transfer_folder path - {}'.format(self.dr_storage)) log_print('start clusters in parallel') futures = [] with ThreadPoolExecutor(max_workers=2) as executor: futures.append(executor.submit(self.start_ignite_grid, 'master')) futures.append(executor.submit(self.start_ignite_grid, 'replica')) self.ignite_master_app = futures[0].result() self.ignite_replica_app = futures[1].result() additional_node_id = self.ignite_master_app.add_additional_nodes( config=Ignite.config_builder.get_config('server', config_set_name='master'), name='base-ignite-2.8.1.b8') self.ignite_master_app.set_node_option(additional_node_id, 'jvm_options', self.get_dr_jvm_options(role='master') ) self.ignite_master_app.start_additional_nodes(additional_node_id) PiClientIgniteUtils.load_data_with_txput_sbt_model(only_caches_batch=None, start_key=0, end_key=100, jvm_options=self.get_dr_jvm_options('master'), config=self.get_client_config('master'), tiden_config=self.ignite_master_app.config, ignite=self.ignite_master_app) print_green('Preload done')
def base_test(self, **kwargs): cache_to_test = 'cache_group_1_001' check_commands = [ '!tables', '!index', '\'select count(*) from \"%s\".ALLTYPESINDEXED;\'' % cache_to_test ] expected = ['COUNT\(\*\)', '1000'] if 'ssl_connection' in kwargs: self.set_current_context('ssl_enabled') self.start_grid(activate_on_particular_node=1) PiClientIgniteUtils.load_data_with_streamer( self.ignite, self.get_client_config('ssl_enabled'), value_type=ModelTypes.VALUE_ALL_TYPES_INDEXED.value, end_key=1000, allow_overwrite=True) sql_tool = Sqlline(self.ignite, **kwargs) output = sql_tool.run_sqlline(check_commands) self.su.check_content_all_required(output, expected) # base on GG-17465 (validate index with secure cluster) self.cu.control_utility('--cache', 'validate_indexes')
def start_piclients(): for _ in range(0, 3): try: PiClientIgniteUtils.load_data_with_putall( self.ignite, self.get_client_config(), value_type=ModelTypes.VALUE_ACCOUNT.value, nodes_num=24, end_key=1000) except Exception as err: print(err)
def calculate_checksums(self, idle_verify=True): if idle_verify: self.diff_idle_verify_dump() checksum_for_master = PiClientIgniteUtils.calc_checksums_distributed(self.ignite_master_app, config=self.get_client_config('master'), jvm_options=self.get_dr_jvm_options( 'master')) checksum_for_replica = PiClientIgniteUtils.calc_checksums_distributed(self.ignite_master_app, config=self.get_client_config('replica'), jvm_options=self.get_dr_jvm_options( 'replica')) if idle_verify: self.diff_idle_verify_dump(raise_flag=True) return checksum_for_master, checksum_for_replica
def test_master_master_master_blinking_blt(self): self.prepare_clusters() client_config = self.preconfigure_cluster_0() iterations = 10 last_loaded_key = START_DATA_SIZE nodes_before = 6 with PiClient(self.clusters[0].grid, client_config, jvm_options=['-ea']) as piclient: PiClientIgniteUtils.load_data_with_streamer( self.clusters[0].grid, client_config, end_key=last_loaded_key, jvm_options=['-ea'], check_clients=False) sleep(60) with TransactionalLoading(self, ignite=self.clusters[0].grid, config_file=client_config, skip_consistency_check=True): for i in range(0, iterations): log_print(f'Current iteration {i + 1} from {iterations}', color='debug') self.clusters[0].grid.kill_node(2) utility_baseline_log = 'control-utility-baseline.log' self.clusters[0].grid.cu.set_current_topology_as_baseline( background=True, log=utility_baseline_log) self.clusters[0].grid.start_node(2, skip_topology_check=True) self.clusters[0].grid.wait_for_topology_snapshot( server_num=6) self.clusters[0].grid.update_started_node_status(2) self.clusters[0].grid.cu.set_current_topology_as_baseline( background=True, log=utility_baseline_log) self.verify_cluster(0, nodes_before, last_loaded_key)
def _setup_with_context(self, context_name, **kwargs): self._start_grid_no_preload(context_name, **kwargs) if self.preloading_size > 0: if self.preloading_with_streamer: PiClientIgniteUtils.load_data_with_streamer( self.ignite, self.get_client_config(), end_key=self.preloading_size) else: PiClientIgniteUtils.load_data_with_putall( self.ignite, self.get_client_config(), end_key=self.preloading_size) self._wait_cluster_ready()
def test_rebalancing_with_ttl_caches(self): """ IGN-13549 (IGNITE-11400) Rebalancing caches with TTL enabled can cause data corruption. :return: """ with PiClient(self.ignite, self.get_client_config()): checksums = create_distributed_checksum_operation().evaluate() self.wait_for_running_clients_num(client_num=0, timeout=120) self.ignite.cu.control_utility( '--cache idle_verify --dump --skip-zeros') log_print('Calculating checksums done: %s' % checksums) self.assert_nodes_alive() self.ignite.kill_node(2) PiClientIgniteUtils.load_data_with_putall( self.ignite, self.get_client_config(), start_key=self.preloading_size, end_key=self.preloading_size + 100000) util_sleep(5 * 60) self.ignite.start_node(2) self.ignite.jmx.wait_for_finish_rebalance(self.rebalance_timeout * 2, self.group_names) with PiClient(self.ignite, self.get_client_config()): # , jvm_options=jvm_options): checksums = create_distributed_checksum_operation().evaluate() self.wait_for_running_clients_num(client_num=0, timeout=120) log_print('Calculating checksums done: %s' % checksums) tiden_assert_equal( 0, self.ignite.find_exception_in_logs('java.lang.AssertionError'), "# of AssertionError")
def common_load(self, start_key, end_key): if self.get_context_variable('sbt_model_enabled'): # add transactional put into 1 cache 2k times total PiClientIgniteUtils.load_data_with_txput_sbt_model( self.config, self.ignite, self.get_client_config(), only_caches_batch=1, start_key=start_key, end_key=start_key + 1) # load normal data into 200 caches 2k times total PiClientIgniteUtils.load_data_with_txput_sbt_model( self.config, self.ignite, self.get_client_config(), only_caches_batch=200, start_key=start_key + 2, end_key=int(end_key * self.load_multiplier)) else: PiClientIgniteUtils.load_data_with_putall( self.ignite, self.get_client_config(), start_key=start_key, end_key=int(end_key * self.load_multiplier))
def _wait_cluster_ready(self): if self.get_context_variable( 'pitr_enabled') and self.get_context_variable( 'snapshots_enabled'): self.su.wait_no_snapshots_activity_in_cluster() self.group_names = PiClientIgniteUtils.collect_cache_group_names( self.ignite, self.get_client_config()) if not self.ignite.jmx.is_started(): self.ignite.jmx.start_utility() # wait for no client on cluster self.ignite.wait_for_topology_snapshot(client_num=0) if is_enabled(self.config.get('disable_baseline_autoadjustment')): log_print("Going to disable baseline autoadjustment", color='green') if self.cu.is_baseline_autoajustment_supported(): self.cu.disable_baseline_autoajustment() log_print("Baseline autoadjustment disabled", color='green') log_print(repr(self.ignite), color='debug')
def load_data_with_streamer(self, *args, **kwargs): PiClientIgniteUtils.load_data_with_streamer(*args, **kwargs)
def test_cycling_restart_grid_dynamic_caches_no_client(self): """ Scenario The Glue (Assertions should be enabled) 1. Start grid, load some data 2. In the loop: 2.1 define node restart timeout (0.5 - 2.0 seconds) 2.2 Load more data 2.3 Restart each node with defined timeout (DOES NOT LOOK ON TOPOLOGY SNAPSHOT) 2.4 Try to activate, check AssertionErrors 2.5 Try to baseline (If 2 operation failed -> PME, kill all nodes, start new test iteration) 2.6 Try to load data 2.7 Try to calculate checksum :return: """ import random PiClient.read_timeout = 240 # sleep_for_time = float(random.randrange(1, 15, 1)) / 5 self.set_current_context('in_memory') self.util_copy_piclient_model_to_libs() self.ignite.set_activation_timeout(240) self.ignite.set_snapshot_timeout(240) self.ignite.set_node_option('*', 'jvm_options', ['-ea']) self.su.clear_snapshots_list() self.start_grid(skip_activation=True) self.start_dynamic_caches_with_node_filter() last_loaded_key = 1000 PiClientIgniteUtils.load_data_with_streamer(self.ignite, self.get_client_config(), end_key=last_loaded_key, jvm_options=['-ea']) nodes_before = self.ignite.get_alive_default_nodes() iterations = 50 last_loaded_key += 1 for i in range(0, iterations): with ExitStack() as stack: # load data before start zk restart thread self.start_dynamic_caches_with_node_filter() # PiClientIgniteUtils.wait_for_running_clients_num(self.ignite, 0, 120) PiClientIgniteUtils.load_data_with_streamer( self.ignite, self.get_client_config(), start_key=last_loaded_key, end_key=last_loaded_key + 500, jvm_options=['-ea'], check_clients=True) last_loaded_key += 500 if self.get_context_variable('zookeeper_enabled') and \ is_enabled(self.config.get('zookeeper_nodes_restart')): stack.enter_context(ZkNodesRestart(self.zoo, 3)) log_print('Current iteration %s from %s' % (i, iterations), color='debug') sleep_for_time = float( self.the_glue_timeout) if self.the_glue_timeout else round( random.uniform(0.5, 2.5), 1) log_print( "In this run we are going to sleep for {} seconds after each node restart" .format(sleep_for_time), color='green') log_print('Trying to load data into created/existing caches', color='yellow') log_print("Round restart") for node_id in self.ignite.get_alive_default_nodes(): self.ignite.kill_node(node_id) self.ignite.start_node(node_id, skip_topology_check=True) sleep(sleep_for_time) log_print("Wait for topology messages") for node_id in self.ignite.get_all_default_nodes(): self.ignite.update_started_node_status(node_id) sleep(15) last_loaded_key = self.verify_cluster(nodes_before, last_loaded_key)
def test_nodes_connecting_to_dead_cluster(self): """ https://ggsystems.atlassian.net/browse/IGN-13800 Two nodes are trying to connect to cluster, meanwhile cluster killed. They should send join request but didn't get self NodeAdded """ PiClient.read_timeout = 240 # sleep_for_time = float(random.randrange(1, 15, 1)) / 5 self.set_current_context('in_memory') self.util_copy_piclient_model_to_libs() self.ignite.set_activation_timeout(240) self.ignite.set_snapshot_timeout(240) self.ignite.set_node_option('*', 'jvm_options', ['-ea']) self.su.clear_snapshots_list() self.start_grid(skip_activation=True) last_loaded_key = 1000 PiClientIgniteUtils.load_data_with_streamer(self.ignite, self.get_client_config(), end_key=last_loaded_key) nodes_before = self.ignite.get_alive_default_nodes() additional_nodes = self.ignite.add_additional_nodes( config=self.get_client_config(), num_nodes=2) def start_piclients(): try: self.ignite.start_additional_nodes(additional_nodes, client_nodes=True, skip_topology_check=True) except Exception as err: print(err) finally: sleep(10) self.ignite.update_started_node_status(additional_nodes) for add_node in additional_nodes: self.ignite.kill_node(add_node) log_print("Starting clients under load", color="green") executor = ThreadPoolExecutor() executor.submit(start_piclients) try: time_to_sleep = round(random.uniform(3.5, 4.9), 1) sleep(time_to_sleep) log_print("Time to sleep: %s" % time_to_sleep, color='green') self.ignite.kill_nodes() sleep(30) self.verify_cluster(nodes_before, 0) except Exception as e: raise e finally: executor.shutdown(wait=True) self.ssh.killall('java')
def test_clients_killed_few_coordinators(self): """ 1. Start grid, load some data 2. Repeat: 2.1. Start clients thread with loading (put all operation) 2.2. Kill first node (coordinator) + second node (possible coordinator) 2.3. Kill 4 next coordinators with some small timeout 2.4. Sleep for 2 minutes to make cluster process failure 2.5. Launch verify procedure 2.6. Stop clients put thread :return: """ self.set_current_context('default') self.util_copy_piclient_model_to_libs() self.util_deploy_sbt_model() self.ignite.set_activation_timeout(240) self.ignite.set_snapshot_timeout(240) self.ignite.set_node_option('*', 'jvm_options', ['-ea']) self.su.clear_snapshots_list() self.start_grid() sleep_for_time = float( self.the_glue_timeout) if self.the_glue_timeout else round( random.uniform(0.1, 2.9), 1) PiClientIgniteUtils.load_data_with_streamer( self.ignite, self.get_client_config(), value_type=ModelTypes.VALUE_ACCOUNT.value, end_key=1000) nodes_before = self.ignite.get_alive_default_nodes() def start_piclients(): for _ in range(0, 3): try: PiClientIgniteUtils.load_data_with_putall( self.ignite, self.get_client_config(), value_type=ModelTypes.VALUE_ACCOUNT.value, nodes_num=24, end_key=1000) except Exception as err: print(err) with PiClient(self.ignite, self.get_client_config()) as load: log_print("Starting clients under load", color="green") executor = ThreadPoolExecutor() executor.submit(start_piclients) sleep(5) try: self.ignite.kill_node(1) for i in range(0, 4): sleep(sleep_for_time) i = int(load.get_ignite().cluster().forOldest().node(). consistentId().replace('node_1_', '')) if self.ignite.nodes[i]['status'] in [ NodeStatus.KILLED, NodeStatus.KILLING ]: sleep(sleep_for_time) i = int(load.get_ignite().cluster().forOldest().node(). consistentId().replace('node_1_', '')) self.ignite.kill_node(i) log_print("Killing node %s" % i) except Exception as e: print(e) sleep(120) self.verify_cluster(nodes_before, 0) executor.shutdown(wait=True)
def test_massive_index_rebuild(self): """ 1) 2 nodes, backupCnt = 1, persistenceEnabled 2) Load (A, B) type into a cache with defined (A, B) types in index config 3) Load new type of data into a cache (C,D) 4) Kill one node 5) Create new index in alive cluster 6) Start node again :return: """ PiClient.read_timeout = 1200 self.set_current_context('indexed_types') self.util_copy_piclient_model_to_libs() self.ignite.set_activation_timeout(240) self.ignite.set_snapshot_timeout(240) self.ignite.set_node_option('*', 'jvm_options', ['-ea']) self.su.clear_snapshots_list() self.start_grid(skip_activation=True) self.ignite.cu.activate(activate_on_particular_node=1) PiClientIgniteUtils.load_data_with_streamer( self.ignite, self.get_client_config(), value_type=ModelTypes.VALUE_ALL_TYPES_30_INDEX.value, end_key=5000) PiClientIgniteUtils.load_data_with_streamer( self.ignite, self.get_client_config(), value_type=ModelTypes.VALUE_ACCOUNT.value, start_key=5000, end_key=10000) PiClientIgniteUtils.load_data_with_streamer( self.ignite, self.get_client_config(), value_type=ModelTypes.VALUE_EXT_ALL_TYPES_30_INDEX.value, start_key=10000, end_key=15000) # PiClientIgniteUtils.load_data_with_streamer(self.ignite, # self.get_client_config(), # cache_names_patterns= # ['cache_group_3'], # value_type=ModelTypes.VALUE_EXT_ALL_TYPES_30_INDEX.value, # end_key=10000) iterations = 50 sqlline = Sqlline(self.ignite) columns = [ 'longCol', 'doubleCol', 'stringCol', 'booleanCol', 'longCol1', # 'doubleCol1', 'stringCol1', 'intCol', 'intCol1', # 'booleanCol1', # 'index', 'longCol2', 'doubleCol2', 'stringCol2', 'booleanCol2', # 'longCol12', 'doubleCol12', 'stringCol12', 'intCol12', 'intCol2', # 'shortCol2', 'longCol3', 'doubleCol3', 'stringCol3', 'booleanCol3', # 'longCol13', 'doubleCol13', 'stringCol13', 'intCol13', 'intCol3', 'shortCol3' ] with PiClient(self.ignite, self.get_client_config()) as piclient: cache_names = piclient.get_ignite().cacheNames().toArray() for i in range(0, iterations): log_print('Current iteration %s from %s' % (i, iterations), color='debug') update_table = [] self.ignite.kill_node(2) indexed_columns = ','.join(columns) for cache_name in cache_names: # self.ssh.exec_on_host('REMOVE') vtype = 'ALLTYPES30INDEX' # if 'cache_group_3' not in cache_name else 'EXTALLTYPES30INDEX' update_table.append( f'\'CREATE INDEX IF NOT EXISTS {cache_name}_{vtype} on ' f'\"{cache_name}\".{vtype}({indexed_columns}) INLINE_SIZE 32 PARALLEL 28;\'' ) update_table.append('!index') sqlline.run_sqlline(update_table) self.ignite.start_node(2) util_sleep_for_a_while(30) self.verify_no_assertion_errors() self.cu.control_utility('--cache validate_indexes', all_required='no issues found.')
def test_full_cluster_blinking(self): """ Enable indexes Start servers with PDS, start clients, start some light tx loading. In loop try to blink with all cluster at the same time. Logically there should be no data loss: full cluster blinking - so there shouldn't be any data loss :return: """ PiClient.read_timeout = 240 self.set_current_context('indexed_types') self.util_copy_piclient_model_to_libs() self.ignite.set_activation_timeout(240) self.ignite.set_snapshot_timeout(240) self.ignite.set_node_option('*', 'jvm_options', ['-ea']) self.su.clear_snapshots_list() self.start_grid(skip_activation=True) self.ignite.cu.activate(activate_on_particular_node=1) PiClientIgniteUtils.load_data_with_streamer(self.ignite, self.get_client_config(), end_key=100000) nodes_before = self.ignite.get_alive_default_nodes() iterations = 50 with PiClient(self.ignite, self.get_client_config()): with TransactionalLoading( self, loading_profile=LoadingProfile(delay=1000)): for i in range(0, iterations): log_print('Current iteration %s from %s' % (i, iterations), color='debug') for node_id in nodes_before: self.ignite.kill_node(node_id) sleep( float(self.the_glue_timeout) if self. the_glue_timeout else round( random.uniform(0.1, 0.5), 1)) for node_id in nodes_before: self.ignite.start_node(node_id, skip_topology_check=True) sleep( float(self.the_glue_timeout) if self. the_glue_timeout else round( random.uniform(0.1, 0.5), 1)) self.ignite.wait_for_topology_snapshot( server_num=len(nodes_before)) for node_id in self.ignite.get_all_default_nodes(): self.ignite.update_started_node_status(node_id) sleep(10) self.cu.control_utility('--cache validate_indexes', all_required='no issues found.') self.verify_cluster(nodes_before, 0)
def do_blink_backups_under_load(self, initial_remove_probability): iteration_size = self.config.get('iteration_size', 80000) start = 0 keep_coordinator_busy = True start_value = 0 # temporary save LFS even on test pass self.need_delete_lfs_on_teardown = False first_node = self.ignite.get_node_consistent_id(1) second_node = self.ignite.get_node_consistent_id(2) if keep_coordinator_busy: other_nodes = list( set(self.ignite.get_all_default_nodes()) - set([1])) else: other_nodes = list( set(self.ignite.get_all_default_nodes()) - set([1, 2])) current_server_num = self.ignite.get_nodes_num('server') tx_caches = [] atomic_caches = [] self.ignite.set_snapshot_timeout(600) with PiClient(self.ignite, self.get_client_config(), nodes_num=1) as piclient: gateway = piclient.get_gateway() ignite = piclient.get_ignite() cache_names = ignite.cacheNames().toArray() for cache_name in cache_names: # run cross cache transfer task only for transactional caches if ignite.getOrCreateCache(cache_name).getConfiguration( gateway.jvm.org.apache.ignite.configuration. CacheConfiguration().getClass()).getAtomicityMode( ).toString() == 'TRANSACTIONAL': tx_caches.append(cache_name) else: atomic_caches.append(cache_name) PiClientIgniteUtils.wait_for_running_clients_num(self.ignite, 0, 120) for iteration in range(0, self.iterations): log_print("Iteration {}/{}".format(str(iteration + 1), str(self.iterations)), color='blue') start_key = start + iteration * iteration_size end_key = start_key + iteration_size if initial_remove_probability > 0.0: remove_probability = initial_remove_probability + iteration / self.iterations / 2.0 else: remove_probability = 0.0 current_client_num = self.ignite.get_nodes_num('client') for i in range(0, 3): with PiClient(self.ignite, self.get_client_config()) as piclient: log_print( "Loading (remove {probability}%) {load} values per cache into {n_caches} caches" .format( probability=remove_probability, load=iteration_size, n_caches=len(tx_caches), )) async_operations = [] for cache_name in tx_caches: node_id = piclient.get_node_id() gateway = piclient.get_gateway(node_id) tx_size = randint(1, 10) log_print( "Client {node_id} -> {cache_name}, tx size {tx_size}" .format( node_id=node_id, cache_name=cache_name, tx_size=tx_size, removeProbability=remove_probability, )) async_operation = create_async_operation( create_put_with_optional_remove_operation, cache_name, start_key, end_key, remove_probability, gateway=gateway, node_consistent_id=first_node if keep_coordinator_busy else second_node, tx_description=TxDescriptor( concurrency='PESSIMISTIC', isolation='REPEATABLE_READ', size=tx_size), use_monotonic_value=True, monotonic_value_seed=start_value, ) start_value = start_value + iteration_size async_operations.append(async_operation) async_operation.evaluate() # little warm up util_sleep(5) node_id = self.ignite.get_random_server_nodes( 1, node_ids=other_nodes)[0] self.ignite.kill_node(node_id) self.ignite.wait_for_topology_snapshot( server_num=current_server_num - 1) # continue load data during node offline util_sleep(15) self.ignite.start_node(node_id) self.ignite.wait_for_topology_snapshot( server_num=current_server_num) PiClientIgniteUtils.wait_for_running_clients_num( self.ignite, current_client_num, 120) self.wait_transactions_finish() self.ignite.jmx.wait_for_finish_rebalance( self.rebalance_timeout, self.group_names) self.idle_verify_check_conflicts_action() self.idle_verify_dump_action()
def base_test_with_all_users(self, ssl_connection): cache_to_test = 'cache_group_1_001' check_commands_read = [ '!tables', '!index', '\'select count(*) from \"%s\".ALLTYPESINDEXED;\'' % cache_to_test ] check_commands_update = [ '!tables', '!index', '\'update \"%s\".ALLTYPESINDEXED set LONGCOL=1;\'' % cache_to_test ] expected_read = ['COUNT\(\*\)', '1000'] expected_update = ['1,000 rows affected'] expected_for_no_access_user = ['Authorization failed'] self.set_current_context('ssl_enabled') self.start_grid(activate_on_particular_node=1) PiClientIgniteUtils.load_data_with_streamer( self.ignite, self.get_client_config('ssl_enabled'), value_type=ModelTypes.VALUE_ALL_TYPES_INDEXED.value, end_key=1000, allow_overwrite=True) users = [ { 'login': '******', 'password': '******', 'read_check': { 'run': check_commands_read, 'expected': expected_read }, 'update_check': { 'run': check_commands_update, 'expected': expected_update } }, { 'login': '******', 'password': '******', 'read_check': { 'run': check_commands_read, 'expected': expected_read }, 'update_check': { 'run': check_commands_update, 'expected': expected_update } }, { 'login': '******', 'password': '******', 'read_check': { 'run': check_commands_read, 'expected': expected_read }, 'update_check': { 'run': check_commands_update, 'expected': expected_for_no_access_user } }, { 'login': '******', 'password': '******', 'read_check': { 'run': check_commands_read, 'expected': expected_for_no_access_user }, 'update_check': { 'run': check_commands_update, 'expected': expected_for_no_access_user } }, ] def check_output(user_info): auth_info = namedtuple('auth_info', 'user password') auth = auth_info(user=user['login'], password=user['password']) sql_tool = Sqlline(self.ignite, auth=auth, ssl_connection=ssl_connection) for operation in ['read_check', 'update_check']: output = sql_tool.run_sqlline(user_info[operation].get('run')) self.su.check_content_all_required( output, user_info[operation].get('expected')) for user in reversed(users): check_output(user) for user in users: check_output(user)
def test_during_rebalance(self): ignite = self.start_ignite_grid(self.ignite_name) ignite.cu.activate(activate_on_particular_node=1) PiClientIgniteUtils.load_data_with_putall(ignite, self.client_config, ) util_sleep_for_a_while(30) with PiClient(ignite, self.client_config) as piclient: cache_to_test = 'test_cache_with_index' # self.create_cache_with_indexed_data(cache_to_test) client_ignite = piclient.get_ignite() gateway = piclient.get_gateway() cache_config = IgniteCacheConfig(gateway) cache_config.set_name('cache_1') cache_config.set_cache_mode('replicated') cache_config.set_atomicity_mode('transactional') cache_config.set_write_synchronization_mode('full_sync') cache_config.set_affinity(False, 32) cache_config.set_group_name('some_new_group') cache_config1 = IgniteCacheConfig(gateway) cache_config1.set_name(cache_to_test) cache_config1.set_cache_mode('replicated') cache_config1.set_atomicity_mode('transactional') cache_config1.set_write_synchronization_mode('full_sync') cache_config1.set_affinity(False, 32) cache_config1.set_group_name('some_new_group') # set query entities caches = gateway.jvm.java.util.ArrayList() caches.add(cache_config.get_config_object()) caches.add(cache_config1.get_config_object()) log_print("Creating caches", color='green') client_ignite.getOrCreateCaches(caches) cache_names = piclient.get_ignite().cacheNames().toArray() if cache_to_test not in cache_names: log_print("Could not find cache in %s" % cache_names, color='red') util_sleep_for_a_while(10) ignite.kill_node(2) log_print("Overwrite values in cache %s" % cache_to_test, color='green') operation = create_put_all_operation(cache_to_test, 1, 1001, 100, value_type=ModelTypes.VALUE_ALL_TYPES_INDEXED.value) operation.evaluate() util_sleep_for_a_while(15) ignite.start_node(2, skip_topology_check=True) util_sleep_for_a_while(5) client_ignite.cache(cache_to_test).destroy() ignite.update_starting_node_attrs() ignite.nodes[3]['status'] = NodeStatus.STARTED client_ignite.cache('cache_1').destroy()
def test_during_loading(self): """ Should be fully fixed in 8.5.8-p1 Scenario: 1. Start 3 server nodes 2. Load 1000 keys into 120 TX caches 3. Start 3 client node and start TX loading (PESSIMISTIC/REPEATABLE_READ, OPTIMISTIC/SERIALIZABLE) (12 transfer operations, 10 caches in each operation, 1000ms between each transaction i.e. ~ 4 tx per second from each client)) 4. In clients try to destroy caches 5. Interesting things happens Fixed in 8.5.8-p1 https://ggsystems.atlassian.net/browse/GG-19179 Issues that was found during this test: https://ggsystems.atlassian.net/browse/GG-19411 https://ggsystems.atlassian.net/browse/GG-19383 :return: """ PiClient.read_timeout = 600 ignite = self.start_ignite_grid(self.ignite_name) ignite.cu.activate(activate_on_particular_node=1) PiClientIgniteUtils.load_data_with_putall(ignite, self.client_config, ) def get_dumps(): for node_id in ignite.nodes.keys(): self.util_get_threads_from_jstack(ignite, node_id, 'END') try: with PiClient(ignite, self.client_config) as piclient: with TransactionalLoading(self, ignite=ignite, config_file=self.client_config, on_exit_action=get_dumps, kill_transactions_on_exit=True, with_exception=False, # do interrupt loading operation if smth happens? skip_consistency_check=True, # we are destroying caches here if you notice loading_profile=LoadingProfile( delay=1000, allowed_transactions=( TxDescriptor(concurrency='OPTIMISTIC', isolation='SERIALIZABLE', ),) )): # allowed_transactions=(TxDescriptor(), ))): # )): node_id = piclient.get_node_id() client_ignite = piclient.get_ignite(node_id) cache_names = client_ignite.cacheNames().toArray() caches_to_kill_num = 50 frags = 0 for cache in cache_names: node_id = piclient.get_node_id() log_print('Destroying cache %s on node %s' % (cache, node_id), color='red') piclient.get_ignite(node_id).cache(cache).destroy() frags += 1 if frags >= caches_to_kill_num: break finally: npe_errors = ignite.find_exception_in_logs(".*java.lang.NullPointerException.*") assertion_errors = ignite.find_exception_in_logs(".*java.lang.AssertionError.*") if npe_errors != 0 or assertion_errors != 0: assert False, "There are errors in logs: NPE - %s, AE - %s" % (npe_errors, assertion_errors)
def verify_cluster(self, cluster_to_verify_id, nodes_before, last_loaded_key=None): client_config = Ignite.config_builder.get_config( 'client', config_set_name='cluster_1_node_without_dr') servers = 0 ignite = self.clusters[cluster_to_verify_id].grid for i in range(3): for res in ignite.last_topology_snapshot(): if res['servers'] > servers: servers = res['servers'] else: break util_sleep_for_a_while(5) if nodes_before != servers: log_print( f"There are missing nodes on cluster: Nodes in cluster: {servers} expecting {nodes_before}", color='yellow') self.verify_no_meaning_errors() log_print("Wait for topology messages again.", color='yellow') for node_id in ignite.get_all_default_nodes(): ignite.update_started_node_status(node_id) log_print("Missing nodes case confirmed. Trying to restart node.", color='red') current_cluster_nodes = ignite.get_nodes_num('server') if nodes_before != current_cluster_nodes: log_print(f"Current nodes in cluster {current_cluster_nodes}") nodes_to_start = [] for node_id in ignite.get_alive_default_nodes(): # assert that node is not dead otherwise kill/restart again if not ignite.check_node_status(node_id): log_print("Restarting node %s" % node_id, color='yellow') nodes_to_start.append(node_id) log_print(f"Going to restart nodes: {nodes_to_start}", color='debug') for node_id in nodes_to_start: ignite.start_node(node_id, skip_nodes_check=True, check_only_servers=True) current_cluster_nodes = ignite.get_nodes_num('server') if nodes_before != current_cluster_nodes: log_print( f"Current amount of nodes in cluster: {current_cluster_nodes}, expecting {nodes_before}", color='debug') for node_id in ignite.get_alive_default_nodes(): self.util_get_threads_from_jstack( ignite, node_id, "FAILED") assert False, "Failed to restart node" ignite.cu.control_utility('--activate') activate_failed = False log_print('Check that there is no Error in activate logs', color='yellow') if 'Error' in ignite.cu.latest_utility_output: activate_failed = True log_print('Failed!', color='red') sleep(5) ignite.cu.control_utility('--baseline') self.verify_no_meaning_errors() log_print('Check that there is no Error in control.sh --baseline logs', color='yellow') if 'Error' in ignite.cu.latest_utility_output: log_print('Failed! Second try after sleep 60 seconds', color='red') sleep(60) ignite.cu.control_utility('--baseline') if 'Error' in ignite.cu.latest_utility_output or activate_failed: log_print('Cluster looks hang.') log_print('Check that there is no AssertionError in logs', color='yellow') self.verify_no_meaning_errors() if last_loaded_key: try: new_last_key = last_loaded_key - int( random.uniform(0, 1) * LOAD_DATA_SIZE) log_print( f'Trying to remove data from survivor caches ({new_last_key}, {last_loaded_key})', color='yellow') PiClientIgniteUtils.remove_data( ignite, client_config, start_key=new_last_key, end_key=last_loaded_key, check_clients=False, ) last_loaded_key = new_last_key except Exception: for node_id in ignite.get_alive_default_nodes(): self.util_get_threads_from_jstack(ignite, node_id, "FAILED") assert False, "Unable to connect client" finally: self.verify_no_meaning_errors() util_sleep_for_a_while(MINI_DR_STABILIZATION_TIMEOUT) checksum_master, checksum_slave = self.calculate_checksum_and_validate( last_loaded_key) tiden_assert(checksum_master == checksum_slave, 'Hash sum master and slave should be equal') return last_loaded_key
def run_stress_restarts(self, cluster_id_to_restart, iterations, nodes_to_restart, time_to_sleep_range): client_config = self.preconfigure_cluster_0() with PiClient(self.clusters[0].grid, client_config, jvm_options=['-ea']) as piclient: ignite = piclient.get_ignite() self.start_dynamic_caches_with_node_filter(client_config) last_loaded_key = START_DATA_SIZE PiClientIgniteUtils.load_data_with_putall(self.clusters[0].grid, client_config, end_key=last_loaded_key, jvm_options=['-ea'], check_clients=False) util_sleep_for_a_while(MINI_DR_STABILIZATION_TIMEOUT) nodes_before = 6 last_loaded_key += 1 for i in range(0, iterations): log_print(f'Current iteration {i + 1} from {iterations}', color='debug') sleep_for_time = random.uniform(time_to_sleep_range[0], time_to_sleep_range[1]) log_print( f'In this run we are going to sleep for {sleep_for_time} seconds after each node restart', color='green') log_print('Trying to load data into created/existing caches', color='yellow') self.start_dynamic_caches_with_node_filter(client_config) PiClientIgniteUtils.load_data_with_putall( self.clusters[0].grid, client_config, start_key=last_loaded_key, end_key=last_loaded_key + LOAD_DATA_SIZE, jvm_options=['-ea'], check_clients=False) last_loaded_key += LOAD_DATA_SIZE self.increment_atomic(ignite) log_print("Round restart") for node_id in nodes_to_restart: self.clusters[cluster_id_to_restart].grid.kill_node( node_id) self.clusters[cluster_id_to_restart].grid.start_node( node_id, skip_topology_check=True) sleep(sleep_for_time) log_print("Wait for topology messages") for node_id in nodes_to_restart: self.clusters[ cluster_id_to_restart].grid.update_started_node_status( node_id) util_sleep_for_a_while(MINI_DR_STABILIZATION_TIMEOUT) last_loaded_key = self.verify_cluster(0, nodes_before, last_loaded_key) util_sleep_for_a_while(DR_STABILIZATION_TIMEOUT) checksum_master1, checksum_slave1 = self.calculate_checksum_and_validate( last_loaded_key) tiden_assert(checksum_master1 == checksum_slave1, 'Hash sum master and slave not match') self.put_data(self.clusters[1], 1, 'cluster_2_node_without_dr') util_sleep_for_a_while(MINI_DR_STABILIZATION_TIMEOUT) checksum_master2, checksum_slave2 = self.calculate_checksum_and_validate( last_loaded_key) tiden_assert(checksum_master2 == checksum_slave2, 'Hash sum master and slave not match')
def run(self, artifact_name): """ Run scenario for defined artifact :param artifact_name: name from artifact configuration file """ super().run(artifact_name) log_print("Running snapshot benchmark with config: %s" % self.config, color='green') version = self.test_class.tiden.config['artifacts'][artifact_name][ 'ignite_version'] time_to_jfr = int(self.config.get('time_to_jfr', 0)) now_plus_300 = datetime.datetime.now() + datetime.timedelta(seconds=1) try: self.test_class.create_app_config_set( Ignite, START_TIME_CONFIG_SET, caches_list_file='caches_%s.xml' % START_TIME_CONFIG_SET, deploy=True, snapshots_enabled=True, logger=False, wal_segment_size=self.test_class.consumption_config.get( 'wal_segment_size', 1024 * 1024 * 1024), logger_path='%s/ignite-log4j2.xml' % self.test_class.tiden.config['rt']['remote'] ['test_module_dir'], disabled_cache_configs=False, zookeeper_enabled=False, checkpoint_read_lock_timeout=self.read_lock_property_value( version), # caches related variables additional_configs=[ 'caches.tmpl.xml', ], part_32=self.config.get('part_32', 10000), # 100), part_64=self.config.get('part_64', 10000), # 100), part_128=self.config.get('part_128', 10000), # 100), # artifact config variables **self.artifact_config_variables, ) version, ignite = self.test_class.start_ignite_grid( artifact_name, activate=True, config_set=START_TIME_CONFIG_SET, jvm_options=self.artifact_jvm_properties) ignite.set_snapshot_timeout(1200) if time_to_jfr: now_plus_300 = datetime.datetime.now() + datetime.timedelta( seconds=350) ignite.make_cluster_jfr(300) PiClientIgniteUtils.load_data_with_putall( ignite, Ignite.config_builder.get_config( 'client', config_set_name=START_TIME_CONFIG_SET), # end_key=int(self.config.get('data_size')) end_key=10000) # kill nodes ignite.kill_nodes() sleep(120) self.start_probes(artifact_name) ignite.start_nodes() self.stop_probes() ignite.kill_nodes() ignite.delete_lfs() # do some calculations finally: # if time_to_jfr: # pause.until(now_plus_300) # remove config set self.test_class.remove_app_config_set(Ignite, START_TIME_CONFIG_SET)
def test_cycling_restart_grid_dynamic_caches_with_atomic_on_restart(self): """ Scenario The Glue (Assertions should be enabled) 1. Start grid, load some data 2. In the loop: 2.1 define node restart timeout (0.5 - 2.0 seconds) 2.2 Load more data 2.3 Restart each node with defined timeout (DOES NOT LOOK ON TOPOLOGY SNAPSHOT) 2.4 Try to activate, check AssertionErrors 2.5 Try to baseline (If 2 operation failed -> PME, kill all nodes, start new test iteration) 2.6 Try to load data 2.7 Try to calculate checksum :return: """ import random PiClient.read_timeout = 240 # sleep_for_time = float(random.randrange(1, 15, 1)) / 5 self.set_current_context('in_memory') self.util_copy_piclient_model_to_libs() self.ignite.set_activation_timeout(240) self.ignite.set_snapshot_timeout(240) self.ignite.set_node_option('*', 'jvm_options', ['-ea']) self.su.clear_snapshots_list() self.start_grid(skip_activation=True) with PiClient(self.ignite, self.get_client_config(), jvm_options=['-ea']) as piclient: # ignite = piclient.get_ignite() self.start_dynamic_caches_with_node_filter() last_loaded_key = 1000 PiClientIgniteUtils.load_data_with_streamer( self.ignite, self.get_client_config(), end_key=last_loaded_key, jvm_options=['-ea']) nodes_before = self.ignite.get_alive_default_nodes() iterations = 50 last_loaded_key += 1 for i in range(0, iterations): log_print('Current iteration %s from %s' % (i, iterations), color='debug') # sleep_for_time = float(self.the_glue_timeout) if self.the_glue_timeout else random.choice([0.7, 0.9, 2.0]) sleep_for_time = float( self.the_glue_timeout) if self.the_glue_timeout else round( random.uniform(0.5, 2.5), 1) log_print( "In this run we are going to sleep for {} seconds after each node restart" .format(sleep_for_time), color='green') log_print('Trying to load data into created/existing caches', color='yellow') self.start_dynamic_caches_with_node_filter() PiClientIgniteUtils.load_data_with_streamer( self.ignite, self.get_client_config(), start_key=last_loaded_key, end_key=last_loaded_key + 500, jvm_options=['-ea']) last_loaded_key += 500 log_print("Round restart") for node_id in self.ignite.get_alive_default_nodes(): self.ignite.kill_node(node_id) self.ignite.start_node(node_id, skip_topology_check=True) sleep(sleep_for_time) try: log_print( "Incrementing atomics using distributed compute") create_async_operation( create_distributed_atomic_long).evaluate() except Exception as e: log_print("Failed to increment atomics") # just print exception (https://issues.apache.org/jira/browse/IGNITE-11535) traceback.print_exc() log_print("Wait for topology messages") for node_id in self.ignite.get_all_default_nodes(): self.ignite.update_started_node_status(node_id) sleep(15) log_print("Validating cluster") last_loaded_key = self.verify_cluster(nodes_before, last_loaded_key)
def run_snapshot(self, artifact_name, snapshot_type): """ Run scenario for defined artifact :param artifact_name: name from artifact configuration file :param snapshot_type: inc/full """ super().run(artifact_name) log_print("Running snapshot benchmark with config: %s" % self.config, color='green') version = self.test_class.tiden.config['artifacts'][artifact_name][ 'ignite_version'] incremental_snapshot = True if snapshot_type == 'inc' else False try: self.test_class.create_app_config_set( Ignite, SNAPSHOT_CONFIG_SET, caches_list_file='caches_%s.xml' % SNAPSHOT_CONFIG_SET, deploy=True, snapshots_enabled=True, logger=False, wal_segment_size=self.test_class.consumption_config.get( 'wal_segment_size', 64 * 1024 * 1024), logger_path='%s/ignite-log4j2.xml' % self.test_class.tiden.config['rt']['remote'] ['test_module_dir'], disabled_cache_configs=False, zookeeper_enabled=False, checkpoint_read_lock_timeout=self.read_lock_property_value( version), # caches related variables additional_configs=[ 'caches.tmpl.xml', ], part_32=self.config.get('part_32', 32), part_64=self.config.get('part_64', 64), part_128=self.config.get('part_64', 128), # artifact config variables **self.artifact_config_variables, ) version, ignite = self.test_class.start_ignite_grid( artifact_name, activate=True, config_set=SNAPSHOT_CONFIG_SET, jvm_options=self.artifact_jvm_properties) time_results = list() directory_size = list() self.start_probes(artifact_name) client_config = Ignite.config_builder.get_config( 'client', config_set_name=SNAPSHOT_CONFIG_SET) PiClientIgniteUtils.load_data_with_putall( ignite, client_config, end_key=int(self.config.get('data_size'))) if incremental_snapshot: ignite.su.snapshot_utility('snapshot', '-type=full') # default times to run # plus warmup times # plus rerun times warmup_runs, prod_runs = self._get_number_of_runs() log_print("Running {} iterations".format(warmup_runs + prod_runs)) for i in range(0, warmup_runs + prod_runs): self.write_time_event('iteration_%s start' % i) warmup_iteration = False if warmup_runs == 0 else i < warmup_runs log_print("Running iteration %s, (%s)" % (i, 'warmup' if warmup_iteration else 'prod')) ignite.su.snapshot_utility('snapshot', f'-type={snapshot_type}') latest_snapshot_id = ignite.su.snapshots[-1]['id'] dir_size = get_nodes_directory_size( ignite, self.test_class.ssh, 'work/snapshot/%s' % list( SnapshotScenario.util_find_snapshot_folders_on_fs( ignite, latest_snapshot_id).values())[0]) m = re.search( 'Command \[SNAPSHOT\] successfully finished in (\d*) seconds', ignite.su.latest_utility_output) if incremental_snapshot: # todo user remove operation after dr-master merge with PiClient(ignite, client_config) as piclient: ignite_instance = piclient.get_ignite() for cache_name in ignite_instance.cacheNames().toArray( ): ignite_instance.cache(cache_name).removeAll() PiClientIgniteUtils.load_data_with_putall( ignite, client_config, end_key=int(self.config.get('data_size'))) # skip some operation as warmup if not warmup_iteration: assert m, 'Unable to get snapshot time execution' time_results.append(int(m.group(1))) directory_size.append(int(dir_size)) self.write_time_event('iteration_%s stop' % i) ignite.cu.deactivate() self.stop_probes(time_results=time_results, avg_snapshot_dir_size=directory_size, seconds=True) self.results['evaluated'] = True ignite.kill_nodes() ignite.delete_lfs() log_put("Cleanup Ignite LFS ... ") commands = {} for node_idx in ignite.nodes.keys(): host = ignite.nodes[node_idx]['host'] if commands.get(host) is None: commands[host] = [ 'rm -rf %s/work/*' % ignite.nodes[node_idx]['ignite_home'] ] else: commands[host].append( 'rm -rf %s/work/*' % ignite.nodes[node_idx]['ignite_home']) results = self.test_class.tiden.ssh.exec(commands) print(results) log_put("Ignite LFS deleted.") log_print() finally: # remove config set self.test_class.remove_app_config_set(Ignite, SNAPSHOT_CONFIG_SET)
def verify_cluster(self, nodes_before, last_loaded_key=None): if len(nodes_before) != self.ignite.get_nodes_num('server'): log_print("There are missing nodes on cluster.", color='yellow') self.verify_no_assertion_errors() log_print("Wait for topology messages again.", color='yellow') for node_id in self.ignite.get_all_default_nodes(): self.ignite.update_started_node_status(node_id) log_print("Missing nodes case confirmed. Trying to restart node.", color='red') if len(nodes_before) != self.ignite.get_nodes_num('server'): nodes_to_start = [] for node_id in self.ignite.get_alive_default_nodes(): # assert that node is not dead otherwise kill/restart again if not self.ignite.check_node_status(node_id): log_print("Restarting node %s" % node_id, color='yellow') nodes_to_start.append(node_id) for node_id in nodes_to_start: self.ignite.start_node(node_id, skip_nodes_check=True, check_only_servers=True) if len(nodes_before) != self.ignite.get_nodes_num('server'): for node_id in self.ignite.get_alive_default_nodes(): self.util_get_threads_from_jstack(node_id, "FAILED") assert False, "Failed to restart node" self.cu.control_utility('--activate') self.verify_no_assertion_errors() activate_failed = False log_print('Check that there is no Error in activate logs', color='yellow') if 'Error' in self.cu.latest_utility_output: activate_failed = True log_print('Failed!', color='red') sleep(5) self.cu.control_utility('--baseline') self.verify_no_assertion_errors() log_print('Check that there is no Error in control.sh --baseline logs', color='yellow') if 'Error' in self.cu.latest_utility_output: log_print('Failed! Second try after sleep 60 seconds', color='red') sleep(60) self.cu.control_utility('--baseline') if 'Error' in self.cu.latest_utility_output or activate_failed: log_print('Cluster looks hang.') log_print('Check that there is no AssertionError in logs', color='yellow') self.verify_no_assertion_errors() if last_loaded_key: try: log_print('Trying to load data into survivor caches', color='yellow') PiClientIgniteUtils.load_data_with_streamer( self.ignite, self.get_client_config(), start_key=last_loaded_key, end_key=last_loaded_key + 500, allow_overwrite=True, check_clients=False, ) last_loaded_key += 500 log_print('Printing checksums of existing caches', color='yellow') print( PiClientIgniteUtils.calc_checksums_distributed( self.ignite, self.get_client_config(), check_clients=False)) log_print('Check that there is no AssertionError in logs', color='yellow') except Exception as e: for node_id in self.ignite.get_alive_default_nodes(): self.util_get_threads_from_jstack(node_id, "FAILED") assert False, "Unable to connect client" finally: self.verify_no_assertion_errors() return last_loaded_key
def test_start_on_lfs(self): PiClient.read_timeout = 3600 SshPool.default_timeout = 1200 self.ignite_old_version = self.start_ignite_grid('old', activate=True) key_map = { 'cache_group_1_001': ModelTypes.KEY_ALL_TYPES_MAPPED.value, 'cache_group_1_002': 'java.lang.Long', 'cache_group_1_003': ModelTypes.KEY_DEFAULT_TABLE.value, 'cache_group_1_004': ModelTypes.KEY_ACCOUNT.value, } batch_size = 10000 PiClientIgniteUtils.load_data_with_putall( self.ignite_old_version, Ignite.config_builder.get_config('client', config_set_name=self.config_name), start_key=0, end_key=250000, batch_size=batch_size, key_map=key_map, value_type=ModelTypes.VALUE_ALL_TYPES_4_INDEX.value) sleep(120) self.ignite_old_version.kill_nodes() cmd = dict() for node_id, dscr in self.ignite_old_version.nodes.items(): ln_to_work = 'ln -s %s/%s %s/%s' % ( dscr['ignite_home'], 'work', self.get_app(self.ignite_app_names['new']).nodes[node_id] ['ignite_home'], 'work', ) if dscr['host'] in cmd: cmd[dscr['host']].append(ln_to_work) else: cmd[dscr['host']] = [ ln_to_work, ] self.tiden.ssh.exec(cmd) sleep(10) self.ignite_old_version = self.start_ignite_grid('new', activate=True, replaced_name='old') sleep(120) self.ignite_old_version.cu.control_utility( '--cache validate_indexes', all_required=['no issues found']) PiClientIgniteUtils.load_data_with_putall( self.ignite_old_version, Ignite.config_builder.get_config('client', config_set_name=self.config_name), start_key=250001, end_key=500001, batch_size=batch_size, key_map=key_map, value_type=ModelTypes.VALUE_ALL_TYPES_4_INDEX.value) sleep(120) self.ignite_old_version.cu.control_utility('--cache', 'idle_verify')