def setup(self): default_context = self.contexts['default'] default_context.add_context_variables( authentication_enabled=False, zookeeper_enabled=is_enabled(self.config.get('zookeeper_enabled')), ) default_context = self.create_test_context('auth_enabled') default_context.add_context_variables( authentication_enabled=True, zookeeper_enabled=is_enabled(self.config.get('zookeeper_enabled')), auth_login='******', auth_password='******', client_auth_login='******', client_auth_password='******', ) context_auth_server = self.create_test_context( 'auth_enabled_server_creds') context_auth_server.add_context_variables( authentication_enabled=True, zookeeper_enabled=is_enabled(self.config.get('zookeeper_enabled')), auth_login='******', auth_password='******', client_auth_login='******', client_auth_password='******', ) super().setup()
def setup(self): default_context = self.contexts['default'] default_context.add_context_variables( authentication_enabled=True, zookeeper_enabled=is_enabled(self.config.get('zookeeper_enabled')), auth_login='******', auth_password='******', ) self.auth_creds = [ 'no_access', 'task_execute', 'admin_ops', 'admin_cache', ] for auth_cred_name in self.auth_creds: context = self.create_test_context(auth_cred_name) context.add_context_variables( authentication_enabled=True, zookeeper_enabled=is_enabled( self.config.get('zookeeper_enabled')), auth_login=auth_cred_name, auth_password=auth_cred_name, ) pp = PrettyPrinter() print_blue('Credentials: \n' + pp.pformat(self.auth_creds)) super().setup()
def test_two_blinking_nodes_clean_lfs(self): with PiClient(self.ignite, self.get_client_config()): with ExitStack() as stack: if is_enabled(self.config.get('zookeeper_enabled')) and \ is_enabled(self.config.get('zookeeper_nodes_restart')): stack.enter_context(ZkNodesRestart(self.zoo, 2)) for iteration in range(0, self.iterations): log_print("Iteration {}/{}".format(str(iteration + 1), str(self.iterations)), color='blue') self.assert_nodes_alive() self.ignite.kill_node(2) self.cleanup_lfs(2) # self.ignite.start_node(2) self.start_node(2) self.ignite.kill_node(3) util_sleep(10) # self.ignite.start_node(3) self.start_node(3) self.ignite.jmx.wait_for_finish_rebalance( self.rebalance_timeout, self.group_names) tiden_assert_equal( 0, self.ignite.find_exception_in_logs('java.lang.AssertionError'), "# of AssertionError")
def setup(self): zookeeper_enabled = is_enabled( self.tiden.config.get('zookeeper_enabled')) self.create_app_config_set( Ignite, 'base', additional_configs=['caches.tmpl.xml'], snapshots_enabled=True, logger=False, caches_list_file='caches_base.xml', logger_path='%s/ignite-log4j2.xml' % self.tiden.config['rt']['remote']['test_module_dir'], disabled_cache_configs=False, zookeeper_enabled=zookeeper_enabled, num_backups=self.pme_config['num_backups'], num_partitions=self.pme_config['num_partitions'], ) # self.create_app_config_set(Ignite, 'snapshot', # snapshots_enabled=False, # logger=False, # logger_path='%s/ignite-log4j2.xml' % self.tiden.config['rt']['remote'][ # 'test_module_dir'], # disabled_cache_configs=False, # zookeeper_enabled=False) self._fix_max_servers() super().setup()
def setup(self): self.ignite.jmx.rebalance_collect_timeout = 5 self.ignite.activate_module('ignite-log4j2') self.the_glue_timeout = self.config.get('the_glue_timeout', None) default_context = self.contexts['default'] default_context.add_context_variables( caches_file='caches.xml', snapshots_enabled=is_enabled(self.config.get('snapshots_enabled')), pitr_enabled=is_enabled(self.config.get('pitr_enabled')), zookeeper_enabled=is_enabled(self.config.get('zookeeper_enabled')), rebalance_pool_size=4, caches=True, logger=True, logger_path='%s/ignite-log4j2.xml' % self.config['rt']['remote']['test_module_dir'], ) indexed_types = self.create_test_context('indexed_types') indexed_types.add_context_variables( caches_file='caches_index.xml', snapshots_enabled=is_enabled(self.config.get('snapshots_enabled')), pitr_enabled=is_enabled(self.config.get('pitr_enabled')), zookeeper_enabled=is_enabled(self.config.get('zookeeper_enabled')), rebalance_pool_size=4, caches=True, logger=True, logger_path='%s/ignite-log4j2.xml' % self.config['rt']['remote']['test_module_dir'], ) in_memory_context = self.create_test_context('in_memory') in_memory_context.add_context_variables( caches_file='caches.xml', snapshots_enabled=is_enabled(self.config.get('snapshots_enabled')), pitr_enabled=is_enabled(self.config.get('pitr_enabled')), zookeeper_enabled=is_enabled(self.config.get('zookeeper_enabled')), rebalance_pool_size=4, in_memory=True, caches=False, logger=True, logger_path='%s/ignite-log4j2.xml' % self.config['rt']['remote']['test_module_dir'], ) super().setup() self.logger = get_logger('tiden') self.logger.set_suite('[TestStressGrid]')
def setup(self): default_context = self.contexts['default'] self.auth_login = self.config.get('auth_login', 'server_user') self.auth_password = self.config.get('auth_password', 'server_password') default_context.add_context_variables( authentication_enabled=True, zookeeper_enabled=is_enabled(self.config.get('zookeeper_enabled')), auth_login=self.auth_login, auth_password=self.auth_password, ) insecure_context = self.create_test_context('insecure') insecure_context.add_context_variables( authentication_enabled=False, zookeeper_enabled=is_enabled(self.config.get('zookeeper_enabled')), ) super().setup()
def test_loading_blinking_node_baseline(self): with PiClient(self.ignite, self.get_client_config()) as piclient: self.wait_for_running_clients_num(piclient.nodes_num, 90) with ExitStack() as stack: stack.enter_context( TransactionalLoading( self, cross_cache_batch=2, skip_atomic=True, post_checksum_action=self.idle_verify_action)) if is_enabled(self.config.get('zookeeper_enabled')) and \ is_enabled(self.config.get('zookeeper_nodes_restart')): stack.enter_context(ZkNodesRestart(self.zoo, 2)) for iteration in range(0, self.iterations): log_print("Iteration {}/{}".format(str(iteration + 1), str(self.iterations)), color='blue') self.assert_nodes_alive() self.ignite.kill_node(2) self.ignite.wait_for_topology_snapshot( server_num=len(self.ignite.get_alive_default_nodes())) self.cu.set_current_topology_as_baseline() util_sleep(5) self.start_node(2) self.ignite.wait_for_topology_snapshot( server_num=len(self.ignite.get_alive_default_nodes())) self.cu.set_current_topology_as_baseline() self.ignite.jmx.wait_for_finish_rebalance( self.rebalance_timeout, self.group_names) tiden_assert_equal( 0, self.ignite.find_exception_in_logs('java.lang.AssertionError'), "# of AssertionError")
def test_loading_blinking_two_nodes_blt_and_extra_node(self): with PiClient(self.ignite, self.get_client_config()): additional_node = self.ignite.add_additional_nodes( self.get_server_config())[0] self.ignite.start_additional_nodes(additional_node) with ExitStack() as stack: stack.enter_context( TransactionalLoading( self, cross_cache_batch=2, skip_atomic=True, post_checksum_action=self.idle_verify_action)) if is_enabled(self.config.get('zookeeper_enabled')) and \ is_enabled(self.config.get('zookeeper_nodes_restart')): stack.enter_context(ZkNodesRestart(self.zoo, 2)) for iteration in range(0, self.iterations): log_print("Iteration {}/{}".format(str(iteration + 1), str(self.iterations)), color='blue') self.assert_nodes_alive() self.ignite.kill_node(2) self.ignite.kill_node(additional_node) # self.ignite.start_node(2) # self.ignite.start_additional_nodes(additional_node) self.start_node(2) self.start_node(additional_node) self.ignite.jmx.wait_for_finish_rebalance( self.rebalance_timeout, self.group_names) tiden_assert_equal( 0, self.ignite.find_exception_in_logs('java.lang.AssertionError'), "# of AssertionError")
def setup(self): self.ignite = IgniteBenchAdapter(self.config, self.ssh) self.cu = self.ignite.cu self.su = self.ignite.su self.zoo = self.ignite.zoo self.consistency_check = is_enabled( self.config.get('consistency_check_enabled', False)) self.optimistic_possibility = self.config.get('optimistic_possibility', 0.5) self.cross_cache_batch = self.config.get('cross_cache_batch', 4) self.tx_delay = self.config.get('tx_delay', 4) self.collect_timeout = self.config.get('collect_timeout', 100) self.collect_timeout_metrics_thread = self.config.get( 'collect_timeout_metrics_thread', 100) zookeeper_enabled = is_enabled(self.config.get('zookeeper_enabled')) sbt_model_enabled = is_enabled(self.config.get('sbt_model_enabled')) default_context = self.contexts['default'] default_context.add_config('caches.xml', 'caches_default.xml') default_context.add_context_variables( caches_file="caches_default.xml", zookeeper_enabled=zookeeper_enabled, sbt_model_enabled=sbt_model_enabled, num_backups=self.num_backups, num_partitions=self.num_partitions, ) if zookeeper_enabled: self.zoo.deploy_zookeeper() default_context.add_context_variables( zoo_connection=self.zoo._get_zkConnectionString(), ) super().setup()
def setup(self): self.zoo = Zookeeper(self.config, self.ssh) zookeeper_enabled = is_enabled(self.config.get('zookeeper_enabled')) default_context = self.contexts['default'] default_context.add_context_variables( caches_file="caches.xml", zookeeper_enabled=zookeeper_enabled, ) if zookeeper_enabled: self.zoo.deploy_zookeeper() default_context.add_context_variables( zoo_connection=self.zoo._get_zkConnectionString(), ) super().setup()
def setup(self): self.auth_creds = { '': { 'name': 'server', 'context': 'default', }, 'read_only': { 'name': 'read_only', 'context': 'read_only', }, 'no_access': { 'name': 'no_access', 'context': 'no_access', }, } pp = PrettyPrinter() for auth_cred_name in self.auth_creds: # aka 'server_login' / 'server_password', etc. default_login = self.auth_creds[auth_cred_name]['name'] + '_user' default_password = self.auth_creds[auth_cred_name][ 'name'] + '_password' self.auth_creds[auth_cred_name].update({ 'login': self.config.get('auth_' + auth_cred_name + 'login', default_login), 'password': self.config.get('auth_' + auth_cred_name + 'password', default_password), }) context_name = self.auth_creds[auth_cred_name]['context'] if context_name != 'default': context = self.create_test_context(context_name) else: context = self.contexts['default'] context.add_context_variables( authentication_enabled=True, zookeeper_enabled=is_enabled( self.config.get('zookeeper_enabled')), auth_login=self.auth_creds[auth_cred_name]['login'], auth_password=self.auth_creds[auth_cred_name]['password'], ) print_blue('Credentials: \n' + pp.pformat(self.auth_creds)) super().setup()
def test_cycling_restart_grid_dynamic_caches_no_client(self): """ Scenario The Glue (Assertions should be enabled) 1. Start grid, load some data 2. In the loop: 2.1 define node restart timeout (0.5 - 2.0 seconds) 2.2 Load more data 2.3 Restart each node with defined timeout (DOES NOT LOOK ON TOPOLOGY SNAPSHOT) 2.4 Try to activate, check AssertionErrors 2.5 Try to baseline (If 2 operation failed -> PME, kill all nodes, start new test iteration) 2.6 Try to load data 2.7 Try to calculate checksum :return: """ import random PiClient.read_timeout = 240 # sleep_for_time = float(random.randrange(1, 15, 1)) / 5 self.set_current_context('in_memory') self.util_copy_piclient_model_to_libs() self.ignite.set_activation_timeout(240) self.ignite.set_snapshot_timeout(240) self.ignite.set_node_option('*', 'jvm_options', ['-ea']) self.su.clear_snapshots_list() self.start_grid(skip_activation=True) self.start_dynamic_caches_with_node_filter() last_loaded_key = 1000 PiClientIgniteUtils.load_data_with_streamer(self.ignite, self.get_client_config(), end_key=last_loaded_key, jvm_options=['-ea']) nodes_before = self.ignite.get_alive_default_nodes() iterations = 50 last_loaded_key += 1 for i in range(0, iterations): with ExitStack() as stack: # load data before start zk restart thread self.start_dynamic_caches_with_node_filter() # PiClientIgniteUtils.wait_for_running_clients_num(self.ignite, 0, 120) PiClientIgniteUtils.load_data_with_streamer( self.ignite, self.get_client_config(), start_key=last_loaded_key, end_key=last_loaded_key + 500, jvm_options=['-ea'], check_clients=True) last_loaded_key += 500 if self.get_context_variable('zookeeper_enabled') and \ is_enabled(self.config.get('zookeeper_nodes_restart')): stack.enter_context(ZkNodesRestart(self.zoo, 3)) log_print('Current iteration %s from %s' % (i, iterations), color='debug') sleep_for_time = float( self.the_glue_timeout) if self.the_glue_timeout else round( random.uniform(0.5, 2.5), 1) log_print( "In this run we are going to sleep for {} seconds after each node restart" .format(sleep_for_time), color='green') log_print('Trying to load data into created/existing caches', color='yellow') log_print("Round restart") for node_id in self.ignite.get_alive_default_nodes(): self.ignite.kill_node(node_id) self.ignite.start_node(node_id, skip_topology_check=True) sleep(sleep_for_time) log_print("Wait for topology messages") for node_id in self.ignite.get_all_default_nodes(): self.ignite.update_started_node_status(node_id) sleep(15) last_loaded_key = self.verify_cluster(nodes_before, last_loaded_key)
def setup(self): self.auth_creds = { '': { 'user': '******', 'pwd': 'qwe123', 'context': 'default', 'description': '{ "defaultAllow": "true" }' }, 'read_only': { 'user': '******', 'pwd': 'qwe123', 'context': 'read_only', 'description': '{' '{ "cache": "cache_ro*", "permissions":["CACHE_READ"] }, ' '{ "cache": "cache_rw*", "permissions":["CACHE_READ", "CACHE_PUT", "CACHE_REMOVE"] }, ' '{ "task": "*", "permissions":["TASK_EXECUTE"] }, ' '{ "system":["ADMIN_CACHE", "CACHE_CREATE"] }, ' '"defaultAllow":"false"' '}' }, 'utility': { 'user': '******', 'pwd': 'qwe123', 'context': 'utility', 'description': '{ "defaultAllow":"true" }' }, } pp = PrettyPrinter() for auth_cred_name in self.auth_creds: context_name = self.auth_creds[auth_cred_name]['context'] if context_name != 'default': context = self.create_test_context(context_name) else: context = self.contexts['default'] context.add_context_variables( ssl_enabled=is_enabled(self.config.get('ssl_enabled')), authentication_enabled=True, client_key_store_file_path='%s/client.jks' % self.config['rt']['remote']['test_module_dir'], server_key_store_file_path='%s/server.jks' % self.config['rt']['remote']['test_module_dir'], trust_store_file_path='%s/trust.jks' % self.config['rt']['remote']['test_module_dir'], zookeeper_enabled=is_enabled( self.config.get('zookeeper_enabled')), pitr_enabled=is_enabled(self.config.get('pitr_enabled')), rolling_updates_enabled=is_enabled( self.config.get('rolling_updates_enabled')), auth_login=self.auth_creds[auth_cred_name]['user'], auth_password=self.auth_creds[auth_cred_name]['pwd'], ) self.cu.enable_authentication('utility', 'qwe123') self.su.enable_authentication('utility', 'qwe123') print_blue('Credentials: \n' + pp.pformat(self.auth_creds)) super().setup()
def _run_iteration(self, ignite, iteration): """ One iteration of server PME benchmark is as follows: 1. start transactional loading, sleep `warmup_servers_delay` so that load stabilize 2. kill random N nodes, measure LEAVE exchange time, sleep `stabilization_delay` 3. restart killed nodes, measure JOIN exchange time, sleep `cooldown_delay` 4. stop load :param ignite: :param iteration: :return: """ log_print( "===> PME {} Server(s) Left-Join Benchmark iteration {}/{} started " .format(self.config['num_servers_to_kill'], iteration, self.config['iterations']), color='green') # if debug: # from pt.util import read_yaml_file # from os.path import join # base_path = 'pt/tests/res/exchanges' # exch_test = iteration # start_exch = read_yaml_file(join(base_path, 'start_exch.%d.yaml' % exch_test)) # finish_exch = read_yaml_file(join(base_path, 'finish_exch.%d.yaml' % exch_test)) # merge_exch = read_yaml_file(join(base_path, 'merge_exch.%d.yaml' % exch_test)) # self.test_class.exchanges = ExchangesCollection.create_from_log_data(start_exch, finish_exch, merge_exch) # self.test_class.new_topVer = 5 # x1_leave_time, x2_time = self.test_class._measurements_after_test('test_leave', skip_exch=1) # self.test_class.new_topVer = 6 # x1_join_time, x2_time = self.test_class._measurements_after_test('test_join', skip_exch=1) # # return x1_leave_time, x1_join_time loading_client_hosts = self._get_loading_client_hosts() num_servers = self._get_num_server_nodes() num_servers_to_kill = self.config['num_servers_to_kill'] kill_coordinator = self.config['kill_coordinator'] metrics = None ex = None x1_join_time = None x1_leave_time = None try: # start loading clients ... with PiClient(ignite, self.test_class.client_config, client_hosts=loading_client_hosts, clients_per_host=self.config.get( 'loading_clients_per_host', 1)): # ... and initiate transactional load with TransactionalLoading( self.test_class, ignite=ignite, kill_transactions_on_exit=self. config['kill_transactions_on_exit'], cross_cache_batch=self.config['cross_cache_batch'], skip_atomic=self.config['skip_atomic'], skip_consistency_check=not self. config['consistency_check_enabled'], loading_profile=LoadingProfile( delay=self.config['tx_delay'], commit_possibility=self. config['commit_possibility'], start_key=1, end_key=self.config['load_factor'] - 1, transaction_timeout=self. config['transaction_timeout']), tx_metrics=[ 'txCreated', 'txCommit', 'txFailed', 'txRollback' ]) as tx_loading: metrics = tx_loading.metrics # pick random server nodes node_ids = ignite.get_random_server_nodes( num_servers_to_kill, use_coordinator=kill_coordinator, node_ids=self.test_class.server_node_ids, ) expected_total_server_num = num_servers - len(node_ids) # ... wait load stabilize util_sleep_for_a_while(self.config['warmup_servers_delay'], "Before LEAVE") if is_enabled(self.config.get('jfr_enabled', False)): ignite.make_cluster_jfr(60) util_sleep_for_a_while(2) self.test_class._prepare_before_test( ignite, tx_loading, 'LEAVE %d server(s)' % len(node_ids)) # ... kill selected random nodes ignite.kill_nodes(*node_ids) ignite.wait_for_topology_snapshot( server_num=expected_total_server_num) tx_loading.metrics_thread.add_custom_event( '%d server(s) left' % len(node_ids)) new_topVer = self.test_class._get_new_top_after_test( ignite) self.test_class._wait_exchange_finished(ignite, new_topVer) x1_leave_time, x2_time = self.test_class._measurements_after_test( 'LEAVE %d server(s)' % len(node_ids), skip_exch=1) if is_enabled(self.config.get('heapdump_enabled', False)): ignite.make_cluster_heapdump( [1], 'after_%d_server_leave' % len(node_ids)) # ... wait exchange stabilize util_sleep_for_a_while(self.config['stabilization_delay'], "After LEAVE, before JOIN") if self.config['measure_restart_nodes']: self.test_class._prepare_before_test( ignite, tx_loading, 'JOIN %d server(s)' % len(node_ids)) # ... restart killed nodes ignite.start_nodes(*node_ids) ignite.wait_for_topology_snapshot( server_num=expected_total_server_num + len(node_ids)) if self.config['measure_restart_nodes']: tx_loading.metrics_thread.add_custom_event( '%d server(s) joined' % len(node_ids)) new_topVer = self.test_class._get_new_top_after_test( ignite) self.test_class._wait_exchange_finished( ignite, new_topVer) x1_join_time, x2_time = self.test_class._measurements_after_test( 'JOIN %d server(s)' % len(node_ids), skip_exch=1) # if is_enabled(self.config.get('heapdump_enabled', False)): # ignite.make_cluster_heapdump([1], 'after_%d_server_join' % len(node_ids)) # ... wait exchange cooldown util_sleep_for_a_while(self.config['cooldown_delay'], "After JOIN") ignite.wait_for_topology_snapshot(client_num=0) except Exception as e: ex = e if metrics: self.test_class.create_loading_metrics_graph( 'pme_%d_servers_left_join_%s_%d' % (num_servers_to_kill, self.run_id, iteration), metrics, dpi_factor=0.75) if ex: raise ex return { 'Exchange Server Join': x1_join_time, 'Exchange Server Leave': x1_leave_time, }
def setup(self): super().setup() self.snapshot_arch_enabled = is_enabled(self.config.get('snapshot_arch')) self.logger = get_logger('tiden') self.logger.set_suite('[TessUtils]')
def setup(self): default_context = self.contexts['default'] authentication_enabled = is_enabled( self.config.get('authentication_enabled')) self.reusable_lfs = is_enabled(self.config.get('reusable_lfs_enable')) default_context.add_context_variables( persistence_enabled=True, snapshots_enabled=is_enabled( self.config.get('snapshots_enabled', True)), pitr_enabled=is_enabled(self.config.get('pitr_enabled')), zookeeper_enabled=is_enabled(self.config.get('zookeeper_enabled')), compaction_enabled=is_enabled( self.config.get('compaction_enabled')), authentication_enabled=authentication_enabled, sbt_model_enabled=is_enabled(self.config.get('sbt_model_enabled')), caches='caches_sbt.xml' if is_enabled( self.config.get('sbt_model_enabled')) else 'caches.xml', snapshot_archive_enabled=is_enabled( self.config.get('snapshot_arch')), dynamic_cache_enabled=is_enabled( self.config.get('dynamic_cache_enabled')), blt_auto_adjust_enabled=is_enabled( self.config.get('blt_auto_adjust_enabled')), community_edition_enabled=is_enabled( self.config.get('community_edition_enabled'))) if authentication_enabled: auth_login = self.config.get('auth_login', 'server_user') auth_password = self.config.get('auth_password', 'server_password') default_context.add_context_variables( auth_login=auth_login, auth_password=auth_password, ) self.cu.enable_authentication(auth_login, auth_password), self.su.enable_authentication(auth_login, auth_password), self.ignite.enable_authentication(auth_login, auth_password), if not is_enabled(self.config.get('community_edition_enabled')): self.gg_version = self.config['artifacts']['ignite'][ 'gridgain_version'] super().setup() self.load_multiplier = float( self.config.get('pitr_load_multiplier', UltimateUtils.load_multiplier)) self.ignite.set_activation_timeout(20) self.ignite.set_snapshot_timeout(60) self.blt_auto_adjust_timeout = 60000 self.logger = get_logger('tiden') self.logger.set_suite('[TestSnapshots]')
def test_indexes_rebuilded(self): """ https://ggsystems.atlassian.net/browse/GG-17428 1. Start cluster. 2. Start transactional loading. 3. Stop one node and remove index.bin files for the caches. 4. Start node and let it finish rebalance. 5. Check indexes are not broken after rebalance. :return: """ self.need_delete_lfs_on_teardown = True debug = False with PiClient(self.ignite, self.get_client_config()) as piclient: self.wait_for_running_clients_num(piclient.nodes_num, 90) with ExitStack() as stack: # todo unreachable code if False: stack.enter_context( TransactionalLoading( self, cross_cache_batch=2, skip_atomic=True, post_checksum_action=self.idle_verify_action)) if is_enabled(self.config.get('zookeeper_enabled')) and \ is_enabled(self.config.get('zookeeper_nodes_restart')): stack.enter_context(ZkNodesRestart(self.zoo, 2)) for iteration in range(0, self.iterations): log_print("Iteration {}/{}".format(str(iteration + 1), str(self.iterations)), color='blue') self.assert_nodes_alive() with TransactionalLoading(self, cross_cache_batch=2, skip_atomic=True): util_sleep(20) self.ignite.kill_node(2) if debug: self.cu.control_utility( '--cache idle_verify --dump --skip-zeros') self.remove_index_bin_files(2) util_sleep(10) if debug: self.cu.control_utility( '--cache idle_verify --dump --skip-zeros') self.start_node(2) self.ignite.jmx.wait_for_finish_rebalance( self.rebalance_timeout, self.group_names) util_sleep(30) log_print("Check indexes") try: if debug: self.cu.control_utility( '--cache idle_verify --dump --skip-zeros') self.idle_verify_action(None) except TidenException: if debug: self.cu.control_utility( '--cache idle_verify --dump --skip-zeros') raise TidenException('validate_index failed') tiden_assert_equal( 0, self.ignite.find_exception_in_logs('java.lang.AssertionError'), "# of AssertionError")