def test_config_add_to_result(self, ignite_version, is_ignite_service): """ Test that the config file is in config directory and Service.logs contains the config directory to add to the result. """ ignite_cfg = IgniteConfiguration(version=IgniteVersion(ignite_version)) if is_ignite_service: ignite = IgniteService(self.test_context, ignite_cfg, num_nodes=1) else: ignite = IgniteApplicationService( self.test_context, ignite_cfg, java_class_name= "org.apache.ignite.internal.ducktest.tests.self_test.TestKillableApplication" ) ignite.start() assert ignite.logs.get('config').get('path') == ignite.config_dir assert ignite.config_file.startswith(ignite.config_dir) ignite.nodes[0].account.ssh( f'ls {ignite.config_dir} | grep {os.path.basename(ignite.config_file)}' ) ignite.nodes[0].account.ssh( f'ls {ignite.config_dir} | grep {os.path.basename(ignite.log_config_file)}' ) ignite.stop()
def __start_tx_app(self, version, servers, *, client_mode=True, **kwargs): app_params = { 'config': IgniteConfiguration(version=IgniteVersion(version), client_mode=client_mode, discovery_spi=from_ignite_cluster(servers)), 'java_class_name': 'org.apache.ignite.internal.ducktest.tests.control_utility' '.LongRunningTransactionsGenerator', 'params': kwargs } app = IgniteApplicationService(self.test_context, **app_params) app.start() return app
def start_app(_from, _to): app = IgniteApplicationService( context, config=config, java_class_name= "org.apache.ignite.internal.ducktest.tests.rebalance.DataGenerationApplication", params={ "backups": rebalance_params.backups, "cacheCount": rebalance_params.cache_count, "entrySize": rebalance_params.entry_size, "from": _from, "to": _to }, shutdown_timeout_sec=timeout) app.start_async() apps.append(app)
def test_simple_services_start_stop(self, ignite_version): """ Tests plain services start and stop (termitation vs self-terination). """ ignites = IgniteService( self.test_context, IgniteConfiguration(version=IgniteVersion(ignite_version)), num_nodes=1) ignites.start() client = IgniteService( self.test_context, IgniteClientConfiguration(version=IgniteVersion(ignite_version)), num_nodes=1) client.start() node1 = IgniteApplicationService( self.test_context, IgniteClientConfiguration( version=IgniteVersion(ignite_version), discovery_spi=from_ignite_cluster(ignites)), java_class_name= "org.apache.ignite.internal.ducktest.tests.self_test.TestKillableApplication", startup_timeout_sec=180) node2 = IgniteApplicationService( self.test_context, IgniteClientConfiguration( version=IgniteVersion(ignite_version), discovery_spi=from_ignite_cluster(ignites)), java_class_name= "org.apache.ignite.internal.ducktest.tests.self_test.TestSelfKillableApplication", startup_timeout_sec=180) node1.start() node2.run() node1.stop() client.stop() ignites.stop()
def start_load_app(test_context, ignite_config, params, modules=None): """ Start loader application. """ IgniteApplicationService( test_context, config=ignite_config, java_class_name= "org.apache.ignite.internal.ducktest.tests.ContinuousDataLoadApplication", modules=modules, # mute spam in log. jvm_opts=["-DIGNITE_DUMP_THREADS_ON_FAILURE=false"], params=params).start()
def test_thin_client_compatibility(self, server_version, thin_client_version): """ Thin client compatibility test. """ server_config = IgniteConfiguration(version=IgniteVersion(server_version), client_connector_configuration=ClientConnectorConfiguration()) ignite = IgniteService(self.test_context, server_config, 1) addresses = ignite.nodes[0].account.hostname + ":" + str(server_config.client_connector_configuration.port) thin_clients = IgniteApplicationService(self.test_context, IgniteThinClientConfiguration(addresses=addresses, version=IgniteVersion( thin_client_version)), java_class_name=self.JAVA_CLIENT_CLASS_NAME, num_nodes=1) ignite.start() thin_clients.run() ignite.stop()
def run_with_creds(self, client_configuration, rest_key: str, name: str, password: str = None, clean=False, free=True): """ Run user modifying command """ app = IgniteApplicationService( self.test_context, client_configuration, java_class_name="org.apache.ignite.internal.ducktest.tests.authentication.UserModifyingApplication", params={"rest_key": rest_key, "username": name, "password": password} ) app.start(clean=clean) app.stop() if free: app.free()
def test_distribution(self, ignite_version): """ Tests Cellular Affinity scenario (partition distribution). """ cell1 = self.start_cell(ignite_version, ['-D' + CellularAffinity.ATTRIBUTE + '=1']) discovery_spi = from_ignite_cluster(cell1) cell2 = self.start_cell(ignite_version, ['-D' + CellularAffinity.ATTRIBUTE + '=2'], discovery_spi) cell3 = self.start_cell( ignite_version, ['-D' + CellularAffinity.ATTRIBUTE + '=XXX', '-DRANDOM=42'], discovery_spi) for cell in [cell1, cell2, cell3]: cell.await_started() ControlUtility(cell1).activate() checker = IgniteApplicationService( self.test_context, IgniteClientConfiguration( version=IgniteVersion(ignite_version), discovery_spi=from_ignite_cluster(cell1)), java_class_name= "org.apache.ignite.internal.ducktest.tests.cellular_affinity_test.DistributionChecker", params={ "cacheName": CellularAffinity.CACHE_NAME, "attr": CellularAffinity.ATTRIBUTE, "nodesPerCell": self.NODES_PER_CELL }) checker.run()
def start_tx_streamer(self, version, cell, discovery_spi, modules): """ Starts transaction streamer. """ return IgniteApplicationService( self.test_context, IgniteClientConfiguration(version=IgniteVersion(version), properties=self.properties(), discovery_spi=discovery_spi), java_class_name= "org.apache.ignite.internal.ducktest.tests.cellular_affinity_test.CellularTxStreamer", params={ "cacheName": CellularAffinity.CACHE_NAME, "attr": CellularAffinity.ATTRIBUTE, "cell": cell, "warmup": 10000 }, modules=modules, startup_timeout_sec=180)
def _check_failed_number(self, failed_nodes, survived_node): """Ensures number of failed nodes is correct.""" cmd = "grep '%s' %s | wc -l" % (node_failed_event_pattern(), survived_node.log_file) failed_cnt = int(IgniteAwareService.exec_command(survived_node, cmd)) # Cache survivor id, do not read each time. surv_id = IgniteApplicationService.node_id(survived_node) if failed_cnt != len(failed_nodes): failed = IgniteAwareService.exec_command( survived_node, "grep '%s' %s" % (node_failed_event_pattern(), survived_node.log_file)) self.logger.warn( "Node '%s' (%s) has detected the following failures:%s%s" % (survived_node.name, surv_id, os.linesep, failed)) raise AssertionError( "Wrong number of failed nodes: %d. Expected: %d. Check the logs." % (failed_cnt, len(failed_nodes)))
def snapshot_test(self, ignite_version): """ Basic snapshot test. """ version = IgniteVersion(ignite_version) ignite_config = IgniteConfiguration( version=version, data_storage=DataStorageConfiguration( default=DataRegionConfiguration(persistent=True)), metric_exporters={ 'org.apache.ignite.spi.metric.jmx.JmxMetricExporterSpi' }) nodes = IgniteService(self.test_context, ignite_config, num_nodes=self.available_cluster_size - 1) nodes.start() control_utility = ControlUtility(nodes) control_utility.activate() loader_config = IgniteConfiguration( client_mode=True, version=version, discovery_spi=from_ignite_cluster(nodes)) loader = IgniteApplicationService( self.test_context, loader_config, java_class_name= "org.apache.ignite.internal.ducktest.tests.snapshot_test.DataLoaderApplication", params={ "start": 0, "cacheName": self.CACHE_NAME, "interval": 500_000, "valueSizeKb": 1 })
def test_ssl_connection(self, ignite_version): """ Test that IgniteService, IgniteApplicationService correctly start and stop with ssl configurations. And check ControlUtility with ssl arguments. """ shared_root = get_shared_root_path(self.test_context.globals) server_ssl = SslParams(shared_root, key_store_jks=DEFAULT_SERVER_KEYSTORE) server_configuration = IgniteConfiguration( version=IgniteVersion(ignite_version), ssl_params=server_ssl, connector_configuration=ConnectorConfiguration( ssl_enabled=True, ssl_params=server_ssl)) ignite = IgniteService(self.test_context, server_configuration, num_nodes=2, startup_timeout_sec=180) client_configuration = server_configuration._replace( client_mode=True, ssl_params=SslParams(shared_root, key_store_jks=DEFAULT_CLIENT_KEYSTORE), connector_configuration=None) app = IgniteApplicationService( self.test_context, client_configuration, java_class_name= "org.apache.ignite.internal.ducktest.tests.smoke_test.SimpleApplication", startup_timeout_sec=180) admin_ssl = SslParams(shared_root, key_store_jks=DEFAULT_ADMIN_KEYSTORE) control_utility = ControlUtility(cluster=ignite, ssl_params=admin_ssl) ignite.start() app.start() control_utility.cluster_state() app.stop() ignite.stop()
def test_assertion_convertion(self, ignite_version): """ Test to make sure Java assertions are converted to python exceptions """ server_configuration = IgniteConfiguration( version=IgniteVersion(ignite_version)) app = IgniteApplicationService( self.test_context, server_configuration, java_class_name= "org.apache.ignite.internal.ducktest.tests.smoke_test.AssertionApplication" ) try: app.start() except IgniteExecutionException as ex: assert str( ex ) == "Java application execution failed. java.lang.AssertionError" else: app.stop() assert False
def test_ignite_app_start_stop(self, ignite_version): """ Test that IgniteService and IgniteApplicationService correctly start and stop """ server_configuration = IgniteConfiguration( version=IgniteVersion(ignite_version)) ignite = IgniteService(self.test_context, server_configuration, num_nodes=1) client_configuration = server_configuration._replace( client_mode=True, discovery_spi=from_ignite_cluster(ignite)) app = IgniteApplicationService( self.test_context, client_configuration, java_class_name= "org.apache.ignite.internal.ducktest.tests.smoke_test.SimpleApplication" ) ignite.start() app.start() app.stop() ignite.stop()
def node_join_historical_test(self, ignite_version, backups, cache_count, entry_count, entry_size, preloaders, thread_pool_size, batch_size, batches_prefetch_count, throttle): """ Test historycal rebalance. """ preload_entries = 10_000 reb_params = RebalanceParams( trigger_event=TriggerEvent.NODE_JOIN, backups=backups, cache_count=cache_count, entry_count=entry_count, entry_size=entry_size, preloaders=preloaders, thread_pool_size=thread_pool_size, batch_size=batch_size, batches_prefetch_count=batches_prefetch_count, throttle=throttle, persistent=True, jvm_opts=[ '-DIGNITE_PDS_WAL_REBALANCE_THRESHOLD=0', '-DIGNITE_PREFER_WAL_REBALANCE=true' ]) ignites = start_ignite(self.test_context, ignite_version, reb_params) control_utility = ControlUtility(ignites) control_utility.activate() preloader_config = ignites.config._replace( client_mode=True, discovery_spi=from_ignite_cluster(ignites)) preloader = IgniteApplicationService( self.test_context, preloader_config, java_class_name= "org.apache.ignite.internal.ducktest.tests.rebalance.DataGenerationApplication", params={ "backups": 1, "cacheCount": 1, "entrySize": 1, "from": 0, "to": preload_entries }) preloader.run() preloader.free() control_utility.deactivate() control_utility.activate() node = ignites.nodes[-1] ignites.stop_node(node) assert ignites.wait_node(node) preload_time = preload_data( self.test_context, ignites.config._replace( client_mode=True, discovery_spi=from_ignite_cluster(ignites)), rebalance_params=reb_params) control_utility.deactivate() control_utility.activate() self.logger.debug( f'DB size before rebalance: {get_database_size_mb(ignites.nodes, ignites.database_dir)}' ) ignites.start_node(node) ignites.await_started() rebalance_nodes = [node] await_and_check_rebalance(ignites, rebalance_nodes, False) result = get_result(rebalance_nodes, preload_time, cache_count, entry_count, entry_size) control_utility.deactivate() self.logger.debug( f'DB size after rebalance: {get_database_size_mb(ignites.nodes, ignites.database_dir)}' ) return result
def ignite_start_stop(self, ignite_version, graceful_shutdown, nodes_num, static_clients_num, temp_client, iteration_count, client_work_time): """ Test for starting and stopping fat clients. """ servers_count = nodes_num - static_clients_num - temp_client current_top_v = servers_count # Topology version after test. fin_top_ver = servers_count + (2 * static_clients_num) + ( 2 * iteration_count * temp_client) server_cfg = IgniteConfiguration( version=IgniteVersion(ignite_version), caches=[ CacheConfiguration(name=self.CACHE_NAME, backups=1, atomicity_mode='TRANSACTIONAL') ]) ignite = IgniteService(self.test_context, server_cfg, num_nodes=servers_count) control_utility = ControlUtility(ignite) client_cfg = server_cfg._replace(client_mode=True) static_clients = IgniteApplicationService( self.test_context, client_cfg, java_class_name=self.JAVA_CLIENT_CLASS_NAME, num_nodes=static_clients_num, params={ "cacheName": self.CACHE_NAME, "pacing": self.PACING }) temp_clients = IgniteApplicationService( self.test_context, client_cfg, java_class_name=self.JAVA_CLIENT_CLASS_NAME, num_nodes=temp_client, params={ "cacheName": self.CACHE_NAME, "pacing": self.PACING }) ignite.start() static_clients.start() current_top_v += static_clients_num check_topology(control_utility, current_top_v) # Start / stop temp_clients node. Check cluster. for i in range(iteration_count): self.logger.info(f'Starting iteration: {i}.') temp_clients.start() current_top_v += temp_client await_event(static_clients, f'ver={current_top_v}, locNode=') check_topology(control_utility, current_top_v) await_event(temp_clients, f'clients={static_clients_num + temp_client}') time.sleep(client_work_time) if graceful_shutdown: temp_clients.stop() else: temp_clients.kill() current_top_v += temp_client await_event(static_clients, f'ver={current_top_v}, locNode=') static_clients.stop() check_topology(control_utility, fin_top_ver)
def await_event(service: IgniteApplicationService, message): """ :param service: target service for wait :param message: message """ service.await_event(message, timeout_sec=80, from_the_beginning=True)
def test(self, ignite_version, load_type): """ Tests PME-free switch scenario (node stop). """ data = {} caches = [CacheConfiguration(name='test-cache', backups=2, atomicity_mode='TRANSACTIONAL')] l_type = LoadType.construct_from(load_type) # Checking PME (before 2.8) vs PME-free (2.8+) switch duration, but # focusing on switch duration (which depends on caches amount) when long_txs is false and # on waiting for previously started txs before the switch (which depends on txs duration) when long_txs of true. if l_type is LoadType.EXTRA_CACHES: for idx in range(1, self.EXTRA_CACHES_AMOUNT): caches.append(CacheConfiguration(name="cache-%d" % idx, backups=2, atomicity_mode='TRANSACTIONAL')) config = IgniteConfiguration(version=IgniteVersion(ignite_version), caches=caches, cluster_state="INACTIVE") num_nodes = len(self.test_context.cluster) - 2 self.test_context.logger.info("Nodes amount calculated as %d." % num_nodes) ignites = IgniteService(self.test_context, config, num_nodes=num_nodes) ignites.start() if IgniteVersion(ignite_version) >= V_2_8_0: ControlUtility(ignites).disable_baseline_auto_adjust() ControlUtility(ignites).activate() client_config = config._replace(client_mode=True, discovery_spi=from_ignite_cluster(ignites, slice(0, num_nodes - 1))) long_tx_streamer = IgniteApplicationService( self.test_context, client_config, java_class_name="org.apache.ignite.internal.ducktest.tests.pme_free_switch_test.LongTxStreamerApplication", params={"cacheName": "test-cache"}, startup_timeout_sec=180) if l_type is LoadType.LONG_TXS: long_tx_streamer.start() single_key_tx_streamer = IgniteApplicationService( self.test_context, client_config, java_class_name="org.apache.ignite.internal.ducktest.tests.pme_free_switch_test." "SingleKeyTxStreamerApplication", params={"cacheName": "test-cache", "warmup": 1000}, startup_timeout_sec=180) single_key_tx_streamer.start() ignites.stop_node(ignites.nodes[num_nodes - 1]) single_key_tx_streamer.await_event("Node left topology", 60, from_the_beginning=True) if l_type is LoadType.LONG_TXS: time.sleep(30) # keeping txs alive for 30 seconds. long_tx_streamer.stop_async() single_key_tx_streamer.await_event("Node left topology", 60, from_the_beginning=True) single_key_tx_streamer.await_event("APPLICATION_STREAMED", 60) # waiting for streaming continuation. single_key_tx_streamer.stop() data["Worst latency (ms)"] = single_key_tx_streamer.extract_result("WORST_LATENCY") data["Streamed txs"] = single_key_tx_streamer.extract_result("STREAMED") data["Measure duration (ms)"] = single_key_tx_streamer.extract_result("MEASURE_DURATION") data["Server nodes"] = num_nodes return data
def test_logging(self, ignite_version): """ Tests logging goes to the correct file (consistency.log) when default AI config used. """ cfg_filename = "ignite-default-log4j.xml" ignites = IgniteApplicationService( self.test_context, IgniteConfiguration( version=IgniteVersion(ignite_version), cluster_state="INACTIVE", properties=self.PROPERTIES, log4j_config=cfg_filename # default AI config (will be generated below) ), java_class_name="org.apache.ignite.internal.ducktest.tests.control_utility.InconsistentNodeApplication", params={ "cacheName": self.CACHE_NAME, "amount": 1024, "parts": 1, "tx": False }, startup_timeout_sec=180, num_nodes=len(self.test_context.cluster)) for node in ignites.nodes: # copying default AI config with log path replacement ignites.init_persistent(node) cfg_file = f"{ignites.config_dir}/{cfg_filename}" ignites.exec_command(node, f"cp {ignites.home_dir}/config/ignite-log4j.xml {cfg_file}") orig = "${IGNITE_HOME}/work/log".replace('/', '\\/') fixed = ignites.log_dir.replace('/', '\\/') ignites.exec_command(node, f"sed -i 's/{orig}/{fixed}/g' {cfg_file}") ignites.start() control_utility = ControlUtility(ignites) control_utility.activate() ignites.await_event("APPLICATION_STREAMING_FINISHED", 60, from_the_beginning=True) try: control_utility.idle_verify() # making sure we have broken data raise IgniteExecutionException("Fail.") except AssertionError: pass control_utility.check_consistency(f"repair {self.CACHE_NAME} 0") # checking/repairing message = "Cache consistency violations recorded." ignites.await_event(message, 60, from_the_beginning=True, log_file="consistency.log") try: ignites.await_event(message, 10, from_the_beginning=True) raise IgniteExecutionException("Fail.") except TimeoutError: pass