def test_config_add_to_result(self, ignite_version, is_ignite_service):
"""
Test that the config file is in config directory
and Service.logs contains the config directory to add to the result.
"""
ignite_cfg = IgniteConfiguration(version=IgniteVersion(ignite_version))
if is_ignite_service:
ignite = IgniteService(self.test_context, ignite_cfg, num_nodes=1)
else:
ignite = IgniteApplicationService(
self.test_context,
ignite_cfg,
java_class_name=
"org.apache.ignite.internal.ducktest.tests.self_test.TestKillableApplication"
)
ignite.start()
assert ignite.logs.get('config').get('path') == ignite.config_dir
assert ignite.config_file.startswith(ignite.config_dir)
ignite.nodes[0].account.ssh(
f'ls {ignite.config_dir} | grep {os.path.basename(ignite.config_file)}'
)
ignite.nodes[0].account.ssh(
f'ls {ignite.config_dir} | grep {os.path.basename(ignite.log_config_file)}'
)
ignite.stop()
def __start_tx_app(self, version, servers, *, client_mode=True, **kwargs):
app_params = {
'config':
IgniteConfiguration(version=IgniteVersion(version),
client_mode=client_mode,
discovery_spi=from_ignite_cluster(servers)),
'java_class_name':
'org.apache.ignite.internal.ducktest.tests.control_utility'
'.LongRunningTransactionsGenerator',
'params':
kwargs
}
app = IgniteApplicationService(self.test_context, **app_params)
app.start()
return app
def start_app(_from, _to):
app = IgniteApplicationService(
context,
config=config,
java_class_name=
"org.apache.ignite.internal.ducktest.tests.rebalance.DataGenerationApplication",
params={
"backups": rebalance_params.backups,
"cacheCount": rebalance_params.cache_count,
"entrySize": rebalance_params.entry_size,
"from": _from,
"to": _to
},
shutdown_timeout_sec=timeout)
app.start_async()
apps.append(app)
def test_simple_services_start_stop(self, ignite_version):
"""
Tests plain services start and stop (termitation vs self-terination).
"""
ignites = IgniteService(
self.test_context,
IgniteConfiguration(version=IgniteVersion(ignite_version)),
num_nodes=1)
ignites.start()
client = IgniteService(
self.test_context,
IgniteClientConfiguration(version=IgniteVersion(ignite_version)),
num_nodes=1)
client.start()
node1 = IgniteApplicationService(
self.test_context,
IgniteClientConfiguration(
version=IgniteVersion(ignite_version),
discovery_spi=from_ignite_cluster(ignites)),
java_class_name=
"org.apache.ignite.internal.ducktest.tests.self_test.TestKillableApplication",
startup_timeout_sec=180)
node2 = IgniteApplicationService(
self.test_context,
IgniteClientConfiguration(
version=IgniteVersion(ignite_version),
discovery_spi=from_ignite_cluster(ignites)),
java_class_name=
"org.apache.ignite.internal.ducktest.tests.self_test.TestSelfKillableApplication",
startup_timeout_sec=180)
node1.start()
node2.run()
node1.stop()
client.stop()
ignites.stop()
def start_load_app(test_context, ignite_config, params, modules=None):
"""
Start loader application.
"""
IgniteApplicationService(
test_context,
config=ignite_config,
java_class_name=
"org.apache.ignite.internal.ducktest.tests.ContinuousDataLoadApplication",
modules=modules,
# mute spam in log.
jvm_opts=["-DIGNITE_DUMP_THREADS_ON_FAILURE=false"],
params=params).start()
def test_thin_client_compatibility(self, server_version, thin_client_version):
"""
Thin client compatibility test.
"""
server_config = IgniteConfiguration(version=IgniteVersion(server_version),
client_connector_configuration=ClientConnectorConfiguration())
ignite = IgniteService(self.test_context, server_config, 1)
addresses = ignite.nodes[0].account.hostname + ":" + str(server_config.client_connector_configuration.port)
thin_clients = IgniteApplicationService(self.test_context,
IgniteThinClientConfiguration(addresses=addresses,
version=IgniteVersion(
thin_client_version)),
java_class_name=self.JAVA_CLIENT_CLASS_NAME,
num_nodes=1)
ignite.start()
thin_clients.run()
ignite.stop()
def run_with_creds(self, client_configuration, rest_key: str, name: str, password: str = None, clean=False,
free=True):
"""
Run user modifying command
"""
app = IgniteApplicationService(
self.test_context,
client_configuration,
java_class_name="org.apache.ignite.internal.ducktest.tests.authentication.UserModifyingApplication",
params={"rest_key": rest_key,
"username": name,
"password": password}
)
app.start(clean=clean)
app.stop()
if free:
app.free()
def test_distribution(self, ignite_version):
"""
Tests Cellular Affinity scenario (partition distribution).
"""
cell1 = self.start_cell(ignite_version,
['-D' + CellularAffinity.ATTRIBUTE + '=1'])
discovery_spi = from_ignite_cluster(cell1)
cell2 = self.start_cell(ignite_version,
['-D' + CellularAffinity.ATTRIBUTE + '=2'],
discovery_spi)
cell3 = self.start_cell(
ignite_version,
['-D' + CellularAffinity.ATTRIBUTE + '=XXX', '-DRANDOM=42'],
discovery_spi)
for cell in [cell1, cell2, cell3]:
cell.await_started()
ControlUtility(cell1).activate()
checker = IgniteApplicationService(
self.test_context,
IgniteClientConfiguration(
version=IgniteVersion(ignite_version),
discovery_spi=from_ignite_cluster(cell1)),
java_class_name=
"org.apache.ignite.internal.ducktest.tests.cellular_affinity_test.DistributionChecker",
params={
"cacheName": CellularAffinity.CACHE_NAME,
"attr": CellularAffinity.ATTRIBUTE,
"nodesPerCell": self.NODES_PER_CELL
})
checker.run()
def start_tx_streamer(self, version, cell, discovery_spi, modules):
"""
Starts transaction streamer.
"""
return IgniteApplicationService(
self.test_context,
IgniteClientConfiguration(version=IgniteVersion(version),
properties=self.properties(),
discovery_spi=discovery_spi),
java_class_name=
"org.apache.ignite.internal.ducktest.tests.cellular_affinity_test.CellularTxStreamer",
params={
"cacheName": CellularAffinity.CACHE_NAME,
"attr": CellularAffinity.ATTRIBUTE,
"cell": cell,
"warmup": 10000
},
modules=modules,
startup_timeout_sec=180)
def _check_failed_number(self, failed_nodes, survived_node):
"""Ensures number of failed nodes is correct."""
cmd = "grep '%s' %s | wc -l" % (node_failed_event_pattern(),
survived_node.log_file)
failed_cnt = int(IgniteAwareService.exec_command(survived_node, cmd))
# Cache survivor id, do not read each time.
surv_id = IgniteApplicationService.node_id(survived_node)
if failed_cnt != len(failed_nodes):
failed = IgniteAwareService.exec_command(
survived_node, "grep '%s' %s" %
(node_failed_event_pattern(), survived_node.log_file))
self.logger.warn(
"Node '%s' (%s) has detected the following failures:%s%s" %
(survived_node.name, surv_id, os.linesep, failed))
raise AssertionError(
"Wrong number of failed nodes: %d. Expected: %d. Check the logs."
% (failed_cnt, len(failed_nodes)))
def snapshot_test(self, ignite_version):
"""
Basic snapshot test.
"""
version = IgniteVersion(ignite_version)
ignite_config = IgniteConfiguration(
version=version,
data_storage=DataStorageConfiguration(
default=DataRegionConfiguration(persistent=True)),
metric_exporters={
'org.apache.ignite.spi.metric.jmx.JmxMetricExporterSpi'
})
nodes = IgniteService(self.test_context,
ignite_config,
num_nodes=self.available_cluster_size - 1)
nodes.start()
control_utility = ControlUtility(nodes)
control_utility.activate()
loader_config = IgniteConfiguration(
client_mode=True,
version=version,
discovery_spi=from_ignite_cluster(nodes))
loader = IgniteApplicationService(
self.test_context,
loader_config,
java_class_name=
"org.apache.ignite.internal.ducktest.tests.snapshot_test.DataLoaderApplication",
params={
"start": 0,
"cacheName": self.CACHE_NAME,
"interval": 500_000,
"valueSizeKb": 1
})
def test_ssl_connection(self, ignite_version):
"""
Test that IgniteService, IgniteApplicationService correctly start and stop with ssl configurations.
And check ControlUtility with ssl arguments.
"""
shared_root = get_shared_root_path(self.test_context.globals)
server_ssl = SslParams(shared_root,
key_store_jks=DEFAULT_SERVER_KEYSTORE)
server_configuration = IgniteConfiguration(
version=IgniteVersion(ignite_version),
ssl_params=server_ssl,
connector_configuration=ConnectorConfiguration(
ssl_enabled=True, ssl_params=server_ssl))
ignite = IgniteService(self.test_context,
server_configuration,
num_nodes=2,
startup_timeout_sec=180)
client_configuration = server_configuration._replace(
client_mode=True,
ssl_params=SslParams(shared_root,
key_store_jks=DEFAULT_CLIENT_KEYSTORE),
connector_configuration=None)
app = IgniteApplicationService(
self.test_context,
client_configuration,
java_class_name=
"org.apache.ignite.internal.ducktest.tests.smoke_test.SimpleApplication",
startup_timeout_sec=180)
admin_ssl = SslParams(shared_root,
key_store_jks=DEFAULT_ADMIN_KEYSTORE)
control_utility = ControlUtility(cluster=ignite, ssl_params=admin_ssl)
ignite.start()
app.start()
control_utility.cluster_state()
app.stop()
ignite.stop()
def test_assertion_convertion(self, ignite_version):
"""
Test to make sure Java assertions are converted to python exceptions
"""
server_configuration = IgniteConfiguration(
version=IgniteVersion(ignite_version))
app = IgniteApplicationService(
self.test_context,
server_configuration,
java_class_name=
"org.apache.ignite.internal.ducktest.tests.smoke_test.AssertionApplication"
)
try:
app.start()
except IgniteExecutionException as ex:
assert str(
ex
) == "Java application execution failed. java.lang.AssertionError"
else:
app.stop()
assert False
def test_ignite_app_start_stop(self, ignite_version):
"""
Test that IgniteService and IgniteApplicationService correctly start and stop
"""
server_configuration = IgniteConfiguration(
version=IgniteVersion(ignite_version))
ignite = IgniteService(self.test_context,
server_configuration,
num_nodes=1)
client_configuration = server_configuration._replace(
client_mode=True, discovery_spi=from_ignite_cluster(ignite))
app = IgniteApplicationService(
self.test_context,
client_configuration,
java_class_name=
"org.apache.ignite.internal.ducktest.tests.smoke_test.SimpleApplication"
)
ignite.start()
app.start()
app.stop()
ignite.stop()
def node_join_historical_test(self, ignite_version, backups, cache_count,
entry_count, entry_size, preloaders,
thread_pool_size, batch_size,
batches_prefetch_count, throttle):
"""
Test historycal rebalance.
"""
preload_entries = 10_000
reb_params = RebalanceParams(
trigger_event=TriggerEvent.NODE_JOIN,
backups=backups,
cache_count=cache_count,
entry_count=entry_count,
entry_size=entry_size,
preloaders=preloaders,
thread_pool_size=thread_pool_size,
batch_size=batch_size,
batches_prefetch_count=batches_prefetch_count,
throttle=throttle,
persistent=True,
jvm_opts=[
'-DIGNITE_PDS_WAL_REBALANCE_THRESHOLD=0',
'-DIGNITE_PREFER_WAL_REBALANCE=true'
])
ignites = start_ignite(self.test_context, ignite_version, reb_params)
control_utility = ControlUtility(ignites)
control_utility.activate()
preloader_config = ignites.config._replace(
client_mode=True, discovery_spi=from_ignite_cluster(ignites))
preloader = IgniteApplicationService(
self.test_context,
preloader_config,
java_class_name=
"org.apache.ignite.internal.ducktest.tests.rebalance.DataGenerationApplication",
params={
"backups": 1,
"cacheCount": 1,
"entrySize": 1,
"from": 0,
"to": preload_entries
})
preloader.run()
preloader.free()
control_utility.deactivate()
control_utility.activate()
node = ignites.nodes[-1]
ignites.stop_node(node)
assert ignites.wait_node(node)
preload_time = preload_data(
self.test_context,
ignites.config._replace(
client_mode=True, discovery_spi=from_ignite_cluster(ignites)),
rebalance_params=reb_params)
control_utility.deactivate()
control_utility.activate()
self.logger.debug(
f'DB size before rebalance: {get_database_size_mb(ignites.nodes, ignites.database_dir)}'
)
ignites.start_node(node)
ignites.await_started()
rebalance_nodes = [node]
await_and_check_rebalance(ignites, rebalance_nodes, False)
result = get_result(rebalance_nodes, preload_time, cache_count,
entry_count, entry_size)
control_utility.deactivate()
self.logger.debug(
f'DB size after rebalance: {get_database_size_mb(ignites.nodes, ignites.database_dir)}'
)
return result
def ignite_start_stop(self, ignite_version, graceful_shutdown, nodes_num,
static_clients_num, temp_client, iteration_count,
client_work_time):
"""
Test for starting and stopping fat clients.
"""
servers_count = nodes_num - static_clients_num - temp_client
current_top_v = servers_count
# Topology version after test.
fin_top_ver = servers_count + (2 * static_clients_num) + (
2 * iteration_count * temp_client)
server_cfg = IgniteConfiguration(
version=IgniteVersion(ignite_version),
caches=[
CacheConfiguration(name=self.CACHE_NAME,
backups=1,
atomicity_mode='TRANSACTIONAL')
])
ignite = IgniteService(self.test_context,
server_cfg,
num_nodes=servers_count)
control_utility = ControlUtility(ignite)
client_cfg = server_cfg._replace(client_mode=True)
static_clients = IgniteApplicationService(
self.test_context,
client_cfg,
java_class_name=self.JAVA_CLIENT_CLASS_NAME,
num_nodes=static_clients_num,
params={
"cacheName": self.CACHE_NAME,
"pacing": self.PACING
})
temp_clients = IgniteApplicationService(
self.test_context,
client_cfg,
java_class_name=self.JAVA_CLIENT_CLASS_NAME,
num_nodes=temp_client,
params={
"cacheName": self.CACHE_NAME,
"pacing": self.PACING
})
ignite.start()
static_clients.start()
current_top_v += static_clients_num
check_topology(control_utility, current_top_v)
# Start / stop temp_clients node. Check cluster.
for i in range(iteration_count):
self.logger.info(f'Starting iteration: {i}.')
temp_clients.start()
current_top_v += temp_client
await_event(static_clients, f'ver={current_top_v}, locNode=')
check_topology(control_utility, current_top_v)
await_event(temp_clients,
f'clients={static_clients_num + temp_client}')
time.sleep(client_work_time)
if graceful_shutdown:
temp_clients.stop()
else:
temp_clients.kill()
current_top_v += temp_client
await_event(static_clients, f'ver={current_top_v}, locNode=')
static_clients.stop()
check_topology(control_utility, fin_top_ver)
def await_event(service: IgniteApplicationService, message):
"""
:param service: target service for wait
:param message: message
"""
service.await_event(message, timeout_sec=80, from_the_beginning=True)
def test(self, ignite_version, load_type):
"""
Tests PME-free switch scenario (node stop).
"""
data = {}
caches = [CacheConfiguration(name='test-cache', backups=2, atomicity_mode='TRANSACTIONAL')]
l_type = LoadType.construct_from(load_type)
# Checking PME (before 2.8) vs PME-free (2.8+) switch duration, but
# focusing on switch duration (which depends on caches amount) when long_txs is false and
# on waiting for previously started txs before the switch (which depends on txs duration) when long_txs of true.
if l_type is LoadType.EXTRA_CACHES:
for idx in range(1, self.EXTRA_CACHES_AMOUNT):
caches.append(CacheConfiguration(name="cache-%d" % idx, backups=2, atomicity_mode='TRANSACTIONAL'))
config = IgniteConfiguration(version=IgniteVersion(ignite_version), caches=caches, cluster_state="INACTIVE")
num_nodes = len(self.test_context.cluster) - 2
self.test_context.logger.info("Nodes amount calculated as %d." % num_nodes)
ignites = IgniteService(self.test_context, config, num_nodes=num_nodes)
ignites.start()
if IgniteVersion(ignite_version) >= V_2_8_0:
ControlUtility(ignites).disable_baseline_auto_adjust()
ControlUtility(ignites).activate()
client_config = config._replace(client_mode=True,
discovery_spi=from_ignite_cluster(ignites, slice(0, num_nodes - 1)))
long_tx_streamer = IgniteApplicationService(
self.test_context,
client_config,
java_class_name="org.apache.ignite.internal.ducktest.tests.pme_free_switch_test.LongTxStreamerApplication",
params={"cacheName": "test-cache"},
startup_timeout_sec=180)
if l_type is LoadType.LONG_TXS:
long_tx_streamer.start()
single_key_tx_streamer = IgniteApplicationService(
self.test_context,
client_config,
java_class_name="org.apache.ignite.internal.ducktest.tests.pme_free_switch_test."
"SingleKeyTxStreamerApplication",
params={"cacheName": "test-cache", "warmup": 1000},
startup_timeout_sec=180)
single_key_tx_streamer.start()
ignites.stop_node(ignites.nodes[num_nodes - 1])
single_key_tx_streamer.await_event("Node left topology", 60, from_the_beginning=True)
if l_type is LoadType.LONG_TXS:
time.sleep(30) # keeping txs alive for 30 seconds.
long_tx_streamer.stop_async()
single_key_tx_streamer.await_event("Node left topology", 60, from_the_beginning=True)
single_key_tx_streamer.await_event("APPLICATION_STREAMED", 60) # waiting for streaming continuation.
single_key_tx_streamer.stop()
data["Worst latency (ms)"] = single_key_tx_streamer.extract_result("WORST_LATENCY")
data["Streamed txs"] = single_key_tx_streamer.extract_result("STREAMED")
data["Measure duration (ms)"] = single_key_tx_streamer.extract_result("MEASURE_DURATION")
data["Server nodes"] = num_nodes
return data
def test_logging(self, ignite_version):
"""
Tests logging goes to the correct file (consistency.log) when default AI config used.
"""
cfg_filename = "ignite-default-log4j.xml"
ignites = IgniteApplicationService(
self.test_context,
IgniteConfiguration(
version=IgniteVersion(ignite_version),
cluster_state="INACTIVE",
properties=self.PROPERTIES,
log4j_config=cfg_filename # default AI config (will be generated below)
),
java_class_name="org.apache.ignite.internal.ducktest.tests.control_utility.InconsistentNodeApplication",
params={
"cacheName": self.CACHE_NAME,
"amount": 1024,
"parts": 1,
"tx": False
},
startup_timeout_sec=180,
num_nodes=len(self.test_context.cluster))
for node in ignites.nodes: # copying default AI config with log path replacement
ignites.init_persistent(node)
cfg_file = f"{ignites.config_dir}/{cfg_filename}"
ignites.exec_command(node, f"cp {ignites.home_dir}/config/ignite-log4j.xml {cfg_file}")
orig = "${IGNITE_HOME}/work/log".replace('/', '\\/')
fixed = ignites.log_dir.replace('/', '\\/')
ignites.exec_command(node, f"sed -i 's/{orig}/{fixed}/g' {cfg_file}")
ignites.start()
control_utility = ControlUtility(ignites)
control_utility.activate()
ignites.await_event("APPLICATION_STREAMING_FINISHED", 60, from_the_beginning=True)
try:
control_utility.idle_verify() # making sure we have broken data
raise IgniteExecutionException("Fail.")
except AssertionError:
pass
control_utility.check_consistency(f"repair {self.CACHE_NAME} 0") # checking/repairing
message = "Cache consistency violations recorded."
ignites.await_event(message, 60, from_the_beginning=True, log_file="consistency.log")
try:
ignites.await_event(message, 10, from_the_beginning=True)
raise IgniteExecutionException("Fail.")
except TimeoutError:
pass