def snapshot_test(self, ignite_version):
"""
Basic snapshot test.
"""
version = IgniteVersion(ignite_version)
ignite_config = IgniteConfiguration(
version=version,
data_storage=DataStorageConfiguration(default=DataRegionConfiguration(persistent=True)),
metric_exporter='org.apache.ignite.spi.metric.jmx.JmxMetricExporterSpi'
)
nodes = IgniteService(self.test_context, ignite_config, num_nodes=len(self.test_context.cluster) - 1)
nodes.start()
control_utility = ControlUtility(nodes)
control_utility.activate()
loader_config = IgniteConfiguration(client_mode=True, version=version, discovery_spi=from_ignite_cluster(nodes))
loader = IgniteApplicationService(
self.test_context,
loader_config,
java_class_name="org.apache.ignite.internal.ducktest.tests.snapshot_test.DataLoaderApplication",
params={"start": 0, "cacheName": self.CACHE_NAME, "interval": 500_000, "valueSizeKb": 1}
)
def start_cell(self,
version,
jvm_opts,
discovery_spi=None,
modules=None,
nodes_cnt=NODES_PER_CELL):
"""
Starts cell.
"""
ignites = IgniteService(
self.test_context,
IgniteConfiguration(
version=IgniteVersion(version),
properties=self.properties(),
cluster_state="INACTIVE",
failure_detection_timeout=self.FAILURE_DETECTION_TIMEOUT,
discovery_spi=TcpDiscoverySpi()
if discovery_spi is None else discovery_spi),
num_nodes=nodes_cnt,
modules=modules,
jvm_opts=jvm_opts,
startup_timeout_sec=180)
ignites.start_async()
return ignites
def __start_ignite_nodes(self,
version,
num_nodes,
timeout_sec=60,
join_cluster=None):
config = IgniteConfiguration(
cluster_state="INACTIVE",
version=IgniteVersion(version),
data_storage=DataStorageConfiguration(
default=DataRegionConfiguration(name='persistent',
persistence_enabled=True),
regions=[
DataRegionConfiguration(name='in-memory',
persistence_enabled=False,
max_size=100 * 1024 * 1024)
]))
if join_cluster:
config._replace(discovery_spi=from_ignite_cluster(join_cluster))
servers = IgniteService(self.test_context,
config=config,
num_nodes=num_nodes,
startup_timeout_sec=timeout_sec)
servers.start()
return servers
def test_node_join(self, ignite_version, backups, cache_count, entry_count,
entry_size, preloaders, thread_pool_size, batch_size,
batches_prefetch_count, throttle):
"""
Tests rebalance on node join.
"""
reb_params = RebalanceParams(
trigger_event=TriggerEvent.NODE_JOIN,
backups=backups,
cache_count=cache_count,
entry_count=entry_count,
entry_size=entry_size,
preloaders=preloaders,
thread_pool_size=thread_pool_size,
batch_size=batch_size,
batches_prefetch_count=batches_prefetch_count,
throttle=throttle,
persistent=True)
ignites = start_ignite(self.test_context, ignite_version, reb_params)
control_utility = ControlUtility(ignites)
control_utility.activate()
preload_time = preload_data(
self.test_context,
ignites.config._replace(
client_mode=True, discovery_spi=from_ignite_cluster(ignites)),
rebalance_params=reb_params)
new_node = IgniteService(
self.test_context,
ignites.config._replace(
discovery_spi=from_ignite_cluster(ignites)),
num_nodes=1)
new_node.start()
control_utility.add_to_baseline(new_node.nodes)
await_and_check_rebalance(new_node)
nodes = ignites.nodes.copy()
nodes.append(new_node.nodes[0])
result = get_result(new_node.nodes, preload_time, cache_count,
entry_count, entry_size)
control_utility.deactivate()
self.logger.debug(
f'DB size after rebalance: {get_database_size_mb(nodes, ignites.database_dir)}'
)
return result
def start_servers(test_context, num_nodes, ignite_config, modules=None):
"""
Start ignite servers.
"""
servers = IgniteService(
test_context,
config=ignite_config,
num_nodes=num_nodes,
modules=modules,
# mute spam in log.
jvm_opts=["-DIGNITE_DUMP_THREADS_ON_FAILURE=false"])
start = monotonic()
servers.start()
return servers, round(monotonic() - start, 1)
def __start_ignite_nodes(self, version, num_nodes, timeout_sec=60):
config = IgniteConfiguration(cluster_state="ACTIVE",
version=IgniteVersion(version),
caches=[
CacheConfiguration(
name=self.CACHE_NAME,
atomicity_mode='TRANSACTIONAL')
])
servers = IgniteService(self.test_context,
config=config,
num_nodes=num_nodes,
startup_timeout_sec=timeout_sec)
servers.start()
return servers
def await_and_check_rebalance(service: IgniteService, rebalance_nodes: list = None, is_full: bool = True):
"""
Check rebalance.
:param service: IgniteService.
:param rebalance_nodes: Ignite nodes in which rebalance will be awaited.
:param is_full: Expected type of rebalancing.
"""
nodes = rebalance_nodes if rebalance_nodes else service.alive_nodes
await_rebalance_start(service)
service.await_rebalance()
check_type_of_rebalancing(nodes, is_full=is_full)
ControlUtility(service).idle_verify()
def test_config_add_to_result(self, ignite_version, is_ignite_service):
"""
Test that the config file is in config directory
and Service.logs contains the config directory to add to the result.
"""
ignite_cfg = IgniteConfiguration(version=IgniteVersion(ignite_version))
if is_ignite_service:
ignite = IgniteService(self.test_context, ignite_cfg, num_nodes=1)
else:
ignite = IgniteApplicationService(
self.test_context,
ignite_cfg,
java_class_name=
"org.apache.ignite.internal.ducktest.tests.self_test.TestKillableApplication"
)
ignite.start()
assert ignite.logs.get('config').get('path') == ignite.config_dir
assert ignite.config_file.startswith(ignite.config_dir)
ignite.nodes[0].account.ssh(
f'ls {ignite.config_dir} | grep {os.path.basename(ignite.config_file)}'
)
ignite.nodes[0].account.ssh(
f'ls {ignite.config_dir} | grep {os.path.basename(ignite.log_config_file)}'
)
ignite.stop()
def test_change_users(self, ignite_version):
"""
Test add, update and remove user
"""
config = IgniteConfiguration(
cluster_state="INACTIVE",
auth_enabled=True,
version=IgniteVersion(ignite_version),
data_storage=DataStorageConfiguration(
default=DataRegionConfiguration(persistent=True)),
client_connector_configuration=ClientConnectorConfiguration())
servers = IgniteService(self.test_context,
config=config,
num_nodes=self.NUM_NODES - 1)
servers.start()
ControlUtility(cluster=servers,
username=DEFAULT_AUTH_USERNAME,
password=DEFAULT_AUTH_PASSWORD).activate()
client_cfg = IgniteThinClientConfiguration(
addresses=servers.nodes[0].account.hostname + ":" +
str(config.client_connector_configuration.port),
version=IgniteVersion(ignite_version),
username=DEFAULT_AUTH_USERNAME,
password=DEFAULT_AUTH_PASSWORD)
# Add new user
check_authenticate(servers, TEST_USERNAME, TEST_PASSWORD, True)
self.run_with_creds(client_cfg, ADD_USER, TEST_USERNAME, TEST_PASSWORD)
check_authenticate(servers, TEST_USERNAME, TEST_PASSWORD)
# Update user password
check_authenticate(servers, TEST_USERNAME, TEST_PASSWORD2, True)
self.run_with_creds(client_cfg, UPDATE_USER, TEST_USERNAME,
TEST_PASSWORD2)
check_authenticate(servers, TEST_USERNAME, TEST_PASSWORD, True)
check_authenticate(servers, TEST_USERNAME, TEST_PASSWORD2)
# Remove user
self.run_with_creds(client_cfg, REMOVE_USER, TEST_USERNAME, free=False)
check_authenticate(servers, TEST_USERNAME, TEST_PASSWORD2, True)
def start_ignite(test_context, ignite_version: str,
rebalance_params: RebalanceParams) -> IgniteService:
"""
Start IgniteService:
:param test_context: Test context.
:param ignite_version: Ignite version.
:param rebalance_params: Rebalance parameters.
:return: IgniteService.
"""
node_count = test_context.available_cluster_size - rebalance_params.preloaders
if rebalance_params.persistent:
data_storage = DataStorageConfiguration(
max_wal_archive_size=2 * rebalance_params.data_region_max_size,
default=DataRegionConfiguration(
persistence_enabled=True,
max_size=rebalance_params.data_region_max_size))
else:
data_storage = DataStorageConfiguration(
default=DataRegionConfiguration(
max_size=rebalance_params.data_region_max_size))
node_config = IgniteConfiguration(
version=IgniteVersion(ignite_version),
data_storage=data_storage,
metric_exporters={
"org.apache.ignite.spi.metric.jmx.JmxMetricExporterSpi"
},
rebalance_thread_pool_size=rebalance_params.thread_pool_size,
rebalance_batch_size=rebalance_params.batch_size,
rebalance_batches_prefetch_count=rebalance_params.
batches_prefetch_count,
rebalance_throttle=rebalance_params.throttle)
ignites = IgniteService(test_context,
config=node_config,
num_nodes=node_count if
rebalance_params.trigger_event else node_count - 1,
jvm_opts=rebalance_params.jvm_opts)
ignites.start()
return ignites
def __run(self, ignite_version, trigger_event, backups, cache_count,
entry_count, entry_size, preloaders, thread_pool_size,
batch_size, batches_prefetch_count, throttle):
"""
Test performs rebalance test which consists of following steps:
* Start cluster.
* Put data to it via IgniteClientApp.
* Triggering a rebalance event and awaits for rebalance to finish.
:param ignite_version: Ignite version.
:param trigger_event: Trigger event.
:param backups: Backup count.
:param cache_count: Cache count.
:param entry_count: Cache entry count.
:param entry_size: Cache entry size.
:param preloaders: Preload application nodes count.
:param thread_pool_size: rebalanceThreadPoolSize config property.
:param batch_size: rebalanceBatchSize config property.
:param batches_prefetch_count: rebalanceBatchesPrefetchCount config property.
:param throttle: rebalanceThrottle config property.
:return: Rebalance and data preload stats.
"""
reb_params = RebalanceParams(
trigger_event=trigger_event,
backups=backups,
cache_count=cache_count,
entry_count=entry_count,
entry_size=entry_size,
preloaders=preloaders,
thread_pool_size=thread_pool_size,
batch_size=batch_size,
batches_prefetch_count=batches_prefetch_count,
throttle=throttle)
ignites = start_ignite(self.test_context, ignite_version, reb_params)
preload_time = preload_data(
self.test_context,
ignites.config._replace(
client_mode=True, discovery_spi=from_ignite_cluster(ignites)),
rebalance_params=reb_params)
if trigger_event:
ignites.stop_node(ignites.nodes[-1])
rebalance_nodes = ignites.nodes[:-1]
else:
ignite = IgniteService(
self.test_context,
ignites.config._replace(
discovery_spi=from_ignite_cluster(ignites)),
num_nodes=1)
ignite.start()
rebalance_nodes = ignite.nodes
await_rebalance_start(ignite)
ignite.await_rebalance()
return get_result(rebalance_nodes, preload_time, cache_count,
entry_count, entry_size)
def test_ignite_start_stop(self, ignite_version):
"""
Test that IgniteService correctly start and stop
"""
ignite = IgniteService(
self.test_context,
IgniteConfiguration(version=IgniteVersion(ignite_version)),
num_nodes=1)
print(self.test_context)
ignite.start()
ignite.stop()
def test_logs_rotation(self, ignite_version):
"""
Test logs rotation after ignite service restart.
"""
def get_log_lines_count(service, filename):
node = service.nodes[0]
log_file = os.path.join(service.log_dir, filename)
log_cnt = list(
node.account.ssh_capture(f'cat {log_file} | wc -l',
callback=int))[0]
return log_cnt
def get_logs_count(service):
node = service.nodes[0]
return list(
node.account.ssh_capture(
f'ls {service.log_dir}/ignite.log* | wc -l',
callback=int))[0]
ignites = IgniteService(
self.test_context,
IgniteConfiguration(version=IgniteVersion(ignite_version)),
num_nodes=1)
ignites.start()
num_restarts = 6
for i in range(num_restarts - 1):
ignites.stop()
old_cnt = get_log_lines_count(ignites, "ignite.log")
assert old_cnt > 0
ignites.start(clean=False)
new_cnt = get_log_lines_count(ignites, "ignite.log")
assert new_cnt > 0
# check that there is no new entry in rotated file
assert old_cnt == get_log_lines_count(ignites,
f"ignite.log.{i + 1}")
assert get_logs_count(ignites) == num_restarts
def await_rebalance_start(service: IgniteService, timeout: int = 30):
"""
Awaits rebalance starting on any test-cache on any node.
:param service: IgniteService in which rebalance start will be awaited.
:param timeout: Rebalance start await timeout.
:return: dictionary of two keypairs with keys "node" and "time",
where "node" contains the first node on which rebalance start was detected
and "time" contains the time when rebalance was started.
"""
for node in service.alive_nodes:
try:
rebalance_start_time = service.get_event_time_on_node(
node, "Starting rebalance routine", timeout=timeout)
except TimeoutError:
continue
else:
return rebalance_start_time
raise RuntimeError("Rebalance start was not detected on any node")
def test_server_config_options(self, ignite_version):
"""
Test to make sure non-default non-trivial ignite node configuration XML file is generated correctly.
"""
ignite = IgniteService(self.test_context,
get_server_config(ignite_version),
1,
jvm_opts="-DCELL=1")
ignite.start()
control_utility = ControlUtility(ignite)
control_utility.activate()
ignite.stop()
def test_ssl_connection(self, ignite_version):
"""
Test that IgniteService, IgniteApplicationService correctly start and stop with ssl configurations.
And check ControlUtility with ssl arguments.
"""
shared_root = get_shared_root_path(self.test_context.globals)
server_ssl = SslParams(shared_root,
key_store_jks=DEFAULT_SERVER_KEYSTORE)
server_configuration = IgniteConfiguration(
version=IgniteVersion(ignite_version),
ssl_params=server_ssl,
connector_configuration=ConnectorConfiguration(
ssl_enabled=True, ssl_params=server_ssl))
ignite = IgniteService(self.test_context,
server_configuration,
num_nodes=2,
startup_timeout_sec=180)
client_configuration = server_configuration._replace(
client_mode=True,
ssl_params=SslParams(shared_root,
key_store_jks=DEFAULT_CLIENT_KEYSTORE),
connector_configuration=None)
app = IgniteApplicationService(
self.test_context,
client_configuration,
java_class_name=
"org.apache.ignite.internal.ducktest.tests.smoke_test.SimpleApplication",
startup_timeout_sec=180)
admin_ssl = SslParams(shared_root,
key_store_jks=DEFAULT_ADMIN_KEYSTORE)
control_utility = ControlUtility(cluster=ignite, ssl_params=admin_ssl)
ignite.start()
app.start()
control_utility.cluster_state()
app.stop()
ignite.stop()
def test_thin_client_compatibility(self, server_version, thin_client_version):
"""
Thin client compatibility test.
"""
server_config = IgniteConfiguration(version=IgniteVersion(server_version),
client_connector_configuration=ClientConnectorConfiguration())
ignite = IgniteService(self.test_context, server_config, 1)
addresses = ignite.nodes[0].account.hostname + ":" + str(server_config.client_connector_configuration.port)
thin_clients = IgniteApplicationService(self.test_context,
IgniteThinClientConfiguration(addresses=addresses,
version=IgniteVersion(
thin_client_version)),
java_class_name=self.JAVA_CLIENT_CLASS_NAME,
num_nodes=1)
ignite.start()
thin_clients.run()
ignite.stop()
def test_ignite_app_start_stop(self, ignite_version):
"""
Test that IgniteService and IgniteApplicationService correctly start and stop
"""
server_configuration = IgniteConfiguration(
version=IgniteVersion(ignite_version))
ignite = IgniteService(self.test_context,
server_configuration,
num_nodes=1)
client_configuration = server_configuration._replace(
client_mode=True, discovery_spi=from_ignite_cluster(ignite))
app = IgniteApplicationService(
self.test_context,
client_configuration,
java_class_name=
"org.apache.ignite.internal.ducktest.tests.smoke_test.SimpleApplication"
)
ignite.start()
app.start()
app.stop()
ignite.stop()
def test_simple_services_start_stop(self, ignite_version):
"""
Tests plain services start and stop (termitation vs self-terination).
"""
ignites = IgniteService(
self.test_context,
IgniteConfiguration(version=IgniteVersion(ignite_version)),
num_nodes=1)
ignites.start()
client = IgniteService(
self.test_context,
IgniteClientConfiguration(version=IgniteVersion(ignite_version)),
num_nodes=1)
client.start()
node1 = IgniteApplicationService(
self.test_context,
IgniteClientConfiguration(
version=IgniteVersion(ignite_version),
discovery_spi=from_ignite_cluster(ignites)),
java_class_name=
"org.apache.ignite.internal.ducktest.tests.self_test.TestKillableApplication",
startup_timeout_sec=180)
node2 = IgniteApplicationService(
self.test_context,
IgniteClientConfiguration(
version=IgniteVersion(ignite_version),
discovery_spi=from_ignite_cluster(ignites)),
java_class_name=
"org.apache.ignite.internal.ducktest.tests.self_test.TestSelfKillableApplication",
startup_timeout_sec=180)
node1.start()
node2.run()
node1.stop()
client.stop()
ignites.stop()
def test(self, ignite_version, load_type):
"""
Tests PME-free switch scenario (node stop).
"""
data = {}
caches = [CacheConfiguration(name='test-cache', backups=2, atomicity_mode='TRANSACTIONAL')]
l_type = LoadType.construct_from(load_type)
# Checking PME (before 2.8) vs PME-free (2.8+) switch duration, but
# focusing on switch duration (which depends on caches amount) when long_txs is false and
# on waiting for previously started txs before the switch (which depends on txs duration) when long_txs of true.
if l_type is LoadType.EXTRA_CACHES:
for idx in range(1, self.EXTRA_CACHES_AMOUNT):
caches.append(CacheConfiguration(name="cache-%d" % idx, backups=2, atomicity_mode='TRANSACTIONAL'))
config = IgniteConfiguration(version=IgniteVersion(ignite_version), caches=caches, cluster_state="INACTIVE")
num_nodes = len(self.test_context.cluster) - 2
self.test_context.logger.info("Nodes amount calculated as %d." % num_nodes)
ignites = IgniteService(self.test_context, config, num_nodes=num_nodes)
ignites.start()
if IgniteVersion(ignite_version) >= V_2_8_0:
ControlUtility(ignites).disable_baseline_auto_adjust()
ControlUtility(ignites).activate()
client_config = config._replace(client_mode=True,
discovery_spi=from_ignite_cluster(ignites, slice(0, num_nodes - 1)))
long_tx_streamer = IgniteApplicationService(
self.test_context,
client_config,
java_class_name="org.apache.ignite.internal.ducktest.tests.pme_free_switch_test.LongTxStreamerApplication",
params={"cacheName": "test-cache"},
startup_timeout_sec=180)
if l_type is LoadType.LONG_TXS:
long_tx_streamer.start()
single_key_tx_streamer = IgniteApplicationService(
self.test_context,
client_config,
java_class_name="org.apache.ignite.internal.ducktest.tests.pme_free_switch_test."
"SingleKeyTxStreamerApplication",
params={"cacheName": "test-cache", "warmup": 1000},
startup_timeout_sec=180)
single_key_tx_streamer.start()
ignites.stop_node(ignites.nodes[num_nodes - 1])
single_key_tx_streamer.await_event("Node left topology", 60, from_the_beginning=True)
if l_type is LoadType.LONG_TXS:
time.sleep(30) # keeping txs alive for 30 seconds.
long_tx_streamer.stop_async()
single_key_tx_streamer.await_event("Node left topology", 60, from_the_beginning=True)
single_key_tx_streamer.await_event("APPLICATION_STREAMED", 60) # waiting for streaming continuation.
single_key_tx_streamer.stop()
data["Worst latency (ms)"] = single_key_tx_streamer.extract_result("WORST_LATENCY")
data["Streamed txs"] = single_key_tx_streamer.extract_result("STREAMED")
data["Measure duration (ms)"] = single_key_tx_streamer.extract_result("MEASURE_DURATION")
data["Server nodes"] = num_nodes
return data
def ignite_start_stop(self, ignite_version, graceful_shutdown, nodes_num,
static_clients_num, temp_client, iteration_count,
client_work_time):
"""
Test for starting and stopping fat clients.
"""
servers_count = nodes_num - static_clients_num - temp_client
current_top_v = servers_count
# Topology version after test.
fin_top_ver = servers_count + (2 * static_clients_num) + (
2 * iteration_count * temp_client)
server_cfg = IgniteConfiguration(
version=IgniteVersion(ignite_version),
caches=[
CacheConfiguration(name=self.CACHE_NAME,
backups=1,
atomicity_mode='TRANSACTIONAL')
])
ignite = IgniteService(self.test_context,
server_cfg,
num_nodes=servers_count)
control_utility = ControlUtility(ignite)
client_cfg = server_cfg._replace(client_mode=True)
static_clients = IgniteApplicationService(
self.test_context,
client_cfg,
java_class_name=self.JAVA_CLIENT_CLASS_NAME,
num_nodes=static_clients_num,
params={
"cacheName": self.CACHE_NAME,
"pacing": self.PACING
})
temp_clients = IgniteApplicationService(
self.test_context,
client_cfg,
java_class_name=self.JAVA_CLIENT_CLASS_NAME,
num_nodes=temp_client,
params={
"cacheName": self.CACHE_NAME,
"pacing": self.PACING
})
ignite.start()
static_clients.start()
current_top_v += static_clients_num
check_topology(control_utility, current_top_v)
# Start / stop temp_clients node. Check cluster.
for i in range(iteration_count):
self.logger.info(f'Starting iteration: {i}.')
temp_clients.start()
current_top_v += temp_client
await_event(static_clients, f'ver={current_top_v}, locNode=')
check_topology(control_utility, current_top_v)
await_event(temp_clients,
f'clients={static_clients_num + temp_client}')
time.sleep(client_work_time)
if graceful_shutdown:
temp_clients.stop()
else:
temp_clients.kill()
current_top_v += temp_client
await_event(static_clients, f'ver={current_top_v}, locNode=')
static_clients.stop()
check_topology(control_utility, fin_top_ver)
