def test_mixed_cluster_load_caches_old_server(self):
"""
1. start mixed cluster (new version servers + old version servers)
2. activate from new version control.sh
3. start old version server
4. add it to baseline
5. smoke check:
5.1. create dynamic caches from old server node
5.2. do some load from old server node
"""
self.ignite_new_version.cu.activate()
created_caches = []
self.server_config = Ignite.config_builder.get_config(
'server', config_set_name='base')
ignite = self.ignite_old_version
with PiClient(ignite, self.server_config, nodes_num=1) as piclient:
ignite.cu.add_node_to_baseline(
ignite.get_node_consistent_id(piclient.node_ids[0]))
dynamic_caches_factory = DynamicCachesFactory()
async_ops = []
for method in dynamic_caches_factory.dynamic_cache_configs:
cache_name = "cache_group_%s" % method
log_print('Loading {}...'.format(cache_name), color='green')
ignite = piclient.get_ignite()
ignite.getOrCreateCache(
getattr(dynamic_caches_factory, method)(cache_name))
async_operation = create_async_operation(
create_put_all_operation,
cache_name,
1,
1001,
10,
value_type=self.data_model)
async_ops.append(async_operation)
async_operation.evaluate()
created_caches.append(cache_name)
log_print('Waiting async results...', color='debug')
# wait for streamer to complete
for async_op in async_ops:
async_op.getResult()
with TransactionalLoading(MixedTestLoadingAdapter(self),
config_file=self.server_config,
loading_profile=LoadingProfile(
delay=1,
transaction_timeout=100000)):
sleep(60)
def test_old_cluster_load_caches_new_client(self):
"""
1. start old version grid
2. activate from old version control.sh
3. start new version client
4. smoke check:
4.1. create dynamic caches
4.2. do some load
"""
created_caches = []
self.ignite_old_version.cu.activate()
with PiClient(self.ignite_new_version, self.client_config,
nodes_num=1) as piclient:
dynamic_caches_factory = DynamicCachesFactory()
async_ops = []
for method in dynamic_caches_factory.dynamic_cache_configs:
cache_name = "cache_group_%s" % method
log_print('Loading {}...'.format(cache_name), color='green')
piclient.get_ignite().getOrCreateCache(
getattr(dynamic_caches_factory, method)(cache_name))
async_operation = create_async_operation(
create_put_all_operation,
cache_name,
1,
1001,
10,
value_type=self.data_model)
async_ops.append(async_operation)
async_operation.evaluate()
created_caches.append(cache_name)
log_print('Waiting async results...', color='debug')
# wait for streamer to complete
for async_op in async_ops:
async_op.getResult()
with TransactionalLoading(MixedTestLoadingAdapter(self),
config_file=self.client_config,
loading_profile=LoadingProfile(
delay=1,
transaction_timeout=100000)):
sleep(60)
def test_pme_bench_dynamic_caches_client(self):
self.start_grid_no_activate()
self.ignite.jmx.activate(1)
self.ignite.cu.set_current_topology_as_baseline()
self.load_data_with_streamer(0,
1 * self.LOAD_FACTOR,
ignite=self.ignite.ignite_cli_load,
nodes_num=self.num_loading_nodes)
expected_total_num_clients = len(self.ignite.get_all_client_nodes() +
self.ignite.get_all_common_nodes())
num_clients = 32
# start num_clients client nodes on 'flaky' hosts
with PiClient(self.ignite.ignite_cli_flaky,
self.get_client_config(),
nodes_num=num_clients,
new_instance=True) as piclient:
self.ignite.ignite_srvs.wait_for_topology_snapshot(
client_num=expected_total_num_clients + num_clients)
from itertools import cycle
client_nodes = cycle(
self.ignite.ignite_cli_flaky.get_all_common_nodes() +
self.ignite.ignite_cli_flaky.get_all_client_nodes())
# data_model = ModelTypes.VALUE_ALL_TYPES.value
# created_caches = []
dynamic_caches_factory = DynamicCachesFactory()
# async_ops = []
for method in dynamic_caches_factory.dynamic_cache_configs:
cache_name = "cache_group_%s" % method
node_id = next(client_nodes)
print('Start %s on node %s ... ' % (cache_name, node_id))
gateway = piclient.get_gateway(node_id)
ignite = piclient.get_ignite(node_id)
ignite.getOrCreateCache(
getattr(dynamic_caches_factory, method)(cache_name,
gateway=gateway))
def _create_dynamic_caches_with_data(self, with_index=False):
log_print("Create dynamic caches and load data")
data_model = ModelTypes.VALUE_ALL_TYPES.value
created_caches = []
with PiClient(self.ignite, self.get_client_config(),
nodes_num=1) as piclient:
dynamic_caches_factory = DynamicCachesFactory()
async_ops = []
for method in dynamic_caches_factory.dynamic_cache_configs:
cache_name = "cache_group_%s" % method
print_green('Loading %s...' % cache_name)
gateway = piclient.get_gateway()
ignite = piclient.get_ignite()
ignite.getOrCreateCache(
getattr(dynamic_caches_factory, method)(cache_name,
gateway=gateway))
if with_index:
data_model = ModelTypes.VALUE_ALL_TYPES.value
async_operation = create_async_operation(
create_streamer_operation,
cache_name,
1,
self.max_key + 2,
value_type=data_model)
async_ops.append(async_operation)
async_operation.evaluate()
created_caches.append(cache_name)
log_print('Waiting async results...', color='blue')
# wait for streamer to complete
for async_op in async_ops:
async_op.getResult()
log_print("Dynamic caches with data created")
return created_caches
def _calc_checksums_over_dynamic_caches(self):
log_print("Calculating checksums")
with PiClient(self.ignite, self.get_client_config()):
dynamic_caches_factory = DynamicCachesFactory()
async_operations = []
for method in dynamic_caches_factory.dynamic_cache_configs:
cache_name = "cache_group_%s" % method
checksum_operation = create_checksum_operation(
cache_name, 1, 1000)
async_operation = create_async_operation(checksum_operation)
async_operations.append(async_operation)
async_operation.evaluate()
checksums = ''
for async_operation in async_operations:
checksums += str(async_operation.getResult())
log_print("Calculating checksums done")
return checksums
def test_24_fitness_two_clients_with_snapshot(self):
"""
"""
created_caches = []
self.ignite_new_version.cu.activate()
with PiClient(self.ignite_new_version, self.client_config,
nodes_num=1) as piclient_new:
dynamic_caches_factory = DynamicCachesFactory()
async_ops = []
for method in dynamic_caches_factory.dynamic_cache_configs:
cache_name = "cache_group_%s" % method
log_print('Loading {}...'.format(cache_name), color='green')
piclient_new.get_ignite().getOrCreateCache(
getattr(dynamic_caches_factory, method)(cache_name))
async_operation = create_async_operation(
create_put_all_operation,
cache_name,
1,
1001,
10,
value_type=self.data_model)
async_ops.append(async_operation)
async_operation.evaluate()
created_caches.append(cache_name)
log_print('Waiting async results...', color='debug')
# wait for streamer to complete
for async_op in async_ops:
async_op.getResult()
with PiClient(self.ignite_old_version,
self.client_config,
nodes_num=2,
new_instance=True) as piclient_old:
log_print('Rolling upgrade', color='green')
self.ignite_new_version.su.snapshot_utility(
'snapshot', '-type=full')
log_print('Snapshot completed', color='debug')
sorted_cache_names = []
for cache_name in piclient_old.get_ignite().cacheNames(
).toArray():
sorted_cache_names.append(cache_name)
sorted_cache_names.sort()
async_operations = []
cache_operation = {}
cache_checksum = {}
for cache_name in sorted_cache_names:
async_operation = create_async_operation(
create_checksum_operation, cache_name, 1, 10000)
async_operations.append(async_operation)
cache_operation[async_operation] = cache_name
async_operation.evaluate()
checksums = ''
for async_operation in async_operations:
result = str(async_operation.getResult())
cache_checksum[cache_operation.get(
async_operation)] = result
checksums += result
log_print(checksums, color='debug')
self.ignite_new_version.su.snapshot_utility(
'restore', '-id={}'.format(
self.ignite_new_version.su.get_created_snapshot_id(1)))
log_print('Test completed', color='debug')
def test_24_fitness_set_baseline_with_properties(self):
"""
This test checks the cluster behaviour with option GG_DISABLE_SNAPSHOT_ON_BASELINE_CHANGE_WITH_ENABLED_PITR
that could be set in different ways:
1. Set at one of the server nodes.
2. Set on some client node/nodes.
"""
created_caches = []
self.ignite_old_version.cu.activate()
# Preloading
with PiClient(self.ignite_new_version, self.client_config,
nodes_num=1) as piclient:
dynamic_caches_factory = DynamicCachesFactory()
async_ops = []
for method in dynamic_caches_factory.dynamic_cache_configs:
cache_name = "cache_group_%s" % method
log_print('Loading {}...'.format(cache_name), color='green')
piclient.get_ignite().getOrCreateCache(
getattr(dynamic_caches_factory, method)(cache_name))
async_operation = create_async_operation(
create_put_all_operation,
cache_name,
1,
1001,
10,
value_type=self.data_model)
async_ops.append(async_operation)
async_operation.evaluate()
created_caches.append(cache_name)
log_print('Waiting async results...', color='debug')
# wait for streamer to complete
for async_op in async_ops:
async_op.getResult()
util_sleep_for_a_while(20)
new_client_config = Ignite.config_builder.get_config(
'client', config_set_name='24_fit_with_consist_id')
jvm_options = self.ignite_new_version.get_jvm_options(1)
jvm_options.append(
'-DGG_DISABLE_SNAPSHOT_ON_BASELINE_CHANGE_WITH_ENABLED_PITR=true')
# with PiClient(self.ignite_new_version, self.client_config, jvm_options=jvm_options, nodes_num=1) as piclient:
with PiClient(self.ignite_new_version, self.client_config,
nodes_num=1) as piclient:
for i in range(1, 5):
self.ignite_old_version.cu.control_utility('--baseline')
log_print('Stopping node {}'.format(i), color='green')
jvm_options = self.ignite_new_version.get_jvm_options(i)
jvm_options.append(
'-DGG_DISABLE_SNAPSHOT_ON_BASELINE_CHANGE_WITH_ENABLED_PITR=false'
)
self.ignite_new_version.set_node_option(
'*', 'config',
Ignite.config_builder.get_config(
'server', config_set_name='24_fit_with_consist_id'))
log_print("Starting node {} with new consistent id".format(i),
color='debug')
self.ignite_new_version.start_nodes(i,
already_nodes=4,
other_nodes=4,
timeout=240)
log_print("Changing baseline", color='debug')
self.ignite_old_version.cu.set_current_topology_as_baseline()
util_sleep_for_a_while(60,
msg='Wait for rebalance to completed')
log_print('Test is done')
def test_24_fitness_rolling_upgrade(self):
"""
This test checks the main rolling upgrade scenario under the load:
1. Old cluster up and running (consistent_id's are not set).
2. First cycle (upgrade to new version and set property
GG_DISABLE_SNAPSHOT_ON_BASELINE_CHANGE_WITH_ENABLED_PITR):
3. Second cycle (set correct consistent_id with adding to baseline topology).
"""
created_caches = []
self.ignite_old_version.cu.activate()
with PiClient(self.ignite_new_version, self.client_config,
nodes_num=1) as piclient:
dynamic_caches_factory = DynamicCachesFactory()
async_ops = []
for method in dynamic_caches_factory.dynamic_cache_configs:
cache_name = "cache_group_%s" % method
log_print('Loading {}...'.format(cache_name), color='green')
piclient.get_ignite().getOrCreateCache(
getattr(dynamic_caches_factory, method)(cache_name))
async_operation = create_async_operation(
create_put_all_operation,
cache_name,
1,
1001,
10,
value_type=self.data_model)
async_ops.append(async_operation)
async_operation.evaluate()
created_caches.append(cache_name)
log_print('Waiting async results...', color='debug')
# wait for streamer to complete
for async_op in async_ops:
async_op.getResult()
util_sleep_for_a_while(60)
with PiClient(self.ignite_old_version, self.client_config,
nodes_num=4) as piclient:
cache_names = piclient.get_ignite().cacheNames()
# Start transaction loading for TTL caches
with TransactionalLoading(MixedTestLoadingAdapter(self),
config_file=self.client_config,
loading_profile=LoadingProfile(
delay=0,
transaction_timeout=100000,
run_for_seconds=600)):
util_sleep_for_a_while(20)
log_print('Rolling upgrade', color='green')
async_ops = []
for cache_name in [
cache_name for cache_name in cache_names.toArray()
if cache_name.startswith("M2_PRODUCT")
]:
async_operation = create_async_operation(
create_put_all_operation,
cache_name,
1001,
400001,
10,
value_type=ModelTypes.VALUE_ALL_TYPES.value)
async_ops.append(async_operation)
async_operation.evaluate()
# First cycle: upgrade version and set property.
for i in range(1, 5):
self.ignite_old_version.cu.control_utility('--baseline')
log_print('Stopping node {}'.format(i), color='green')
self.ignite_old_version.kill_nodes(i)
self.ignite_new_version.cleanup_work_dir(i)
folder = self.ignite_old_version.get_work_dir(i)
log_print(folder, color='debug')
self.ignite_new_version.copy_work_dir_from(i, folder)
jvm_options = self.ignite_new_version.get_jvm_options(i)
jvm_options.append(
'-DGG_DISABLE_SNAPSHOT_ON_BASELINE_CHANGE_WITH_ENABLED_PITR=true'
)
util_sleep_for_a_while(10)
self.ignite_new_version.start_nodes(i,
already_nodes=(4 - i),
other_nodes=(4 - i),
timeout=240)
self.ignite_new_version.cu.control_utility('--baseline')
for async_op in async_ops:
async_op.getResult()
util_sleep_for_a_while(30)
log_print('Change consistent ID', color='green')
self.ignite_new_version.set_node_option(
'*', 'config',
Ignite.config_builder.get_config(
'server', config_set_name='24_fit_with_consist_id'))
# Second cycle - change consistent_id and add to baseline topology.
for i in range(1, 5):
self.ignite_new_version.cu.control_utility('--baseline')
log_print('Stopping node {}'.format(i), color='green')
self.ignite_new_version.kill_nodes(i)
log_print(
"Starting node {} with new consistent id".format(i),
color='debug')
self.ignite_new_version.start_nodes(i, timeout=240)
log_print("Changing baseline", color='debug')
self.ignite_new_version.cu.set_current_topology_as_baseline(
)
util_sleep_for_a_while(
60, msg='Wait for rebalance to completed')
log_print('Transactional loading done', color='green')
# Just to check client node still can interact with cluster - calculate checksum from client node.
sorted_cache_names = []
for cache_name in piclient.get_ignite().cacheNames().toArray():
sorted_cache_names.append(cache_name)
sorted_cache_names.sort()
async_operations = []
cache_operation = {}
for cache_name in sorted_cache_names:
async_operation = create_async_operation(
create_checksum_operation, cache_name, 1, 10000)
async_operations.append(async_operation)
cache_operation[async_operation] = cache_name
async_operation.evaluate()
checksums = ''
cache_checksum = {}
for async_operation in async_operations:
result = str(async_operation.getResult())
cache_checksum[cache_operation.get(async_operation)] = result
checksums += result
log_print('Calculating checksums done')