def state_nodes_leave(self, options='nodes_leave'):
"""
Kill nodes
"""
default_options = self.context['options']['actions'][options]
nodes_to_leave = int(default_options['nodes_count'])
killed_nodes = []
for i in range(nodes_to_leave):
nodes_to_delete = self.cluster.get_alive_default_nodes()
if self.context.get('temp_action_options'):
node_to_kill = int(self.context.get('temp_action_options')[0])
else:
node_to_kill = choice(nodes_to_delete)
self.context['attachments'].append({
'name': 'Kill info',
'source':
f'Kill node: {node_to_kill}\nKill time: {self.current_time_pretty}',
'type': 'text/plain'
})
killed_nodes.append(node_to_kill)
self.cluster.kill_node(node_to_kill)
server_nodes = len(self.cluster.get_alive_default_nodes()) + len(
self.cluster.get_alive_additional_nodes())
self.cluster.wait_for_topology_snapshot(server_num=server_nodes,
check_only_servers=True,
timeout=60)
util_sleep_for_a_while(5)
return killed_nodes
def _prepare_before_test(self, tx_loading, custom_event_name='test'):
util_sleep_for_a_while(self.warmup_time)
self.last_top = self.ignite.ignite_srvs.last_topology_snapshot()
self.last_topVer = max([_['ver'] for _ in self.last_top])
print_blue("Last topology version before %s: %d" %
(custom_event_name, self.last_topVer))
tx_loading.metrics_thread.add_custom_event(custom_event_name)
def state_nodes_join_existed(self,
options='nodes_join_existed',
action=None):
"""
Kill node and start again
"""
if action is None:
action = lambda node_id: None
self.context['attachments'].append({
'name': 'Join info',
'source': f'Join time: {self.current_time_pretty}',
'type': 'text/plain'
})
killed_nodes = self.state_nodes_leave(options)
util_sleep_for_a_while(5)
hosts = len(self.tiden.config['environment']["server_hosts"])
nodes_per_host = self.tiden.config['environment']["servers_per_host"]
for idx, killed_node in enumerate(killed_nodes):
action(killed_node)
self.cluster.start_node(killed_node, force=True)
self.cluster.wait_for_topology_snapshot(
server_num=hosts * nodes_per_host - len(killed_nodes) + idx +
1,
timeout=80,
check_only_servers=True)
def _wait_for_same_caches_size(self, piclient_master, piclient_replica, how_long=300, predicate=None):
from datetime import datetime
start = datetime.now()
iteration = 0
delay = 5
while True:
cache_mask = lambda x: '' in x
if predicate:
cache_mask = predicate
master_sizes = self.get_caches_size(cache_mask, piclient=piclient_master, debug=True)
replica_sizes = self.get_caches_size(cache_mask, piclient=piclient_replica, debug=True)
if master_sizes == replica_sizes:
break
self._compare_dicts(master_sizes, replica_sizes, debug=False)
util_sleep_for_a_while(delay)
iteration += 1
log_print('Waiting for {} seconds. Master size={}, replica size={}'.format(iteration * delay, master_sizes,
replica_sizes))
if (datetime.now() - start).seconds > how_long:
self._compare_dicts(master_sizes, replica_sizes)
raise TidenException('Caches size were not sync for {} seconds.'.format(how_long))
execution_time = (datetime.now() - start).seconds
log_print('Caches size have had sync for {} seconds.'.format(execution_time))
def util_get_restore_point(cls, seconds_ago=1):
from datetime import datetime, timedelta
from time import timezone
util_sleep_for_a_while(2)
time_format = "%Y-%m-%d-%H:%M:%S.%f"
restore_point = (datetime.now() - timedelta(seconds=seconds_ago))
# Hack to handle UTC timezone
if int(timezone / -(60 * 60)) == 0:
restore_point = restore_point + timedelta(hours=3)
return restore_point.strftime(time_format)[:-3]
def _measurements_after_test(self,
custom_event_name='test',
skip_exch=0,
skip_minor_exch=-1,
max_tries=100,
sleep_between_tries=10,
num_partitions=1024):
util_sleep_for_a_while(self.stabilization_time)
n_tries = 0
max_time = -1
while n_tries < max_tries:
n_tries += 1
max_time = self._get_last_exchange_time()
if self.exchange_finished:
break
util_sleep_for_a_while(sleep_between_tries)
self.new_top = self.ignite.ignite_srvs.last_topology_snapshot()
self.new_topVer = max([_['ver'] for _ in self.new_top])
print_blue("New topology version after %s: %d" %
(custom_event_name, self.new_topVer))
if max_time > 0:
for topVer in range(self.last_topVer + skip_exch,
self.new_topVer + 1):
for exch_topVer, exch_data in self.agg_exch_x1.items():
exch_maj_topVer = int(exch_topVer / 10000)
exch_min_topVer = exch_topVer - exch_maj_topVer * 10000
if exch_maj_topVer == topVer and exch_min_topVer > skip_minor_exch:
x1_time = self.agg_exch_x1[exch_topVer]['max_duration']
x2_time = self.agg_exch_x2[exch_topVer]['max_duration']
self._dump_exchange_time(
x1_time,
x2_time,
"%s [%d, %d]" % (custom_event_name,
exch_maj_topVer, exch_min_topVer),
num_partitions=num_partitions)
print_red(
"Exchange [%d, %d] during %s: %d msec, %d msec" %
(exch_maj_topVer, exch_min_topVer,
custom_event_name, x1_time, x2_time))
def test_during_rebalance(self):
ignite = self.start_ignite_grid(self.ignite_name)
ignite.cu.activate(activate_on_particular_node=1)
PiClientIgniteUtils.load_data_with_putall(ignite, self.client_config, )
util_sleep_for_a_while(30)
with PiClient(ignite, self.client_config) as piclient:
cache_to_test = 'test_cache_with_index'
# self.create_cache_with_indexed_data(cache_to_test)
client_ignite = piclient.get_ignite()
gateway = piclient.get_gateway()
cache_config = IgniteCacheConfig(gateway)
cache_config.set_name('cache_1')
cache_config.set_cache_mode('replicated')
cache_config.set_atomicity_mode('transactional')
cache_config.set_write_synchronization_mode('full_sync')
cache_config.set_affinity(False, 32)
cache_config.set_group_name('some_new_group')
cache_config1 = IgniteCacheConfig(gateway)
cache_config1.set_name(cache_to_test)
cache_config1.set_cache_mode('replicated')
cache_config1.set_atomicity_mode('transactional')
cache_config1.set_write_synchronization_mode('full_sync')
cache_config1.set_affinity(False, 32)
cache_config1.set_group_name('some_new_group')
# set query entities
caches = gateway.jvm.java.util.ArrayList()
caches.add(cache_config.get_config_object())
caches.add(cache_config1.get_config_object())
log_print("Creating caches", color='green')
client_ignite.getOrCreateCaches(caches)
cache_names = piclient.get_ignite().cacheNames().toArray()
if cache_to_test not in cache_names:
log_print("Could not find cache in %s" % cache_names, color='red')
util_sleep_for_a_while(10)
ignite.kill_node(2)
log_print("Overwrite values in cache %s" % cache_to_test, color='green')
operation = create_put_all_operation(cache_to_test, 1, 1001, 100,
value_type=ModelTypes.VALUE_ALL_TYPES_INDEXED.value)
operation.evaluate()
util_sleep_for_a_while(15)
ignite.start_node(2, skip_topology_check=True)
util_sleep_for_a_while(5)
client_ignite.cache(cache_to_test).destroy()
ignite.update_starting_node_attrs()
ignite.nodes[3]['status'] = NodeStatus.STARTED
client_ignite.cache('cache_1').destroy()
def calculate_checksum_and_validate(self, last_loaded_key, iteration=0):
diff_keys = {}
client_config = Ignite.config_builder.get_config(
'client', config_set_name='cluster_1_node_without_dr')
with PiClient(self.clusters[0].grid, client_config,
new_instance=True) as piclient:
checksum_master = create_distributed_checksum_operation().evaluate(
)
if COLLECT_KEYS:
cache_names = piclient.get_ignite().cacheNames().toArray()
diff_keys['master'] = {}
for cache in cache_names:
diff_keys['master'][cache] = []
iterator = piclient.get_ignite().cache(cache).iterator()
while iterator.hasNext():
diff_keys['master'][cache].append(
iterator.next().getKey())
diff_keys['master'][cache].sort()
client_config = Ignite.config_builder.get_config(
'client', config_set_name='cluster_2_node_without_dr')
with PiClient(self.clusters[1].grid, client_config,
new_instance=True) as piclient:
checksum_slave = create_distributed_checksum_operation().evaluate()
if COLLECT_KEYS:
cache_names = piclient.get_ignite().cacheNames().toArray()
diff_keys['replica'] = {}
for cache in cache_names:
diff_keys['replica'][cache] = []
iterator = piclient.get_ignite().cache(cache).iterator()
while iterator.hasNext():
diff_keys['replica'][cache].append(
iterator.next().getKey())
diff_keys['replica'][cache].sort()
if checksum_master != checksum_slave:
print(f"Checksum master\n{checksum_master}")
print(f"Checksum slave\n{checksum_slave}")
# if COLLECT_KEYS:
# pprint(diff_keys, width=240)
comparison = ''.join(difflib.Differ().compare(
checksum_master.splitlines(True),
checksum_slave.splitlines(True)))
if iteration > 3:
print(comparison)
# log_print('Unable to get checksum equality on both clusters after FST', color='blue')
#
# return
tiden_assert(
False,
'Unable to get checksum equality on both clusters after FST'
)
if iteration > 2:
if DO_FST_ON_DIFFERENCE:
caches_with_diff = []
for line in comparison.split('\n'):
val = None
if line.startswith('-'):
m = search('Cache.*\'(.*)\'.rows', line)
if m:
val = m.group(1)
if val:
caches_with_diff.append(val)
log_print(f'Difference detected in some caches.',
color='red')
log_print(f'Starting clear() caches: {caches_with_diff}.',
color='red')
client_config = Ignite.config_builder.get_config(
'client', config_set_name='cluster_2_node_without_dr')
with PiClient(self.clusters[1].grid,
client_config,
new_instance=True) as piclient:
ignite = piclient.get_ignite()
for cache in caches_with_diff:
ignite.cache(cache).clear()
# cache_names = piclient.get_ignite().cacheNames().toArray()
#
# print(list(
# create_checksum_operation(cache_name, 1, last_loaded_key).evaluate() for cache_name in
# cache_names))
util_sleep_for_a_while(RECHECK_CHECKSUM_TIMEOUT)
log_print(
f'Starting full state transfer on caches: {caches_with_diff}.',
color='red')
client_config = Ignite.config_builder.get_config(
'client', config_set_name='cluster_1_node_without_dr')
with PiClient(self.clusters[0].grid,
client_config,
new_instance=True) as piclient:
ignite = piclient.get_ignite()
try:
futs = []
for cache in caches_with_diff:
# TODO https://ggsystems.atlassian.net/browse/GG-22669
ignite.cache(cache)
futs.append(
ignite.plugin(
'GridGain').dr().stateTransfer(
cache, bytes([2])))
for fut in futs:
fut.get()
except Exception as e:
log_print(
'Exception caught on on FST\n{}'.format(e),
color='red')
log_print('Going to restart replication with FST',
color='yellow')
futs = []
for cache in caches_with_diff:
ignite.cache(cache)
ignite.plugin(
"GridGain").dr().startReplication(cache)
futs.append(
ignite.plugin(
'GridGain').dr().stateTransfer(
cache, bytes([2])))
for fut in futs:
fut.get()
util_sleep_for_a_while(FST_TIMEOUT)
log_print(
f'Going to collect checksum again after timeout - {RECHECK_CHECKSUM_TIMEOUT} seconds',
color='red')
util_sleep_for_a_while(RECHECK_CHECKSUM_TIMEOUT)
return self.calculate_checksum_and_validate(
last_loaded_key, iteration + 1)
return checksum_master, checksum_slave
def verify_cluster(self,
cluster_to_verify_id,
nodes_before,
last_loaded_key=None):
client_config = Ignite.config_builder.get_config(
'client', config_set_name='cluster_1_node_without_dr')
servers = 0
ignite = self.clusters[cluster_to_verify_id].grid
for i in range(3):
for res in ignite.last_topology_snapshot():
if res['servers'] > servers:
servers = res['servers']
else:
break
util_sleep_for_a_while(5)
if nodes_before != servers:
log_print(
f"There are missing nodes on cluster: Nodes in cluster: {servers} expecting {nodes_before}",
color='yellow')
self.verify_no_meaning_errors()
log_print("Wait for topology messages again.", color='yellow')
for node_id in ignite.get_all_default_nodes():
ignite.update_started_node_status(node_id)
log_print("Missing nodes case confirmed. Trying to restart node.",
color='red')
current_cluster_nodes = ignite.get_nodes_num('server')
if nodes_before != current_cluster_nodes:
log_print(f"Current nodes in cluster {current_cluster_nodes}")
nodes_to_start = []
for node_id in ignite.get_alive_default_nodes():
# assert that node is not dead otherwise kill/restart again
if not ignite.check_node_status(node_id):
log_print("Restarting node %s" % node_id,
color='yellow')
nodes_to_start.append(node_id)
log_print(f"Going to restart nodes: {nodes_to_start}",
color='debug')
for node_id in nodes_to_start:
ignite.start_node(node_id,
skip_nodes_check=True,
check_only_servers=True)
current_cluster_nodes = ignite.get_nodes_num('server')
if nodes_before != current_cluster_nodes:
log_print(
f"Current amount of nodes in cluster: {current_cluster_nodes}, expecting {nodes_before}",
color='debug')
for node_id in ignite.get_alive_default_nodes():
self.util_get_threads_from_jstack(
ignite, node_id, "FAILED")
assert False, "Failed to restart node"
ignite.cu.control_utility('--activate')
activate_failed = False
log_print('Check that there is no Error in activate logs',
color='yellow')
if 'Error' in ignite.cu.latest_utility_output:
activate_failed = True
log_print('Failed!', color='red')
sleep(5)
ignite.cu.control_utility('--baseline')
self.verify_no_meaning_errors()
log_print('Check that there is no Error in control.sh --baseline logs',
color='yellow')
if 'Error' in ignite.cu.latest_utility_output:
log_print('Failed! Second try after sleep 60 seconds', color='red')
sleep(60)
ignite.cu.control_utility('--baseline')
if 'Error' in ignite.cu.latest_utility_output or activate_failed:
log_print('Cluster looks hang.')
log_print('Check that there is no AssertionError in logs',
color='yellow')
self.verify_no_meaning_errors()
if last_loaded_key:
try:
new_last_key = last_loaded_key - int(
random.uniform(0, 1) * LOAD_DATA_SIZE)
log_print(
f'Trying to remove data from survivor caches ({new_last_key}, {last_loaded_key})',
color='yellow')
PiClientIgniteUtils.remove_data(
ignite,
client_config,
start_key=new_last_key,
end_key=last_loaded_key,
check_clients=False,
)
last_loaded_key = new_last_key
except Exception:
for node_id in ignite.get_alive_default_nodes():
self.util_get_threads_from_jstack(ignite, node_id,
"FAILED")
assert False, "Unable to connect client"
finally:
self.verify_no_meaning_errors()
util_sleep_for_a_while(MINI_DR_STABILIZATION_TIMEOUT)
checksum_master, checksum_slave = self.calculate_checksum_and_validate(
last_loaded_key)
tiden_assert(checksum_master == checksum_slave,
'Hash sum master and slave should be equal')
return last_loaded_key
def run_stress_restarts(self, cluster_id_to_restart, iterations,
nodes_to_restart, time_to_sleep_range):
client_config = self.preconfigure_cluster_0()
with PiClient(self.clusters[0].grid,
client_config,
jvm_options=['-ea']) as piclient:
ignite = piclient.get_ignite()
self.start_dynamic_caches_with_node_filter(client_config)
last_loaded_key = START_DATA_SIZE
PiClientIgniteUtils.load_data_with_putall(self.clusters[0].grid,
client_config,
end_key=last_loaded_key,
jvm_options=['-ea'],
check_clients=False)
util_sleep_for_a_while(MINI_DR_STABILIZATION_TIMEOUT)
nodes_before = 6
last_loaded_key += 1
for i in range(0, iterations):
log_print(f'Current iteration {i + 1} from {iterations}',
color='debug')
sleep_for_time = random.uniform(time_to_sleep_range[0],
time_to_sleep_range[1])
log_print(
f'In this run we are going to sleep for {sleep_for_time} seconds after each node restart',
color='green')
log_print('Trying to load data into created/existing caches',
color='yellow')
self.start_dynamic_caches_with_node_filter(client_config)
PiClientIgniteUtils.load_data_with_putall(
self.clusters[0].grid,
client_config,
start_key=last_loaded_key,
end_key=last_loaded_key + LOAD_DATA_SIZE,
jvm_options=['-ea'],
check_clients=False)
last_loaded_key += LOAD_DATA_SIZE
self.increment_atomic(ignite)
log_print("Round restart")
for node_id in nodes_to_restart:
self.clusters[cluster_id_to_restart].grid.kill_node(
node_id)
self.clusters[cluster_id_to_restart].grid.start_node(
node_id, skip_topology_check=True)
sleep(sleep_for_time)
log_print("Wait for topology messages")
for node_id in nodes_to_restart:
self.clusters[
cluster_id_to_restart].grid.update_started_node_status(
node_id)
util_sleep_for_a_while(MINI_DR_STABILIZATION_TIMEOUT)
last_loaded_key = self.verify_cluster(0, nodes_before,
last_loaded_key)
util_sleep_for_a_while(DR_STABILIZATION_TIMEOUT)
checksum_master1, checksum_slave1 = self.calculate_checksum_and_validate(
last_loaded_key)
tiden_assert(checksum_master1 == checksum_slave1,
'Hash sum master and slave not match')
self.put_data(self.clusters[1], 1, 'cluster_2_node_without_dr')
util_sleep_for_a_while(MINI_DR_STABILIZATION_TIMEOUT)
checksum_master2, checksum_slave2 = self.calculate_checksum_and_validate(
last_loaded_key)
tiden_assert(checksum_master2 == checksum_slave2,
'Hash sum master and slave not match')
def calculate_rebalance_speed(self, ignite, prod_runs, warmup_runs, last_end_key, keys_to_load,
tx_loading=None, version=None):
"""
:param ignite: Ignite app instance
:param prod_runs: production runs
:param warmup_runs: warmup runs
:param last_end_key: last inserted key (for historical rebalance)
:param tx_loading: TransactionalLoading instance (for rebalance under loading)
:param version: Ignite version (for non historical rebalance - full with/ and w/o loading)
:param keys_to_load: number of key to load 5kk ~ 4.5 Gb traffic
:return:
"""
assert ignite or self.historical_rebalance, \
'Error in scenario code! Ignite or historical_rebalance should be defined'
netstat = Netstat('dft', {}, self.test_class.ssh) # run netstat to collect network activity
rebalance_speed = list() # list of results for various runs
jmx = None
# Here we use artifact name to store LFS.
# This needed to measure same versions with different JVM options/configs
stored_lfs_name = 'rebalance_%s' % self.artifact_name
try:
warmup_time = 0
# Be careful here - JMX utility must be properly killed after test
jmx = JmxUtility(ignite)
jmx.start_utility()
for i in range(0, warmup_runs + prod_runs):
host = ignite.nodes[NODE_TO_REBALANCE]['host']
group_path = 'work/db/{}/cacheGroup-{}'.format(
ignite.get_node_consistent_id(NODE_TO_REBALANCE),
CACHE_GROUP
)
warmup_iteration = False if warmup_runs == 0 else i < warmup_runs
log_print("Running iteration %s (%s)" % (i, 'warmup' if warmup_iteration else 'prod'))
dir_size_before = get_nodes_directory_size(
ignite,
self.test_class.ssh,
group_path,
nodes=[NODE_TO_REBALANCE]
)
self.rebalance_event_write(i, tx_loading, 'iteration_{}: restart node'.format(i))
ignite.kill_node(NODE_TO_REBALANCE)
if self.historical_rebalance:
# For historical rebalance we store LFS for all cluster and then use this LFS to continue test
# There is some issues appears while restoring Ignite from stored LFS
# For 2.5.8 version for example there may be some failures
list_nodes_to_stop = list(node_id for node_id in ignite.nodes.keys()
if node_id < 100000 and node_id != NODE_TO_REBALANCE)
if not ignite.exists_stored_lfs(stored_lfs_name):
self.load_amount_of_data(ignite, GAGE_NODE, last_end_key, keys_to_load)
# Avoid topology changes by client exiting
# https://ggsystems.atlassian.net/browse/GG-21629
ignite.wait_for_topology_snapshot(client_num=0)
util_sleep_for_a_while(30)
ignite.cu.deactivate()
util_sleep_for_a_while(30)
ignite.kill_nodes(*list_nodes_to_stop)
ignite.save_lfs(stored_lfs_name)
ignite.start_nodes()
ignite.cu.activate()
else:
util_sleep_for_a_while(30)
ignite.cu.deactivate()
util_sleep_for_a_while(30)
ignite.kill_nodes(*list_nodes_to_stop)
ignite.delete_lfs()
ignite.restore_lfs(stored_lfs_name)
ignite.start_nodes()
else:
self._cleanup_lfs(ignite, NODE_TO_REBALANCE)
# sleep after cleanup LFS
sleep(10)
start_rcvd_bytes = netstat.network()[host]['total']['RX']['bytes']
start_sent_bytes = netstat.network()[host]['total']['TX']['bytes']
ignite.start_node(NODE_TO_REBALANCE)
log_print("Waiting for finish rebalance")
# To collect JFRs we need to define time to collect JFRs.
# Here we use warmup iteration rebalance time to set time
if warmup_time and not warmup_iteration and self.jfr_settings:
ignite.make_cluster_jfr(int(warmup_time) + LOADING_STABLE_TIME, self.jfr_settings)
rebalance_time = jmx.wait_for_finish_rebalance(
MAX_REBALANCE_TIMEOUT,
[CACHE_GROUP],
calculate_exact_time=True,
time_for_node=NODE_TO_REBALANCE,
log=False
)
self.rebalance_event_write(i, tx_loading, 'rebalance finished')
finish_rcvd_bytes = netstat.network()[host]['total']['RX']['bytes']
finish_sent_bytes = netstat.network()[host]['total']['TX']['bytes']
self.write_time_event('iteration_%s rebalance finished' % i)
# wait for checkpoint
sleep(CHECKPOINT_SLEEP)
dir_size_after = get_nodes_directory_size(
ignite,
self.test_class.ssh,
group_path,
nodes=[NODE_TO_REBALANCE]
)
if version:
log_print('Values for version: {}'.format(version))
rcvd_bytes = finish_rcvd_bytes - start_rcvd_bytes
avg_speed = float(rcvd_bytes) / rebalance_time
log_print("Current rebalance time in {} seconds (Directory size: before/after - {}/{}, "
"Network activity: send/received - {}/{})\nRebalance speed (received network/time): {}/sec"
.format(rebalance_time,
convert_size(dir_size_before),
convert_size(dir_size_after),
convert_size(finish_sent_bytes - start_sent_bytes, start_byte=True),
convert_size(rcvd_bytes, start_byte=True),
convert_size(avg_speed, start_byte=True)
),
color='yellow'
)
if not warmup_iteration:
if tx_loading:
sleep(LOADING_STABLE_TIME)
self.write_time_event('~JFR collection ended')
sleep(LOADING_STABLE_TIME)
rebalance_speed.append(avg_speed)
else:
warmup_time = rebalance_time
finally:
if jmx:
jmx.kill_utility()
return rebalance_speed
def assert_results(self, allowed_asserts: list, cycle_start: datetime,
start_time: datetime, base_name: str):
"""
Get all exceptions from all files on remote host
:param allowed_asserts: filters list [(method, args), (method, args), ... ]
:param cycle_start: latest full cluster clean
:param start_time: case start time
:param base_name: local directory name
"""
log_print('assert')
util_sleep_for_a_while(10)
to_write_all = []
# find all log files on remote host
logs_to_check = []
for host, results in self.cluster.ssh.exec(
[f"ls {self.tiden.config['rt']['remote']['test_dir']}"]).items():
for file_name in results[0].split('\n'):
if file_name.endswith('.log'):
logs_to_check.append({
'log_path':
f"{self.tiden.config['rt']['remote']['test_dir']}/{file_name}",
'name': file_name,
'host': host
})
# go through all patterns with errors
found_fails = self.cluster.find_fails(
files_to_check=logs_to_check,
store_files=self.tiden.config['rt']['test_module_dir'],
ignore_node_ids=True)
for file_name, fail_info in found_fails.items():
exceptions_on_this_cycle = False
for exceptions_info in fail_info['exceptions']:
found_time: datetime = exceptions_info.get('time')
if found_time and found_time > start_time:
if not exceptions_on_this_cycle:
exceptions_on_this_cycle = True
valid_exception = False
for assert_item in allowed_asserts:
if assert_item['method'](*assert_item['args'],
exceptions_info['exception']):
valid_exception = True
break
if not valid_exception:
to_write_all.append({
'time':
found_time,
'file':
file_name,
'text':
'\n'.join(exceptions_info['exception'])
})
elif exceptions_on_this_cycle:
to_write_all.append({
'file':
file_name,
'text':
'\n'.join(exceptions_info['exception'])
})
if to_write_all:
# aggregate fails in one file
self.context['step_failed'] = 'Found exceptions in logs!'
self.context['report']['status'] = 'failed'
log_print("Found exceptions!", color='red')
fails_text = ''
with_time = [i for i in to_write_all if i.get('time')]
with_time.sort(key=lambda i: i['time'])
for exception_with_time in with_time:
fails_text += f'\n\n{exception_with_time["file"]} {exception_with_time["time"].isoformat()}\n' \
f'{exception_with_time["text"]}\n'
without_time = [i for i in to_write_all if not i.get('time')]
without_time.sort(key=lambda i: i['file'])
for exception_without_time in without_time:
fails_text += f'\n\n{exception_without_time["file"]}\n' \
f'{exception_without_time["text"]}\n'
if self.tiden.config['combinations'].get('save_attachments'):
run_dir = self.ensure_dir(
join(self.tiden.config['rt']['test_module_dir'],
cycle_start.isoformat()))
combine_run_dir = self.ensure_dir(join(run_dir, base_name))
with open(join(combine_run_dir, 'fails.log'), 'w') as f:
f.write(fails_text)
files_unique_name = f'{uuid4()}_fails.log'
files_receiver_url = self.tiden.config['environment'].get(
'report_files_url')
if files_receiver_url:
post(f'{files_receiver_url}/files/add',
headers={'filename': files_unique_name},
files={'file': fails_text.encode('utf-8')})
self.context['attachments'].append({
'name': 'fails.log',
'source': files_unique_name,
'type': 'file'
})
self.download_run_logs()
log_print('end up with assert')