def test_another_cache_with_same_id():
"""
title: Test for creating another cache device with the same ID.
description: |
Checking if adding another cache device and setting
the same cache ID as the previous one fails.
pass_criteria:
- No additional cache device added.
"""
with TestRun.step("Start cache with ID = 1"):
cache_dev_1 = TestRun.disks["cache_1"]
cache_dev_1.create_partitions([Size(2, Unit.GibiByte)])
TestRun.executor.run_expect_success(
cli.start_cmd(cache_dev_1.partitions[0].system_path,
cache_id="1",
force=True))
with TestRun.step("Try to start another cache with the same ID = 1"):
cache_dev_2 = TestRun.disks["cache_2"]
cache_dev_2.create_partitions([Size(2, Unit.GibiByte)])
TestRun.executor.run_expect_fail(
cli.start_cmd(cache_dev_2.partitions[0].system_path,
cache_id="1",
force=True))
with TestRun.step("Stop all caches"):
casadm.stop_all_caches()
def test_fault_power_hit_init(cache_mode):
"""
title: Test with power hit and verification of metadata initialization after it.
description: |
Test if there will be metadata initialization after wake up
- when starting cache with initialization.
pass_criteria:
- Start cache with initialization works correctly after power hit.
"""
with TestRun.step("Prepare CAS device."):
cache_disk = TestRun.disks['cache']
core_disk = TestRun.disks['core']
cache_disk.create_partitions([Size(2, Unit.GibiByte)])
core_disk.create_partitions([Size(2, Unit.GibiByte)])
cache_dev = cache_disk.partitions[0]
core_dev = core_disk.partitions[0]
cache = casadm.start_cache(cache_dev, cache_mode, force=True)
core = cache.add_core(core_dev)
cache_device_link = cache_dev.get_device_link("/dev/disk/by-id")
with TestRun.step("Hard reset."):
power_control = TestRun.plugin_manager.get_plugin('power_control')
power_control.power_cycle()
with TestRun.step("Start cache with re-initialization."):
cache_dev.system_path = cache_device_link.get_target()
TestRun.executor.run_expect_success(
cli.start_cmd(cache_dev=str(cache_dev.system_path),
cache_mode=str(cache_mode.name.lower()),
force=True,
load=False))
TestRun.LOGGER.info(
f"Successful assembly cache device with initialization")
def test_cli_load_and_force():
"""
title: Test if it is possible to use start command with 'load' and 'force' flag at once
description: |
Try to start cache with 'load' and 'force' options at the same time
and check if it is not possible to do
pass_criteria:
- Start cache command with both 'force' and 'load' options should fail
- Proper message should be received
"""
with TestRun.step("Prepare cache."):
cache_device = TestRun.disks['cache']
cache_device.create_partitions([Size(50, Unit.MebiByte)])
cache_device = cache_device.partitions[0]
cache = casadm.start_cache(cache_device)
with TestRun.step("Stop cache."):
cache.stop()
with TestRun.step("Try to load cache with 'force'."):
output = TestRun.executor.run(
start_cmd(cache_dev=cache_device.system_path,
force=True,
load=True))
if output.exit_code == 0:
TestRun.fail("Loading cache with 'force' option should fail.")
cli_messages.check_stderr_msg(output, cli_messages.load_and_force)
def test_stop_no_flush_load_cache(cache_mode, filesystem):
"""
title: Test to check that 'stop --no-data-flush' command works correctly.
description: |
Negative test of the ability of CAS to load unflushed cache on core device
with filesystem. Test uses lazy flush cache modes.
pass_criteria:
- No system crash while load cache.
- Starting cache without loading metadata fails.
- Starting cache with loading metadata finishes with success.
"""
with TestRun.step("Prepare cache and core devices."):
cache_part, core_part = prepare()
with TestRun.step("Start cache."):
cache = casadm.start_cache(cache_part, cache_mode, force=True)
with TestRun.step("Change cleaning policy to NOP."):
cache.set_cleaning_policy(CleaningPolicy.nop)
with TestRun.step(
f"Add core with {filesystem.name} filesystem to cache and mount it."
):
core_part.create_filesystem(filesystem)
core = cache.add_core(core_part)
core.mount(mount_point)
with TestRun.step(
f"Create test file in mount point of exported object and check its md5 sum."
):
test_file = fs_utils.create_random_test_file(test_file_path,
Size(48, Unit.MebiByte))
test_file_md5_before = test_file.md5sum()
with TestRun.step("Unmount exported object."):
core.unmount()
with TestRun.step("Count dirty blocks on exported object."):
dirty_blocks_before = core.get_dirty_blocks()
with TestRun.step("Stop cache with option '--no-data-flush'."):
cache.stop(no_data_flush=True)
caches_count = len(casadm_parser.get_caches())
if caches_count != 0:
TestRun.fail(
f"Expected caches count: 0; Actual caches count: {caches_count}."
)
with TestRun.step("Try to start cache without loading metadata."):
output = TestRun.executor.run_expect_fail(
cli.start_cmd(cache_dev=str(cache_part.path),
cache_mode=str(cache_mode.name.lower()),
force=False,
load=False))
cli_messages.check_stderr_msg(
output, cli_messages.start_cache_with_existing_metadata)
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache.cache_device)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(
f"Expected caches count: 1 Actual caches count: {caches_count}."
)
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 1:
TestRun.fail(
f"Expected cores count: 1; Actual cores count: {cores_count}.")
with TestRun.step(
"Compare dirty blocks number before and after loading cache."):
if dirty_blocks_before != core.get_dirty_blocks():
TestRun.LOGGER.error(
"Dirty blocks number is different than before loading cache.")
with TestRun.step("Mount exported object."):
core.mount(mount_point)
with TestRun.step(
"Compare md5 sum of test file before and after loading cache."):
if test_file_md5_before != test_file.md5sum():
TestRun.LOGGER.error(
"Test file's md5 sum is different than before loading cache.")
with TestRun.step("Unmount exported object."):
core.unmount()
with TestRun.step("Stop cache."):
casadm.stop_all_caches()
def test_stop_no_flush_load_cache_no_fs(cache_mode):
"""
title: Test to check that 'stop --no-data-flush' command works correctly.
description: |
Negative test of the ability of CAS to load unflushed cache on core device
without filesystem. Test uses lazy flush cache modes.
pass_criteria:
- No system crash while load cache.
- Starting cache without loading metadata fails.
- Starting cache with loading metadata finishes with success.
"""
with TestRun.step("Prepare cache and core devices."):
cache_part, core_part = prepare()
with TestRun.step("Start cache with --force option."):
cache = casadm.start_cache(cache_part, cache_mode, force=True)
with TestRun.step("Change cleaning policy to NOP."):
cache.set_cleaning_policy(CleaningPolicy.nop)
with TestRun.step("Add core device without filesystem."):
core_part.wipe_filesystem()
core = cache.add_core(core_part)
with TestRun.step("Fill exported object with data."):
dd = (Dd().input("/dev/zero").output(core.path).block_size(
Size(1, Unit.Blocks4096)).oflag("direct"))
dd.run()
with TestRun.step("Count dirty blocks on exported object."):
dirty_blocks_before = core.get_dirty_blocks()
with TestRun.step("Stop cache with option '--no-data-flush'."):
cache.stop(no_data_flush=True)
caches_count = len(casadm_parser.get_caches())
if caches_count != 0:
TestRun.fail(
f"Expected caches count: 0; Actual caches count: {caches_count}."
)
with TestRun.step("Try to start cache without loading metadata."):
output = TestRun.executor.run_expect_fail(
cli.start_cmd(cache_dev=str(cache_part.path),
cache_mode=str(cache_mode.name.lower()),
force=False,
load=False))
cli_messages.check_stderr_msg(
output, cli_messages.start_cache_with_existing_metadata)
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache.cache_device)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(
f"Expected caches count: 1 Actual caches count: {caches_count}."
)
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 1:
TestRun.fail(
f"Expected cores count: 1; Actual cores count: {cores_count}.")
with TestRun.step(
"Compare dirty blocks number before and after loading cache."):
if dirty_blocks_before != core.get_dirty_blocks():
TestRun.LOGGER.error(
"Dirty blocks number is different than before loading cache.")
with TestRun.step("Stop cache."):
casadm.stop_all_caches()
def test_user_cli():
"""
title: Test that OpenCAS does not allow to change parameters in CLI by non-root user.
description: |
Checking if changing parameters in CLI by non-root user is forbidden by OpenCAS,
but is permitted with 'sudo' command.
pass_criteria:
- Non-root user can only print help and CAS version.
- Sudoer user is allowed to change OpenCAS parameters in CLI with sudo.
"""
with TestRun.step("Prepare cache and core devices."):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions([Size(256, Unit.MebiByte)])
cache_dev = cache_dev.partitions[0]
core_dev = TestRun.disks['core']
core_dev.create_partitions([Size(1, Unit.GibiByte), Size(256, Unit.MebiByte)])
core_part1 = core_dev.partitions[0]
core_part2 = core_dev.partitions[1]
with TestRun.step("Start cache."):
cache = casadm.start_cache(cache_dev, force=True)
with TestRun.step("Add core to cache and mount it."):
core_part1.create_filesystem(Filesystem.ext3)
core = cache.add_core(core_part1)
core.mount(mount_point)
with TestRun.step(f"Copy casadm bin from {system_casadm_bin_path} "
f"to {user_casadm_bin_dest_path}."):
casadm_bin = fs_utils.parse_ls_output(fs_utils.ls_item(f"{system_casadm_bin_path}"))[0]
casadm_bin_copy = casadm_bin.copy(user_casadm_bin_dest_path, True)
casadm_bin_copy.chmod_numerical(777)
with TestRun.step("Copy IO class config."):
io_conf = fs_utils.parse_ls_output(fs_utils.ls_item(f"{ioclass_config_path}"))[0]
io_conf_copy = io_conf.copy(ioclass_config_copy_path, force=True)
with TestRun.step("Unmount core."):
core.unmount()
with TestRun.step("Stop cache."):
casadm.stop_all_caches()
with TestRun.step("Add non-root user account."):
TestRun.executor.run(f"useradd -N -r -l {user_name}")
user_home_dir = fs_utils.parse_ls_output(fs_utils.ls_item(f"/home/{user_name}"))[0]
user_home_dir.chmod_numerical(777, True)
with TestRun.step("Try to start cache."):
try:
output = run_as_other_user(cli.start_cmd(cache_dev.path), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Starting cache should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot start cache.")
with TestRun.step("Start cache again."):
casadm.load_cache(cache_dev)
with TestRun.step("Try to stop cache."):
try:
output = run_as_other_user(cli.stop_cmd(str(cache.cache_id)), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Stopping cache should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot stop cache.")
with TestRun.step("Try to set cache mode."):
try:
output = run_as_other_user(cli.set_cache_mode_cmd(CacheMode.WB,
str(cache.cache_id)), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Setting cache mode should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot set cache mode.")
with TestRun.step("Try to add core to cache."):
try:
output = run_as_other_user(cli.add_core_cmd(str(cache.cache_id),
core_part2.path), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Adding core to cache should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot add core.")
with TestRun.step("Try to remove core from cache."):
try:
output = run_as_other_user(cli.remove_core_cmd(str(cache.cache_id),
str(core.core_id)), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Removing core from cache should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot remove core.")
with TestRun.step("Try to zero metadata."):
try:
output = run_as_other_user(cli.zero_metadata_cmd(str(cache_dev)), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Zeroing metadata should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot zero metadata.")
with TestRun.step("Try to list caches."):
try:
output = run_as_other_user(cli.list_cmd(), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Listing caches should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot list caches.")
with TestRun.step("Try to print stats."):
try:
output = run_as_other_user(cli.print_statistics_cmd(str(cache.cache_id)), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Printing stats should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot print statistics.")
with TestRun.step("Try to reset stats."):
try:
output = run_as_other_user(cli.reset_counters_cmd(str(cache.cache_id)), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Resetting stats should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot reset statistics.")
with TestRun.step("Try to flush cache."):
try:
output = run_as_other_user(cli.flush_cache_cmd(str(cache.cache_id)), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Flushing cache should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot flush cache.")
with TestRun.step("Try to flush core."):
try:
output = run_as_other_user(cli.flush_core_cmd(str(cache.cache_id),
str(core.core_id)), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Flushing core should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot flush core.")
with TestRun.step("Try to set cleaning policy and its parameters."):
try:
output = run_as_other_user(cli.set_param_cleaning_cmd(
str(cache.cache_id), "nop"), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Setting cleaning policy should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot set cleaning policy as nop.")
try:
output = run_as_other_user(cli.set_param_cleaning_cmd(
str(cache.cache_id), "alru"), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Setting cleaning policy should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot set cleaning policy as alru.")
try:
output = run_as_other_user(cli.set_param_cleaning_alru_cmd(str(cache.cache_id),
"15",
"60",
"1000",
"8000"), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Setting cleaning policy parameters should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot set alru cleaning policy parameters.")
try:
output = run_as_other_user(cli.set_param_cleaning_cmd(
str(cache.cache_id), "acp"), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Setting cleaning policy should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot set cleaning policy as acp.")
try:
output = run_as_other_user(cli.set_param_cleaning_acp_cmd(str(cache.cache_id),
"15",
"1000"), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Setting cleaning policy parameters should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot set acp cleaning policy parameters.")
with TestRun.step("Try to list IO class configuration."):
try:
output = run_as_other_user(cli.list_io_classes_cmd(
str(cache.cache_id), OutputFormat.table.name), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Listing IO class configuration should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot list IO class configuration.")
with TestRun.step("Try to load IO class configuration."):
try:
output = run_as_other_user(cli.load_io_classes_cmd(
str(cache.cache_id), io_conf_copy), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Loading IO class configuration should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot load IO class configuration.")
with TestRun.step("Try to print help for casadm."):
try:
run_as_other_user(cli.help_cmd(), user_name)
except CmdException:
TestRun.LOGGER.error("Non-root user should be able to print help for casadm.")
with TestRun.step("Try to print version of OpenCAS."):
try:
run_as_other_user(cli.version_cmd(), user_name)
except CmdException:
TestRun.LOGGER.error("Non-root user should be able to print version of OpenCAS.")
with TestRun.step("Add non-root user account to sudoers group."):
TestRun.executor.run(f'echo "{user_name} ALL = (root) NOPASSWD:ALL" '
f'| sudo tee /etc/sudoers.d/{user_name}')
with TestRun.step("Try to stop cache with 'sudo'."):
try:
run_as_other_user(cli.stop_cmd(str(cache.cache_id)), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to stop cache.")
with TestRun.step("Try to start cache with 'sudo'."):
try:
run_as_other_user(cli.start_cmd(cache_dev.path, force=True), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to start cache.")
with TestRun.step("Try to set cache mode with 'sudo'."):
try:
run_as_other_user(
cli.set_cache_mode_cmd(str(CacheMode.WB.name).lower(), str(cache.cache_id)),
user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to set cache mode.")
with TestRun.step("Try to add core to cache with 'sudo'."):
try:
run_as_other_user(cli.add_core_cmd(str(cache.cache_id),
core_part1.path), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to add core to cache.")
with TestRun.step("Try to list caches with 'sudo'."):
try:
run_as_other_user(cli.list_cmd(), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to list caches.")
with TestRun.step("Try to print stats with 'sudo'."):
try:
run_as_other_user(cli.print_statistics_cmd(str(cache.cache_id)), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to print stats.")
with TestRun.step("Try to reset stats with 'sudo'."):
try:
run_as_other_user(cli.reset_counters_cmd(str(cache.cache_id)), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to reset stats.")
with TestRun.step("Try to flush cache with 'sudo'."):
try:
run_as_other_user(cli.flush_cache_cmd(str(cache.cache_id)), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to flush cache.")
with TestRun.step("Try to flush core with 'sudo'."):
try:
run_as_other_user(cli.flush_core_cmd(str(cache.cache_id),
str(core.core_id)), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to flush core.")
with TestRun.step("Try to set cleaning policy and its parameters with 'sudo'."):
try:
run_as_other_user(cli.set_param_cleaning_cmd(str(cache.cache_id), "nop"),
user_name, True)
run_as_other_user(cli.set_param_cleaning_cmd(str(cache.cache_id), "alru"),
user_name, True)
try:
run_as_other_user(cli.set_param_cleaning_alru_cmd(str(cache.cache_id),
"15",
"60",
"1000",
"8000"), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to "
"set alru cleaning policy parameters.")
run_as_other_user(cli.set_param_cleaning_cmd(str(cache.cache_id), "acp"),
user_name, True)
try:
run_as_other_user(cli.set_param_cleaning_acp_cmd(str(cache.cache_id),
"15",
"1000"), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to "
"set acp cleaning policy parameters.")
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to "
"set cleaning policy and its parameters.")
with TestRun.step("Try to list IO class with 'sudo'."):
try:
run_as_other_user(cli.list_io_classes_cmd(str(cache.cache_id), OutputFormat.table.name),
user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to list IO class.")
with TestRun.step("Try to load IO class configuration with 'sudo'."):
try:
run_as_other_user(cli.load_io_classes_cmd(str(cache.cache_id), io_conf_copy),
user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to "
"load IO class configuration.")
with TestRun.step("Try to remove core from cache with 'sudo'."):
try:
run_as_other_user(cli.remove_core_cmd(str(cache.cache_id), str(core.core_id)),
user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to remove core from cache.")
with TestRun.step("Try to zero metadata with 'sudo'."):
try:
run_as_other_user(cli.zero_metadata_cmd(str(cache_dev)),
user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to zero metadata.")
with TestRun.step("Try to print help for casadm with 'sudo'."):
try:
run_as_other_user(cli.help_cmd(), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to print help for casadm.")
with TestRun.step("Try to print version of OpenCAS with 'sudo'."):
try:
run_as_other_user(cli.version_cmd(), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to print version of OpenCAS.")
with TestRun.step("Stop caches."):
casadm.stop_all_caches()
with TestRun.step("Remove user account."):
TestRun.executor.run(f"userdel -r -Z {user_name}")
def test_fault_power_hit(cache_mode):
"""
title: Test with power hit.
description: |
Test if there will be no metadata initialization after wake up
- when starting cache without initialization.
pass_criteria:
- Start cache without re-initialization failed.
- Start cache with load works correctly.
- Expected message found in log.
"""
with TestRun.step("Prepare CAS device."):
cache_disk = TestRun.disks['cache']
core_disk = TestRun.disks['core']
cache_disk.create_partitions([Size(2, Unit.GibiByte)])
core_disk.create_partitions([Size(2, Unit.GibiByte)])
cache_dev = cache_disk.partitions[0]
core_dev = core_disk.partitions[0]
cache = casadm.start_cache(cache_dev, cache_mode, force=True)
core = cache.add_core(core_dev)
with TestRun.step("Mark log lines for later validation of new entries."):
last_read_line = 1
log_lines = TestRun.executor.run_expect_success(
f"tail -qn +{last_read_line} {log_path}").stdout.splitlines()
last_read_line += len(log_lines)
with TestRun.step("Hard reset."):
power_control = TestRun.plugin_manager.get_plugin('power_control')
power_control.power_cycle()
with TestRun.step("Start cache without re-initialization."):
output = TestRun.executor.run_expect_fail(
cli.start_cmd(cache_dev=str(cache_dev.path),
cache_mode=str(cache_mode.name.lower()),
force=False,
load=False))
if cli_messages.check_stderr_msg(output, cli_messages.error_inserting_cache) and \
cli_messages.check_stderr_msg(output,
cli_messages.reinitialize_with_force_or_recovery):
TestRun.LOGGER.info(
f"Found expected exception: {cli_messages.error_inserting_cache}"
f" and {cli_messages.reinitialize_with_force_or_recovery}")
with TestRun.step("Start cache with load."):
try:
cache = casadm.load_cache(cache_dev)
TestRun.LOGGER.info(
f"Cache device loaded correctly (as expected).")
except CmdException as e:
TestRun.LOGGER.fail(
f"Failed to load cache device. Exception: {e.output}")
time.sleep(wait_short_time)
message_found = check_log(
last_read_line, cli_messages.reinitialize_with_force_or_recovery)
# check log second time in case that operation logging would take some more time
if not message_found:
time.sleep(wait_long_time)
result = check_log(
last_read_line,
cli_messages.reinitialize_with_force_or_recovery)
if not result:
TestRun.LOGGER.fail(
f"Haven't found expected message in the log.")
