def test_cli_load_and_force():
"""
title: Test if it is possible to use start command with 'load' and 'force' flag at once
description: |
Try to start cache with 'load' and 'force' options at the same time
and check if it is not possible to do
pass_criteria:
- Start cache command with both 'force' and 'load' options should fail
- Proper message should be received
"""
with TestRun.step("Prepare cache."):
cache_device = TestRun.disks['cache']
cache_device.create_partitions([Size(50, Unit.MebiByte)])
cache_device = cache_device.partitions[0]
cache = casadm.start_cache(cache_device)
with TestRun.step("Stop cache."):
cache.stop()
with TestRun.step("Try to load cache with 'force'."):
output = TestRun.executor.run(
start_cmd(cache_dev=cache_device.system_path,
force=True,
load=True))
if output.exit_code == 0:
TestRun.fail("Loading cache with 'force' option should fail.")
cli_messages.check_stderr_msg(output, cli_messages.load_and_force)
def compare_capabilities(cache_device, core_device, cache, core, msg):
if core is None:
cli_messages.check_stderr_msg(msg,
cli_messages.try_add_core_sector_size_mismatch)
else:
core_dev_sectors_num = \
disk_utils.get_size(core_device.get_device_id()) / disk_utils.get_block_size(
core_device.get_device_id())
core_sectors_num = disk_utils.get_size(core.get_device_id()) / disk_utils.get_block_size(
core.get_device_id())
if core_dev_sectors_num != core_sectors_num:
TestRun.LOGGER.error(
"Number of sectors in CAS device and attached core device is different.")
cache.stop()
return
cas_capabilities = measure_capabilities(core)
cache_dev_capabilities = measure_capabilities(cache_device)
core_dev_capabilities = measure_capabilities(core_device)
for (capability, method) in capabilities.items():
cas_val = cas_capabilities[capability]
cache_val = cache_dev_capabilities[capability]
core_val = core_dev_capabilities[capability]
comparison_val = method(core_val, cache_val) if method is not None else core_val
if comparison_val != cas_val:
TestRun.LOGGER.error(f"Cas device {capability} is not set properly. Is: {cas_val}, "
f"should be {comparison_val} (cache: {cache_val}, "
f"core: {core_val})")
continue
TestRun.LOGGER.info(f"Cas device {capability} has proper value: {cas_val} "
f"(cache: {cache_val}, core: {core_val})")
cache.stop()
def try_stop_incomplete_cache(cache):
try:
cache.stop()
except CmdException as e:
TestRun.LOGGER.info(
"Stopping cache without 'no data flush' option is blocked as expected."
)
cli_messages.check_stderr_msg(e.output,
cli_messages.stop_cache_incomplete)
def test_ioclass_stats_basic(random_cls):
"""
title: Basic test for retrieving IO class statistics.
description: |
Check if statistics are retrieved only for configured IO classes.
pass_criteria:
- Statistics are retrieved for configured IO classes.
- Error is displayed when retrieving statistics for non-configured IO class.
- Error is displayed when retrieving statistics for out of range IO class id.
"""
min_ioclass_id = 11
max_ioclass_id = 21
with TestRun.step("Test prepare"):
prepare(random_cls)
with TestRun.step("Prepare IO class config file"):
ioclass_list = []
for class_id in range(min_ioclass_id, max_ioclass_id):
ioclass_list.append(IoClass(
class_id=class_id,
rule=f"file_size:le:{4096 * class_id}&done",
priority=22
))
IoClass.save_list_to_config_file(ioclass_list, True)
with TestRun.step("Load IO class config file"):
casadm.load_io_classes(cache_id, file=ioclass_config.default_config_file_path)
with TestRun.step("Try retrieving IO class stats for all allowed id values "
"and one out of range id"):
for class_id in range(ioclass_config.MAX_IO_CLASS_ID + 2):
out_of_range = " out of range" if class_id > ioclass_config.MAX_IO_CLASS_ID else ""
with TestRun.group(f"Checking{out_of_range} IO class id {class_id}..."):
expected = class_id == 0 or class_id in range(min_ioclass_id, max_ioclass_id)
try:
casadm.print_statistics(
cache_id=cache_id,
io_class_id=class_id,
per_io_class=True)
if not expected:
TestRun.LOGGER.error(
f"Stats retrieved for not configured IO class {class_id}")
except CmdException as e:
if expected:
TestRun.LOGGER.error(f"Stats not retrieved for IO class id: {class_id}")
elif class_id <= ioclass_config.MAX_IO_CLASS_ID:
if not check_stderr_msg(e.output, get_stats_ioclass_id_not_configured):
TestRun.LOGGER.error(
f"Wrong message for unused IO class id: {class_id}")
elif not check_stderr_msg(e.output, get_stats_ioclass_id_out_of_range):
TestRun.LOGGER.error(
f"Wrong message for out of range IO class id: {class_id}")
def compare_capabilities(cache_device, core_device, cache, core, msg):
if core is None:
cli_messages.check_stderr_msg(
msg, cli_messages.try_add_core_sector_size_mismatch)
else:
core_dev_sectors_num = \
disk_utils.get_size(core_device.get_device_id()) / disk_utils.get_block_size(
core_device.get_device_id())
core_sectors_num = disk_utils.get_size(
core.get_device_id()) / disk_utils.get_block_size(
core.get_device_id())
if core_dev_sectors_num != core_sectors_num:
TestRun.LOGGER.error(
"Number of sectors in CAS device and attached core device is different."
)
cache.stop()
return
cas_capabilities = measure_capabilities(core)
cache_dev_capabilities = measure_capabilities(cache_device)
core_dev_capabilities = measure_capabilities(core_device)
for (capability, method) in capabilities.items():
cas_val = cas_capabilities[capability]
cache_val = cache_dev_capabilities[capability]
core_val = core_dev_capabilities[capability]
expected_val = method(
core_val, cache_val) if method is not None else core_val
if capability in ["max_sectors_kb", "max_hw_sectors_kb"
] and expected_val != cas_val:
# On the newer kernels this trait is rounded. Instead of checking for
# the current kernel version, assume that both values are acceptable.
SECTOR_SHIFT = 9
lbs = measure_capabilities(core)["logical_block_size"]
# The original uint is kb, but number of sectors is needed
new_expected_val = expected_val * 2
round_val = lbs >> SECTOR_SHIFT
new_expected_val -= new_expected_val % round_val
# Restore the original unit
expected_val = new_expected_val // 2
if expected_val != cas_val:
TestRun.LOGGER.error(
f"Cas device {capability} is not set properly. Is: {cas_val}, "
f"should be {expected_val} (cache: {cache_val}, "
f"core: {core_val})")
continue
TestRun.LOGGER.info(
f"Cas device {capability} has proper value: {cas_val} "
f"(cache: {cache_val}, core: {core_val})")
cache.stop()
def test_add_cached_core(cache_mode):
"""
title: Fault injection test for adding already used core to a cache.
description: |
Negative test of the ability to add the core to the cache twice to the same cache
and while the core is already used by the another cache instance.
pass_criteria:
- No system crash.
- Adding already used core to another cache instance fails.
- The same core device cannot be used twice in one cache instance.
"""
with TestRun.step("Prepare two caches and one core device."):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions(
[Size(2, Unit.GibiByte),
Size(2, Unit.GibiByte)])
cache_part1 = cache_dev.partitions[0]
cache_part2 = cache_dev.partitions[1]
core_dev = TestRun.disks['core']
core_dev.create_partitions([Size(4, Unit.GibiByte)])
core_part = core_dev.partitions[0]
with TestRun.step("Start the first cache instance"):
cache1 = casadm.start_cache(cache_part1, cache_mode, force=True)
with TestRun.step("Add core device to first cache instance."):
core = cache1.add_core(core_part)
with TestRun.step("Start the second cache instance"):
cache2 = casadm.start_cache(cache_part2, cache_mode, force=True)
with TestRun.step(
"Try adding the same core device to the second cache instance."):
output = TestRun.executor.run_expect_fail(
cli.add_core_cmd(cache_id=str(cache2.cache_id),
core_dev=str(core_part.path),
core_id=str(core.core_id)))
cli_messages.check_stderr_msg(output, cli_messages.add_cached_core)
with TestRun.step(
"Try adding the same core device to the same cache for the second time."
):
output = TestRun.executor.run_expect_fail(
cli.add_core_cmd(cache_id=str(cache1.cache_id),
core_dev=str(core_part.path)))
cli_messages.check_stderr_msg(output, cli_messages.add_cached_core)
with TestRun.step("Stop caches."):
casadm.stop_all_caches()
def test_start_neg_cli_flags():
"""
title: Blocking standby start command with mutually exclusive flags
description: |
Try executing the standby start command with different combinations of mutually
exclusive flags.
pass_criteria:
- The command execution is unsuccessful for commands with mutually exclusive flags
- A proper error message is displayed
"""
with TestRun.step("Prepare the device for the cache."):
cache_device = TestRun.disks["cache"]
cache_device.create_partitions([Size(500, Unit.MebiByte)])
cache_device = cache_device.partitions[0]
cache_id = 1
cache_line_size = 32
with TestRun.step(
"Try to start standby cache with mutually exclusive parameters"):
init_required_params = f' --cache-device {cache_device.path}' \
f' --cache-id {cache_id}' \
f' --cache-line-size {cache_line_size}'
mutually_exclusive_cmd_init = f"{casadm_bin} --standby --init --load" \
f" {init_required_params}"
output = TestRun.executor.run_expect_fail(mutually_exclusive_cmd_init)
if not check_stderr_msg(output, mutually_exclusive_params_init):
TestRun.LOGGER.error(f'Expected error message in format '
f'"{mutually_exclusive_params_init[0]}"'
f'Got "{output.stderr}" instead.')
mutually_exclusive_cmd_load = [
f"{casadm_bin} --standby --load --cache-device {cache_device.path}"
f" --cache-id {cache_id}",
f"{casadm_bin} --standby --load --cache-device {cache_device.path}"
f" --cache-line-size {cache_line_size}",
f"{casadm_bin} --standby --load --cache-device {cache_device.path}"
f" --force"
]
for cmd in mutually_exclusive_cmd_load:
output = TestRun.executor.run_expect_fail(cmd)
if not check_stderr_msg(output, mutually_exclusive_params_load):
TestRun.LOGGER.error(f'Expected error message in format '
f'"{mutually_exclusive_params_load[0]}"'
f'Got "{output.stderr}" instead.')
def test_zero_metadata_filesystem(filesystem):
"""
title: Test for '--zero-metadata' and filesystem.
description: |
Test for '--zero-metadata' on drive with filesystem.
pass_criteria:
- Zeroing metadata on device with filesystem failed and not removed filesystem.
- Zeroing metadata on mounted device failed.
"""
mount_point = "/mnt"
with TestRun.step("Prepare devices."):
cache_dev, core_disk, cache_disk = prepare_devices()
with TestRun.step("Create filesystem on core device."):
core_disk.create_filesystem(filesystem)
with TestRun.step("Start cache and add core."):
cache = casadm.start_cache(cache_dev, force=True)
core = cache.add_core(core_disk)
with TestRun.step(
"Zeroing metadata on core device and validating filesystem"):
try:
casadm.zero_metadata(core)
TestRun.LOGGER.error("Zeroing metadata should fail!")
except CmdException as e:
cli_messages.check_stderr_msg(e.output,
cli_messages.no_cas_metadata)
file_system = get_device_filesystem_type(core.get_device_id())
if file_system != filesystem:
TestRun.LOGGER.error(
f"Incorrect filesystem: {file_system}; expected: {filesystem}")
with TestRun.step("Mount core."):
core.mount(mount_point)
with TestRun.step(
"Zeroing metadata on mounted core device and validate result"):
try:
casadm.zero_metadata(core)
TestRun.LOGGER.error("Zeroing metadata should fail!")
except CmdException as e:
cli_messages.check_stderr_msg(e.output,
cli_messages.unavailable_device)
def test_load_cache_with_inactive_core():
"""
title: Load cache with unavailable core devices.
description: Check if it is possible to load cache with unavailable core devices.
pass_criteria:
- No kernel error
- It is possible to perform cache load operation with unavailable devices.
- Warning message about not available core device should appear.
- Cache status should change to active after plugging missing core device.
"""
with TestRun.step("Prepare devices."):
devices = prepare_devices([("cache", 1), ("core", 1)])
cache_dev = devices["cache"].partitions[0]
core_dev = devices["core"].partitions[0]
plug_device = devices["core"]
with TestRun.step("Start cache and add core."):
cache = casadm.start_cache(cache_dev, force=True)
core = cache.add_core(core_dev)
with TestRun.step(
"Create init config file using current CAS configuration."):
InitConfig.create_init_config_from_running_configuration()
with TestRun.step("Stop cache."):
cache.stop()
with TestRun.step("Unplug core device."):
plug_device.unplug()
with TestRun.step("Load cache."):
output = TestRun.executor.run(cli.load_cmd(cache_dev.path))
cli_messages.check_stderr_msg(output,
cli_messages.load_inactive_core_missing)
with TestRun.step("Plug missing device and stop cache."):
plug_device.plug()
time.sleep(1)
core.wait_for_status_change(CoreStatus.active)
cache_status = cache.get_status()
if cache_status != CacheStatus.running:
TestRun.fail(
f"Cache did not change status to 'running' after plugging core device. "
f"Actual state: {cache_status}.")
cache.stop()
def test_stop_cache_with_mounted_partition(cache_mode):
"""
title: Fault injection test for removing core and stopping cache with mounted core.
description: |
Negative test of the ability of CAS to remove core and stop cache while core
is still mounted.
pass_criteria:
- No system crash.
- Unable to stop cache when partition is mounted.
- Unable to remove core when partition is mounted.
"""
with TestRun.step("Prepare cache and core devices. Start CAS."):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions([Size(1, Unit.GibiByte)])
cache_part = cache_dev.partitions[0]
core_dev = TestRun.disks['core']
core_dev.create_partitions([Size(4, Unit.GibiByte)])
core_part = core_dev.partitions[0]
cache = casadm.start_cache(cache_part, cache_mode, force=True)
with TestRun.step("Add core device with xfs filesystem and mount it."):
core_part.create_filesystem(Filesystem.xfs)
core = cache.add_core(core_part)
core.mount(mount_point)
with TestRun.step("Try to remove core from cache."):
output = TestRun.executor.run_expect_fail(
cli.remove_core_cmd(cache_id=str(cache.cache_id),
core_id=str(core.core_id)))
cli_messages.check_stderr_msg(output, cli_messages.remove_mounted_core)
with TestRun.step("Try to stop CAS."):
output = TestRun.executor.run_expect_fail(
cli.stop_cmd(cache_id=str(cache.cache_id)))
cli_messages.check_stderr_msg(output,
cli_messages.stop_cache_mounted_core)
with TestRun.step("Unmount core device."):
core.unmount()
with TestRun.step("Stop cache."):
casadm.stop_all_caches()
def test_remove_multilevel_core():
"""
title: Test of the ability to remove a core used in a multilevel cache.
description: |
Negative test if OpenCAS does not allow to remove a core when the related exported object
is used as a core device for another cache instance.
pass_criteria:
- No system crash.
- OpenCAS does not allow removing a core used in a multilevel cache instance.
"""
with TestRun.step("Prepare two devices for cache and one for core."):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions([Size(512, Unit.MebiByte)] * 2)
cache_part1 = cache_dev.partitions[0]
cache_part2 = cache_dev.partitions[1]
core_dev = TestRun.disks['core']
core_dev.create_partitions([Size(1, Unit.GibiByte)])
core_dev = core_dev.partitions[0]
with TestRun.step("Start the first cache instance"):
cache1 = casadm.start_cache(cache_part1, force=True)
with TestRun.step("Add a core device to the first cache instance."):
core1 = cache1.add_core(core_dev)
with TestRun.step("Start the second cache instance"):
cache2 = casadm.start_cache(cache_part2, force=True)
with TestRun.step("Add the first cache's exported object as a core "
"to the second cache instance."):
cache2.add_core(core1)
with TestRun.step("Try to remove core from the first level cache."):
output = TestRun.executor.run_expect_fail(
cli.remove_core_cmd(cache_id=str(cache1.cache_id),
core_id=str(core1.core_id),
force=True))
cli_messages.check_stderr_msg(output,
cli_messages.remove_multilevel_core)
with TestRun.step("Stop cache."):
casadm.stop_all_caches()
def test_activate_neg_cli_params():
"""
title: Verifying parameters for activating a standby cache instance.
description: |
Try executing the standby activate command with required arguments missing or unallowed
arguments present.
pass_criteria:
-The execution is unsuccessful for all improper argument combinations
-A proper error message is displayed for unsuccessful executions
"""
with TestRun.step("Prepare the device for the cache."):
cache_device = TestRun.disks["cache"]
cache_device.create_partitions([Size(500, Unit.MebiByte)])
cache_device = cache_device.partitions[0]
cache_id = 1
cache_line_size = 32
with TestRun.step("Init standby cache"):
cache_dev = Device(cache_device.path)
cache = standby_init(cache_dev=cache_dev,
cache_id=cache_id,
cache_line_size=cache_line_size,
force=True)
with TestRun.step("Detach standby cache"):
cache.standby_detach()
# Test standby activate
with TestRun.step(
"Prepare config for testing standby activate with required params"
):
standby_activate_required_params = dict([("--cache-device",
cache_device.path),
("--cache-id", cache_id)])
# Prepare full valid `standby activate` command
valid_cmd = casadm_bin + " --standby --activate"
for name, value in standby_activate_required_params.items():
valid_cmd += f" {name} {value}"
for name, value in standby_activate_required_params.items():
with TestRun.step(
f'Try to standby activate instance without "{name}" param'
):
tested_param = f"{name} {value}"
tested_cmd = valid_cmd.replace(tested_param, "")
output = TestRun.executor.run(tested_cmd)
if output.exit_code == 0:
TestRun.LOGGER.error(
f'"{tested_cmd}" command succeeded despite missing obligatory'
f' "{name}" paramter!')
if not check_stderr_msg(
output, missing_param) or name not in output.stderr:
TestRun.LOGGER.error(
f'Expected error message in format "{missing_param[0]}" with "{name}" '
f'(the missing param). Got "{output.stderr}" instead.')
with TestRun.step(
"Prepare config for testing standby activate with disallowed params"
):
activate_disallowed_params = dict([
("--core-device", "/dev/disk/by-id/core_dev_id"),
("--core-id", 5),
("--cache-mode", 32),
("--file", "/etc/opencas/ioclass-config.csv"),
("--io-class-id", "0"),
("--cache-line-size", 32),
])
for name, value in activate_disallowed_params.items():
with TestRun.step(
f'Try to activate standby instance with disallowed "{name}" param'
):
tested_param = f"{name} {value}"
tested_cmd = f"{valid_cmd} {tested_param}"
output = TestRun.executor.run(tested_cmd)
if output.exit_code == 0:
TestRun.LOGGER.error(
f'"{tested_cmd}" command succeeded despite disallowed "{name}" paramter!'
)
if not check_stderr_msg(output, disallowed_param):
TestRun.LOGGER.error(
f'Expected error message in format "{disallowed_param[0]}" '
f'Got "{output.stderr}" instead.')
def test_stop_no_flush_load_cache_no_fs(cache_mode):
"""
title: Test to check that 'stop --no-data-flush' command works correctly.
description: |
Negative test of the ability of CAS to load unflushed cache on core device
without filesystem. Test uses lazy flush cache modes.
pass_criteria:
- No system crash while load cache.
- Starting cache without loading metadata fails.
- Starting cache with loading metadata finishes with success.
"""
with TestRun.step("Prepare cache and core devices."):
cache_part, core_part = prepare()
with TestRun.step("Start cache with --force option."):
cache = casadm.start_cache(cache_part, cache_mode, force=True)
with TestRun.step("Change cleaning policy to NOP."):
cache.set_cleaning_policy(CleaningPolicy.nop)
with TestRun.step("Add core device without filesystem."):
core_part.wipe_filesystem()
core = cache.add_core(core_part)
with TestRun.step("Fill exported object with data."):
dd = (Dd().input("/dev/zero").output(core.path).block_size(
Size(1, Unit.Blocks4096)).oflag("direct"))
dd.run()
with TestRun.step("Count dirty blocks on exported object."):
dirty_blocks_before = core.get_dirty_blocks()
with TestRun.step("Stop cache with option '--no-data-flush'."):
cache.stop(no_data_flush=True)
caches_count = len(casadm_parser.get_caches())
if caches_count != 0:
TestRun.fail(
f"Expected caches count: 0; Actual caches count: {caches_count}."
)
with TestRun.step("Try to start cache without loading metadata."):
output = TestRun.executor.run_expect_fail(
cli.start_cmd(cache_dev=str(cache_part.path),
cache_mode=str(cache_mode.name.lower()),
force=False,
load=False))
cli_messages.check_stderr_msg(
output, cli_messages.start_cache_with_existing_metadata)
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache.cache_device)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(
f"Expected caches count: 1 Actual caches count: {caches_count}."
)
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 1:
TestRun.fail(
f"Expected cores count: 1; Actual cores count: {cores_count}.")
with TestRun.step(
"Compare dirty blocks number before and after loading cache."):
if dirty_blocks_before != core.get_dirty_blocks():
TestRun.LOGGER.error(
"Dirty blocks number is different than before loading cache.")
with TestRun.step("Stop cache."):
casadm.stop_all_caches()
def test_standby_neg_cli_management():
"""
title: Blocking management commands in standby state
description: |
Try executing management commands for a cache in standby state
pass_criteria:
- The execution is unsuccessful for blocked management commands
- The execution is successful for allowed management commands
- A proper error message is displayed for unsuccessful executions
"""
with TestRun.step("Prepare the device for the cache."):
device = TestRun.disks["cache"]
device.create_partitions(
[Size(500, Unit.MebiByte),
Size(500, Unit.MebiByte)])
cache_device = device.partitions[0]
core_device = device.partitions[1]
with TestRun.step("Prepare the standby instance"):
cache_id = 1
cache = casadm.standby_init(cache_dev=cache_device,
cache_id=cache_id,
cache_line_size=32,
force=True)
ioclass_config_path = "/tmp/standby_cli_neg_mngt_test_ioclass_config_file.csv"
TestRun.executor.run(f"rm -rf {ioclass_config_path}")
random_ioclass_config = IoClass.generate_random_ioclass_list(5)
IoClass.save_list_to_config_file(
random_ioclass_config, ioclass_config_path=ioclass_config_path)
blocked_mngt_commands = [
cli.get_param_cutoff_cmd(str(cache_id), "1"),
cli.get_param_cleaning_cmd(str(cache_id)),
cli.get_param_cleaning_alru_cmd(str(cache_id)),
cli.get_param_cleaning_acp_cmd(str(cache_id)),
cli.set_param_cutoff_cmd(str(cache_id), "1", threshold="1"),
cli.set_param_cutoff_cmd(str(cache_id), policy="never"),
cli.set_param_cleaning_cmd(str(cache_id), policy="nop"),
cli.set_param_cleaning_alru_cmd(str(cache_id), wake_up="30"),
cli.set_param_cleaning_acp_cmd(str(cache_id), wake_up="100"),
cli.set_param_promotion_cmd(str(cache_id), policy="nhit"),
cli.set_param_promotion_nhit_cmd(str(cache_id), threshold="5"),
cli.set_cache_mode_cmd("wb", str(cache_id)),
cli.add_core_cmd(str(cache_id), core_device.path),
cli.remove_core_cmd(str(cache_id), "1"),
cli.remove_inactive_cmd(str(cache_id), "1"),
cli.reset_counters_cmd(str(cache_id)),
cli.flush_cache_cmd(str(cache_id)),
cli.flush_core_cmd(str(cache_id), "1"),
cli.load_io_classes_cmd(str(cache_id), ioclass_config_path),
cli.list_io_classes_cmd(str(cache_id), output_format="csv"),
cli.script_try_add_cmd(str(cache_id), core_device.path, core_id=1),
cli.script_purge_cache_cmd(str(cache_id)),
cli.script_purge_core_cmd(str(cache_id), "1"),
cli.script_detach_core_cmd(str(cache_id), "1"),
cli.script_remove_core_cmd(str(cache_id), "1"),
]
with TestRun.step(
"Try to execute forbidden management commands in standby mode"):
for cmd in blocked_mngt_commands:
TestRun.LOGGER.info(f"Verify {cmd}")
output = TestRun.executor.run_expect_fail(cmd)
if not check_stderr_msg(output, operation_forbiden_in_standby):
TestRun.LOGGER.error(
f'Expected the following error message "{operation_forbiden_in_standby[0]}" '
f'Got "{output.stderr}" instead.')
with TestRun.step("Stop the standby instance"):
TestRun.executor.run(f"rm -rf {ioclass_config_path}")
cache.stop()
def test_activate_neg_cache_id():
"""
title: Blocking cache with mismatching cache ID activation
description: |
Try restoring cache operations from a replicated cache that was initialized
with different cache ID than the original cache
pass_criteria:
- The activation is cancelled
- The cache remains in Standby detached state after an unsuccessful activation
- The cache exported object is present after an unsuccessful activation
- A proper error message is displayed
"""
with TestRun.step("Prepare two partitions of the same size"):
disk = TestRun.disks["cache"]
disk.create_partitions(
[Size(200, Unit.MebiByte),
Size(200, Unit.MebiByte)])
active_dev = disk.partitions[0]
standby_dev = disk.partitions[1]
with TestRun.step(
"Start a regular cache instance explicitly providing a valid cache id and stop it"
):
active_cache_id = 5
standby_cache_id = 15
cache_exp_obj_name = f"cas-cache-{standby_cache_id}"
cls = CacheLineSize.LINE_32KiB
cache = casadm.start_cache(active_dev,
cache_id=active_cache_id,
cache_line_size=cls,
force=True)
cache.stop()
with TestRun.step(
"On the second partition initialize standby instance with different cache id"
):
standby_cache = casadm.standby_init(
standby_dev,
cache_line_size=int(cls.value.value / Unit.KibiByte.value),
cache_id=standby_cache_id,
force=True,
)
with TestRun.step(
"Copy contents of the first partition into the cache exported object"
):
Dd().input(active_dev.path).output(f"/dev/{cache_exp_obj_name}").run()
with TestRun.step(
"Verify if the cache exported object appeared in the system"):
output = TestRun.executor.run_expect_success(
f"ls -la /dev/ | grep {cache_exp_obj_name}")
if output.stdout[0] != "b":
TestRun.fail("The cache exported object is not a block device")
with TestRun.step("Detach the standby instance"):
standby_cache.standby_detach()
with TestRun.step(
"Try to activate the standby cache instance with different cache id"
):
output = TestRun.executor.run_expect_fail(
standby_activate_cmd(cache_dev=standby_dev.path,
cache_id=str(standby_cache_id)))
if not check_stderr_msg(output, activate_with_different_cache_id):
TestRun.LOGGER.error(
f"Invalid error message. Expected {activate_with_different_cache_id}."
f"Got {output.stderr}")
status = standby_cache.get_status()
if status != CacheStatus.standby_detached:
TestRun.LOGGER.error(
"The standby cache instance is in an invalid state "
f"Expected {CacheStatus.standby_detached}. Got {status}")
def test_fault_power_hit(cache_mode):
"""
title: Test with power hit.
description: |
Test if there will be no metadata initialization after wake up
- when starting cache without initialization.
pass_criteria:
- Start cache without re-initialization failed.
- Start cache with load works correctly.
- Expected message found in log.
"""
with TestRun.step("Prepare CAS device."):
cache_disk = TestRun.disks['cache']
core_disk = TestRun.disks['core']
cache_disk.create_partitions([Size(2, Unit.GibiByte)])
core_disk.create_partitions([Size(2, Unit.GibiByte)])
cache_dev = cache_disk.partitions[0]
core_dev = core_disk.partitions[0]
cache = casadm.start_cache(cache_dev, cache_mode, force=True)
core = cache.add_core(core_dev)
with TestRun.step("Mark log lines for later validation of new entries."):
last_read_line = 1
log_lines = TestRun.executor.run_expect_success(
f"tail -qn +{last_read_line} {log_path}").stdout.splitlines()
last_read_line += len(log_lines)
with TestRun.step("Hard reset."):
power_control = TestRun.plugin_manager.get_plugin('power_control')
power_control.power_cycle()
with TestRun.step("Start cache without re-initialization."):
output = TestRun.executor.run_expect_fail(
cli.start_cmd(cache_dev=str(cache_dev.path),
cache_mode=str(cache_mode.name.lower()),
force=False,
load=False))
if cli_messages.check_stderr_msg(output, cli_messages.error_inserting_cache) and \
cli_messages.check_stderr_msg(output,
cli_messages.reinitialize_with_force_or_recovery):
TestRun.LOGGER.info(
f"Found expected exception: {cli_messages.error_inserting_cache}"
f" and {cli_messages.reinitialize_with_force_or_recovery}")
with TestRun.step("Start cache with load."):
try:
cache = casadm.load_cache(cache_dev)
TestRun.LOGGER.info(
f"Cache device loaded correctly (as expected).")
except CmdException as e:
TestRun.LOGGER.fail(
f"Failed to load cache device. Exception: {e.output}")
time.sleep(wait_short_time)
message_found = check_log(
last_read_line, cli_messages.reinitialize_with_force_or_recovery)
# check log second time in case that operation logging would take some more time
if not message_found:
time.sleep(wait_long_time)
result = check_log(
last_read_line,
cli_messages.reinitialize_with_force_or_recovery)
if not result:
TestRun.LOGGER.fail(
f"Haven't found expected message in the log.")
def test_zero_metadata_dirty_data():
"""
title: Test for '--zero-metadata' and dirty data scenario.
description: |
Test for '--zero-metadata' with and without 'force' option if there are dirty data
on cache.
pass_criteria:
- Zeroing metadata without force failed on cache with dirty data.
- Zeroing metadata with force ran successfully on cache with dirty data.
- Cache started successfully after zeroing metadata on cache with dirty data.
"""
with TestRun.step("Prepare cache and core devices."):
cache_dev, core_disk, cache_disk = prepare_devices()
with TestRun.step("Start cache."):
cache = casadm.start_cache(cache_dev, CacheMode.WB, force=True)
core = cache.add_core(core_disk)
cache.set_cleaning_policy(CleaningPolicy.nop)
with TestRun.step("Run workload on CAS"):
fio_run_fill = Fio().create_command()
fio_run_fill.io_engine(IoEngine.libaio)
fio_run_fill.direct()
fio_run_fill.read_write(ReadWrite.randwrite)
fio_run_fill.io_depth(16)
fio_run_fill.block_size(Size(1, Unit.MebiByte))
fio_run_fill.target(core.path)
fio_run_fill.run_time(timedelta(seconds=5))
fio_run_fill.time_based()
fio_run_fill.run()
with TestRun.step("Stop cache without flushing dirty data."):
cache.stop(no_data_flush=True)
with TestRun.step("Start cache (expect to fail)."):
try:
cache = casadm.start_cache(cache_dev, CacheMode.WB)
except CmdException:
TestRun.LOGGER.info("Start cache failed as expected.")
with TestRun.step("Zeroing metadata on CAS device without force"):
try:
casadm.zero_metadata(cache_dev)
TestRun.LOGGER.error("Zeroing metadata without force should fail!")
except CmdException as e:
cli_messages.check_stderr_msg(e.output,
cli_messages.cache_dirty_data)
with TestRun.step("Zeroing metadata on cache device with force"):
try:
casadm.zero_metadata(cache_dev, force=True)
TestRun.LOGGER.info("Zeroing metadata with force successful!")
except CmdException as e:
TestRun.LOGGER.error(
f"Zeroing metadata with force should work for cache device!"
f"Error message: {e.output}")
with TestRun.step("Start cache without 'force' option."):
try:
cache = casadm.start_cache(cache_dev, CacheMode.WB)
TestRun.LOGGER.info("Cache started successfully.")
except CmdException:
TestRun.LOGGER.error("Start cache failed.")
def test_cli_help(shortcut):
"""
title: Test for 'help' command.
description: Test if help for commands displays.
pass_criteria:
- Proper help displays for every command.
"""
TestRun.LOGGER.info("Run 'help' for every 'casadm' command.")
output = casadm.help(shortcut)
check_stdout_msg(output, casadm_help)
output = TestRun.executor.run("casadm" +
(" -S" if shortcut else " --start-cache") +
(" -H" if shortcut else " --help"))
check_stdout_msg(output, start_cache_help)
output = TestRun.executor.run("casadm" +
(" -T" if shortcut else " --stop-cache") +
(" -H" if shortcut else " --help"))
check_stdout_msg(output, stop_cache_help)
output = TestRun.executor.run("casadm" +
(" -X" if shortcut else " --set-param") +
(" -H" if shortcut else " --help"))
check_stdout_msg(output, set_params_help)
output = TestRun.executor.run("casadm" +
(" -G" if shortcut else " --get-param") +
(" -H" if shortcut else " --help"))
check_stdout_msg(output, get_params_help)
output = TestRun.executor.run(
"casadm" + (" -Q" if shortcut else " --set-cache-mode") +
(" -H" if shortcut else " --help"))
check_stdout_msg(output, set_cache_mode_help)
output = TestRun.executor.run("casadm" +
(" -A" if shortcut else " --add-core") +
(" -H" if shortcut else " --help"))
check_stdout_msg(output, add_core_help)
output = TestRun.executor.run("casadm" +
(" -R" if shortcut else " --remove-core") +
(" -H" if shortcut else " --help"))
check_stdout_msg(output, remove_core_help)
output = TestRun.executor.run("casadm" + " --remove-detached" +
(" -H" if shortcut else " --help"))
check_stdout_msg(output, remove_detached_help)
output = TestRun.executor.run("casadm" +
(" -L" if shortcut else " --list-caches") +
(" -H" if shortcut else " --help"))
check_stdout_msg(output, list_help)
output = TestRun.executor.run("casadm" +
(" -P" if shortcut else " --stats") +
(" -H" if shortcut else " --help"))
check_stdout_msg(output, stats_help)
output = TestRun.executor.run(
"casadm" + (" -Z" if shortcut else " --reset-counters") +
(" -H" if shortcut else " --help"))
check_stdout_msg(output, reset_counters_help)
output = TestRun.executor.run("casadm" +
(" -F" if shortcut else " --flush-cache") +
(" -H" if shortcut else " --help"))
check_stdout_msg(output, flush_cache_help)
output = TestRun.executor.run("casadm" +
(" -E" if shortcut else " --flush-core") +
(" -H" if shortcut else " --help"))
check_stdout_msg(output, flush_core_help)
output = TestRun.executor.run("casadm" +
(" -C" if shortcut else " --io-class") +
(" -H" if shortcut else " --help"))
check_stdout_msg(output, ioclass_help)
output = TestRun.executor.run("casadm" +
(" -V" if shortcut else " --version") +
(" -H" if shortcut else " --help"))
check_stdout_msg(output, version_help)
output = TestRun.executor.run("casadm" +
(" -H" if shortcut else " --help") +
(" -H" if shortcut else " --help"))
check_stdout_msg(output, help_help)
output = TestRun.executor.run("casadm" + " --zero-metadata" +
(" -H" if shortcut else " --help"))
check_stdout_msg(output, zero_metadata_help)
output = TestRun.executor.run("casadm" +
(" -Y" if shortcut else " --yell") +
(" -H" if shortcut else " --help"))
check_stderr_msg(output, unrecognized_stderr)
check_stdout_msg(output, unrecognized_stdout)
def test_zero_metadata_dirty_shutdown():
"""
title: Test for '--zero-metadata' and dirty shutdown scenario.
description: |
Test for '--zero-metadata' with and without 'force' option on cache which had been dirty
shut down before.
pass_criteria:
- Zeroing metadata without force failed on cache after dirty shutdown.
- Zeroing metadata with force ran successfully on cache after dirty shutdown.
- Cache started successfully after dirty shutdown and zeroing metadata on cache.
"""
with TestRun.step("Prepare cache and core devices."):
cache_dev, core_disk, cache_disk = prepare_devices()
with TestRun.step("Start cache."):
cache = casadm.start_cache(cache_dev, CacheMode.WT, force=True)
core = cache.add_core(core_disk)
with TestRun.step("Unplug cache device."):
cache_disk.unplug()
with TestRun.step("Stop cache without flush."):
try:
cache.stop(no_data_flush=True)
except CmdException:
TestRun.LOGGER.info("This could ended with error (expected)")
with TestRun.step("Plug cache device."):
cache_disk.plug()
time.sleep(1)
with TestRun.step("Start cache (expect to fail)."):
try:
cache = casadm.start_cache(cache_dev, CacheMode.WT)
TestRun.LOGGER.error("Starting cache should fail!")
except CmdException:
TestRun.LOGGER.info("Start cache failed as expected.")
with TestRun.step("Zeroing metadata on CAS device without force"):
try:
casadm.zero_metadata(cache_dev)
TestRun.LOGGER.error("Zeroing metadata without force should fail!")
except CmdException as e:
cli_messages.check_stderr_msg(e.output,
cli_messages.cache_dirty_shutdown)
with TestRun.step("Zeroing metadata on cache device with force"):
try:
casadm.zero_metadata(cache_dev, force=True)
TestRun.LOGGER.info("Zeroing metadata with force successful!")
except CmdException as e:
TestRun.LOGGER.error(
f"Zeroing metadata with force should work for cache device!"
f"Error message: {e.output}")
with TestRun.step("Start cache."):
try:
cache = casadm.start_cache(cache_dev, CacheMode.WT)
TestRun.LOGGER.info("Cache started successfully.")
except CmdException:
TestRun.LOGGER.error("Start cache failed.")
def test_zero_metadata_negative_cases():
"""
title: Test for '--zero-metadata' negative cases.
description: |
Test for '--zero-metadata' scenarios with expected failures.
pass_criteria:
- Zeroing metadata without '--force' failed when run on cache.
- Zeroing metadata with '--force' failed when run on cache.
- Zeroing metadata failed when run on system drive.
- Load cache command failed after successfully zeroing metadata on the cache device.
"""
with TestRun.step("Prepare cache and core devices."):
cache_dev, core_dev, cache_disk = prepare_devices()
with TestRun.step("Start cache."):
cache = casadm.start_cache(cache_dev, force=True)
with TestRun.step("Try to zero metadata and validate error message."):
try:
casadm.zero_metadata(cache_dev)
TestRun.LOGGER.error("Zeroing metadata should fail!")
except CmdException as e:
cli_messages.check_stderr_msg(e.output,
cli_messages.unavailable_device)
with TestRun.step(
"Try to zero metadata with '--force' option and validate error message."
):
try:
casadm.zero_metadata(cache_dev, force=True)
TestRun.LOGGER.error(
"Zeroing metadata with '--force' option should fail!")
except CmdException as e:
cli_messages.check_stderr_msg(e.output,
cli_messages.unavailable_device)
with TestRun.step("Try to zeroing metadata on system disk."):
os_disks = get_system_disks()
for os_disk in os_disks:
output = TestRun.executor.run(cli.zero_metadata_cmd(str(os_disk)))
if output.exit_code != 0:
cli_messages.check_stderr_msg(output,
cli_messages.error_handling)
else:
TestRun.LOGGER.error("Zeroing metadata should fail!")
with TestRun.step("Stop cache."):
casadm.stop_all_caches()
with TestRun.step("Zeroing metadata."):
try:
casadm.zero_metadata(cache_dev)
TestRun.LOGGER.info("Zeroing metadata successful!")
except CmdException as e:
TestRun.LOGGER.error(
f"Zeroing metadata should work for cache device after stopping "
f"cache! Error message: {e.output}")
with TestRun.step("Load cache."):
try:
cache = casadm.load_cache(cache_dev)
TestRun.LOGGER.error("Loading cache should fail.")
except CmdException:
TestRun.LOGGER.info("Loading cache failed as expected.")
async def test_data_integrity_unplug(cache_mode):
"""
title: Test if data integrity is maintained in a power failure scenario.
description: |
The test checks if the data written to the cache device is saved correctly in a power
failure scenario, which is simulated by unplugging the cache device.
FIO is interrupted when the cache device is unplugged. The test determines how many
writes each FIO job was able to perform before the unplug and then checks if the data
on the cache device matches FIO output up to the unplug (bearing in mind that the last
write might have been interrupted).
pass_criteria:
- No system crash.
- Data on the cache device are consistent with the data sent from FIO.
"""
global fio_seed, tmp_dir, ram_disk
cache_dev = TestRun.disks["cache"]
core_dev = TestRun.disks["core"]
sleep_max_s = timedelta(seconds=10)
with TestRun.step("Test prepare"):
random.seed(TestRun.random_seed)
fio_seed = random.randint(0, 2**32)
TestRun.LOGGER.info(f"FIO seed: {fio_seed}")
tmp_dir = Directory.create_temp_directory()
TestRun.LOGGER.info(f"Temporary directory: {tmp_dir.full_path}")
ram_disk = RamDisk.create(Size(1, Unit.GiB), 1)[0]
# csums[j][i] is csum for i-th io of j-th job
csums = [{} for _ in range(num_jobs)]
with TestRun.step("Test iterations:"):
for cache_line_size in TestRun.iteration(CacheLineSize):
with TestRun.step("Prefill the core device."):
write_device(core_dev.path)
data_prefill_cs = read_device_md5s(core_dev.path)
# csums_rev is a reverse mapping to identify job, sector and seqno of I/O
# with given csum
csums_rev = {}
for j in range(num_jobs):
for b in range(job_workset_blocks):
cs = data_prefill_cs[j][b]
csums_rev[cs] = get_data_name(j, b, -1)
with TestRun.step(
"Start a cache, add a core and set cache cleaning policy to NOP"
):
cache = casadm.start_cache(cache_dev,
cache_mode,
cache_line_size,
force=True)
exported_object = cache.add_core(core_dev)
cache.set_cleaning_policy(CleaningPolicy.nop)
with TestRun.step("Start FIO to the exported object"):
fio = prepare_base_fio() \
.target(exported_object.path) \
.run_time(100 * sleep_max_s)
for i in range(num_jobs):
fio.add_job(f"di_{i}") \
.offset(job_workset_size * i) \
.io_size(Size(100, Unit.GiB))
fio_task = start_async_func(fio.fio.run)
with TestRun.step("Hot unplug the cache device after random time"):
wait_time_s = random.randint(5,
int(sleep_max_s.total_seconds()))
sleep(wait_time_s)
cache_dev.unplug()
with TestRun.step("Analyze FIO execution after hot unplug"):
fio_output = await fio_task
if fio_output.exit_code == 0:
TestRun.LOGGER.warning(
"Unexpectedly successful fio - check if the device was unplugged correctly."
)
results = fio.get_results(
TestRun.executor.run(f"cat {fio.fio.fio_file}").stdout)
ios = [r.job.write.total_ios for r in results]
with TestRun.step("Stop cache without flushing data"):
try:
cache.stop(no_data_flush=True)
except CmdException as e:
if not cli_messages.check_stderr_msg(
e.output, cli_messages.stop_cache_errors):
raise
with TestRun.step("Plug back the cache device"):
cache_dev.plug()
with TestRun.step("Load cache"):
cache = casadm.load_cache(cache_dev)
with TestRun.step("Check data"):
csums_actual = read_device_md5s(exported_object.path)
# The last I/O in each job is interrupted by the unplug. It could have made it
# to the medium or not. So the last I/O we expect to actually hit the disk
# is 'num_io-2' or 'num_io-1' for each job. Below 'n1_' refers to 'num_io-1'
# and 'n2_' refers to 'num_io-2'
# seqno[j] is the last I/O seqno for given job (entire workset)
n2_seqno = [io - 2 for io in ios]
n1_seqno = [io - 1 for io in ios]
# pattern[j][b] is the last I/O seqno for job j block b
n2_pattern = get_pattern(n2_seqno)
n1_pattern = get_pattern(n1_seqno)
# Make sure we know data checksums for I/O that we expect to have
# been committed assuming either n2_seqno or n1_seqno is the last
# I/O committed by each job.
gen_csums(ram_disk.path, n1_seqno, n1_pattern, csums,
csums_rev)
gen_csums(ram_disk.path, n2_seqno, n2_pattern, csums,
csums_rev)
fail = False
for j in range(num_jobs):
for b in range(job_workset_blocks):
# possible checksums assuming n2_pattern or n1_pattern
cs_n2 = get_data_csum(j, b, n2_pattern,
data_prefill_cs, csums)
cs_n1 = get_data_csum(j, b, n1_pattern,
data_prefill_cs, csums)
# actual checksum read from CAS
cs_actual = csums_actual[j][b]
if cs_actual != cs_n2 and cs_actual != cs_n1:
fail = True
# attempt to identify erroneous data by comparing its checksum
# against the known checksums
identity = csums_rev[cs_actual] if cs_actual in csums_rev else \
f"UNKNOWN ({cs_actual[:8]})"
TestRun.LOGGER.error(
f"MISMATCH job {j} block {b} contains {identity} "
f"expected {get_data_name(j, b, n2_pattern[j][b])} "
f"or {get_data_name(j, b, n1_pattern[j][b]) }")
if fail:
break
cache.stop(no_data_flush=True)
def test_stop_no_flush_load_cache(cache_mode, filesystem):
"""
title: Test to check that 'stop --no-data-flush' command works correctly.
description: |
Negative test of the ability of CAS to load unflushed cache on core device
with filesystem. Test uses lazy flush cache modes.
pass_criteria:
- No system crash while load cache.
- Starting cache without loading metadata fails.
- Starting cache with loading metadata finishes with success.
"""
with TestRun.step("Prepare cache and core devices."):
cache_part, core_part = prepare()
with TestRun.step("Start cache."):
cache = casadm.start_cache(cache_part, cache_mode, force=True)
with TestRun.step("Change cleaning policy to NOP."):
cache.set_cleaning_policy(CleaningPolicy.nop)
with TestRun.step(
f"Add core with {filesystem.name} filesystem to cache and mount it."
):
core_part.create_filesystem(filesystem)
core = cache.add_core(core_part)
core.mount(mount_point)
with TestRun.step(
f"Create test file in mount point of exported object and check its md5 sum."
):
test_file = fs_utils.create_random_test_file(test_file_path,
Size(48, Unit.MebiByte))
test_file_md5_before = test_file.md5sum()
with TestRun.step("Unmount exported object."):
core.unmount()
with TestRun.step("Count dirty blocks on exported object."):
dirty_blocks_before = core.get_dirty_blocks()
with TestRun.step("Stop cache with option '--no-data-flush'."):
cache.stop(no_data_flush=True)
caches_count = len(casadm_parser.get_caches())
if caches_count != 0:
TestRun.fail(
f"Expected caches count: 0; Actual caches count: {caches_count}."
)
with TestRun.step("Try to start cache without loading metadata."):
output = TestRun.executor.run_expect_fail(
cli.start_cmd(cache_dev=str(cache_part.path),
cache_mode=str(cache_mode.name.lower()),
force=False,
load=False))
cli_messages.check_stderr_msg(
output, cli_messages.start_cache_with_existing_metadata)
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache.cache_device)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(
f"Expected caches count: 1 Actual caches count: {caches_count}."
)
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 1:
TestRun.fail(
f"Expected cores count: 1; Actual cores count: {cores_count}.")
with TestRun.step(
"Compare dirty blocks number before and after loading cache."):
if dirty_blocks_before != core.get_dirty_blocks():
TestRun.LOGGER.error(
"Dirty blocks number is different than before loading cache.")
with TestRun.step("Mount exported object."):
core.mount(mount_point)
with TestRun.step(
"Compare md5 sum of test file before and after loading cache."):
if test_file_md5_before != test_file.md5sum():
TestRun.LOGGER.error(
"Test file's md5 sum is different than before loading cache.")
with TestRun.step("Unmount exported object."):
core.unmount()
with TestRun.step("Stop cache."):
casadm.stop_all_caches()
def test_recovery_unplug_cache_fs(cache_mode, cls, filesystem, direct):
"""
title: Test for recovery after cache drive removal - test with filesystem.
description: |
Verify that unflushed data can be safely recovered after, when SSD drive is removed
after write completion - test with filesystem.
pass_criteria:
- CAS recovers successfully after cache drive unplug
- No data corruption
"""
with TestRun.step("Prepare devices"):
cache_disk = TestRun.disks['cache']
core_disk = TestRun.disks['core']
cache_disk.create_partitions([Size(2, Unit.GibiByte)])
core_disk.create_partitions([Size(16, Unit.GibiByte)])
cache_device = cache_disk.partitions[0]
core_device = core_disk.partitions[0]
with TestRun.step("Create test files."):
source_file, target_file = create_test_files(test_file_size)
source_file_md5 = source_file.md5sum()
with TestRun.step("Create filesystem on core device."):
core_device.create_filesystem(filesystem)
with TestRun.step("Start cache and add core."):
cache = casadm.start_cache(cache_device, cache_mode, cls)
core = cache.add_core(core_device)
with TestRun.step("Mount CAS device."):
core.mount(mount_point)
with TestRun.step("Copy file to CAS."):
copy_file(source=source_file.full_path, target=test_file_path,
size=test_file_size, direct="oflag" if direct else None)
TestRun.LOGGER.info(str(core.get_statistics()))
with TestRun.step("Unmount CAS device."):
core.unmount()
with TestRun.step("Unplug cache device."):
cache_disk.unplug()
TestRun.LOGGER.info(f"List caches:\n{casadm.list_caches().stdout}")
TestRun.LOGGER.info(f"Dirty blocks on cache: "
f"{cache.get_dirty_blocks().get_value(Unit.Blocks4096)}")
with TestRun.step("Stop cache."):
try:
cache.stop()
TestRun.fail("Stopping the cache should be aborted without --no-flush flag.")
except CmdException as e:
TestRun.LOGGER.info(str(e.output))
try:
cache.stop(no_data_flush=True)
TestRun.LOGGER.warning("Expected stopping cache with errors with --no-flush flag.")
except CmdException as e1:
cli_messages.check_stderr_msg(e1.output, cli_messages.stop_cache_errors)
with TestRun.step("Plug missing cache device."):
TestRun.LOGGER.info(str(casadm.list_caches(by_id_path=False)))
cache_disk.plug()
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache_device)
TestRun.LOGGER.info(f"Dirty blocks on cache: "
f"{cache.get_dirty_blocks().get_value(Unit.Blocks4096)}")
with TestRun.step("Stop cache with data flush."):
cache.stop()
with TestRun.step("Mount core device."):
core_device.mount(mount_point)
with TestRun.step("Copy file from core device and check md5sum."):
copy_file(source=test_file_path, target=target_file.full_path,
size=test_file_size, direct="iflag" if direct else None)
target_file_md5 = target_file.md5sum()
compare_files(source_file_md5, target_file_md5)
with TestRun.step("Unmount core device and remove files."):
core_device.unmount()
try:
target_file.remove()
source_file.remove()
except Exception:
# On some OSes files at /tmp location are automatically removed after DUT hard reset
pass
def test_remove_inactive_devices():
"""
title: Validate removing inactive CAS devices.
description: |
Validate that it is possible to remove inactive CAS devices when there are no dirty
cache lines associated with them and that removing CAS devices is prevented otherwise
(unless ‘force’ option is used).
pass_criteria:
- No kernel error
- Removing CAS devices without dirty data is successful.
- Removing CAS devices with dirty data without ‘force’ option is blocked.
- Removing CAS devices with dirty data with ‘force’ option is successful.
"""
with TestRun.step("Prepare devices."):
devices = prepare_devices([("cache", 1), ("core", 4)])
cache_dev = devices["cache"].partitions[0]
core_devs = devices["core"].partitions
plug_device = devices["core"]
with TestRun.step("Start cache and add four cores."):
cache = casadm.start_cache(cache_dev,
cache_mode=CacheMode.WB,
force=True)
cores = []
for d in core_devs:
cores.append(cache.add_core(d))
with TestRun.step(
"Create init config file using current CAS configuration."):
InitConfig.create_init_config_from_running_configuration()
with TestRun.step("Run random writes to all CAS devices."):
run_fio([c.system_path for c in cores])
with TestRun.step(
"Flush dirty data from two CAS devices and verify than other two "
"contain dirty data."):
for core in cores:
if core.core_id % 2 == 0:
core.flush_core()
if core.get_dirty_blocks() != Size.zero():
TestRun.fail("Failed to flush CAS device.")
elif core.get_dirty_blocks() == Size.zero():
TestRun.fail("There should be dirty data on CAS device.")
with TestRun.step("Stop cache without flushing dirty data."):
cache.stop(no_data_flush=True)
with TestRun.step("Unplug core disk."):
plug_device.unplug()
with TestRun.step("Load cache."):
casadm.load_cache(cache_dev)
with TestRun.step(
"Verify that all previously created CAS devices are listed with "
"proper status."):
for core in cores:
if core.get_status() != CoreStatus.inactive:
TestRun.fail(f"Each core should be in inactive state. "
f"Actual states:\n{casadm.list_caches().stdout}")
with TestRun.step(
"Try removing CAS device without ‘force’ option. Verify that for "
"dirty CAS devices operation is blocked, proper message is displayed "
"and device is still listed."):
shuffle(cores)
for core in cores:
try:
dirty_blocks = core.get_dirty_blocks()
core.remove_core()
if dirty_blocks != Size.zero():
TestRun.fail(
"Removing dirty CAS device should be impossible but remove "
"command executed without any error.")
TestRun.LOGGER.info(
"Removing core with force option skipped for clean CAS device."
)
except CmdException as e:
if dirty_blocks == Size.zero():
TestRun.fail(
"Removing clean CAS device should be possible but remove "
"command returned an error.")
TestRun.LOGGER.info(
"Remove operation without force option is blocked for "
"dirty CAS device as expected.")
cli_messages.check_stderr_msg(
e.output, cli_messages.remove_inactive_core)
output = casadm.list_caches().stdout
if core.system_path not in output:
TestRun.fail(
f"CAS device is not listed in casadm list output but it should be."
f"\n{output}")
core.remove_core(force=True)
with TestRun.step("Plug missing disk and stop cache."):
plug_device.plug()
casadm.stop_all_caches()