def test_remove_core_during_io():
"""
title: Test for removing core device during IO.
description: |
Creating CAS device, running fio on it and checking
if core can be removed during IO operations.
pass_criteria:
- Core device is not removed.
"""
with TestRun.step("Start cache and add core"):
cache, core = prepare()
with TestRun.step("Running 'fio'"):
fio = (Fio().create_command().io_engine(IoEngine.libaio).block_size(
Size(4, Unit.KibiByte)).read_write(
ReadWrite.randrw).target(f"{core.path}").direct(1).run_time(
timedelta(minutes=4)).time_based())
fio_pid = fio.run_in_background()
time.sleep(10)
with TestRun.step("Try to remove core during 'fio'"):
TestRun.executor.run_expect_fail(
cli.remove_core_cmd(f"{core.cache_id}", f"{core.core_id}"))
with TestRun.step("Stopping 'fio'"):
TestRun.executor.kill_process(fio_pid)
with TestRun.step("Stop all caches"):
casadm.stop_all_caches()
def test_remove_multilevel_core():
"""
title: Test of the ability to remove a core used in a multilevel cache.
description: |
Negative test if OpenCAS does not allow to remove a core when the related exported object
is used as a core device for another cache instance.
pass_criteria:
- No system crash.
- OpenCAS does not allow removing a core used in a multilevel cache instance.
"""
with TestRun.step("Prepare two devices for cache and one for core."):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions([Size(512, Unit.MebiByte)] * 2)
cache_part1 = cache_dev.partitions[0]
cache_part2 = cache_dev.partitions[1]
core_dev = TestRun.disks['core']
core_dev.create_partitions([Size(1, Unit.GibiByte)])
core_dev = core_dev.partitions[0]
with TestRun.step("Start the first cache instance"):
cache1 = casadm.start_cache(cache_part1, force=True)
with TestRun.step("Add a core device to the first cache instance."):
core1 = cache1.add_core(core_dev)
with TestRun.step("Start the second cache instance"):
cache2 = casadm.start_cache(cache_part2, force=True)
with TestRun.step("Add the first cache's exported object as a core "
"to the second cache instance."):
cache2.add_core(core1)
with TestRun.step("Try to remove core from the first level cache."):
output = TestRun.executor.run_expect_fail(
cli.remove_core_cmd(cache_id=str(cache1.cache_id),
core_id=str(core1.core_id),
force=True))
cli_messages.check_stderr_msg(output,
cli_messages.remove_multilevel_core)
with TestRun.step("Stop cache."):
casadm.stop_all_caches()
def test_stop_cache_with_mounted_partition(cache_mode):
"""
title: Fault injection test for removing core and stopping cache with mounted core.
description: |
Negative test of the ability of CAS to remove core and stop cache while core
is still mounted.
pass_criteria:
- No system crash.
- Unable to stop cache when partition is mounted.
- Unable to remove core when partition is mounted.
"""
with TestRun.step("Prepare cache and core devices. Start CAS."):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions([Size(1, Unit.GibiByte)])
cache_part = cache_dev.partitions[0]
core_dev = TestRun.disks['core']
core_dev.create_partitions([Size(4, Unit.GibiByte)])
core_part = core_dev.partitions[0]
cache = casadm.start_cache(cache_part, cache_mode, force=True)
with TestRun.step("Add core device with xfs filesystem and mount it."):
core_part.create_filesystem(Filesystem.xfs)
core = cache.add_core(core_part)
core.mount(mount_point)
with TestRun.step("Try to remove core from cache."):
output = TestRun.executor.run_expect_fail(
cli.remove_core_cmd(cache_id=str(cache.cache_id),
core_id=str(core.core_id)))
cli_messages.check_stderr_msg(output, cli_messages.remove_mounted_core)
with TestRun.step("Try to stop CAS."):
output = TestRun.executor.run_expect_fail(
cli.stop_cmd(cache_id=str(cache.cache_id)))
cli_messages.check_stderr_msg(output,
cli_messages.stop_cache_mounted_core)
with TestRun.step("Unmount core device."):
core.unmount()
with TestRun.step("Stop cache."):
casadm.stop_all_caches()
def test_standby_neg_cli_management():
"""
title: Blocking management commands in standby state
description: |
Try executing management commands for a cache in standby state
pass_criteria:
- The execution is unsuccessful for blocked management commands
- The execution is successful for allowed management commands
- A proper error message is displayed for unsuccessful executions
"""
with TestRun.step("Prepare the device for the cache."):
device = TestRun.disks["cache"]
device.create_partitions(
[Size(500, Unit.MebiByte),
Size(500, Unit.MebiByte)])
cache_device = device.partitions[0]
core_device = device.partitions[1]
with TestRun.step("Prepare the standby instance"):
cache_id = 1
cache = casadm.standby_init(cache_dev=cache_device,
cache_id=cache_id,
cache_line_size=32,
force=True)
ioclass_config_path = "/tmp/standby_cli_neg_mngt_test_ioclass_config_file.csv"
TestRun.executor.run(f"rm -rf {ioclass_config_path}")
random_ioclass_config = IoClass.generate_random_ioclass_list(5)
IoClass.save_list_to_config_file(
random_ioclass_config, ioclass_config_path=ioclass_config_path)
blocked_mngt_commands = [
cli.get_param_cutoff_cmd(str(cache_id), "1"),
cli.get_param_cleaning_cmd(str(cache_id)),
cli.get_param_cleaning_alru_cmd(str(cache_id)),
cli.get_param_cleaning_acp_cmd(str(cache_id)),
cli.set_param_cutoff_cmd(str(cache_id), "1", threshold="1"),
cli.set_param_cutoff_cmd(str(cache_id), policy="never"),
cli.set_param_cleaning_cmd(str(cache_id), policy="nop"),
cli.set_param_cleaning_alru_cmd(str(cache_id), wake_up="30"),
cli.set_param_cleaning_acp_cmd(str(cache_id), wake_up="100"),
cli.set_param_promotion_cmd(str(cache_id), policy="nhit"),
cli.set_param_promotion_nhit_cmd(str(cache_id), threshold="5"),
cli.set_cache_mode_cmd("wb", str(cache_id)),
cli.add_core_cmd(str(cache_id), core_device.path),
cli.remove_core_cmd(str(cache_id), "1"),
cli.remove_inactive_cmd(str(cache_id), "1"),
cli.reset_counters_cmd(str(cache_id)),
cli.flush_cache_cmd(str(cache_id)),
cli.flush_core_cmd(str(cache_id), "1"),
cli.load_io_classes_cmd(str(cache_id), ioclass_config_path),
cli.list_io_classes_cmd(str(cache_id), output_format="csv"),
cli.script_try_add_cmd(str(cache_id), core_device.path, core_id=1),
cli.script_purge_cache_cmd(str(cache_id)),
cli.script_purge_core_cmd(str(cache_id), "1"),
cli.script_detach_core_cmd(str(cache_id), "1"),
cli.script_remove_core_cmd(str(cache_id), "1"),
]
with TestRun.step(
"Try to execute forbidden management commands in standby mode"):
for cmd in blocked_mngt_commands:
TestRun.LOGGER.info(f"Verify {cmd}")
output = TestRun.executor.run_expect_fail(cmd)
if not check_stderr_msg(output, operation_forbiden_in_standby):
TestRun.LOGGER.error(
f'Expected the following error message "{operation_forbiden_in_standby[0]}" '
f'Got "{output.stderr}" instead.')
with TestRun.step("Stop the standby instance"):
TestRun.executor.run(f"rm -rf {ioclass_config_path}")
cache.stop()
def test_user_cli():
"""
title: Test that OpenCAS does not allow to change parameters in CLI by non-root user.
description: |
Checking if changing parameters in CLI by non-root user is forbidden by OpenCAS,
but is permitted with 'sudo' command.
pass_criteria:
- Non-root user can only print help and CAS version.
- Sudoer user is allowed to change OpenCAS parameters in CLI with sudo.
"""
with TestRun.step("Prepare cache and core devices."):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions([Size(256, Unit.MebiByte)])
cache_dev = cache_dev.partitions[0]
core_dev = TestRun.disks['core']
core_dev.create_partitions([Size(1, Unit.GibiByte), Size(256, Unit.MebiByte)])
core_part1 = core_dev.partitions[0]
core_part2 = core_dev.partitions[1]
with TestRun.step("Start cache."):
cache = casadm.start_cache(cache_dev, force=True)
with TestRun.step("Add core to cache and mount it."):
core_part1.create_filesystem(Filesystem.ext3)
core = cache.add_core(core_part1)
core.mount(mount_point)
with TestRun.step(f"Copy casadm bin from {system_casadm_bin_path} "
f"to {user_casadm_bin_dest_path}."):
casadm_bin = fs_utils.parse_ls_output(fs_utils.ls_item(f"{system_casadm_bin_path}"))[0]
casadm_bin_copy = casadm_bin.copy(user_casadm_bin_dest_path, True)
casadm_bin_copy.chmod_numerical(777)
with TestRun.step("Copy IO class config."):
io_conf = fs_utils.parse_ls_output(fs_utils.ls_item(f"{ioclass_config_path}"))[0]
io_conf_copy = io_conf.copy(ioclass_config_copy_path, force=True)
with TestRun.step("Unmount core."):
core.unmount()
with TestRun.step("Stop cache."):
casadm.stop_all_caches()
with TestRun.step("Add non-root user account."):
TestRun.executor.run(f"useradd -N -r -l {user_name}")
user_home_dir = fs_utils.parse_ls_output(fs_utils.ls_item(f"/home/{user_name}"))[0]
user_home_dir.chmod_numerical(777, True)
with TestRun.step("Try to start cache."):
try:
output = run_as_other_user(cli.start_cmd(cache_dev.path), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Starting cache should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot start cache.")
with TestRun.step("Start cache again."):
casadm.load_cache(cache_dev)
with TestRun.step("Try to stop cache."):
try:
output = run_as_other_user(cli.stop_cmd(str(cache.cache_id)), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Stopping cache should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot stop cache.")
with TestRun.step("Try to set cache mode."):
try:
output = run_as_other_user(cli.set_cache_mode_cmd(CacheMode.WB,
str(cache.cache_id)), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Setting cache mode should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot set cache mode.")
with TestRun.step("Try to add core to cache."):
try:
output = run_as_other_user(cli.add_core_cmd(str(cache.cache_id),
core_part2.path), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Adding core to cache should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot add core.")
with TestRun.step("Try to remove core from cache."):
try:
output = run_as_other_user(cli.remove_core_cmd(str(cache.cache_id),
str(core.core_id)), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Removing core from cache should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot remove core.")
with TestRun.step("Try to zero metadata."):
try:
output = run_as_other_user(cli.zero_metadata_cmd(str(cache_dev)), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Zeroing metadata should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot zero metadata.")
with TestRun.step("Try to list caches."):
try:
output = run_as_other_user(cli.list_cmd(), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Listing caches should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot list caches.")
with TestRun.step("Try to print stats."):
try:
output = run_as_other_user(cli.print_statistics_cmd(str(cache.cache_id)), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Printing stats should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot print statistics.")
with TestRun.step("Try to reset stats."):
try:
output = run_as_other_user(cli.reset_counters_cmd(str(cache.cache_id)), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Resetting stats should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot reset statistics.")
with TestRun.step("Try to flush cache."):
try:
output = run_as_other_user(cli.flush_cache_cmd(str(cache.cache_id)), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Flushing cache should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot flush cache.")
with TestRun.step("Try to flush core."):
try:
output = run_as_other_user(cli.flush_core_cmd(str(cache.cache_id),
str(core.core_id)), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Flushing core should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot flush core.")
with TestRun.step("Try to set cleaning policy and its parameters."):
try:
output = run_as_other_user(cli.set_param_cleaning_cmd(
str(cache.cache_id), "nop"), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Setting cleaning policy should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot set cleaning policy as nop.")
try:
output = run_as_other_user(cli.set_param_cleaning_cmd(
str(cache.cache_id), "alru"), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Setting cleaning policy should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot set cleaning policy as alru.")
try:
output = run_as_other_user(cli.set_param_cleaning_alru_cmd(str(cache.cache_id),
"15",
"60",
"1000",
"8000"), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Setting cleaning policy parameters should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot set alru cleaning policy parameters.")
try:
output = run_as_other_user(cli.set_param_cleaning_cmd(
str(cache.cache_id), "acp"), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Setting cleaning policy should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot set cleaning policy as acp.")
try:
output = run_as_other_user(cli.set_param_cleaning_acp_cmd(str(cache.cache_id),
"15",
"1000"), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Setting cleaning policy parameters should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot set acp cleaning policy parameters.")
with TestRun.step("Try to list IO class configuration."):
try:
output = run_as_other_user(cli.list_io_classes_cmd(
str(cache.cache_id), OutputFormat.table.name), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Listing IO class configuration should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot list IO class configuration.")
with TestRun.step("Try to load IO class configuration."):
try:
output = run_as_other_user(cli.load_io_classes_cmd(
str(cache.cache_id), io_conf_copy), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Loading IO class configuration should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot load IO class configuration.")
with TestRun.step("Try to print help for casadm."):
try:
run_as_other_user(cli.help_cmd(), user_name)
except CmdException:
TestRun.LOGGER.error("Non-root user should be able to print help for casadm.")
with TestRun.step("Try to print version of OpenCAS."):
try:
run_as_other_user(cli.version_cmd(), user_name)
except CmdException:
TestRun.LOGGER.error("Non-root user should be able to print version of OpenCAS.")
with TestRun.step("Add non-root user account to sudoers group."):
TestRun.executor.run(f'echo "{user_name} ALL = (root) NOPASSWD:ALL" '
f'| sudo tee /etc/sudoers.d/{user_name}')
with TestRun.step("Try to stop cache with 'sudo'."):
try:
run_as_other_user(cli.stop_cmd(str(cache.cache_id)), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to stop cache.")
with TestRun.step("Try to start cache with 'sudo'."):
try:
run_as_other_user(cli.start_cmd(cache_dev.path, force=True), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to start cache.")
with TestRun.step("Try to set cache mode with 'sudo'."):
try:
run_as_other_user(
cli.set_cache_mode_cmd(str(CacheMode.WB.name).lower(), str(cache.cache_id)),
user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to set cache mode.")
with TestRun.step("Try to add core to cache with 'sudo'."):
try:
run_as_other_user(cli.add_core_cmd(str(cache.cache_id),
core_part1.path), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to add core to cache.")
with TestRun.step("Try to list caches with 'sudo'."):
try:
run_as_other_user(cli.list_cmd(), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to list caches.")
with TestRun.step("Try to print stats with 'sudo'."):
try:
run_as_other_user(cli.print_statistics_cmd(str(cache.cache_id)), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to print stats.")
with TestRun.step("Try to reset stats with 'sudo'."):
try:
run_as_other_user(cli.reset_counters_cmd(str(cache.cache_id)), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to reset stats.")
with TestRun.step("Try to flush cache with 'sudo'."):
try:
run_as_other_user(cli.flush_cache_cmd(str(cache.cache_id)), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to flush cache.")
with TestRun.step("Try to flush core with 'sudo'."):
try:
run_as_other_user(cli.flush_core_cmd(str(cache.cache_id),
str(core.core_id)), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to flush core.")
with TestRun.step("Try to set cleaning policy and its parameters with 'sudo'."):
try:
run_as_other_user(cli.set_param_cleaning_cmd(str(cache.cache_id), "nop"),
user_name, True)
run_as_other_user(cli.set_param_cleaning_cmd(str(cache.cache_id), "alru"),
user_name, True)
try:
run_as_other_user(cli.set_param_cleaning_alru_cmd(str(cache.cache_id),
"15",
"60",
"1000",
"8000"), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to "
"set alru cleaning policy parameters.")
run_as_other_user(cli.set_param_cleaning_cmd(str(cache.cache_id), "acp"),
user_name, True)
try:
run_as_other_user(cli.set_param_cleaning_acp_cmd(str(cache.cache_id),
"15",
"1000"), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to "
"set acp cleaning policy parameters.")
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to "
"set cleaning policy and its parameters.")
with TestRun.step("Try to list IO class with 'sudo'."):
try:
run_as_other_user(cli.list_io_classes_cmd(str(cache.cache_id), OutputFormat.table.name),
user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to list IO class.")
with TestRun.step("Try to load IO class configuration with 'sudo'."):
try:
run_as_other_user(cli.load_io_classes_cmd(str(cache.cache_id), io_conf_copy),
user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to "
"load IO class configuration.")
with TestRun.step("Try to remove core from cache with 'sudo'."):
try:
run_as_other_user(cli.remove_core_cmd(str(cache.cache_id), str(core.core_id)),
user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to remove core from cache.")
with TestRun.step("Try to zero metadata with 'sudo'."):
try:
run_as_other_user(cli.zero_metadata_cmd(str(cache_dev)),
user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to zero metadata.")
with TestRun.step("Try to print help for casadm with 'sudo'."):
try:
run_as_other_user(cli.help_cmd(), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to print help for casadm.")
with TestRun.step("Try to print version of OpenCAS with 'sudo'."):
try:
run_as_other_user(cli.version_cmd(), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to print version of OpenCAS.")
with TestRun.step("Stop caches."):
casadm.stop_all_caches()
with TestRun.step("Remove user account."):
TestRun.executor.run(f"userdel -r -Z {user_name}")
def test_interrupt_core_remove(cache_mode, filesystem):
"""
title: Test if OpenCAS works correctly after core's removing interruption.
description: |
Negative test of the ability of OpenCAS to handle core's removing interruption.
pass_criteria:
- No system crash.
- Core would not be removed from cache after interruption.
- Flushing would be stopped after interruption.
- Md5sum are correct during all test steps.
- Dirty blocks quantity after interruption is equal or lower.
"""
with TestRun.step("Prepare cache and core."):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions([cache_size])
cache_part = cache_dev.partitions[0]
core_dev = TestRun.disks['core']
core_dev.create_partitions([cache_size * 2])
core_part = core_dev.partitions[0]
for _ in TestRun.iteration(
range(iterations_per_config),
f"Reload cache configuration {iterations_per_config} times."):
with TestRun.step("Start cache."):
cache = casadm.start_cache(cache_part, cache_mode, force=True)
with TestRun.step("Set cleaning policy to NOP."):
cache.set_cleaning_policy(CleaningPolicy.nop)
with TestRun.step(
f"Add core device with {filesystem} filesystem and mount it."):
core_part.create_filesystem(filesystem)
core = cache.add_core(core_part)
core.mount(mount_point)
with TestRun.step(
f"Create test file in mount point of exported object."):
test_file = create_test_file()
with TestRun.step("Check md5 sum of test file."):
test_file_md5sum_before = test_file.md5sum()
with TestRun.step(
"Get number of dirty data on exported object before core removal interruption."
):
os_utils.sync()
os_utils.drop_caches(DropCachesMode.ALL)
cache_dirty_blocks_before = cache.get_dirty_blocks()
with TestRun.step("Unmount core."):
core.unmount()
with TestRun.step("Start removing core device."):
flush_pid = TestRun.executor.run_in_background(
cli.remove_core_cmd(str(cache.cache_id), str(core.core_id)))
sleep(2)
with TestRun.step("Interrupt core removing"):
percentage = casadm_parser.get_flushing_progress(
cache.cache_id, core.core_id)
while percentage < 50:
percentage = casadm_parser.get_flushing_progress(
cache.cache_i, core.core_id)
TestRun.executor.run(f"kill -s SIGINT {flush_pid}")
with TestRun.step(
"Check number of dirty data on exported object after core removal interruption."
):
cache_dirty_blocks_after = cache.get_dirty_blocks()
if cache_dirty_blocks_after >= cache_dirty_blocks_before:
TestRun.LOGGER.error(
"Quantity of dirty lines after core removal interruption "
"should be lower.")
if int(cache_dirty_blocks_after) == 0:
TestRun.LOGGER.error(
"Quantity of dirty lines after core removal interruption "
"should not be zero.")
with TestRun.step("Remove core from cache."):
core.remove_core()
with TestRun.step("Stop cache."):
cache.stop()
with TestRun.step("Mount core device."):
core_part.mount(mount_point)
with TestRun.step("Check md5 sum of test file again."):
if test_file_md5sum_before != test_file.md5sum():
TestRun.LOGGER.error(
"Md5 sums before and after interrupting core removal are different."
)
with TestRun.step("Unmount core device."):
core_part.unmount()
