def test_stop_cache_during_io():
"""
title: Test for stopping cache during IO.
description: |
Creating CAS device, running fio on it and checking
if cache can be stopped during IO operations.
pass_criteria:
- Cache is not stopped.
"""
with TestRun.step("Start cache and add core"):
cache, core = prepare()
with TestRun.step("Running 'fio'"):
fio = (
Fio()
.create_command()
.io_engine(IoEngine.libaio)
.block_size(Size(4, Unit.KibiByte))
.read_write(ReadWrite.randrw)
.target(f"{core.system_path}")
.direct(1)
.run_time(timedelta(minutes=4))
.time_based()
)
fio_pid = fio.run_in_background()
time.sleep(10)
with TestRun.step("Try to stop cache during 'fio'"):
TestRun.executor.run_expect_fail(cli.stop_cmd(f"{cache.cache_id}"))
with TestRun.step("Stopping 'fio'"):
TestRun.executor.kill_process(fio_pid)
with TestRun.step("Stop all caches"):
casadm.stop_all_caches()
def test_flush_over_640_gibibytes_raw_device(cache_mode):
"""
title: Test of the ability to flush huge amount of dirty data on raw device.
description: |
Flush cache when amount of dirty data in cache exceeds 640 GiB.
pass_criteria:
- Flushing completes successfully without any errors.
"""
with TestRun.step("Prepare devices for cache and core."):
cache_dev = TestRun.disks['cache']
check_disk_size(cache_dev)
cache_dev.create_partitions([required_disk_size])
cache_part = cache_dev.partitions[0]
core_dev = TestRun.disks['core']
check_disk_size(core_dev)
Udev.disable()
with TestRun.step(f"Start cache in {cache_mode} mode."):
cache = casadm.start_cache(cache_part, cache_mode)
with TestRun.step(f"Add core to cache."):
core = cache.add_core(core_dev)
with TestRun.step("Disable cleaning and sequential cutoff."):
cache.set_cleaning_policy(CleaningPolicy.nop)
cache.set_seq_cutoff_policy(SeqCutOffPolicy.never)
with TestRun.step("Create test file"):
fio = (Fio().create_command().io_engine(IoEngine.libaio).read_write(
ReadWrite.write).block_size(bs).direct().io_depth(256).target(
core).size(file_size))
fio.default_run_time = timedelta(
hours=4) # timeout for non-time-based fio
fio.run()
with TestRun.step(f"Check if dirty data exceeded {file_size * 0.98} GiB."):
minimum_4KiB_blocks = int(
(file_size * 0.98).get_value(Unit.Blocks4096))
if int(cache.get_statistics().usage_stats.dirty) < minimum_4KiB_blocks:
TestRun.fail("There is not enough dirty data in the cache!")
with TestRun.step("Stop cache with flush."):
# this operation could take few hours, depending on core disk
output = TestRun.executor.run(stop_cmd(str(cache.cache_id)),
timedelta(hours=12))
if output.exit_code != 0:
TestRun.fail(f"Stopping cache with flush failed!\n{output.stderr}")
def test_stop_cache_with_mounted_partition(cache_mode):
"""
title: Fault injection test for removing core and stopping cache with mounted core.
description: |
Negative test of the ability of CAS to remove core and stop cache while core
is still mounted.
pass_criteria:
- No system crash.
- Unable to stop cache when partition is mounted.
- Unable to remove core when partition is mounted.
"""
with TestRun.step("Prepare cache and core devices. Start CAS."):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions([Size(1, Unit.GibiByte)])
cache_part = cache_dev.partitions[0]
core_dev = TestRun.disks['core']
core_dev.create_partitions([Size(4, Unit.GibiByte)])
core_part = core_dev.partitions[0]
cache = casadm.start_cache(cache_part, cache_mode, force=True)
with TestRun.step("Add core device with xfs filesystem and mount it."):
core_part.create_filesystem(Filesystem.xfs)
core = cache.add_core(core_part)
core.mount(mount_point)
with TestRun.step("Try to remove core from cache."):
output = TestRun.executor.run_expect_fail(
cli.remove_core_cmd(cache_id=str(cache.cache_id),
core_id=str(core.core_id)))
cli_messages.check_stderr_msg(output, cli_messages.remove_mounted_core)
with TestRun.step("Try to stop CAS."):
output = TestRun.executor.run_expect_fail(
cli.stop_cmd(cache_id=str(cache.cache_id)))
cli_messages.check_stderr_msg(output,
cli_messages.stop_cache_mounted_core)
with TestRun.step("Unmount core device."):
core.unmount()
with TestRun.step("Stop cache."):
casadm.stop_all_caches()
def test_user_cli():
"""
title: Test that OpenCAS does not allow to change parameters in CLI by non-root user.
description: |
Checking if changing parameters in CLI by non-root user is forbidden by OpenCAS,
but is permitted with 'sudo' command.
pass_criteria:
- Non-root user can only print help and CAS version.
- Sudoer user is allowed to change OpenCAS parameters in CLI with sudo.
"""
with TestRun.step("Prepare cache and core devices."):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions([Size(256, Unit.MebiByte)])
cache_dev = cache_dev.partitions[0]
core_dev = TestRun.disks['core']
core_dev.create_partitions([Size(1, Unit.GibiByte), Size(256, Unit.MebiByte)])
core_part1 = core_dev.partitions[0]
core_part2 = core_dev.partitions[1]
with TestRun.step("Start cache."):
cache = casadm.start_cache(cache_dev, force=True)
with TestRun.step("Add core to cache and mount it."):
core_part1.create_filesystem(Filesystem.ext3)
core = cache.add_core(core_part1)
core.mount(mount_point)
with TestRun.step(f"Copy casadm bin from {system_casadm_bin_path} "
f"to {user_casadm_bin_dest_path}."):
casadm_bin = fs_utils.parse_ls_output(fs_utils.ls_item(f"{system_casadm_bin_path}"))[0]
casadm_bin_copy = casadm_bin.copy(user_casadm_bin_dest_path, True)
casadm_bin_copy.chmod_numerical(777)
with TestRun.step("Copy IO class config."):
io_conf = fs_utils.parse_ls_output(fs_utils.ls_item(f"{ioclass_config_path}"))[0]
io_conf_copy = io_conf.copy(ioclass_config_copy_path, force=True)
with TestRun.step("Unmount core."):
core.unmount()
with TestRun.step("Stop cache."):
casadm.stop_all_caches()
with TestRun.step("Add non-root user account."):
TestRun.executor.run(f"useradd -N -r -l {user_name}")
user_home_dir = fs_utils.parse_ls_output(fs_utils.ls_item(f"/home/{user_name}"))[0]
user_home_dir.chmod_numerical(777, True)
with TestRun.step("Try to start cache."):
try:
output = run_as_other_user(cli.start_cmd(cache_dev.path), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Starting cache should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot start cache.")
with TestRun.step("Start cache again."):
casadm.load_cache(cache_dev)
with TestRun.step("Try to stop cache."):
try:
output = run_as_other_user(cli.stop_cmd(str(cache.cache_id)), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Stopping cache should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot stop cache.")
with TestRun.step("Try to set cache mode."):
try:
output = run_as_other_user(cli.set_cache_mode_cmd(CacheMode.WB,
str(cache.cache_id)), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Setting cache mode should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot set cache mode.")
with TestRun.step("Try to add core to cache."):
try:
output = run_as_other_user(cli.add_core_cmd(str(cache.cache_id),
core_part2.path), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Adding core to cache should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot add core.")
with TestRun.step("Try to remove core from cache."):
try:
output = run_as_other_user(cli.remove_core_cmd(str(cache.cache_id),
str(core.core_id)), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Removing core from cache should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot remove core.")
with TestRun.step("Try to zero metadata."):
try:
output = run_as_other_user(cli.zero_metadata_cmd(str(cache_dev)), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Zeroing metadata should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot zero metadata.")
with TestRun.step("Try to list caches."):
try:
output = run_as_other_user(cli.list_cmd(), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Listing caches should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot list caches.")
with TestRun.step("Try to print stats."):
try:
output = run_as_other_user(cli.print_statistics_cmd(str(cache.cache_id)), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Printing stats should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot print statistics.")
with TestRun.step("Try to reset stats."):
try:
output = run_as_other_user(cli.reset_counters_cmd(str(cache.cache_id)), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Resetting stats should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot reset statistics.")
with TestRun.step("Try to flush cache."):
try:
output = run_as_other_user(cli.flush_cache_cmd(str(cache.cache_id)), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Flushing cache should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot flush cache.")
with TestRun.step("Try to flush core."):
try:
output = run_as_other_user(cli.flush_core_cmd(str(cache.cache_id),
str(core.core_id)), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Flushing core should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot flush core.")
with TestRun.step("Try to set cleaning policy and its parameters."):
try:
output = run_as_other_user(cli.set_param_cleaning_cmd(
str(cache.cache_id), "nop"), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Setting cleaning policy should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot set cleaning policy as nop.")
try:
output = run_as_other_user(cli.set_param_cleaning_cmd(
str(cache.cache_id), "alru"), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Setting cleaning policy should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot set cleaning policy as alru.")
try:
output = run_as_other_user(cli.set_param_cleaning_alru_cmd(str(cache.cache_id),
"15",
"60",
"1000",
"8000"), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Setting cleaning policy parameters should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot set alru cleaning policy parameters.")
try:
output = run_as_other_user(cli.set_param_cleaning_cmd(
str(cache.cache_id), "acp"), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Setting cleaning policy should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot set cleaning policy as acp.")
try:
output = run_as_other_user(cli.set_param_cleaning_acp_cmd(str(cache.cache_id),
"15",
"1000"), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Setting cleaning policy parameters should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot set acp cleaning policy parameters.")
with TestRun.step("Try to list IO class configuration."):
try:
output = run_as_other_user(cli.list_io_classes_cmd(
str(cache.cache_id), OutputFormat.table.name), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Listing IO class configuration should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot list IO class configuration.")
with TestRun.step("Try to load IO class configuration."):
try:
output = run_as_other_user(cli.load_io_classes_cmd(
str(cache.cache_id), io_conf_copy), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Loading IO class configuration should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot load IO class configuration.")
with TestRun.step("Try to print help for casadm."):
try:
run_as_other_user(cli.help_cmd(), user_name)
except CmdException:
TestRun.LOGGER.error("Non-root user should be able to print help for casadm.")
with TestRun.step("Try to print version of OpenCAS."):
try:
run_as_other_user(cli.version_cmd(), user_name)
except CmdException:
TestRun.LOGGER.error("Non-root user should be able to print version of OpenCAS.")
with TestRun.step("Add non-root user account to sudoers group."):
TestRun.executor.run(f'echo "{user_name} ALL = (root) NOPASSWD:ALL" '
f'| sudo tee /etc/sudoers.d/{user_name}')
with TestRun.step("Try to stop cache with 'sudo'."):
try:
run_as_other_user(cli.stop_cmd(str(cache.cache_id)), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to stop cache.")
with TestRun.step("Try to start cache with 'sudo'."):
try:
run_as_other_user(cli.start_cmd(cache_dev.path, force=True), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to start cache.")
with TestRun.step("Try to set cache mode with 'sudo'."):
try:
run_as_other_user(
cli.set_cache_mode_cmd(str(CacheMode.WB.name).lower(), str(cache.cache_id)),
user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to set cache mode.")
with TestRun.step("Try to add core to cache with 'sudo'."):
try:
run_as_other_user(cli.add_core_cmd(str(cache.cache_id),
core_part1.path), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to add core to cache.")
with TestRun.step("Try to list caches with 'sudo'."):
try:
run_as_other_user(cli.list_cmd(), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to list caches.")
with TestRun.step("Try to print stats with 'sudo'."):
try:
run_as_other_user(cli.print_statistics_cmd(str(cache.cache_id)), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to print stats.")
with TestRun.step("Try to reset stats with 'sudo'."):
try:
run_as_other_user(cli.reset_counters_cmd(str(cache.cache_id)), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to reset stats.")
with TestRun.step("Try to flush cache with 'sudo'."):
try:
run_as_other_user(cli.flush_cache_cmd(str(cache.cache_id)), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to flush cache.")
with TestRun.step("Try to flush core with 'sudo'."):
try:
run_as_other_user(cli.flush_core_cmd(str(cache.cache_id),
str(core.core_id)), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to flush core.")
with TestRun.step("Try to set cleaning policy and its parameters with 'sudo'."):
try:
run_as_other_user(cli.set_param_cleaning_cmd(str(cache.cache_id), "nop"),
user_name, True)
run_as_other_user(cli.set_param_cleaning_cmd(str(cache.cache_id), "alru"),
user_name, True)
try:
run_as_other_user(cli.set_param_cleaning_alru_cmd(str(cache.cache_id),
"15",
"60",
"1000",
"8000"), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to "
"set alru cleaning policy parameters.")
run_as_other_user(cli.set_param_cleaning_cmd(str(cache.cache_id), "acp"),
user_name, True)
try:
run_as_other_user(cli.set_param_cleaning_acp_cmd(str(cache.cache_id),
"15",
"1000"), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to "
"set acp cleaning policy parameters.")
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to "
"set cleaning policy and its parameters.")
with TestRun.step("Try to list IO class with 'sudo'."):
try:
run_as_other_user(cli.list_io_classes_cmd(str(cache.cache_id), OutputFormat.table.name),
user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to list IO class.")
with TestRun.step("Try to load IO class configuration with 'sudo'."):
try:
run_as_other_user(cli.load_io_classes_cmd(str(cache.cache_id), io_conf_copy),
user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to "
"load IO class configuration.")
with TestRun.step("Try to remove core from cache with 'sudo'."):
try:
run_as_other_user(cli.remove_core_cmd(str(cache.cache_id), str(core.core_id)),
user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to remove core from cache.")
with TestRun.step("Try to zero metadata with 'sudo'."):
try:
run_as_other_user(cli.zero_metadata_cmd(str(cache_dev)),
user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to zero metadata.")
with TestRun.step("Try to print help for casadm with 'sudo'."):
try:
run_as_other_user(cli.help_cmd(), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to print help for casadm.")
with TestRun.step("Try to print version of OpenCAS with 'sudo'."):
try:
run_as_other_user(cli.version_cmd(), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to print version of OpenCAS.")
with TestRun.step("Stop caches."):
casadm.stop_all_caches()
with TestRun.step("Remove user account."):
TestRun.executor.run(f"userdel -r -Z {user_name}")
def test_interrupt_cache_stop(cache_mode, filesystem):
"""
title: Test if OpenCAS works correctly after cache stopping interruption.
description: |
Negative test of the ability of OpenCAS to handle cache's stop interruption.
pass_criteria:
- No system crash.
- Flushing would be stopped after interruption.
- Md5sum are correct during all test steps.
- Dirty blocks quantity after interruption is lower but non-zero.
"""
with TestRun.step("Prepare cache and core."):
cache_part, core_part = prepare()
for _ in TestRun.iteration(
range(iterations_per_config),
f"Reload cache configuration {iterations_per_config} times."):
with TestRun.step("Start cache."):
cache = casadm.start_cache(cache_part, cache_mode, force=True)
with TestRun.step("Set cleaning policy to NOP."):
cache.set_cleaning_policy(CleaningPolicy.nop)
with TestRun.step(
f"Add core device with {filesystem} filesystem and mount it."):
core_part.create_filesystem(filesystem)
core = cache.add_core(core_part)
core.mount(mount_point)
with TestRun.step(
f"Create test file in mount point of exported object."):
test_file = create_test_file()
with TestRun.step("Check md5 sum of test file."):
test_file_md5sum_before = test_file.md5sum()
with TestRun.step(
"Get number of dirty data on exported object before interruption."
):
os_utils.sync()
os_utils.drop_caches(DropCachesMode.ALL)
cache_dirty_blocks_before = cache.get_dirty_blocks()
with TestRun.step("Unmount core."):
core.unmount()
with TestRun.step("Start stopping cache."):
flush_pid = TestRun.executor.run_in_background(
cli.stop_cmd(str(cache.cache_id)))
sleep(2)
with TestRun.step("Interrupt cache stopping."):
percentage = casadm_parser.get_flushing_progress(
cache.cache_id, core.core_id)
while percentage < 50:
percentage = casadm_parser.get_flushing_progress(
cache.cache_id, core.core_id)
TestRun.executor.run(f"kill -s SIGINT {flush_pid}")
with TestRun.step(
"Check number of dirty data on exported object after interruption."
):
cache_dirty_blocks_after = cache.get_dirty_blocks()
if cache_dirty_blocks_after >= cache_dirty_blocks_before:
TestRun.LOGGER.error(
"Quantity of dirty lines after cache stop interruption "
"should be lower.")
if int(cache_dirty_blocks_after) == 0:
TestRun.LOGGER.error(
"Quantity of dirty lines after cache stop interruption "
"should not be zero.")
with TestRun.step("Stop cache."):
cache.stop()
with TestRun.step("Mount core device."):
core_part.mount(mount_point)
with TestRun.step("Check md5 sum of test file again."):
if test_file_md5sum_before != test_file.md5sum():
TestRun.LOGGER.error("Md5 sums before and after interrupting"
" cache stop are different.")
with TestRun.step("Unmount core device."):
core_part.unmount()
def test_flush_over_640_gibibytes_with_fs(cache_mode, fs):
"""
title: Test of the ability to flush huge amount of dirty data on device with filesystem.
description: |
Flush cache when amount of dirty data in cache with core with filesystem exceeds 640 GiB.
pass_criteria:
- Flushing completes successfully without any errors.
"""
with TestRun.step("Prepare devices for cache and core."):
cache_dev = TestRun.disks['cache']
check_disk_size(cache_dev)
cache_dev.create_partitions([required_disk_size])
cache_part = cache_dev.partitions[0]
core_dev = TestRun.disks['core']
check_disk_size(core_dev)
Udev.disable()
with TestRun.step(f"Start cache in {cache_mode} mode."):
cache = casadm.start_cache(cache_part, cache_mode)
with TestRun.step(
f"Add core with {fs.name} filesystem to cache and mount it."):
core_dev.create_filesystem(fs)
core = cache.add_core(core_dev)
core.mount(mnt_point)
with TestRun.step("Disable cleaning and sequential cutoff."):
cache.set_cleaning_policy(CleaningPolicy.nop)
cache.set_seq_cutoff_policy(SeqCutOffPolicy.never)
with TestRun.step("Create test file"):
test_file_main = File.create_file("/tmp/test_file_main")
fio = (Fio().create_command().io_engine(IoEngine.libaio).read_write(
ReadWrite.write).block_size(bs).direct().io_depth(256).target(
test_file_main.full_path).size(file_size))
fio.default_run_time = timedelta(
hours=4) # timeout for non-time-based fio
fio.run()
test_file_main.refresh_item()
with TestRun.step("Validate test file and read its md5 sum."):
if test_file_main.size != file_size:
TestRun.fail("Created test file hasn't reached its target size.")
test_file_md5sum_main = test_file_main.md5sum()
with TestRun.step("Write data to exported object."):
test_file_copy = test_file_main.copy(mnt_point + "test_file_copy")
test_file_copy.refresh_item()
sync()
with TestRun.step(f"Check if dirty data exceeded {file_size * 0.98} GiB."):
minimum_4KiB_blocks = int(
(file_size * 0.98).get_value(Unit.Blocks4096))
if int(cache.get_statistics().usage_stats.dirty) < minimum_4KiB_blocks:
TestRun.fail("There is not enough dirty data in the cache!")
with TestRun.step("Unmount core and stop cache with flush."):
core.unmount()
# this operation could take few hours, depending on core disk
output = TestRun.executor.run(stop_cmd(str(cache.cache_id)),
timedelta(hours=12))
if output.exit_code != 0:
TestRun.fail(f"Stopping cache with flush failed!\n{output.stderr}")
with TestRun.step(
"Mount core device and check md5 sum of test file copy."):
core_dev.mount(mnt_point)
if test_file_md5sum_main != test_file_copy.md5sum():
TestRun.LOGGER.error("Md5 sums should be equal.")
with TestRun.step("Delete test files."):
test_file_main.remove(True)
test_file_copy.remove(True)
with TestRun.step("Unmount core device."):
core_dev.unmount()
remove(mnt_point, True, True, True)
