def test_stop_cache_during_io(): """ title: Test for stopping cache during IO. description: | Creating CAS device, running fio on it and checking if cache can be stopped during IO operations. pass_criteria: - Cache is not stopped. """ with TestRun.step("Start cache and add core"): cache, core = prepare() with TestRun.step("Running 'fio'"): fio = ( Fio() .create_command() .io_engine(IoEngine.libaio) .block_size(Size(4, Unit.KibiByte)) .read_write(ReadWrite.randrw) .target(f"{core.system_path}") .direct(1) .run_time(timedelta(minutes=4)) .time_based() ) fio_pid = fio.run_in_background() time.sleep(10) with TestRun.step("Try to stop cache during 'fio'"): TestRun.executor.run_expect_fail(cli.stop_cmd(f"{cache.cache_id}")) with TestRun.step("Stopping 'fio'"): TestRun.executor.kill_process(fio_pid) with TestRun.step("Stop all caches"): casadm.stop_all_caches()
def test_flush_over_640_gibibytes_raw_device(cache_mode): """ title: Test of the ability to flush huge amount of dirty data on raw device. description: | Flush cache when amount of dirty data in cache exceeds 640 GiB. pass_criteria: - Flushing completes successfully without any errors. """ with TestRun.step("Prepare devices for cache and core."): cache_dev = TestRun.disks['cache'] check_disk_size(cache_dev) cache_dev.create_partitions([required_disk_size]) cache_part = cache_dev.partitions[0] core_dev = TestRun.disks['core'] check_disk_size(core_dev) Udev.disable() with TestRun.step(f"Start cache in {cache_mode} mode."): cache = casadm.start_cache(cache_part, cache_mode) with TestRun.step(f"Add core to cache."): core = cache.add_core(core_dev) with TestRun.step("Disable cleaning and sequential cutoff."): cache.set_cleaning_policy(CleaningPolicy.nop) cache.set_seq_cutoff_policy(SeqCutOffPolicy.never) with TestRun.step("Create test file"): fio = (Fio().create_command().io_engine(IoEngine.libaio).read_write( ReadWrite.write).block_size(bs).direct().io_depth(256).target( core).size(file_size)) fio.default_run_time = timedelta( hours=4) # timeout for non-time-based fio fio.run() with TestRun.step(f"Check if dirty data exceeded {file_size * 0.98} GiB."): minimum_4KiB_blocks = int( (file_size * 0.98).get_value(Unit.Blocks4096)) if int(cache.get_statistics().usage_stats.dirty) < minimum_4KiB_blocks: TestRun.fail("There is not enough dirty data in the cache!") with TestRun.step("Stop cache with flush."): # this operation could take few hours, depending on core disk output = TestRun.executor.run(stop_cmd(str(cache.cache_id)), timedelta(hours=12)) if output.exit_code != 0: TestRun.fail(f"Stopping cache with flush failed!\n{output.stderr}")
def test_stop_cache_with_mounted_partition(cache_mode): """ title: Fault injection test for removing core and stopping cache with mounted core. description: | Negative test of the ability of CAS to remove core and stop cache while core is still mounted. pass_criteria: - No system crash. - Unable to stop cache when partition is mounted. - Unable to remove core when partition is mounted. """ with TestRun.step("Prepare cache and core devices. Start CAS."): cache_dev = TestRun.disks['cache'] cache_dev.create_partitions([Size(1, Unit.GibiByte)]) cache_part = cache_dev.partitions[0] core_dev = TestRun.disks['core'] core_dev.create_partitions([Size(4, Unit.GibiByte)]) core_part = core_dev.partitions[0] cache = casadm.start_cache(cache_part, cache_mode, force=True) with TestRun.step("Add core device with xfs filesystem and mount it."): core_part.create_filesystem(Filesystem.xfs) core = cache.add_core(core_part) core.mount(mount_point) with TestRun.step("Try to remove core from cache."): output = TestRun.executor.run_expect_fail( cli.remove_core_cmd(cache_id=str(cache.cache_id), core_id=str(core.core_id))) cli_messages.check_stderr_msg(output, cli_messages.remove_mounted_core) with TestRun.step("Try to stop CAS."): output = TestRun.executor.run_expect_fail( cli.stop_cmd(cache_id=str(cache.cache_id))) cli_messages.check_stderr_msg(output, cli_messages.stop_cache_mounted_core) with TestRun.step("Unmount core device."): core.unmount() with TestRun.step("Stop cache."): casadm.stop_all_caches()
def test_user_cli(): """ title: Test that OpenCAS does not allow to change parameters in CLI by non-root user. description: | Checking if changing parameters in CLI by non-root user is forbidden by OpenCAS, but is permitted with 'sudo' command. pass_criteria: - Non-root user can only print help and CAS version. - Sudoer user is allowed to change OpenCAS parameters in CLI with sudo. """ with TestRun.step("Prepare cache and core devices."): cache_dev = TestRun.disks['cache'] cache_dev.create_partitions([Size(256, Unit.MebiByte)]) cache_dev = cache_dev.partitions[0] core_dev = TestRun.disks['core'] core_dev.create_partitions([Size(1, Unit.GibiByte), Size(256, Unit.MebiByte)]) core_part1 = core_dev.partitions[0] core_part2 = core_dev.partitions[1] with TestRun.step("Start cache."): cache = casadm.start_cache(cache_dev, force=True) with TestRun.step("Add core to cache and mount it."): core_part1.create_filesystem(Filesystem.ext3) core = cache.add_core(core_part1) core.mount(mount_point) with TestRun.step(f"Copy casadm bin from {system_casadm_bin_path} " f"to {user_casadm_bin_dest_path}."): casadm_bin = fs_utils.parse_ls_output(fs_utils.ls_item(f"{system_casadm_bin_path}"))[0] casadm_bin_copy = casadm_bin.copy(user_casadm_bin_dest_path, True) casadm_bin_copy.chmod_numerical(777) with TestRun.step("Copy IO class config."): io_conf = fs_utils.parse_ls_output(fs_utils.ls_item(f"{ioclass_config_path}"))[0] io_conf_copy = io_conf.copy(ioclass_config_copy_path, force=True) with TestRun.step("Unmount core."): core.unmount() with TestRun.step("Stop cache."): casadm.stop_all_caches() with TestRun.step("Add non-root user account."): TestRun.executor.run(f"useradd -N -r -l {user_name}") user_home_dir = fs_utils.parse_ls_output(fs_utils.ls_item(f"/home/{user_name}"))[0] user_home_dir.chmod_numerical(777, True) with TestRun.step("Try to start cache."): try: output = run_as_other_user(cli.start_cmd(cache_dev.path), user_name) if output.exit_code == 0: TestRun.LOGGER.error("Starting cache should fail!") except CmdException: TestRun.LOGGER.info("Non-root user cannot start cache.") with TestRun.step("Start cache again."): casadm.load_cache(cache_dev) with TestRun.step("Try to stop cache."): try: output = run_as_other_user(cli.stop_cmd(str(cache.cache_id)), user_name) if output.exit_code == 0: TestRun.LOGGER.error("Stopping cache should fail!") except CmdException: TestRun.LOGGER.info("Non-root user cannot stop cache.") with TestRun.step("Try to set cache mode."): try: output = run_as_other_user(cli.set_cache_mode_cmd(CacheMode.WB, str(cache.cache_id)), user_name) if output.exit_code == 0: TestRun.LOGGER.error("Setting cache mode should fail!") except CmdException: TestRun.LOGGER.info("Non-root user cannot set cache mode.") with TestRun.step("Try to add core to cache."): try: output = run_as_other_user(cli.add_core_cmd(str(cache.cache_id), core_part2.path), user_name) if output.exit_code == 0: TestRun.LOGGER.error("Adding core to cache should fail!") except CmdException: TestRun.LOGGER.info("Non-root user cannot add core.") with TestRun.step("Try to remove core from cache."): try: output = run_as_other_user(cli.remove_core_cmd(str(cache.cache_id), str(core.core_id)), user_name) if output.exit_code == 0: TestRun.LOGGER.error("Removing core from cache should fail!") except CmdException: TestRun.LOGGER.info("Non-root user cannot remove core.") with TestRun.step("Try to zero metadata."): try: output = run_as_other_user(cli.zero_metadata_cmd(str(cache_dev)), user_name) if output.exit_code == 0: TestRun.LOGGER.error("Zeroing metadata should fail!") except CmdException: TestRun.LOGGER.info("Non-root user cannot zero metadata.") with TestRun.step("Try to list caches."): try: output = run_as_other_user(cli.list_cmd(), user_name) if output.exit_code == 0: TestRun.LOGGER.error("Listing caches should fail!") except CmdException: TestRun.LOGGER.info("Non-root user cannot list caches.") with TestRun.step("Try to print stats."): try: output = run_as_other_user(cli.print_statistics_cmd(str(cache.cache_id)), user_name) if output.exit_code == 0: TestRun.LOGGER.error("Printing stats should fail!") except CmdException: TestRun.LOGGER.info("Non-root user cannot print statistics.") with TestRun.step("Try to reset stats."): try: output = run_as_other_user(cli.reset_counters_cmd(str(cache.cache_id)), user_name) if output.exit_code == 0: TestRun.LOGGER.error("Resetting stats should fail!") except CmdException: TestRun.LOGGER.info("Non-root user cannot reset statistics.") with TestRun.step("Try to flush cache."): try: output = run_as_other_user(cli.flush_cache_cmd(str(cache.cache_id)), user_name) if output.exit_code == 0: TestRun.LOGGER.error("Flushing cache should fail!") except CmdException: TestRun.LOGGER.info("Non-root user cannot flush cache.") with TestRun.step("Try to flush core."): try: output = run_as_other_user(cli.flush_core_cmd(str(cache.cache_id), str(core.core_id)), user_name) if output.exit_code == 0: TestRun.LOGGER.error("Flushing core should fail!") except CmdException: TestRun.LOGGER.info("Non-root user cannot flush core.") with TestRun.step("Try to set cleaning policy and its parameters."): try: output = run_as_other_user(cli.set_param_cleaning_cmd( str(cache.cache_id), "nop"), user_name) if output.exit_code == 0: TestRun.LOGGER.error("Setting cleaning policy should fail!") except CmdException: TestRun.LOGGER.info("Non-root user cannot set cleaning policy as nop.") try: output = run_as_other_user(cli.set_param_cleaning_cmd( str(cache.cache_id), "alru"), user_name) if output.exit_code == 0: TestRun.LOGGER.error("Setting cleaning policy should fail!") except CmdException: TestRun.LOGGER.info("Non-root user cannot set cleaning policy as alru.") try: output = run_as_other_user(cli.set_param_cleaning_alru_cmd(str(cache.cache_id), "15", "60", "1000", "8000"), user_name) if output.exit_code == 0: TestRun.LOGGER.error("Setting cleaning policy parameters should fail!") except CmdException: TestRun.LOGGER.info("Non-root user cannot set alru cleaning policy parameters.") try: output = run_as_other_user(cli.set_param_cleaning_cmd( str(cache.cache_id), "acp"), user_name) if output.exit_code == 0: TestRun.LOGGER.error("Setting cleaning policy should fail!") except CmdException: TestRun.LOGGER.info("Non-root user cannot set cleaning policy as acp.") try: output = run_as_other_user(cli.set_param_cleaning_acp_cmd(str(cache.cache_id), "15", "1000"), user_name) if output.exit_code == 0: TestRun.LOGGER.error("Setting cleaning policy parameters should fail!") except CmdException: TestRun.LOGGER.info("Non-root user cannot set acp cleaning policy parameters.") with TestRun.step("Try to list IO class configuration."): try: output = run_as_other_user(cli.list_io_classes_cmd( str(cache.cache_id), OutputFormat.table.name), user_name) if output.exit_code == 0: TestRun.LOGGER.error("Listing IO class configuration should fail!") except CmdException: TestRun.LOGGER.info("Non-root user cannot list IO class configuration.") with TestRun.step("Try to load IO class configuration."): try: output = run_as_other_user(cli.load_io_classes_cmd( str(cache.cache_id), io_conf_copy), user_name) if output.exit_code == 0: TestRun.LOGGER.error("Loading IO class configuration should fail!") except CmdException: TestRun.LOGGER.info("Non-root user cannot load IO class configuration.") with TestRun.step("Try to print help for casadm."): try: run_as_other_user(cli.help_cmd(), user_name) except CmdException: TestRun.LOGGER.error("Non-root user should be able to print help for casadm.") with TestRun.step("Try to print version of OpenCAS."): try: run_as_other_user(cli.version_cmd(), user_name) except CmdException: TestRun.LOGGER.error("Non-root user should be able to print version of OpenCAS.") with TestRun.step("Add non-root user account to sudoers group."): TestRun.executor.run(f'echo "{user_name} ALL = (root) NOPASSWD:ALL" ' f'| sudo tee /etc/sudoers.d/{user_name}') with TestRun.step("Try to stop cache with 'sudo'."): try: run_as_other_user(cli.stop_cmd(str(cache.cache_id)), user_name, True) except CmdException: TestRun.LOGGER.error("Non-root sudoer user should be able to stop cache.") with TestRun.step("Try to start cache with 'sudo'."): try: run_as_other_user(cli.start_cmd(cache_dev.path, force=True), user_name, True) except CmdException: TestRun.LOGGER.error("Non-root sudoer user should be able to start cache.") with TestRun.step("Try to set cache mode with 'sudo'."): try: run_as_other_user( cli.set_cache_mode_cmd(str(CacheMode.WB.name).lower(), str(cache.cache_id)), user_name, True) except CmdException: TestRun.LOGGER.error("Non-root sudoer user should be able to set cache mode.") with TestRun.step("Try to add core to cache with 'sudo'."): try: run_as_other_user(cli.add_core_cmd(str(cache.cache_id), core_part1.path), user_name, True) except CmdException: TestRun.LOGGER.error("Non-root sudoer user should be able to add core to cache.") with TestRun.step("Try to list caches with 'sudo'."): try: run_as_other_user(cli.list_cmd(), user_name, True) except CmdException: TestRun.LOGGER.error("Non-root sudoer user should be able to list caches.") with TestRun.step("Try to print stats with 'sudo'."): try: run_as_other_user(cli.print_statistics_cmd(str(cache.cache_id)), user_name, True) except CmdException: TestRun.LOGGER.error("Non-root sudoer user should be able to print stats.") with TestRun.step("Try to reset stats with 'sudo'."): try: run_as_other_user(cli.reset_counters_cmd(str(cache.cache_id)), user_name, True) except CmdException: TestRun.LOGGER.error("Non-root sudoer user should be able to reset stats.") with TestRun.step("Try to flush cache with 'sudo'."): try: run_as_other_user(cli.flush_cache_cmd(str(cache.cache_id)), user_name, True) except CmdException: TestRun.LOGGER.error("Non-root sudoer user should be able to flush cache.") with TestRun.step("Try to flush core with 'sudo'."): try: run_as_other_user(cli.flush_core_cmd(str(cache.cache_id), str(core.core_id)), user_name, True) except CmdException: TestRun.LOGGER.error("Non-root sudoer user should be able to flush core.") with TestRun.step("Try to set cleaning policy and its parameters with 'sudo'."): try: run_as_other_user(cli.set_param_cleaning_cmd(str(cache.cache_id), "nop"), user_name, True) run_as_other_user(cli.set_param_cleaning_cmd(str(cache.cache_id), "alru"), user_name, True) try: run_as_other_user(cli.set_param_cleaning_alru_cmd(str(cache.cache_id), "15", "60", "1000", "8000"), user_name, True) except CmdException: TestRun.LOGGER.error("Non-root sudoer user should be able to " "set alru cleaning policy parameters.") run_as_other_user(cli.set_param_cleaning_cmd(str(cache.cache_id), "acp"), user_name, True) try: run_as_other_user(cli.set_param_cleaning_acp_cmd(str(cache.cache_id), "15", "1000"), user_name, True) except CmdException: TestRun.LOGGER.error("Non-root sudoer user should be able to " "set acp cleaning policy parameters.") except CmdException: TestRun.LOGGER.error("Non-root sudoer user should be able to " "set cleaning policy and its parameters.") with TestRun.step("Try to list IO class with 'sudo'."): try: run_as_other_user(cli.list_io_classes_cmd(str(cache.cache_id), OutputFormat.table.name), user_name, True) except CmdException: TestRun.LOGGER.error("Non-root sudoer user should be able to list IO class.") with TestRun.step("Try to load IO class configuration with 'sudo'."): try: run_as_other_user(cli.load_io_classes_cmd(str(cache.cache_id), io_conf_copy), user_name, True) except CmdException: TestRun.LOGGER.error("Non-root sudoer user should be able to " "load IO class configuration.") with TestRun.step("Try to remove core from cache with 'sudo'."): try: run_as_other_user(cli.remove_core_cmd(str(cache.cache_id), str(core.core_id)), user_name, True) except CmdException: TestRun.LOGGER.error("Non-root sudoer user should be able to remove core from cache.") with TestRun.step("Try to zero metadata with 'sudo'."): try: run_as_other_user(cli.zero_metadata_cmd(str(cache_dev)), user_name, True) except CmdException: TestRun.LOGGER.error("Non-root sudoer user should be able to zero metadata.") with TestRun.step("Try to print help for casadm with 'sudo'."): try: run_as_other_user(cli.help_cmd(), user_name, True) except CmdException: TestRun.LOGGER.error("Non-root sudoer user should be able to print help for casadm.") with TestRun.step("Try to print version of OpenCAS with 'sudo'."): try: run_as_other_user(cli.version_cmd(), user_name, True) except CmdException: TestRun.LOGGER.error("Non-root sudoer user should be able to print version of OpenCAS.") with TestRun.step("Stop caches."): casadm.stop_all_caches() with TestRun.step("Remove user account."): TestRun.executor.run(f"userdel -r -Z {user_name}")
def test_interrupt_cache_stop(cache_mode, filesystem): """ title: Test if OpenCAS works correctly after cache stopping interruption. description: | Negative test of the ability of OpenCAS to handle cache's stop interruption. pass_criteria: - No system crash. - Flushing would be stopped after interruption. - Md5sum are correct during all test steps. - Dirty blocks quantity after interruption is lower but non-zero. """ with TestRun.step("Prepare cache and core."): cache_part, core_part = prepare() for _ in TestRun.iteration( range(iterations_per_config), f"Reload cache configuration {iterations_per_config} times."): with TestRun.step("Start cache."): cache = casadm.start_cache(cache_part, cache_mode, force=True) with TestRun.step("Set cleaning policy to NOP."): cache.set_cleaning_policy(CleaningPolicy.nop) with TestRun.step( f"Add core device with {filesystem} filesystem and mount it."): core_part.create_filesystem(filesystem) core = cache.add_core(core_part) core.mount(mount_point) with TestRun.step( f"Create test file in mount point of exported object."): test_file = create_test_file() with TestRun.step("Check md5 sum of test file."): test_file_md5sum_before = test_file.md5sum() with TestRun.step( "Get number of dirty data on exported object before interruption." ): os_utils.sync() os_utils.drop_caches(DropCachesMode.ALL) cache_dirty_blocks_before = cache.get_dirty_blocks() with TestRun.step("Unmount core."): core.unmount() with TestRun.step("Start stopping cache."): flush_pid = TestRun.executor.run_in_background( cli.stop_cmd(str(cache.cache_id))) sleep(2) with TestRun.step("Interrupt cache stopping."): percentage = casadm_parser.get_flushing_progress( cache.cache_id, core.core_id) while percentage < 50: percentage = casadm_parser.get_flushing_progress( cache.cache_id, core.core_id) TestRun.executor.run(f"kill -s SIGINT {flush_pid}") with TestRun.step( "Check number of dirty data on exported object after interruption." ): cache_dirty_blocks_after = cache.get_dirty_blocks() if cache_dirty_blocks_after >= cache_dirty_blocks_before: TestRun.LOGGER.error( "Quantity of dirty lines after cache stop interruption " "should be lower.") if int(cache_dirty_blocks_after) == 0: TestRun.LOGGER.error( "Quantity of dirty lines after cache stop interruption " "should not be zero.") with TestRun.step("Stop cache."): cache.stop() with TestRun.step("Mount core device."): core_part.mount(mount_point) with TestRun.step("Check md5 sum of test file again."): if test_file_md5sum_before != test_file.md5sum(): TestRun.LOGGER.error("Md5 sums before and after interrupting" " cache stop are different.") with TestRun.step("Unmount core device."): core_part.unmount()
def test_flush_over_640_gibibytes_with_fs(cache_mode, fs): """ title: Test of the ability to flush huge amount of dirty data on device with filesystem. description: | Flush cache when amount of dirty data in cache with core with filesystem exceeds 640 GiB. pass_criteria: - Flushing completes successfully without any errors. """ with TestRun.step("Prepare devices for cache and core."): cache_dev = TestRun.disks['cache'] check_disk_size(cache_dev) cache_dev.create_partitions([required_disk_size]) cache_part = cache_dev.partitions[0] core_dev = TestRun.disks['core'] check_disk_size(core_dev) Udev.disable() with TestRun.step(f"Start cache in {cache_mode} mode."): cache = casadm.start_cache(cache_part, cache_mode) with TestRun.step( f"Add core with {fs.name} filesystem to cache and mount it."): core_dev.create_filesystem(fs) core = cache.add_core(core_dev) core.mount(mnt_point) with TestRun.step("Disable cleaning and sequential cutoff."): cache.set_cleaning_policy(CleaningPolicy.nop) cache.set_seq_cutoff_policy(SeqCutOffPolicy.never) with TestRun.step("Create test file"): test_file_main = File.create_file("/tmp/test_file_main") fio = (Fio().create_command().io_engine(IoEngine.libaio).read_write( ReadWrite.write).block_size(bs).direct().io_depth(256).target( test_file_main.full_path).size(file_size)) fio.default_run_time = timedelta( hours=4) # timeout for non-time-based fio fio.run() test_file_main.refresh_item() with TestRun.step("Validate test file and read its md5 sum."): if test_file_main.size != file_size: TestRun.fail("Created test file hasn't reached its target size.") test_file_md5sum_main = test_file_main.md5sum() with TestRun.step("Write data to exported object."): test_file_copy = test_file_main.copy(mnt_point + "test_file_copy") test_file_copy.refresh_item() sync() with TestRun.step(f"Check if dirty data exceeded {file_size * 0.98} GiB."): minimum_4KiB_blocks = int( (file_size * 0.98).get_value(Unit.Blocks4096)) if int(cache.get_statistics().usage_stats.dirty) < minimum_4KiB_blocks: TestRun.fail("There is not enough dirty data in the cache!") with TestRun.step("Unmount core and stop cache with flush."): core.unmount() # this operation could take few hours, depending on core disk output = TestRun.executor.run(stop_cmd(str(cache.cache_id)), timedelta(hours=12)) if output.exit_code != 0: TestRun.fail(f"Stopping cache with flush failed!\n{output.stderr}") with TestRun.step( "Mount core device and check md5 sum of test file copy."): core_dev.mount(mnt_point) if test_file_md5sum_main != test_file_copy.md5sum(): TestRun.LOGGER.error("Md5 sums should be equal.") with TestRun.step("Delete test files."): test_file_main.remove(True) test_file_copy.remove(True) with TestRun.step("Unmount core device."): core_dev.unmount() remove(mnt_point, True, True, True)