def test_recovery_flush_reset_fs(cache_mode, fs): """ title: Recovery after reset during cache flushing - test on filesystem. description: | Verify that unflushed data can be safely recovered, when reset was pressed during data flushing on filesystem. pass_criteria: - CAS recovers successfully after reboot - No data corruption """ with TestRun.step("Prepare cache and core devices."): cache_disk = TestRun.disks['cache'] core_disk = TestRun.disks['core'] cache_disk.create_partitions([Size(2, Unit.GibiByte)]) core_disk.create_partitions([Size(16, Unit.GibiByte)] * 2) cache_device = cache_disk.partitions[0] core_device = core_disk.partitions[0] core_device_link = core_device.get_device_link("/dev/disk/by-id") cache_device_link = cache_device.get_device_link("/dev/disk/by-id") with TestRun.step(f"Create {fs} filesystem on core."): core_device.create_filesystem(fs) with TestRun.step("Create test files."): source_file, target_file = create_test_files(test_file_size) with TestRun.step("Setup cache and add core."): cache = casadm.start_cache(cache_device, cache_mode) Udev.disable() core = cache.add_core(core_device) cache.set_cleaning_policy(CleaningPolicy.nop) cache.set_seq_cutoff_policy(SeqCutOffPolicy.never) with TestRun.step("Mount CAS device."): core.mount(mount_point) with TestRun.step("Copy file to CAS."): copy_file(source=source_file.full_path, target=os.path.join(mount_point, "source_test_file"), size=test_file_size, direct="oflag") with TestRun.step("Unmount CAS device."): core.unmount() with TestRun.step("Trigger flush."): TestRun.executor.run_in_background( cli.flush_cache_cmd(f"{cache.cache_id}")) with TestRun.step("Hard reset DUT during data flushing."): power_cycle_dut(True, core_device) cache_device.path = cache_device_link.get_target() core_device.path = core_device_link.get_target() with TestRun.step("Load cache."): cache = casadm.load_cache(cache_device) if cache.get_dirty_blocks() == Size.zero(): TestRun.fail("There are no dirty blocks on cache device.") with TestRun.step("Stop cache with dirty data flush."): core_writes_before = core_device.get_io_stats().sectors_written cache.stop() if core_writes_before >= core_device.get_io_stats().sectors_written: TestRun.fail( "No data was flushed after stopping cache started with load option." ) with TestRun.step("Mount core device."): core_device.mount(mount_point) with TestRun.step( "Copy test file from core device to temporary location. " "Compare it with the first version – they should be the same."): copy_file(source=os.path.join(mount_point, "source_test_file"), target=target_file.full_path, size=test_file_size, direct="iflag") compare_files(source_file, target_file) with TestRun.step("Unmount core device and remove test files."): core_device.unmount() target_file.remove() source_file.remove() Udev.enable()
def test_recovery_flush_reset_raw(cache_mode): """ title: Recovery after reset during cache flushing - test on raw device. description: | Verify that unflushed data can be safely recovered, when reset was pressed during data flushing on raw device. pass_criteria: - CAS recovers successfully after reboot - No data corruption """ with TestRun.step("Prepare cache and core devices."): cache_disk = TestRun.disks['cache'] core_disk = TestRun.disks['core'] cache_disk.create_partitions([Size(2, Unit.GibiByte)]) core_disk.create_partitions([Size(16, Unit.GibiByte)] * 2) cache_device = cache_disk.partitions[0] core_device = core_disk.partitions[0] core_device_link = core_device.get_device_link("/dev/disk/by-id") cache_device_link = cache_device.get_device_link("/dev/disk/by-id") with TestRun.step("Create test files."): source_file, target_file = create_test_files(test_file_size) with TestRun.step("Setup cache and add core."): cache = casadm.start_cache(cache_device, cache_mode) core = cache.add_core(core_device) cache.set_cleaning_policy(CleaningPolicy.nop) cache.set_seq_cutoff_policy(SeqCutOffPolicy.never) with TestRun.step("Copy file to CAS."): copy_file(source=source_file.full_path, target=core.path, size=test_file_size, direct="oflag") with TestRun.step("Sync and flush buffers."): os_utils.sync() output = TestRun.executor.run(f"hdparm -f {core.path}") if output.exit_code != 0: raise CmdException("Error during hdparm", output) with TestRun.step("Trigger flush."): TestRun.executor.run_in_background( cli.flush_cache_cmd(f"{cache.cache_id}")) with TestRun.step("Hard reset DUT during data flushing."): power_cycle_dut(wait_for_flush_begin=True, core_device=core_device) cache_device.path = cache_device_link.get_target() core_device.path = core_device_link.get_target() with TestRun.step( "Copy file from core and check if current md5sum is different than " "before restart."): copy_file(source=core_device_link.get_target(), target=target_file.full_path, size=test_file_size, direct="iflag") compare_files(source_file, target_file, should_differ=True) with TestRun.step("Load cache."): cache = casadm.load_cache(cache_device) if cache.get_dirty_blocks() == Size.zero(): TestRun.fail("There are no dirty blocks on cache device.") with TestRun.step("Stop cache with dirty data flush."): core_writes_before = core_device.get_io_stats().sectors_written cache.stop() if core_writes_before >= core_device.get_io_stats().sectors_written: TestRun.fail( "No data was flushed after stopping cache started with load option." ) with TestRun.step( "Copy test file from core device to temporary location. " "Compare it with the first version – they should be the same."): copy_file(source=core_device_link.get_target(), target=target_file.full_path, size=test_file_size, direct="iflag") compare_files(source_file, target_file) with TestRun.step("Cleanup core device and remove test files."): target_file.remove() source_file.remove()
def test_recovery_all_options(cache_mode, cache_line_size, cleaning_policy, filesystem): """ title: Test for recovery after reset with various cache options. description: Verify that unflushed data can be safely recovered after reset. pass_criteria: - CAS recovers successfully after reboot - No data corruption """ with TestRun.step("Prepare cache and core devices."): cache_disk = TestRun.disks['cache'] core_disk = TestRun.disks['core'] cache_disk.create_partitions([Size(200, Unit.MebiByte)]) core_disk.create_partitions([Size(2000, Unit.MebiByte)] * 2) cache_device = cache_disk.partitions[0] core_device = core_disk.partitions[0] test_file = File(os.path.join(mount_point, filename)) file_operation(test_file.full_path, pattern, ReadWrite.write) file_md5 = test_file.md5sum() with TestRun.step(f"Make {filesystem} on core device."): core_device.create_filesystem(filesystem) with TestRun.step("Mount core device."): core_device.mount(mount_point) file_operation(test_file.full_path, other_pattern, ReadWrite.write) os_utils.drop_caches(DropCachesMode.ALL) with TestRun.step("Unmount core device."): core_device.unmount() with TestRun.step( f"Start cache in {cache_mode.name} with given configuration."): cache = casadm.start_cache(cache_device, cache_mode, cache_line_size, force=True) cache.set_cleaning_policy(cleaning_policy) if cleaning_policy == CleaningPolicy.acp: cache.set_params_acp( FlushParametersAcp(wake_up_time=Time(seconds=1))) with TestRun.step("Add core."): core = cache.add_core(core_device) with TestRun.step("Mount CAS device."): core.mount(mount_point) file_operation(test_file.full_path, pattern, ReadWrite.write) with TestRun.step( "Change cache mode to Write-Through without flush option."): cache.set_cache_mode(CacheMode.WT, flush=False) with TestRun.step("Reset platform."): os_utils.sync() core.unmount() TestRun.LOGGER.info( f"Number of dirty blocks in cache: {cache.get_dirty_blocks()}") power_cycle_dut() with TestRun.step("Try to start cache without load and force option."): try: casadm.start_cache(cache_device, cache_mode, cache_line_size) TestRun.fail("Cache started without load or force option.") except Exception: TestRun.LOGGER.info( "Cache did not start without load and force option.") with TestRun.step("Load cache and stop it with flush."): cache = casadm.load_cache(cache_device) cache.stop() with TestRun.step("Check md5sum of tested file on core device."): core_device.mount(mount_point) cas_md5 = test_file.md5sum() core_device.unmount() if cas_md5 == file_md5: TestRun.LOGGER.info( "Source and target file checksums are identical.") else: TestRun.fail("Source and target file checksums are different.")