def setup_workload(target: str, runtime: timedelta, io_depth=128, verify=True, block_size=int(Size(4, Unit.KibiByte)), num_jobs=1, method=ReadWrite.randrw, io_engine=IoEngine.libaio): fio_run = Fio().create_command() fio_run.io_engine(io_engine) fio_run.direct() fio_run.time_based() fio_run.run_time(runtime) fio_run.io_depth(io_depth) if verify: fio_run.do_verify() fio_run.verify(VerifyMethod.meta) fio_run.verify_dump() fio_run.read_write(method) fio_run.target(target) fio_run.block_size(block_size) for i in range(num_jobs): fio_run.add_job() return fio_run
def test_data_integrity_5d(): """ title: | Data integrity test on three cas instances with different cache modes with duration time equal to 5 days description: | Create 3 cache instances with different cache modes on caches equal to 50GB and cores equal to 150GB, and run workload with data verification. pass_criteria: - System does not crash. - All operations complete successfully. - Data consistency is preserved. """ with TestRun.step("Prepare cache and core devices"): cache_devices, core_devices = prepare_devices() with TestRun.step( "Run 4 cache instances in different cache modes, add single core to each" ): cache_modes = [CacheMode.WT, CacheMode.WB, CacheMode.WA, CacheMode.WO] caches = [] cores = [] for i in range(4): cache, core = start_instance(cache_devices[i], core_devices[i], cache_modes[i]) caches.append(cache) cores.append(core) with TestRun.step("Run test workloads with verification"): fio_run = Fio().create_command() fio_run.io_engine(IoEngine.libaio) fio_run.direct() fio_run.time_based() fio_run.do_verify() fio_run.verify(VerifyMethod.md5) fio_run.verify_dump() fio_run.run_time(runtime) fio_run.read_write(ReadWrite.randrw) fio_run.io_depth(128) fio_run.blocksize_range([(start_size, stop_size)]) for core in cores: fio_job = fio_run.add_job() fio_job.target(core) fio_run.run() with TestRun.step("Calculate md5 for each core"): core_md5s = [File(core.full_path).md5sum() for core in cores] with TestRun.step("Stop caches"): for cache in caches: cache.stop() with TestRun.step("Calculate md5 for each core"): dev_md5s = [File(dev.full_path).md5sum() for dev in core_devices] with TestRun.step("Compare md5 sums for cores and core devices"): for core_md5, dev_md5, mode in zip(core_md5s, dev_md5s, cache_modes): if core_md5 != dev_md5: TestRun.fail(f"MD5 sums of core and core device do not match! " f"Cache mode: {mode}")
def fill_cache(target): fio_run_fill = Fio().create_command() fio_run_fill.io_engine(IoEngine.libaio) fio_run_fill.direct() fio_run_fill.read_write(ReadWrite.write) fio_run_fill.io_depth(16) fio_run_fill.block_size(Size(1, Unit.MebiByte)) fio_run_fill.target(target) fio_run_fill.run()
def fio_workload(target: str, runtime: timedelta): fio_run = Fio().create_command() fio_run.io_engine(IoEngine.libaio) fio_run.direct() fio_run.time_based() fio_run.run_time(runtime) fio_run.io_depth(64) fio_run.read_write(ReadWrite.readwrite) fio_run.target(target) fio_run.block_size(int(Size(4, Unit.KibiByte))) return fio_run
def run_workload(target): fio_run = Fio().create_command() fio_run.io_engine(IoEngine.libaio) fio_run.direct() fio_run.time_based() fio_run.do_verify() fio_run.verify(VerifyMethod.meta) fio_run.verify_dump() fio_run.run_time(runtime) fio_run.read_write(ReadWrite.randrw) fio_run.io_depth(128) fio_run.target(target) for block_size in range(start_size, stop_size + 1, step): fio_job = fio_run.add_job() fio_job.stonewall() fio_job.block_size(block_size) fio_run.run()
def test_ioclass_direct(filesystem): """ title: Direct IO classification. description: Check if direct requests are properly cached. pass_criteria: - No kernel bug. - Data from direct IO should be cached. - Data from buffered IO should not be cached and if performed to/from already cached data should cause reclassification to unclassified IO class. """ ioclass_id = 1 io_size = Size(random.randint(1000, 2000), Unit.Blocks4096) with TestRun.step("Prepare cache and core. Disable udev."): cache, core = prepare() Udev.disable() with TestRun.step("Create and load IO class config."): # direct IO class ioclass_config.add_ioclass( ioclass_id=ioclass_id, eviction_priority=1, allocation=True, rule="direct", ioclass_config_path=ioclass_config_path, ) casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path) with TestRun.step("Prepare fio command."): fio = Fio().create_command() \ .io_engine(IoEngine.libaio) \ .size(io_size).offset(io_size) \ .read_write(ReadWrite.write) \ .target(f"{mountpoint}/tmp_file" if filesystem else core.path) with TestRun.step("Prepare filesystem."): if filesystem: TestRun.LOGGER.info( f"Preparing {filesystem.name} filesystem and mounting {core.path} at" f" {mountpoint}" ) core.create_filesystem(filesystem) core.mount(mountpoint) sync() else: TestRun.LOGGER.info("Testing on raw exported object.") with TestRun.step(f"Run buffered writes to {'file' if filesystem else 'device'}"): base_occupancy = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy fio.run() sync() with TestRun.step("Check if buffered writes are not cached."): new_occupancy = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy if new_occupancy != base_occupancy: TestRun.fail("Buffered writes were cached!\n" f"Expected: {base_occupancy}, actual: {new_occupancy}") with TestRun.step(f"Run direct writes to {'file' if filesystem else 'device'}"): fio.direct() fio.run() sync() with TestRun.step("Check if direct writes are cached."): new_occupancy = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy if new_occupancy != base_occupancy + io_size: TestRun.fail("Wrong number of direct writes was cached!\n" f"Expected: {base_occupancy + io_size}, actual: {new_occupancy}") with TestRun.step(f"Run buffered reads from {'file' if filesystem else 'device'}"): fio.remove_param("readwrite").remove_param("direct") fio.read_write(ReadWrite.read) fio.run() sync() with TestRun.step("Check if buffered reads caused reclassification."): new_occupancy = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy if new_occupancy != base_occupancy: TestRun.fail("Buffered reads did not cause reclassification!" f"Expected occupancy: {base_occupancy}, actual: {new_occupancy}") with TestRun.step(f"Run direct reads from {'file' if filesystem else 'device'}"): fio.direct() fio.run() sync() with TestRun.step("Check if direct reads are cached."): new_occupancy = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy if new_occupancy != base_occupancy + io_size: TestRun.fail("Wrong number of direct reads was cached!\n" f"Expected: {base_occupancy + io_size}, actual: {new_occupancy}")
def test_data_integrity_5d_dss(filesystems): """ title: | Data integrity test on three cas instances with different file systems with duration time equal to 5 days description: | Create 3 cache instances on caches equal to 50GB and cores equal to 150GB with different file systems, and run workload with data verification. pass_criteria: - System does not crash. - All operations complete successfully. - Data consistency is being preserved. """ with TestRun.step("Prepare cache and core devices"): cache_devices, core_devices = prepare_devices() with TestRun.step( "Run 4 cache instances in different cache modes, add single core to each" ): cache_modes = [CacheMode.WT, CacheMode.WB, CacheMode.WA, CacheMode.WO] caches = [] cores = [] for i in range(4): cache, core = start_instance(cache_devices[i], core_devices[i], cache_modes[i]) caches.append(cache) cores.append(core) with TestRun.step("Load default io class config for each cache"): for cache in caches: cache.load_io_class("/etc/opencas/ioclass-config.csv") with TestRun.step("Create filesystems and mount cores"): for i, core in enumerate(cores): mount_point = core.path.replace('/dev/', '/mnt/') if not fs_utils.check_if_directory_exists(mount_point): fs_utils.create_directory(mount_point) TestRun.LOGGER.info( f"Create filesystem {filesystems[i].name} on {core.path}") core.create_filesystem(filesystems[i]) TestRun.LOGGER.info( f"Mount filesystem {filesystems[i].name} on {core.path} to " f"{mount_point}") core.mount(mount_point) sync() with TestRun.step("Run test workloads on filesystems with verification"): fio_run = Fio().create_command() fio_run.io_engine(IoEngine.libaio) fio_run.direct() fio_run.time_based() fio_run.nr_files(4096) fio_run.file_size_range([(file_min_size, file_max_size)]) fio_run.do_verify() fio_run.verify(VerifyMethod.md5) fio_run.verify_dump() fio_run.run_time(runtime) fio_run.read_write(ReadWrite.randrw) fio_run.io_depth(128) fio_run.blocksize_range([(start_size, stop_size)]) for core in cores: fio_job = fio_run.add_job() fio_job.directory(core.mount_point) fio_job.size(core.size) fio_run.run() with TestRun.step("Unmount cores"): for core in cores: core.unmount() with TestRun.step("Calculate md5 for each core"): core_md5s = [File(core.full_path).md5sum() for core in cores] with TestRun.step("Stop caches"): for cache in caches: cache.stop() with TestRun.step("Calculate md5 for each core"): dev_md5s = [File(dev.full_path).md5sum() for dev in core_devices] with TestRun.step("Compare md5 sums for cores and core devices"): for core_md5, dev_md5, mode, fs in zip(core_md5s, dev_md5s, cache_modes, filesystems): if core_md5 != dev_md5: TestRun.fail(f"MD5 sums of core and core device do not match! " f"Cache mode: {mode} Filesystem: {fs}")
def test_zero_metadata_dirty_data(): """ title: Test for '--zero-metadata' and dirty data scenario. description: | Test for '--zero-metadata' with and without 'force' option if there are dirty data on cache. pass_criteria: - Zeroing metadata without force failed on cache with dirty data. - Zeroing metadata with force ran successfully on cache with dirty data. - Cache started successfully after zeroing metadata on cache with dirty data. """ with TestRun.step("Prepare cache and core devices."): cache_dev, core_disk, cache_disk = prepare_devices() with TestRun.step("Start cache."): cache = casadm.start_cache(cache_dev, CacheMode.WB, force=True) core = cache.add_core(core_disk) cache.set_cleaning_policy(CleaningPolicy.nop) with TestRun.step("Run workload on CAS"): fio_run_fill = Fio().create_command() fio_run_fill.io_engine(IoEngine.libaio) fio_run_fill.direct() fio_run_fill.read_write(ReadWrite.randwrite) fio_run_fill.io_depth(16) fio_run_fill.block_size(Size(1, Unit.MebiByte)) fio_run_fill.target(core.path) fio_run_fill.run_time(timedelta(seconds=5)) fio_run_fill.time_based() fio_run_fill.run() with TestRun.step("Stop cache without flushing dirty data."): cache.stop(no_data_flush=True) with TestRun.step("Start cache (expect to fail)."): try: cache = casadm.start_cache(cache_dev, CacheMode.WB) except CmdException: TestRun.LOGGER.info("Start cache failed as expected.") with TestRun.step("Zeroing metadata on CAS device without force"): try: casadm.zero_metadata(cache_dev) TestRun.LOGGER.error("Zeroing metadata without force should fail!") except CmdException as e: cli_messages.check_stderr_msg(e.output, cli_messages.cache_dirty_data) with TestRun.step("Zeroing metadata on cache device with force"): try: casadm.zero_metadata(cache_dev, force=True) TestRun.LOGGER.info("Zeroing metadata with force successful!") except CmdException as e: TestRun.LOGGER.error( f"Zeroing metadata with force should work for cache device!" f"Error message: {e.output}") with TestRun.step("Start cache without 'force' option."): try: cache = casadm.start_cache(cache_dev, CacheMode.WB) TestRun.LOGGER.info("Cache started successfully.") except CmdException: TestRun.LOGGER.error("Start cache failed.")