def create_partitions(device, sizes: [], partition_table_type=PartitionTable.gpt): from storage_devices.partition import Partition if create_partition_table(device, partition_table_type): device.partition_table = partition_table_type partition_type = PartitionType.primary partition_number_offset = 0 for s in sizes: size = Size( s.get_value(device.block_size) - device.block_size.value, device.block_size) if partition_table_type == PartitionTable.msdos and \ len(sizes) > 4 and len(device.partitions) == 3: create_partition(device, Size.zero(), 4, PartitionType.extended, Unit.MebiByte, True) partition_type = PartitionType.logical partition_number_offset = 1 partition_number = len( device.partitions) + 1 + partition_number_offset if create_partition(device, size, partition_number, partition_type, Unit.MebiByte, True): new_part = Partition(device, partition_type, partition_number) dd = Dd().input("/dev/zero") \ .output(new_part.system_path) \ .count(1) \ .block_size(Size(1, Unit.Blocks4096)) \ .oflag("direct") dd.run() device.partitions.append(new_part)
def run_io_dir_read(path): dd = Dd().output("/dev/null").input(f"{path}") output = dd.run() if output.exit_code != 0: TestRun.fail(f"Failed to execute dd.\n {output.stdout}\n{output.stderr}") sync() drop_caches(DropCachesMode.ALL)
def test_ioclass_file_offset(prepare_and_cleanup): cache, core = prepare() ioclass_id = 1 iterations = 100 dd_size = Size(4, Unit.KibiByte) dd_count = 1 min_cached_offset = 16384 max_cached_offset = 65536 ioclass_config.add_ioclass( ioclass_id=ioclass_id, eviction_priority=1, allocation=True, rule= f"file_offset:gt:{min_cached_offset}&file_offset:lt:{max_cached_offset}&done", ioclass_config_path=ioclass_config_path, ) casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path) TestProperties.LOGGER.info( f"Preparing filesystem and mounting {core.system_path} at {mountpoint}" ) core.create_filesystem(Filesystem.ext3) core.mount(mountpoint) cache.flush_cache() # Since ioclass rule consists of strict inequalities, 'seek' can't be set to first # nor last sector min_seek = int( (min_cached_offset + Unit.Blocks4096.value) / Unit.Blocks4096.value) max_seek = int( (max_cached_offset - min_cached_offset - Unit.Blocks4096.value) / Unit.Blocks4096.value) TestProperties.LOGGER.info(f"Writing to file within cached offset range") for i in range(iterations): file_offset = random.choice(range(min_seek, max_seek)) dd = (Dd().input("/dev/zero").output(f"{mountpoint}/tmp_file").count( dd_count).block_size(dd_size).seek(file_offset)) dd.run() sync() stats = cache.get_cache_statistics(io_class_id=ioclass_id) assert (stats["dirty"].get_value( Unit.Blocks4096) == 1), f"Offset not cached: {file_offset}" cache.flush_cache() min_seek = 0 max_seek = int(min_cached_offset / Unit.Blocks4096.value) TestProperties.LOGGER.info( f"Writing to file outside of cached offset range") for i in range(iterations): file_offset = random.choice(range(min_seek, max_seek)) dd = (Dd().input("/dev/zero").output(f"{mountpoint}/tmp_file").count( dd_count).block_size(dd_size).seek(file_offset)) dd.run() sync() stats = cache.get_cache_statistics(io_class_id=ioclass_id) assert (stats["dirty"].get_value(Unit.Blocks4096) == 0 ), f"Inappropriately cached offset: {file_offset}"
def create_files_with_classification_delay_check(directory: Directory, ioclass_id: int): start_time = datetime.now() occupancy_after = cache.get_statistics_deprecated( io_class_id=ioclass_id)["occupancy"] dd_blocks = 10 dd_size = Size(dd_blocks, Unit.Blocks4096) file_counter = 0 unclassified_files = [] time_from_start = datetime.now() - start_time while time_from_start < ioclass_config.MAX_CLASSIFICATION_DELAY: occupancy_before = occupancy_after file_path = f"{directory.full_path}/test_file_{file_counter}" file_counter += 1 time_from_start = datetime.now() - start_time (Dd().input( "/dev/zero").output(file_path).oflag("sync").block_size( Size(1, Unit.Blocks4096)).count(dd_blocks).run()) occupancy_after = cache.get_statistics_deprecated( io_class_id=ioclass_id)["occupancy"] if occupancy_after - occupancy_before < dd_size: unclassified_files.append(file_path) if len(unclassified_files) == file_counter: pytest.xfail( "No files were properly classified within max delay time!") if len(unclassified_files): TestRun.LOGGER.info("Rewriting unclassified test files...") for file_path in unclassified_files: (Dd().input("/dev/zero").output( file_path).oflag("sync").block_size( Size(1, Unit.Blocks4096)).count(dd_blocks).run())
def test_ioclass_lba(prepare_and_cleanup): """Write data to random lba and check if it is cached according to range defined in ioclass rule""" cache, core = prepare() ioclass_id = 1 min_cached_lba = 56 max_cached_lba = 200 iterations = 100 dd_size = Size(1, Unit.Blocks512) dd_count = 1 # Prepare ioclass config ioclass_config.add_ioclass( ioclass_id=ioclass_id, eviction_priority=1, allocation=True, rule=f"lba:ge:{min_cached_lba}&lba:le:{max_cached_lba}&done", ioclass_config_path=ioclass_config_path, ) # Prepare cache for test casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path) cache.flush_cache() # Check if lbas from defined range are cached dirty_count = 0 # '8' step is set to prevent writting cache line more than once TestProperties.LOGGER.info( f"Writing to one sector in each cache line from range.") for lba in range(min_cached_lba, max_cached_lba, 8): dd = (Dd().input("/dev/zero").output(f"{core.system_path}").count( dd_count).block_size(dd_size).seek(lba)) dd.run() sync() dirty_count += 1 stats = cache.get_cache_statistics(per_io_class=True, io_class_id=ioclass_id) assert (stats["dirty"].get_value( Unit.Blocks4096) == dirty_count), f"LBA {lba} not cached" cache.flush_cache() # Check if lba outside of defined range are not cached TestProperties.LOGGER.info( f"Writing to random sectors outside of cached range.") for i in range(iterations): rand_lba = random.randrange(2000) if min_cached_lba <= rand_lba <= max_cached_lba: continue dd = (Dd().input("/dev/zero").output(f"{core.system_path}").count( dd_count).block_size(dd_size).seek(rand_lba)) dd.run() sync() stats = cache.get_cache_statistics(per_io_class=True, io_class_id=ioclass_id) assert (stats["dirty"].get_value( Unit.Blocks4096) == 0), f"Inappropriately cached lba: {rand_lba}"
def test_activate_corrupted_after_dump(): """ title: Activate cache instance on metadata corrupted after the detach description: | Initialize standby cache, populate it with metadata, detach cache, corrupt metadata on the cache-to-be device and try to activate. pass_criteria: - Kernel panic doesn't occur """ with TestRun.step("Prepare devices for the cache and core."): cache_device = TestRun.disks["cache"] cache_device.create_partitions([Size(200, Unit.MebiByte)]) cache_device = cache_device.partitions[0] core_device = TestRun.disks["core"] core_device.create_partitions([Size(500, Unit.MebiByte)]) core_device = core_device.partitions[0] with TestRun.step("Prepare metadata dump"): cache_id = 1 cls = CacheLineSize.LINE_32KiB md_dump = prepare_md_dump(cache_device, core_device, cls, cache_id) for offset in get_offsets_to_corrupt(md_dump.size, block_size): with TestRun.step("Prepare standby instance"): cache = casadm.standby_init( cache_dev=cache_device, cache_line_size=int(cls.value.value / Unit.KibiByte.value), cache_id=cache_id, force=True, ) with TestRun.step( f"Populate the passive instance with valid metadata"): Dd().input( md_dump.full_path).output(f"/dev/cas-cache-{cache_id}").run() sync() with TestRun.step(f"Standby detach"): cache.standby_detach() with TestRun.step( f"Corrupt {block_size} on the offset {offset*block_size}"): corrupted_md = prepare_corrupted_md(md_dump, offset, block_size) with TestRun.step(f"Copy corrupted metadata to the passive instance"): Dd().input(corrupted_md.full_path).output(cache_device.path).run() sync() with TestRun.step("Try to activate cache instance"): output = TestRun.executor.run( standby_activate_cmd(cache_dev=cache_device.path, cache_id=str(cache_id))) with TestRun.step("Per iteration cleanup"): cache.stop() corrupted_md.remove(force=True, ignore_errors=True) with TestRun.step("Test cleanup"): md_dump.remove()
def test_ioclass_file_extension(prepare_and_cleanup): cache, core = prepare() iterations = 50 ioclass_id = 1 tested_extension = "tmp" wrong_extensions = ["tm", "tmpx", "txt", "t", "", "123", "tmp.xx"] dd_size = Size(4, Unit.KibiByte) dd_count = 10 ioclass_config.add_ioclass( ioclass_id=ioclass_id, eviction_priority=1, allocation=True, rule=f"extension:{tested_extension}&done", ioclass_config_path=ioclass_config_path, ) casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path) TestRun.LOGGER.info( f"Preparing filesystem and mounting {core.system_path} at {mountpoint}" ) core.create_filesystem(Filesystem.ext3) core.mount(mountpoint) cache.flush_cache() # Check if file with proper extension is cached dd = ( Dd() .input("/dev/zero") .output(f"{mountpoint}/test_file.{tested_extension}") .count(dd_count) .block_size(dd_size) ) TestRun.LOGGER.info(f"Writing to file with cached extension.") for i in range(iterations): dd.run() sync() stats = cache.get_cache_statistics(io_class_id=ioclass_id) assert stats["dirty"].get_value(Unit.Blocks4096) == (i + 1) * dd_count cache.flush_cache() # Check if file with improper extension is not cached TestRun.LOGGER.info(f"Writing to file with no cached extension.") for ext in wrong_extensions: dd = ( Dd() .input("/dev/zero") .output(f"{mountpoint}/test_file.{ext}") .count(dd_count) .block_size(dd_size) ) dd.run() sync() stats = cache.get_cache_statistics(io_class_id=ioclass_id) assert stats["dirty"].get_value(Unit.Blocks4096) == 0
def run_dd(target_path, count, seek): dd = Dd() \ .input("/dev/zero") \ .output(target_path) \ .block_size(Size(1, Unit.Blocks4096)) \ .count(count) \ .oflag("direct") \ .seek(seek) dd.run() TestRun.LOGGER.info(f"dd command:\n{dd}")
def read_files_with_reclassification_check(cache, target_ioclass_id: int, source_ioclass_id: int, directory: Directory, with_delay: bool): start_time = datetime.now() target_occupancy_after = cache.get_io_class_statistics( io_class_id=target_ioclass_id).usage_stats.occupancy source_occupancy_after = cache.get_io_class_statistics( io_class_id=source_ioclass_id).usage_stats.occupancy files_to_reclassify = [] target_ioclass_is_enabled = ioclass_is_enabled(cache, target_ioclass_id) for file in [item for item in directory.ls() if isinstance(item, File)]: target_occupancy_before = target_occupancy_after source_occupancy_before = source_occupancy_after time_from_start = datetime.now() - start_time dd = Dd().input(file.full_path).output("/dev/null").block_size( Size(1, Unit.Blocks4096)) dd.run() target_occupancy_after = cache.get_io_class_statistics( io_class_id=target_ioclass_id).usage_stats.occupancy source_occupancy_after = cache.get_io_class_statistics( io_class_id=source_ioclass_id).usage_stats.occupancy if target_ioclass_is_enabled: if target_occupancy_after < target_occupancy_before: TestRun.LOGGER.error("Target IO class occupancy lowered!") elif target_occupancy_after - target_occupancy_before < file.size: files_to_reclassify.append(file) if with_delay and time_from_start <= ioclass_config.MAX_CLASSIFICATION_DELAY: continue TestRun.LOGGER.error( "Target IO class occupancy not changed properly!\n" f"Expected: {file.size + target_occupancy_before}\n" f"Actual: {target_occupancy_after}") elif target_occupancy_after > target_occupancy_before and with_delay: files_to_reclassify.append(file) if source_occupancy_after >= source_occupancy_before: if file not in files_to_reclassify: files_to_reclassify.append(file) if with_delay and time_from_start <= ioclass_config.MAX_CLASSIFICATION_DELAY: continue TestRun.LOGGER.error( "Source IO class occupancy not changed properly!\n" f"Before: {source_occupancy_before}\n" f"After: {source_occupancy_after}") if len(files_to_reclassify): TestRun.LOGGER.info("Rereading unclassified test files...") sync() drop_caches(DropCachesMode.ALL) for file in files_to_reclassify: (Dd().input(file.full_path).output("/dev/null").block_size( Size(1, Unit.Blocks4096)).run())
def create_partition( device, part_size, part_number, part_type: PartitionType = PartitionType.primary, unit=Unit.MebiByte, aligned: bool = True): TestRun.LOGGER.info( f"Creating {part_type.name} partition on device: {device.path}") begin = get_first_partition_offset(device, aligned) for part in device.partitions: begin += part.size if part.type == PartitionType.logical: begin += Size(1, Unit.MebiByte if not aligned else device.block_size) if part_type == PartitionType.logical: begin += Size(1, Unit.MebiByte if not aligned else device.block_size) end = (begin + part_size) if part_size != Size.zero() else '100%' cmd = f'parted --script {device.path} mkpart ' \ f'{part_type.name} ' \ f'{begin.get_value(unit)}{unit_to_string(unit)} ' \ f'{end.get_value(unit)}{unit_to_string(unit)}' output = TestRun.executor.run(cmd) if output.exit_code != 0: TestRun.executor.run_expect_success("partprobe") TestRun.executor.run_expect_success("udevadm settle") if not check_partition_after_create( part_size, part_number, device.path, part_type, aligned): raise Exception("Could not create partition!") if part_type != PartitionType.extended: from storage_devices.partition import Partition new_part = Partition(device, part_type, part_number, begin, end if type(end) is Size else device.size) dd = Dd().input("/dev/zero") \ .output(new_part.path) \ .count(1) \ .block_size(Size(1, Unit.Blocks4096)) \ .oflag("direct") dd.run() device.partitions.append(new_part) TestRun.LOGGER.info(f"Successfully created {part_type.name} partition on {device.path}")
def test_ioclass_request_size(): cache, core = prepare() ioclass_id = 1 iterations = 100 ioclass_config.add_ioclass( ioclass_id=ioclass_id, eviction_priority=1, allocation=True, rule=f"request_size:ge:8192&request_size:le:16384&done", ioclass_config_path=ioclass_config_path, ) casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path) Udev.disable() # Check if requests with appropriate size are cached TestRun.LOGGER.info( f"Check if requests with size within defined range are cached") cached_req_sizes = [Size(2, Unit.Blocks4096), Size(4, Unit.Blocks4096)] for i in range(iterations): cache.flush_cache() req_size = random.choice(cached_req_sizes) dd = (Dd().input("/dev/zero").output( core.system_path).count(1).block_size(req_size).oflag("direct")) dd.run() dirty = cache.get_io_class_statistics( io_class_id=ioclass_id).usage_stats.dirty if dirty.get_value( Unit.Blocks4096) != req_size.value / Unit.Blocks4096.value: TestRun.fail("Incorrect number of dirty blocks!") cache.flush_cache() # Check if requests with inappropriate size are not cached TestRun.LOGGER.info( f"Check if requests with size outside defined range are not cached") not_cached_req_sizes = [ Size(1, Unit.Blocks4096), Size(8, Unit.Blocks4096), Size(16, Unit.Blocks4096), ] for i in range(iterations): req_size = random.choice(not_cached_req_sizes) dd = (Dd().input("/dev/zero").output( core.system_path).count(1).block_size(req_size).oflag("direct")) dd.run() dirty = cache.get_io_class_statistics( io_class_id=ioclass_id).usage_stats.dirty if dirty.get_value(Unit.Blocks4096) != 0: TestRun.fail("Dirty data present!")
def create_test_file(): from test_utils.filesystem.file import File from test_tools.dd import Dd bs = Size(512, Unit.KibiByte) cnt = int(cache_size.value / bs.value) test_file = File.create_file(test_file_path) dd = Dd().output(test_file_path) \ .input("/dev/zero") \ .block_size(bs) \ .count(cnt) dd.run() test_file.refresh_item() return test_file
def test_ioclass_file_extension_preexisting_filesystem(): """Create files on filesystem, add device with filesystem as a core, write data to files and check if they are cached properly""" cache, core = prepare() ioclass_id = 1 extensions = ["tmp", "tm", "out", "txt", "log", "123"] dd_size = Size(4, Unit.KibiByte) dd_count = 10 TestRun.LOGGER.info(f"Preparing files on raw block device") casadm.remove_core(cache.cache_id, core_id=core.core_id) core.core_device.create_filesystem(Filesystem.ext3) core.core_device.mount(mountpoint) # Prepare files for ext in extensions: dd = (Dd().input("/dev/zero").output(f"{mountpoint}/test_file.{ext}"). count(dd_count).block_size(dd_size)) dd.run() core.core_device.unmount() # Prepare ioclass config rule = "|".join([f"extension:{ext}" for ext in extensions]) ioclass_config.add_ioclass( ioclass_id=ioclass_id, eviction_priority=1, allocation=True, rule=f"{rule}&done", ioclass_config_path=ioclass_config_path, ) # Prepare cache for test TestRun.LOGGER.info(f"Adding device with preexisting data as a core") core = casadm.add_core(cache, core_dev=core.core_device) casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path) core.mount(mountpoint) cache.flush_cache() # Check if files with proper extensions are cached TestRun.LOGGER.info(f"Writing to file with cached extension.") for ext in extensions: dd = (Dd().input("/dev/zero").output(f"{mountpoint}/test_file.{ext}"). count(dd_count).block_size(dd_size)) dd.run() sync() dirty = cache.get_io_class_statistics( io_class_id=ioclass_id).usage_stats.dirty if dirty.get_value( Unit.Blocks4096) != (extensions.index(ext) + 1) * dd_count: TestRun.LOGGER.error(f"Wrong amount of dirty data ({dirty}).")
def test_ioclass_request_size(prepare_and_cleanup): cache, core = prepare() ioclass_id = 1 iterations = 100 ioclass_config.add_ioclass( ioclass_id=ioclass_id, eviction_priority=1, allocation=True, rule=f"request_size:ge:8192&request_size:le:16384&done", ioclass_config_path=ioclass_config_path, ) casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path) Udev.disable() # Check if requests with appropriate size are cached TestProperties.LOGGER.info( f"Check if requests with size within defined range are cached") cached_req_sizes = [Size(2, Unit.Blocks4096), Size(4, Unit.Blocks4096)] for i in range(iterations): cache.flush_cache() req_size = random.choice(cached_req_sizes) dd = (Dd().input("/dev/zero").output( core.system_path).count(1).block_size(req_size).oflag("direct")) dd.run() stats = cache.get_cache_statistics(per_io_class=True, io_class_id=ioclass_id) assert (stats["dirty"].get_value(Unit.Blocks4096) == req_size.value / Unit.Blocks4096.value) cache.flush_cache() # Check if requests with inappropriate size are not cached TestProperties.LOGGER.info( f"Check if requests with size outside defined range are not cached") not_cached_req_sizes = [ Size(1, Unit.Blocks4096), Size(8, Unit.Blocks4096), Size(16, Unit.Blocks4096), ] for i in range(iterations): req_size = random.choice(not_cached_req_sizes) dd = (Dd().input("/dev/zero").output( core.system_path).count(1).block_size(req_size).oflag("direct")) dd.run() stats = cache.get_cache_statistics(per_io_class=True, io_class_id=ioclass_id) assert stats["dirty"].get_value(Unit.Blocks4096) == 0
def create_random_test_file(target_file_path: str, file_size: Size = Size(1, Unit.MebiByte), random: bool = True): from test_utils.filesystem.file import File bs = Size(512, Unit.KibiByte) cnt = math.ceil(file_size.value / bs.value) file = File.create_file(target_file_path) dd = Dd().output(target_file_path) \ .input("/dev/urandom" if random else "/dev/zero") \ .block_size(bs) \ .count(cnt) \ .oflag("direct") dd.run() file.refresh_item() return file
def run_io_dir(path, size_4k): dd = (Dd().input("/dev/zero").output(f"{path}").count(size_4k).block_size( Size(1, Unit.Blocks4096))) TestRun.LOGGER.info(f"{dd}") dd.run() sync() drop_caches(DropCachesMode.ALL)
def remove_files_classification(): TestRun.LOGGER.info("Moving all files to 'unclassified' IO class") ioclass_config.remove_ioclass_config( ioclass_config_path=ioclass_config_path) ioclass_config.create_ioclass_config( add_default_rule=False, ioclass_config_path=ioclass_config_path) ioclass_config.add_ioclass( ioclass_id=0, eviction_priority=22, allocation=False, rule="unclassified", ioclass_config_path=ioclass_config_path, ) casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path) occupancy_before = cache.get_statistics_deprecated( io_class_id=0)["occupancy"] for file in test_files: Dd().input(file.full_path).output("/dev/null").block_size( file.size).run() occupancy_after = cache.get_statistics_deprecated( io_class_id=0)["occupancy"] if occupancy_after != occupancy_before + file.size: pytest.xfail("File not reclassified properly!\n" f"Expected {occupancy_before + file.size}\n" f"Actual {occupancy_after}") occupancy_before = occupancy_after sync() drop_caches(DropCachesMode.ALL)
def test_ioclass_process_name(prepare_and_cleanup): """Check if data generated by process with particular name is cached""" cache, core = prepare() ioclass_id = 1 dd_size = Size(4, Unit.KibiByte) dd_count = 1 iterations = 100 ioclass_config.add_ioclass( ioclass_id=ioclass_id, eviction_priority=1, allocation=True, rule=f"process_name:dd&done", ioclass_config_path=ioclass_config_path, ) casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path) cache.flush_cache() Udev.disable() TestProperties.LOGGER.info( f"Check if all data generated by dd process is cached.") for i in range(iterations): dd = (Dd().input("/dev/zero").output( core.system_path).count(dd_count).block_size(dd_size).seek(i)) dd.run() sync() time.sleep(0.1) stats = cache.get_cache_statistics(io_class_id=ioclass_id) assert stats["dirty"].get_value(Unit.Blocks4096) == (i + 1) * dd_count
def prepare_md_dump(cache_device, core_device, cls, cache_id): with TestRun.step("Setup WB cache instance with one core"): cache = casadm.start_cache( cache_dev=cache_device, cache_line_size=cls, cache_mode=CacheMode.WB, cache_id=cache_id, force=True, ) cache.add_core(core_device) with TestRun.step("Get metadata size"): dmesg_out = TestRun.executor.run_expect_success("dmesg").stdout md_size = dmesg.get_metadata_size(dmesg_out) with TestRun.step("Dump the metadata of the cache"): dump_file_path = "/tmp/test_activate_corrupted.dump" md_dump = File(dump_file_path) md_dump.remove(force=True, ignore_errors=True) dd_count = int(md_size / Size(1, Unit.MebiByte)) + 1 (Dd().input(cache_device.path).output(md_dump.full_path).block_size( Size(1, Unit.MebiByte)).count(dd_count).run()) md_dump.refresh_item() with TestRun.step("Stop cache device"): cache.stop() return md_dump
def dd_builder(cache_mode: CacheMode, dev: Core, size: Size): blocks = int(size.value / block_size.value) dd = (Dd().block_size(block_size).count(blocks)) if CacheModeTrait.InsertRead in CacheMode.get_traits(cache_mode): dd.input(dev.path).output("/dev/null") else: dd.input("/dev/urandom").output(dev.path) return dd
def test_ioclass_pid(): cache, core = prepare() ioclass_id = 1 iterations = 20 dd_count = 100 dd_size = Size(4, Unit.KibiByte) Udev.disable() # Since 'dd' has to be executed right after writing pid to 'ns_last_pid', # 'dd' command is created and is appended to 'echo' command instead of running it dd_command = str( Dd() .input("/dev/zero") .output(core.system_path) .count(dd_count) .block_size(dd_size) ) for i in range(iterations): cache.flush_cache() output = TestRun.executor.run("cat /proc/sys/kernel/ns_last_pid") if output.exit_code != 0: raise Exception( f"Failed to retrieve pid. stdout: {output.stdout} \n stderr :{output.stderr}" ) # Few pids might be used by system during test preparation pid = int(output.stdout) + 50 ioclass_config.add_ioclass( ioclass_id=ioclass_id, eviction_priority=1, allocation=True, rule=f"pid:eq:{pid}&done", ioclass_config_path=ioclass_config_path, ) casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path) TestRun.LOGGER.info(f"Running dd with pid {pid}") # pid saved in 'ns_last_pid' has to be smaller by one than target dd pid dd_and_pid_command = ( f"echo {pid-1} > /proc/sys/kernel/ns_last_pid && {dd_command}" ) output = TestRun.executor.run(dd_and_pid_command) if output.exit_code != 0: raise Exception( f"Failed to run dd with target pid. " f"stdout: {output.stdout} \n stderr :{output.stderr}" ) sync() dirty = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.dirty if dirty.get_value(Unit.Blocks4096) != dd_count: TestRun.LOGGER.error(f"Wrong amount of dirty data ({dirty}).") ioclass_config.remove_ioclass(ioclass_id)
def remove_partitions(device): if device.is_mounted(): device.unmount() for partition in device.partitions: unmount(partition) TestProperties.LOGGER.info(f"Removing partitions from device: {device.system_path}.") dd = Dd().input("/dev/zero") \ .output(device.system_path) \ .count(1) \ .block_size(Size(device.block_size.value, Unit.Byte)) dd.run() output = TestProperties.executor.execute(f"ls {device.system_path}* -1") if len(output.stdout.split('\n')) > 1: TestProperties.LOGGER.error(f"Could not remove partitions from device {device.system_path}") return False return True
def dd_builder(cache_mode, cache_line_size, count, device): dd = (Dd().block_size(cache_line_size.value).count(count)) if CacheModeTrait.InsertRead in CacheMode.get_traits(cache_mode): dd.input(device.path).output("/dev/null").iflag("direct") else: dd.input("/dev/urandom").output(device.path).oflag("direct") return dd
def read_files_with_reclassification_check(target_ioclass_id: int, source_ioclass_id: int, directory: Directory, with_delay: bool): start_time = datetime.now() target_occupancy_after = cache.get_statistics_deprecated( io_class_id=target_ioclass_id)["occupancy"] source_occupancy_after = cache.get_statistics_deprecated( io_class_id=source_ioclass_id)["occupancy"] unclassified_files = [] for file in [ item for item in directory.ls() if isinstance(item, File) ]: target_occupancy_before = target_occupancy_after source_occupancy_before = source_occupancy_after time_from_start = datetime.now() - start_time (Dd().input(file.full_path).output("/dev/null").block_size( Size(1, Unit.Blocks4096)).run()) target_occupancy_after = cache.get_statistics_deprecated( io_class_id=target_ioclass_id)["occupancy"] source_occupancy_after = cache.get_statistics_deprecated( io_class_id=source_ioclass_id)["occupancy"] if target_occupancy_after < target_occupancy_before: pytest.xfail("Target IO class occupancy lowered!") elif target_occupancy_after - target_occupancy_before < file.size: unclassified_files.append(file) if with_delay and time_from_start <= ioclass_config.MAX_CLASSIFICATION_DELAY: continue pytest.xfail("Target IO class occupancy not changed properly!") if source_occupancy_after >= source_occupancy_before: if file not in unclassified_files: unclassified_files.append(file) if with_delay and time_from_start <= ioclass_config.MAX_CLASSIFICATION_DELAY: continue pytest.xfail("Source IO class occupancy not changed properly!") if len(unclassified_files): TestRun.LOGGER.info("Rereading unclassified test files...") sync() drop_caches(DropCachesMode.ALL) for file in unclassified_files: (Dd().input(file.full_path).output("/dev/null").block_size( Size(1, Unit.Blocks4096)).run())
def test_purge(purge_target): """ title: Call purge without and with `--script` switch description: | Check if purge is called only when `--script` switch is used. pass_criteria: - casadm returns an error when `--script` is missing - cache is wiped when purge command is used properly """ with TestRun.step("Prepare devices"): cache_device = TestRun.disks["cache"] core_device = TestRun.disks["core"] cache_device.create_partitions([Size(500, Unit.MebiByte)]) core_device.create_partitions([Size(500, Unit.MebiByte)]) cache_device = cache_device.partitions[0] core_device = core_device.partitions[0] with TestRun.step("Prepare cache instance"): cache = casadm.start_cache(cache_device, force=True) core = casadm.add_core(cache, core_device) with TestRun.step("Trigger IO to prepared cache instance"): dd = (Dd().input("/dev/zero").output(core.path).count(100).block_size( Size(1, Unit.Blocks512)).oflag("direct")) dd.run() sync() with TestRun.step( f"Try to call purge-{purge_target} without `--script` switch"): original_occupancy = cache.get_statistics().usage_stats.occupancy purge_params = f"--cache-id {cache.cache_id} " if purge_target == "core": purge_params += f"--core-id {core.core_id}" TestRun.executor.run_expect_fail( f"casadm --purge-{purge_target} {purge_params}") if cache.get_statistics().usage_stats.occupancy != original_occupancy: TestRun.fail( f"Purge {purge_target} should not be possible to use without `--script` switch!" ) with TestRun.step( f"Try to call purge-{purge_target} with `--script` switch"): TestRun.executor.run_expect_success( f"casadm --script --purge-{purge_target} {purge_params}") if cache.get_statistics().usage_stats.occupancy.get_value() != 0: TestRun.fail( f"{cache.get_statistics().usage_stats.occupancy.get_value()}") TestRun.fail( f"Purge {purge_target} should invalidate all cache lines!") with TestRun.step(f"Stop cache"): casadm.stop_all_caches()
def copy_file(source, target, size, direct=None): dd = Dd() \ .input(source) \ .output(target) \ .block_size(Size(1, Unit.Blocks4096)) \ .count(int(size.get_value(Unit.Blocks4096))) if direct == "oflag": dd.oflag("direct") elif direct == "iflag": dd.iflag("direct") dd.run()
def run_io_dir(path, size_4k): dd = (Dd().input("/dev/zero").output(f"{path}").count(size_4k).block_size( Size(1, Unit.Blocks4096))) TestRun.LOGGER.info(f"{dd}") output = dd.run() if output.exit_code != 0: TestRun.fail( f"Failed to execute dd.\n {output.stdout}\n{output.stderr}") sync() drop_caches(DropCachesMode.ALL)
def prepare_corrupted_md(md_dump, offset_to_corrupt, bs): invalid_dump_path = "/tmp/test_activate_corrupted.invalid_dump" dd_count = offset_to_corrupt + 1 md_dump.copy(destination=invalid_dump_path, force=True) corrupted_md = File(invalid_dump_path) (Dd().input("/dev/urandom").output(corrupted_md.full_path).block_size( bs).count(dd_count).seek(offset_to_corrupt).conv("notrunc").run()) corrupted_md.refresh_item() return corrupted_md
def run_io_dir(path, num_ios): dd = ( Dd() .input("/dev/zero") .output(f"{path}/tmp_file") .count(num_ios) .block_size(Size(1, Unit.Blocks4096)) ) dd.run() sync() drop_caches(DropCachesMode.ALL)
def allocate_memory(size: Size): """Allocates given amount of memory""" mount_ramfs() TestRun.LOGGER.info( f"Allocating {size.get_value(Unit.MiB):0.2f} MiB of memory.") bs = Size(1, Unit.Blocks512) dd = (Dd().block_size(bs).count(math.ceil( size / bs)).input("/dev/zero").output(f"{MEMORY_MOUNT_POINT}/data")) output = dd.run() if output.exit_code != 0: raise CmdException("Allocating memory failed.", output)