def test_ioclass_directory_file_operations(filesystem):
"""
title: Test IO classification by file operations.
description: |
Test if directory classification works properly after file operations like move or rename.
pass_criteria:
- No kernel bug.
- The operations themselves should not cause reclassification but IO after those
operations should be reclassified to proper IO class.
"""
test_dir_path = f"{mountpoint}/test_dir"
nested_dir_path = f"{test_dir_path}/nested_dir"
dd_blocks = random.randint(5, 50)
with TestRun.step("Prepare cache and core."):
cache, core = prepare(default_allocation="1.00")
Udev.disable()
with TestRun.step("Create and load IO class config file."):
ioclass_id = random.randint(2, ioclass_config.MAX_IO_CLASS_ID)
ioclass_config.add_ioclass(ioclass_id=1,
eviction_priority=1,
allocation="1.00",
rule="metadata",
ioclass_config_path=ioclass_config_path)
# directory IO class
ioclass_config.add_ioclass(
ioclass_id=ioclass_id,
eviction_priority=1,
allocation="1.00",
rule=f"directory:{test_dir_path}",
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
with TestRun.step(f"Prepare {filesystem.name} filesystem "
f"and mounting {core.path} at {mountpoint}."):
core.create_filesystem(fs_type=filesystem)
core.mount(mount_point=mountpoint)
sync()
with TestRun.step(f"Create directory {nested_dir_path}."):
Directory.create_directory(path=nested_dir_path, parents=True)
sync()
drop_caches(DropCachesMode.ALL)
with TestRun.step("Create test file."):
classified_before = cache.get_io_class_statistics(
io_class_id=ioclass_id).usage_stats.occupancy
file_path = f"{test_dir_path}/test_file"
(Dd().input("/dev/urandom").output(file_path).oflag("sync")
.block_size(Size(1, Unit.MebiByte)).count(dd_blocks).run())
sync()
drop_caches(DropCachesMode.ALL)
test_file = File(file_path).refresh_item()
with TestRun.step("Check classified occupancy."):
classified_after = cache.get_io_class_statistics(
io_class_id=ioclass_id).usage_stats.occupancy
check_occupancy(classified_before + test_file.size, classified_after)
with TestRun.step("Move test file out of classified directory."):
classified_before = classified_after
non_classified_before = cache.get_io_class_statistics(io_class_id=0).usage_stats.occupancy
test_file.move(destination=mountpoint)
sync()
drop_caches(DropCachesMode.ALL)
with TestRun.step("Check classified occupancy."):
classified_after = cache.get_io_class_statistics(
io_class_id=ioclass_id).usage_stats.occupancy
check_occupancy(classified_before, classified_after)
TestRun.LOGGER.info("Checking non-classified occupancy")
non_classified_after = cache.get_io_class_statistics(io_class_id=0).usage_stats.occupancy
check_occupancy(non_classified_before, non_classified_after)
with TestRun.step("Read test file."):
classified_before = classified_after
non_classified_before = non_classified_after
(Dd().input(test_file.full_path).output("/dev/null")
.block_size(Size(1, Unit.Blocks4096)).run())
with TestRun.step("Check classified occupancy."):
classified_after = cache.get_io_class_statistics(
io_class_id=ioclass_id).usage_stats.occupancy
check_occupancy(classified_before - test_file.size, classified_after)
TestRun.LOGGER.info("Checking non-classified occupancy")
non_classified_after = cache.get_io_class_statistics(io_class_id=0).usage_stats.occupancy
check_occupancy(non_classified_before + test_file.size, non_classified_after)
with TestRun.step(f"Move test file to {nested_dir_path}."):
classified_before = classified_after
non_classified_before = non_classified_after
test_file.move(destination=nested_dir_path)
sync()
drop_caches(DropCachesMode.ALL)
with TestRun.step("Check classified occupancy."):
classified_after = cache.get_io_class_statistics(
io_class_id=ioclass_id).usage_stats.occupancy
check_occupancy(classified_before, classified_after)
TestRun.LOGGER.info("Checking non-classified occupancy")
non_classified_after = cache.get_io_class_statistics(io_class_id=0).usage_stats.occupancy
check_occupancy(non_classified_before, non_classified_after)
with TestRun.step("Read test file."):
classified_before = classified_after
non_classified_before = non_classified_after
(Dd().input(test_file.full_path).output("/dev/null")
.block_size(Size(1, Unit.Blocks4096)).run())
with TestRun.step("Check classified occupancy."):
classified_after = cache.get_io_class_statistics(
io_class_id=ioclass_id).usage_stats.occupancy
check_occupancy(classified_before + test_file.size, classified_after)
with TestRun.step("Check non-classified occupancy."):
non_classified_after = cache.get_io_class_statistics(io_class_id=0).usage_stats.occupancy
check_occupancy(non_classified_before - test_file.size, non_classified_after)
def test_flush_over_640_gibibytes_with_fs(cache_mode, fs):
"""
title: Test of the ability to flush huge amount of dirty data on device with filesystem.
description: |
Flush cache when amount of dirty data in cache with core with filesystem exceeds 640 GiB.
pass_criteria:
- Flushing completes successfully without any errors.
"""
with TestRun.step("Prepare devices for cache and core."):
cache_dev = TestRun.disks['cache']
check_disk_size(cache_dev)
cache_dev.create_partitions([required_disk_size])
cache_part = cache_dev.partitions[0]
core_dev = TestRun.disks['core']
check_disk_size(core_dev)
Udev.disable()
with TestRun.step(f"Start cache in {cache_mode} mode."):
cache = casadm.start_cache(cache_part, cache_mode)
with TestRun.step(
f"Add core with {fs.name} filesystem to cache and mount it."):
core_dev.create_filesystem(fs)
core = cache.add_core(core_dev)
core.mount(mnt_point)
with TestRun.step("Disable cleaning and sequential cutoff."):
cache.set_cleaning_policy(CleaningPolicy.nop)
cache.set_seq_cutoff_policy(SeqCutOffPolicy.never)
with TestRun.step("Create test file"):
test_file_main = File.create_file("/tmp/test_file_main")
fio = (Fio().create_command().io_engine(IoEngine.libaio).read_write(
ReadWrite.write).block_size(bs).direct().io_depth(256).target(
test_file_main.full_path).size(file_size))
fio.default_run_time = timedelta(
hours=4) # timeout for non-time-based fio
fio.run()
test_file_main.refresh_item()
with TestRun.step("Validate test file and read its md5 sum."):
if test_file_main.size != file_size:
TestRun.fail("Created test file hasn't reached its target size.")
test_file_md5sum_main = test_file_main.md5sum()
with TestRun.step("Write data to exported object."):
test_file_copy = test_file_main.copy(mnt_point + "test_file_copy")
test_file_copy.refresh_item()
sync()
with TestRun.step(f"Check if dirty data exceeded {file_size * 0.98} GiB."):
minimum_4KiB_blocks = int(
(file_size * 0.98).get_value(Unit.Blocks4096))
if int(cache.get_statistics().usage_stats.dirty) < minimum_4KiB_blocks:
TestRun.fail("There is not enough dirty data in the cache!")
with TestRun.step("Unmount core and stop cache with flush."):
core.unmount()
# this operation could take few hours, depending on core disk
output = TestRun.executor.run(stop_cmd(str(cache.cache_id)),
timedelta(hours=12))
if output.exit_code != 0:
TestRun.fail(f"Stopping cache with flush failed!\n{output.stderr}")
with TestRun.step(
"Mount core device and check md5 sum of test file copy."):
core_dev.mount(mnt_point)
if test_file_md5sum_main != test_file_copy.md5sum():
TestRun.LOGGER.error("Md5 sums should be equal.")
with TestRun.step("Delete test files."):
test_file_main.remove(True)
test_file_copy.remove(True)
with TestRun.step("Unmount core device."):
core_dev.unmount()
remove(mnt_point, True, True, True)
def test_ioclass_metadata(prepare_and_cleanup, filesystem):
"""
Perform operations on files that cause metadata update.
Determine if every such operation results in increased writes to cached metadata.
Exact values may not be tested as each file system has different metadata structure.
"""
cache, core = prepare()
Udev.disable()
ioclass_id = random.randint(1, ioclass_config.MAX_IO_CLASS_ID)
# metadata IO class
ioclass_config.add_ioclass(
ioclass_id=ioclass_id,
eviction_priority=1,
allocation=True,
rule="metadata&done",
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
TestRun.LOGGER.info(f"Preparing {filesystem.name} filesystem "
f"and mounting {core.system_path} at {mountpoint}")
core.create_filesystem(filesystem)
core.mount(mountpoint)
sync()
requests_to_metadata_before = cache.get_cache_statistics(
io_class_id=ioclass_id)["write total"]
TestRun.LOGGER.info("Creating 20 test files")
files = []
for i in range(1, 21):
file_path = f"{mountpoint}/test_file_{i}"
dd = (Dd().input("/dev/urandom").output(file_path).count(
random.randint(5,
50)).block_size(Size(1,
Unit.MebiByte)).oflag("sync"))
dd.run()
files.append(File(file_path))
TestRun.LOGGER.info("Checking requests to metadata")
requests_to_metadata_after = cache.get_cache_statistics(
io_class_id=ioclass_id)["write total"]
if requests_to_metadata_after == requests_to_metadata_before:
pytest.xfail("No requests to metadata while creating files!")
requests_to_metadata_before = requests_to_metadata_after
TestRun.LOGGER.info("Renaming all test files")
for file in files:
file.move(f"{file.full_path}_renamed")
sync()
TestRun.LOGGER.info("Checking requests to metadata")
requests_to_metadata_after = cache.get_cache_statistics(
io_class_id=ioclass_id)["write total"]
if requests_to_metadata_after == requests_to_metadata_before:
pytest.xfail("No requests to metadata while renaming files!")
requests_to_metadata_before = requests_to_metadata_after
test_dir_path = f"{mountpoint}/test_dir"
TestRun.LOGGER.info(f"Creating directory {test_dir_path}")
fs_utils.create_directory(path=test_dir_path)
TestRun.LOGGER.info(f"Moving test files into {test_dir_path}")
for file in files:
file.move(test_dir_path)
sync()
TestRun.LOGGER.info("Checking requests to metadata")
requests_to_metadata_after = cache.get_cache_statistics(
io_class_id=ioclass_id)["write total"]
if requests_to_metadata_after == requests_to_metadata_before:
pytest.xfail("No requests to metadata while moving files!")
TestRun.LOGGER.info(f"Removing {test_dir_path}")
fs_utils.remove(path=test_dir_path, force=True, recursive=True)
TestRun.LOGGER.info("Checking requests to metadata")
requests_to_metadata_after = cache.get_cache_statistics(
io_class_id=ioclass_id)["write total"]
if requests_to_metadata_after == requests_to_metadata_before:
pytest.xfail(
"No requests to metadata while deleting directory with files!")
def test_cleaning_policies_in_write_through(cleaning_policy):
"""
title: Test for cleaning policy operation in Write-Through cache mode.
description: |
Check if ALRU, NOP and ACP cleaning policies preserve their
parameters when changed and if they flush dirty data properly
in Write-Through cache mode.
pass_criteria:
- Flush parameters preserve their values when changed.
- Dirty data is flushed or not according to the policy used.
"""
with TestRun.step("Partition cache and core devices"):
cache_dev, core_dev = storage_prepare()
Udev.disable()
with TestRun.step(
f"Start cache in Write-Through mode with {cleaning_policy} cleaning policy"
):
cache = casadm.start_cache(cache_dev.partitions[0],
CacheMode.WT,
force=True)
set_cleaning_policy_and_params(cache, cleaning_policy)
with TestRun.step("Check for running CAS cleaner"):
if TestRun.executor.run(
f"pgrep {cas_cleaner_process_name}").exit_code != 0:
TestRun.fail("CAS cleaner process is not running!")
with TestRun.step(f"Add {cores_count} cores to the cache"):
core = []
for i in range(cores_count):
core.append(cache.add_core(core_dev.partitions[i]))
with TestRun.step("Change cache mode to Write-Back"):
cache.set_cache_mode(CacheMode.WB)
with TestRun.step("Run 'fio'"):
fio = fio_prepare()
for i in range(cores_count):
fio.add_job().target(core[i].path)
fio.run()
time.sleep(3)
with TestRun.step("Change cache mode back to Write-Through"):
cache.set_cache_mode(CacheMode.WT, flush=False)
core_writes_before_wait_for_cleaning = (
cache.get_statistics().block_stats.core.writes)
with TestRun.step(f"Wait {time_to_wait} seconds"):
time.sleep(time_to_wait)
with TestRun.step("Check write statistics for core device"):
core_writes_after_wait_for_cleaning = (
cache.get_statistics().block_stats.core.writes)
check_cleaning_policy_operation(
cleaning_policy,
core_writes_before_wait_for_cleaning,
core_writes_after_wait_for_cleaning,
)
with TestRun.step("Stop all caches"):
casadm.stop_all_caches()
Udev.enable()
def test_ioclass_directory_dir_operations(filesystem):
"""
Test if directory classification works properly after directory operations like move or rename.
The operations themselves should not cause reclassification but IO after those operations
should be reclassified to proper IO class.
Directory classification may work with a delay after loading IO class configuration or
move/rename operations. Test checks if maximum delay is not exceeded.
"""
def create_files_with_classification_delay_check(directory: Directory, ioclass_id: int):
start_time = datetime.now()
occupancy_after = cache.get_io_class_statistics(
io_class_id=ioclass_id).usage_stats.occupancy
dd_blocks = 10
dd_size = Size(dd_blocks, Unit.Blocks4096)
file_counter = 0
unclassified_files = []
time_from_start = datetime.now() - start_time
while time_from_start < ioclass_config.MAX_CLASSIFICATION_DELAY:
occupancy_before = occupancy_after
file_path = f"{directory.full_path}/test_file_{file_counter}"
file_counter += 1
time_from_start = datetime.now() - start_time
(Dd().input("/dev/zero").output(file_path).oflag("sync")
.block_size(Size(1, Unit.Blocks4096)).count(dd_blocks).run())
occupancy_after = cache.get_io_class_statistics(
io_class_id=ioclass_id).usage_stats.occupancy
if occupancy_after - occupancy_before < dd_size:
unclassified_files.append(file_path)
if len(unclassified_files) == file_counter:
pytest.xfail("No files were properly classified within max delay time!")
if len(unclassified_files):
TestRun.LOGGER.info("Rewriting unclassified test files...")
for file_path in unclassified_files:
(Dd().input("/dev/zero").output(file_path).oflag("sync")
.block_size(Size(1, Unit.Blocks4096)).count(dd_blocks).run())
def read_files_with_reclassification_check(
target_ioclass_id: int, source_ioclass_id: int, directory: Directory, with_delay: bool):
start_time = datetime.now()
target_occupancy_after = cache.get_io_class_statistics(
io_class_id=target_ioclass_id).usage_stats.occupancy
source_occupancy_after = cache.get_io_class_statistics(
io_class_id=source_ioclass_id).usage_stats.occupancy
unclassified_files = []
for file in [item for item in directory.ls() if isinstance(item, File)]:
target_occupancy_before = target_occupancy_after
source_occupancy_before = source_occupancy_after
time_from_start = datetime.now() - start_time
(Dd().input(file.full_path).output("/dev/null")
.block_size(Size(1, Unit.Blocks4096)).run())
target_occupancy_after = cache.get_io_class_statistics(
io_class_id=target_ioclass_id).usage_stats.occupancy
source_occupancy_after = cache.get_io_class_statistics(
io_class_id=source_ioclass_id).usage_stats.occupancy
if target_occupancy_after < target_occupancy_before:
pytest.xfail("Target IO class occupancy lowered!")
elif target_occupancy_after - target_occupancy_before < file.size:
unclassified_files.append(file)
if with_delay and time_from_start <= ioclass_config.MAX_CLASSIFICATION_DELAY:
continue
pytest.xfail("Target IO class occupancy not changed properly!")
if source_occupancy_after >= source_occupancy_before:
if file not in unclassified_files:
unclassified_files.append(file)
if with_delay and time_from_start <= ioclass_config.MAX_CLASSIFICATION_DELAY:
continue
pytest.xfail("Source IO class occupancy not changed properly!")
if len(unclassified_files):
TestRun.LOGGER.info("Rereading unclassified test files...")
sync()
drop_caches(DropCachesMode.ALL)
for file in unclassified_files:
(Dd().input(file.full_path).output("/dev/null")
.block_size(Size(1, Unit.Blocks4096)).run())
cache, core = prepare()
Udev.disable()
proper_ids = random.sample(range(1, ioclass_config.MAX_IO_CLASS_ID + 1), 2)
ioclass_id_1 = proper_ids[0]
classified_dir_path_1 = f"{mountpoint}/dir_{ioclass_id_1}"
ioclass_id_2 = proper_ids[1]
classified_dir_path_2 = f"{mountpoint}/dir_{ioclass_id_2}"
# directory IO classes
ioclass_config.add_ioclass(
ioclass_id=ioclass_id_1,
eviction_priority=1,
allocation=True,
rule=f"directory:{classified_dir_path_1}",
ioclass_config_path=ioclass_config_path,
)
ioclass_config.add_ioclass(
ioclass_id=ioclass_id_2,
eviction_priority=1,
allocation=True,
rule=f"directory:{classified_dir_path_2}",
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
TestRun.LOGGER.info(f"Preparing {filesystem.name} filesystem "
f"and mounting {core.system_path} at {mountpoint}")
core.create_filesystem(fs_type=filesystem)
core.mount(mount_point=mountpoint)
sync()
non_classified_dir_path = f"{mountpoint}/non_classified"
TestRun.LOGGER.info(
f"Creating a non-classified directory: {non_classified_dir_path}")
dir_1 = Directory.create_directory(path=non_classified_dir_path)
TestRun.LOGGER.info(f"Renaming {non_classified_dir_path} to {classified_dir_path_1}")
dir_1.move(destination=classified_dir_path_1)
TestRun.LOGGER.info("Creating files with delay check")
create_files_with_classification_delay_check(directory=dir_1, ioclass_id=ioclass_id_1)
TestRun.LOGGER.info(f"Creating {classified_dir_path_2}/subdir")
dir_2 = Directory.create_directory(path=f"{classified_dir_path_2}/subdir", parents=True)
TestRun.LOGGER.info("Creating files with delay check")
create_files_with_classification_delay_check(directory=dir_2, ioclass_id=ioclass_id_2)
sync()
drop_caches(DropCachesMode.ALL)
TestRun.LOGGER.info(f"Moving {dir_2.full_path} to {classified_dir_path_1}")
dir_2.move(destination=classified_dir_path_1)
TestRun.LOGGER.info("Reading files with reclassification check")
read_files_with_reclassification_check(
target_ioclass_id=ioclass_id_1, source_ioclass_id=ioclass_id_2,
directory=dir_2, with_delay=False)
sync()
drop_caches(DropCachesMode.ALL)
TestRun.LOGGER.info(f"Moving {dir_2.full_path} to {mountpoint}")
dir_2.move(destination=mountpoint)
TestRun.LOGGER.info("Reading files with reclassification check")
read_files_with_reclassification_check(
target_ioclass_id=0, source_ioclass_id=ioclass_id_1,
directory=dir_2, with_delay=False)
TestRun.LOGGER.info(f"Removing {classified_dir_path_2}")
fs_utils.remove(path=classified_dir_path_2, force=True, recursive=True)
sync()
drop_caches(DropCachesMode.ALL)
TestRun.LOGGER.info(f"Renaming {classified_dir_path_1} to {classified_dir_path_2}")
dir_1.move(destination=classified_dir_path_2)
TestRun.LOGGER.info("Reading files with reclassification check")
read_files_with_reclassification_check(
target_ioclass_id=ioclass_id_2, source_ioclass_id=ioclass_id_1,
directory=dir_1, with_delay=True)
TestRun.LOGGER.info(f"Renaming {classified_dir_path_2} to {non_classified_dir_path}")
dir_1.move(destination=non_classified_dir_path)
TestRun.LOGGER.info("Reading files with reclassification check")
read_files_with_reclassification_check(
target_ioclass_id=0, source_ioclass_id=ioclass_id_2,
directory=dir_1, with_delay=True)
def test_ioclass_directory_file_operations(filesystem):
"""
Test if directory classification works properly after file operations like move or rename.
The operations themselves should not cause reclassification but IO after those operations
should be reclassified to proper IO class.
"""
def check_occupancy(expected: Size, actual: Size):
if expected != actual:
pytest.xfail("Occupancy check failed!\n"
f"Expected: {expected}, actual: {actual}")
cache, core = prepare()
Udev.disable()
test_dir_path = f"{mountpoint}/test_dir"
nested_dir_path = f"{test_dir_path}/nested_dir"
dd_blocks = random.randint(5, 50)
ioclass_id = random.randint(1, ioclass_config.MAX_IO_CLASS_ID)
# directory IO class
ioclass_config.add_ioclass(
ioclass_id=ioclass_id,
eviction_priority=1,
allocation=True,
rule=f"directory:{test_dir_path}",
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
TestRun.LOGGER.info(f"Preparing {filesystem.name} filesystem "
f"and mounting {core.system_path} at {mountpoint}")
core.create_filesystem(fs_type=filesystem)
core.mount(mount_point=mountpoint)
sync()
TestRun.LOGGER.info(f"Creating directory {nested_dir_path}")
Directory.create_directory(path=nested_dir_path, parents=True)
sync()
drop_caches(DropCachesMode.ALL)
TestRun.LOGGER.info("Creating test file")
classified_before = cache.get_io_class_statistics(
io_class_id=ioclass_id).usage_stats.occupancy
file_path = f"{test_dir_path}/test_file"
(Dd().input("/dev/urandom").output(file_path).oflag("sync")
.block_size(Size(1, Unit.MebiByte)).count(dd_blocks).run())
sync()
drop_caches(DropCachesMode.ALL)
test_file = File(file_path).refresh_item()
TestRun.LOGGER.info("Checking classified occupancy")
classified_after = cache.get_io_class_statistics(
io_class_id=ioclass_id).usage_stats.occupancy
check_occupancy(classified_before + test_file.size, classified_after)
TestRun.LOGGER.info("Moving test file out of classified directory")
classified_before = classified_after
non_classified_before = cache.get_io_class_statistics(io_class_id=0).usage_stats.occupancy
test_file.move(destination=mountpoint)
sync()
drop_caches(DropCachesMode.ALL)
TestRun.LOGGER.info("Checking classified occupancy")
classified_after = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy
check_occupancy(classified_before, classified_after)
TestRun.LOGGER.info("Checking non-classified occupancy")
non_classified_after = cache.get_io_class_statistics(io_class_id=0).usage_stats.occupancy
check_occupancy(non_classified_before, non_classified_after)
TestRun.LOGGER.info("Reading test file")
classified_before = classified_after
non_classified_before = non_classified_after
(Dd().input(test_file.full_path).output("/dev/null")
.block_size(Size(1, Unit.MebiByte)).run())
TestRun.LOGGER.info("Checking classified occupancy")
classified_after = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy
check_occupancy(classified_before - test_file.size, classified_after)
TestRun.LOGGER.info("Checking non-classified occupancy")
non_classified_after = cache.get_io_class_statistics(io_class_id=0).usage_stats.occupancy
check_occupancy(non_classified_before + test_file.size, non_classified_after)
TestRun.LOGGER.info(f"Moving test file to {nested_dir_path}")
classified_before = classified_after
non_classified_before = non_classified_after
test_file.move(destination=nested_dir_path)
sync()
drop_caches(DropCachesMode.ALL)
TestRun.LOGGER.info("Checking classified occupancy")
classified_after = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy
check_occupancy(classified_before, classified_after)
TestRun.LOGGER.info("Checking non-classified occupancy")
non_classified_after = cache.get_io_class_statistics(io_class_id=0).usage_stats.occupancy
check_occupancy(non_classified_before, non_classified_after)
TestRun.LOGGER.info("Reading test file")
classified_before = classified_after
non_classified_before = non_classified_after
(Dd().input(test_file.full_path).output("/dev/null")
.block_size(Size(1, Unit.MebiByte)).run())
TestRun.LOGGER.info("Checking classified occupancy")
classified_after = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy
check_occupancy(classified_before + test_file.size, classified_after)
TestRun.LOGGER.info("Checking non-classified occupancy")
non_classified_after = cache.get_io_class_statistics(io_class_id=0).usage_stats.occupancy
check_occupancy(non_classified_before - test_file.size, non_classified_after)
def test_ioclass_directory_dir_operations(filesystem):
"""
title: Test IO classification by directory operations.
description: |
Test if directory classification works properly after directory operations like move or
rename.
pass_criteria:
- No kernel bug.
- The operations themselves should not cause reclassification but IO after those
operations should be reclassified to proper IO class.
- Directory classification may work with a delay after loading IO class configuration or
move/rename operations. Test checks if maximum delay is not exceeded.
"""
non_classified_dir_path = f"{mountpoint}/non_classified"
with TestRun.step("Prepare cache and core."):
cache, core = prepare(default_allocation="1.00")
Udev.disable()
with TestRun.step("Create and load IO class config file."):
proper_ids = random.sample(
range(1, ioclass_config.MAX_IO_CLASS_ID + 1), 2)
ioclass_id_1 = proper_ids[0]
classified_dir_path_1 = f"{mountpoint}/dir_{ioclass_id_1}"
ioclass_id_2 = proper_ids[1]
classified_dir_path_2 = f"{mountpoint}/dir_{ioclass_id_2}"
# directory IO classes
ioclass_config.add_ioclass(
ioclass_id=ioclass_id_1,
eviction_priority=1,
allocation="1.00",
rule=f"directory:{classified_dir_path_1}",
ioclass_config_path=ioclass_config_path,
)
ioclass_config.add_ioclass(
ioclass_id=ioclass_id_2,
eviction_priority=1,
allocation="1.00",
rule=f"directory:{classified_dir_path_2}",
ioclass_config_path=ioclass_config_path,
)
ioclass_config.add_ioclass(
ioclass_id=32,
eviction_priority=255,
allocation="0.00",
rule=f"metadata",
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id,
file=ioclass_config_path)
with TestRun.step(f"Prepare {filesystem.name} filesystem "
f"and mount {core.path} at {mountpoint}."):
core.create_filesystem(fs_type=filesystem)
core.mount(mount_point=mountpoint)
sync()
with TestRun.step(
f"Create a non-classified directory: {non_classified_dir_path}."):
dir_1 = Directory.create_directory(path=non_classified_dir_path)
with TestRun.step(
f"Rename {non_classified_dir_path} to {classified_dir_path_1}."):
dir_1.move(destination=classified_dir_path_1)
with TestRun.step("Create files with delay check."):
create_files_with_classification_delay_check(cache,
directory=dir_1,
ioclass_id=ioclass_id_1)
with TestRun.step(f"Create {classified_dir_path_2}/subdir."):
dir_2 = Directory.create_directory(
path=f"{classified_dir_path_2}/subdir", parents=True)
with TestRun.step("Create files with delay check."):
create_files_with_classification_delay_check(cache,
directory=dir_2,
ioclass_id=ioclass_id_2)
sync()
drop_caches(DropCachesMode.ALL)
with TestRun.step(f"Move {dir_2.full_path} to {classified_dir_path_1}."):
dir_2.move(destination=classified_dir_path_1)
with TestRun.step("Read files with reclassification check."):
read_files_with_reclassification_check(cache,
target_ioclass_id=ioclass_id_1,
source_ioclass_id=ioclass_id_2,
directory=dir_2,
with_delay=False)
sync()
drop_caches(DropCachesMode.ALL)
with TestRun.step(f"Move {dir_2.full_path} to {mountpoint}."):
dir_2.move(destination=mountpoint)
with TestRun.step("Read files with reclassification check."):
read_files_with_reclassification_check(cache,
target_ioclass_id=0,
source_ioclass_id=ioclass_id_1,
directory=dir_2,
with_delay=True)
with TestRun.step(f"Remove {classified_dir_path_2}."):
fs_utils.remove(path=classified_dir_path_2, force=True, recursive=True)
sync()
drop_caches(DropCachesMode.ALL)
with TestRun.step(
f"Rename {classified_dir_path_1} to {classified_dir_path_2}."):
dir_1.move(destination=classified_dir_path_2)
with TestRun.step("Read files with reclassification check."):
read_files_with_reclassification_check(cache,
target_ioclass_id=ioclass_id_2,
source_ioclass_id=ioclass_id_1,
directory=dir_1,
with_delay=True)
with TestRun.step(
f"Rename {classified_dir_path_2} to {non_classified_dir_path}."):
dir_1.move(destination=non_classified_dir_path)
with TestRun.step("Read files with reclassification check."):
read_files_with_reclassification_check(cache,
target_ioclass_id=0,
source_ioclass_id=ioclass_id_2,
directory=dir_1,
with_delay=True)
def test_ioclass_id_as_condition(filesystem):
"""
title: IO class as a condition.
description: |
Load config in which IO class ids are used as conditions in other IO class definitions.
pass_criteria:
- No kernel bug.
- IO is classified properly as described in IO class config.
"""
base_dir_path = f"{mountpoint}/base_dir"
ioclass_file_size = Size(random.randint(25, 50), Unit.MebiByte)
ioclass_file_size_bytes = int(ioclass_file_size.get_value(Unit.Byte))
with TestRun.step("Prepare cache and core. Disable udev."):
cache, core = prepare()
Udev.disable()
with TestRun.step("Create and load IO class config file."):
# directory condition
ioclass_config.add_ioclass(
ioclass_id=1,
eviction_priority=1,
allocation="1.00",
rule=f"directory:{base_dir_path}",
ioclass_config_path=ioclass_config_path,
)
# file size condition
ioclass_config.add_ioclass(
ioclass_id=2,
eviction_priority=1,
allocation="1.00",
rule=f"file_size:eq:{ioclass_file_size_bytes}",
ioclass_config_path=ioclass_config_path,
)
# direct condition
ioclass_config.add_ioclass(
ioclass_id=3,
eviction_priority=1,
allocation="1.00",
rule="direct",
ioclass_config_path=ioclass_config_path,
)
# IO class 1 OR 2 condition
ioclass_config.add_ioclass(
ioclass_id=4,
eviction_priority=1,
allocation="1.00",
rule="io_class:1|io_class:2",
ioclass_config_path=ioclass_config_path,
)
# IO class 4 AND file size condition (same as IO class 2)
ioclass_config.add_ioclass(
ioclass_id=5,
eviction_priority=1,
allocation="1.00",
rule=f"io_class:4&file_size:eq:{ioclass_file_size_bytes}",
ioclass_config_path=ioclass_config_path,
)
# IO class 3 condition
ioclass_config.add_ioclass(
ioclass_id=6,
eviction_priority=1,
allocation="1.00",
rule="io_class:3",
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
with TestRun.step(f"Prepare {filesystem.name} filesystem "
f"and mount {core.path} at {mountpoint}."):
core.create_filesystem(filesystem)
core.mount(mountpoint)
fs_utils.create_directory(base_dir_path)
sync()
with TestRun.step("Run IO fulfilling IO class 1 condition (and not IO class 2) and check if "
"it is classified properly."):
# Should be classified as IO class 4
base_occupancy = cache.get_io_class_statistics(io_class_id=4).usage_stats.occupancy
non_ioclass_file_size = Size(random.randrange(1, 25), Unit.MebiByte)
(Fio().create_command()
.io_engine(IoEngine.libaio)
.size(non_ioclass_file_size)
.read_write(ReadWrite.write)
.target(f"{base_dir_path}/test_file_1")
.run())
sync()
new_occupancy = cache.get_io_class_statistics(io_class_id=4).usage_stats.occupancy
if new_occupancy != base_occupancy + non_ioclass_file_size:
TestRun.fail("Writes were not properly cached!\n"
f"Expected: {base_occupancy + non_ioclass_file_size}, "
f"actual: {new_occupancy}")
with TestRun.step("Run IO fulfilling IO class 2 condition (and not IO class 1) and check if "
"it is classified properly."):
# Should be classified as IO class 5
base_occupancy = cache.get_io_class_statistics(io_class_id=5).usage_stats.occupancy
(Fio().create_command()
.io_engine(IoEngine.libaio)
.size(ioclass_file_size)
.read_write(ReadWrite.write)
.target(f"{mountpoint}/test_file_2")
.run())
sync()
new_occupancy = cache.get_io_class_statistics(io_class_id=5).usage_stats.occupancy
if new_occupancy != base_occupancy + ioclass_file_size:
TestRun.fail("Writes were not properly cached!\n"
f"Expected: {base_occupancy + ioclass_file_size}, actual: {new_occupancy}")
with TestRun.step("Run IO fulfilling IO class 1 and 2 conditions and check if "
"it is classified properly."):
# Should be classified as IO class 5
base_occupancy = new_occupancy
(Fio().create_command()
.io_engine(IoEngine.libaio)
.size(ioclass_file_size)
.read_write(ReadWrite.write)
.target(f"{base_dir_path}/test_file_3")
.run())
sync()
new_occupancy = cache.get_io_class_statistics(io_class_id=5).usage_stats.occupancy
if new_occupancy != base_occupancy + ioclass_file_size:
TestRun.fail("Writes were not properly cached!\n"
f"Expected: {base_occupancy + ioclass_file_size}, actual: {new_occupancy}")
with TestRun.step("Run direct IO fulfilling IO class 1 and 2 conditions and check if "
"it is classified properly."):
# Should be classified as IO class 6
base_occupancy = cache.get_io_class_statistics(io_class_id=6).usage_stats.occupancy
(Fio().create_command()
.io_engine(IoEngine.libaio)
.size(ioclass_file_size)
.read_write(ReadWrite.write)
.target(f"{base_dir_path}/test_file_3")
.direct()
.run())
sync()
new_occupancy = cache.get_io_class_statistics(io_class_id=6).usage_stats.occupancy
if new_occupancy != base_occupancy + ioclass_file_size:
TestRun.fail("Writes were not properly cached!\n"
f"Expected: {base_occupancy + ioclass_file_size}, actual: {new_occupancy}")
def test_ioclass_occupancy_directory_write(io_size_multiplication, cache_mode,
cache_line_size):
"""
title: Test for max occupancy set for ioclass based on directory
description: |
Create ioclass for 3 different directories, each with different
max cache occupancy configured. Run IO against each directory and see
if occupancy limit is repected.
pass_criteria:
- Max occupancy is set correctly for each ioclass
- Each ioclass does not exceed max occupancy
"""
with TestRun.step("Prepare CAS device"):
cache, core = prepare(cache_mode=cache_mode,
cache_line_size=cache_line_size)
cache_size = cache.get_statistics().config_stats.cache_size
with TestRun.step("Disable udev"):
Udev.disable()
with TestRun.step(
f"Prepare filesystem and mount {core.path} at {mountpoint}"):
filesystem = Filesystem.xfs
core.create_filesystem(filesystem)
core.mount(mountpoint)
sync()
with TestRun.step("Prepare test dirs"):
IoclassConfig = namedtuple("IoclassConfig",
"id eviction_prio max_occupancy dir_path")
io_classes = [
IoclassConfig(1, 3, 0.10, f"{mountpoint}/A"),
IoclassConfig(2, 4, 0.20, f"{mountpoint}/B"),
IoclassConfig(3, 5, 0.30, f"{mountpoint}/C"),
]
for io_class in io_classes:
fs_utils.create_directory(io_class.dir_path, parents=True)
with TestRun.step("Remove old ioclass config"):
ioclass_config.remove_ioclass_config()
ioclass_config.create_ioclass_config(False)
with TestRun.step("Add default ioclasses"):
ioclass_config.add_ioclass(*str(IoClass.default(
allocation="0.00")).split(","))
with TestRun.step("Add ioclasses for all dirs"):
for io_class in io_classes:
ioclass_config.add_ioclass(
io_class.id,
f"directory:{io_class.dir_path}&done",
io_class.eviction_prio,
f"{io_class.max_occupancy:0.2f}",
)
casadm.load_io_classes(cache_id=cache.cache_id,
file=ioclass_config_path)
with TestRun.step("Reset cache stats"):
cache.purge_cache()
cache.reset_counters()
with TestRun.step("Check initial occupancy"):
for io_class in io_classes:
occupancy = get_io_class_occupancy(cache, io_class.id)
if occupancy.get_value() != 0:
TestRun.LOGGER.error(
f"Incorrect inital occupancy for ioclass id: {io_class.id}."
f" Expected 0, got: {occupancy}")
with TestRun.step(
f"To each directory perform IO with size of {io_size_multiplication} max io_class occupancy"
):
for io_class in io_classes:
original_occupacies = {}
tmp_io_class_list = [i for i in io_classes if i != io_class]
for i in tmp_io_class_list:
original_occupacies[i.id] = get_io_class_occupancy(cache, i.id)
run_io_dir(
f"{io_class.dir_path}/tmp_file",
int((io_class.max_occupancy * cache_size) / Unit.Blocks4096 *
io_size_multiplication),
)
actuall_occupancy = get_io_class_occupancy(cache, io_class.id)
io_size = io_class.max_occupancy * cache_size
if io_size_multiplication < 1:
io_size *= io_size_multiplication
io_size.set_unit(Unit.Blocks4096)
if not isclose(io_size.value, actuall_occupancy.value,
rel_tol=0.1):
TestRun.LOGGER.error(
f"Occupancy for ioclass {i.id} should be equal {io_size} "
f"but is {actuall_occupancy} instead!")
for i in tmp_io_class_list:
actuall_occupancy = get_io_class_occupancy(cache, i.id)
if original_occupacies[i.id] != actuall_occupancy:
TestRun.LOGGER.error(
f"Occupancy for ioclass {i.id} should not change "
f"during IO to ioclass {io_class.id}. Original value: "
f"{original_occupacies[i.id]}, actuall: {actuall_occupancy}"
)
with TestRun.step(
"Check if none of ioclasses did not exceed specified occupancy"):
for io_class in io_classes:
actuall_occupancy = get_io_class_occupancy(cache, io_class.id)
occupancy_limit = ((io_class.max_occupancy * cache_size).align_up(
Unit.Blocks4096.get_value()).set_unit(Unit.Blocks4096))
# Divergency may be casued be rounding max occupancy
if actuall_occupancy > occupancy_limit + Size(
100, Unit.Blocks4096):
TestRun.LOGGER.error(
f"Occupancy for ioclass id exceeded: {io_class.id}. "
f"Limit: {occupancy_limit}, actuall: {actuall_occupancy}")
def test_ioclass_occupancy_sum_cache():
"""
title: Test for ioclasses occupancy sum
description: |
Create ioclass for 3 different directories, each with different
max cache occupancy configured. Trigger IO to each ioclass and check
if sum of their Usage stats is equal to cache Usage stats.
pass_criteria:
- Max occupancy is set correctly for each ioclass
- Sum of ioclassess stats is equal to cache stats
"""
with TestRun.step("Prepare CAS device"):
cache, core = prepare()
cache_size = cache.get_statistics().config_stats.cache_size
with TestRun.step("Disable udev"):
Udev.disable()
with TestRun.step(
f"Prepare filesystem and mount {core.path} at {mountpoint}"):
filesystem = Filesystem.xfs
core.create_filesystem(filesystem)
core.mount(mountpoint)
sync()
with TestRun.step("Prepare test dirs"):
default_ioclass_id = 0
IoclassConfig = namedtuple("IoclassConfig",
"id eviction_prio max_occupancy dir_path")
io_classes = [
IoclassConfig(1, 3, 0.10, f"{mountpoint}/A"),
IoclassConfig(2, 4, 0.20, f"{mountpoint}/B"),
IoclassConfig(3, 5, 0.30, f"{mountpoint}/C"),
]
for io_class in io_classes:
fs_utils.create_directory(io_class.dir_path, parents=True)
with TestRun.step("Remove old ioclass config"):
ioclass_config.remove_ioclass_config()
ioclass_config.create_ioclass_config(False)
with TestRun.step("Add default ioclasses"):
ioclass_config.add_ioclass(*str(IoClass.default(
allocation="0.00")).split(","))
with TestRun.step("Add ioclasses for all dirs"):
for io_class in io_classes:
ioclass_config.add_ioclass(
io_class.id,
f"directory:{io_class.dir_path}&done",
io_class.eviction_prio,
f"{io_class.max_occupancy:0.2f}",
)
casadm.load_io_classes(cache_id=cache.cache_id,
file=ioclass_config_path)
with TestRun.step("Purge cache"):
cache.purge_cache()
with TestRun.step("Verify stats before IO"):
usage_stats_sum = IoClassUsageStats(Size(0), Size(0), Size(0))
for i in io_classes:
usage_stats_sum += get_io_class_usage(cache, i.id)
usage_stats_sum += get_io_class_usage(cache, default_ioclass_id)
cache_stats = cache.get_statistics().usage_stats
cache_stats.free = Size(0)
if (cache_stats.occupancy != usage_stats_sum.occupancy
or cache_stats.clean != usage_stats_sum.clean
or cache_stats.dirty != usage_stats_sum.dirty):
TestRun.LOGGER.error(
"Initial cache usage stats doesn't match sum of ioclasses stats\n"
f"cache stats: {cache_stats}, sumed up stats {usage_stats_sum}\n"
f"particular stats {[get_io_class_usage(cache, i.id) for i in io_classes]}"
)
with TestRun.step(f"Trigger IO to each directory"):
for io_class in io_classes:
run_io_dir(
f"{io_class.dir_path}/tmp_file",
int((io_class.max_occupancy * cache_size) / Unit.Blocks4096),
)
with TestRun.step("Verify stats after IO"):
usage_stats_sum = IoClassUsageStats(Size(0), Size(0), Size(0))
for i in io_classes:
usage_stats_sum += get_io_class_usage(cache, i.id)
usage_stats_sum += get_io_class_usage(cache, default_ioclass_id)
cache_stats = cache.get_statistics().usage_stats
cache_stats.free = Size(0)
if (cache_stats.occupancy != usage_stats_sum.occupancy
or cache_stats.clean != usage_stats_sum.clean
or cache_stats.dirty != usage_stats_sum.dirty):
TestRun.LOGGER.error(
"Cache usage stats doesn't match sum of ioclasses stats\n"
f"cache stats: {cache_stats}, sumed up stats {usage_stats_sum}\n"
f"particular stats {[get_io_class_usage(cache, i.id) for i in io_classes]}"
)
def test_trim_eviction(cache_mode, cache_line_size, filesystem, cleaning):
"""
title: Test verifying if trim requests do not cause eviction on CAS device.
description: |
When trim requests enabled and files are being added and removed from CAS device,
there is no eviction (no reads from cache).
pass_criteria:
- Reads from cache device are the same before and after removing test file.
"""
mount_point = "/mnt"
test_file_path = os.path.join(mount_point, "test_file")
with TestRun.step("Prepare devices."):
cache_disk = TestRun.disks['cache']
cache_disk.create_partitions([Size(1, Unit.GibiByte)])
cache_dev = cache_disk.partitions[0]
core_disk = TestRun.disks['core']
core_disk.create_partitions([Size(1, Unit.GibiByte)])
core_dev = core_disk.partitions[0]
cache_block_size = disk_utils.get_block_size(cache_disk)
with TestRun.step("Start cache on device supporting trim and add core."):
cache = casadm.start_cache(cache_dev,
cache_mode,
cache_line_size,
force=True)
cache.set_cleaning_policy(cleaning)
Udev.disable()
core = cache.add_core(core_dev)
with TestRun.step("Create filesystem on CAS device and mount it."):
core.create_filesystem(filesystem)
core.mount(mount_point, ["discard"])
with TestRun.step("Create ioclass config."):
ioclass_config.create_ioclass_config()
ioclass_config.add_ioclass(ioclass_id=1,
eviction_priority=1,
allocation="0.00",
rule=f"metadata")
casadm.load_io_classes(cache_id=cache.cache_id,
file=ioclass_config.default_config_file_path)
with TestRun.step("Create random file using ddrescue."):
test_file = fs_utils.create_random_test_file(test_file_path,
core_dev.size * 0.9)
create_file_with_ddrescue(core_dev, test_file)
os_utils.sync()
os_utils.drop_caches()
with TestRun.step("Remove file and create a new one."):
cache_iostats_before = cache_dev.get_io_stats()
data_reads_before = cache.get_io_class_statistics(
io_class_id=0).block_stats.cache.reads
metadata_reads_before = cache.get_io_class_statistics(
io_class_id=1).block_stats.cache.reads
test_file.remove()
os_utils.sync()
os_utils.drop_caches()
create_file_with_ddrescue(core_dev, test_file)
with TestRun.step(
"Check using iostat that reads from cache did not occur."):
cache_iostats_after = cache_dev.get_io_stats()
data_reads_after = cache.get_io_class_statistics(
io_class_id=0).block_stats.cache.reads
metadata_reads_after = cache.get_io_class_statistics(
io_class_id=1).block_stats.cache.reads
reads_before = cache_iostats_before.sectors_read
reads_after = cache_iostats_after.sectors_read
metadata_reads_diff = metadata_reads_after - metadata_reads_before
data_reads_diff = data_reads_after - data_reads_before
iostat_diff = (reads_after - reads_before) * cache_block_size
if iostat_diff > int(metadata_reads_diff) or int(data_reads_diff) > 0:
TestRun.fail(
f"Number of reads from cache before and after removing test file "
f"differs. Sectors read before: {reads_before}, sectors read after: {reads_after}."
f"Data read from cache before {data_reads_before}, after {data_reads_after}."
f"Metadata read from cache before {metadata_reads_before}, "
f"after {metadata_reads_after}.")
else:
TestRun.LOGGER.info(
"Number of reads from cache before and after removing test file is the same."
)
def test_ioclass_request_size():
cache, core = prepare()
ioclass_id = 1
iterations = 100
ioclass_config.add_ioclass(
ioclass_id=ioclass_id,
eviction_priority=1,
allocation=True,
rule=f"request_size:ge:8192&request_size:le:16384&done",
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
Udev.disable()
# Check if requests with appropriate size are cached
TestRun.LOGGER.info(
f"Check if requests with size within defined range are cached"
)
cached_req_sizes = [Size(2, Unit.Blocks4096), Size(4, Unit.Blocks4096)]
for i in range(iterations):
cache.flush_cache()
req_size = random.choice(cached_req_sizes)
dd = (
Dd()
.input("/dev/zero")
.output(core.system_path)
.count(1)
.block_size(req_size)
.oflag("direct")
)
dd.run()
dirty = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.dirty
if dirty.get_value(Unit.Blocks4096) != req_size.value / Unit.Blocks4096.value:
TestRun.fail("Incorrect number of dirty blocks!")
cache.flush_cache()
# Check if requests with inappropriate size are not cached
TestRun.LOGGER.info(
f"Check if requests with size outside defined range are not cached"
)
not_cached_req_sizes = [
Size(1, Unit.Blocks4096),
Size(8, Unit.Blocks4096),
Size(16, Unit.Blocks4096),
]
for i in range(iterations):
req_size = random.choice(not_cached_req_sizes)
dd = (
Dd()
.input("/dev/zero")
.output(core.system_path)
.count(1)
.block_size(req_size)
.oflag("direct")
)
dd.run()
dirty = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.dirty
if dirty.get_value(Unit.Blocks4096) != 0:
TestRun.fail("Dirty data present!")
def test_stat_max_cache():
"""
title: CAS statistics values for maximum cache devices.
description: |
Check CAS ability to display correct values in statistics
for 16 cache devices per cache mode.
pass_criteria:
- Core's statistics matches cache's statistics.
"""
caches_per_cache_mode = 16
cores_per_cache = 1
caches_count = caches_per_cache_mode * len(CacheMode)
with TestRun.step(f"Create {caches_count} cache and "
f"{cores_per_cache * caches_count} core partitions"):
cache_dev = TestRun.disks["cache"]
cache_parts = [cache_size] * caches_count
cache_dev.create_partitions(cache_parts)
core_dev = TestRun.disks["core"]
core_parts = [core_size] * cores_per_cache * caches_count
core_dev.create_partitions(core_parts)
Udev.disable()
with TestRun.step(
f"Start {caches_count} caches ({caches_per_cache_mode} for "
f"every cache mode) and add {cores_per_cache} core(s) per cache"):
caches = []
cores = [[] for i in range(caches_count)]
for i, cache_mode in enumerate(CacheMode):
for j in range(caches_per_cache_mode):
cache_partition_number = i * caches_per_cache_mode + j
caches.append(
casadm.start_cache(
cache_dev.partitions[cache_partition_number],
cache_mode=cache_mode,
force=True))
for i in range(caches_count):
caches[i].set_cleaning_policy(CleaningPolicy.nop)
for j in range(cores_per_cache):
core_partition_number = i * cores_per_cache + j
cores[i].append(caches[i].add_core(
core_dev.partitions[core_partition_number]))
with TestRun.step("Run 'fio'"):
fio = fio_prepare()
for i in range(caches_count):
for j in range(cores_per_cache):
fio.add_job().target(cores[i][j].path)
fio.run()
sleep(3)
with TestRun.step("Check if cache's statistics matches core's statistics"):
for i in range(caches_count):
cache_stats = caches[i].get_statistics_flat(
stat_filter=stat_filter)
cores_stats = [
cores[i][j].get_statistics_flat(stat_filter=stat_filter)
for j in range(cores_per_cache)
]
fail_message = f"For cache ID {caches[i].cache_id} "
stats_compare(cache_stats, cores_stats, cores_per_cache,
fail_message)
def test_ioclass_direct(filesystem):
"""
title: Direct IO classification.
description: Check if direct requests are properly cached.
pass_criteria:
- No kernel bug.
- Data from direct IO should be cached.
- Data from buffered IO should not be cached and if performed to/from already cached data
should cause reclassification to unclassified IO class.
"""
ioclass_id = 1
io_size = Size(random.randint(1000, 2000), Unit.Blocks4096)
with TestRun.step("Prepare cache and core. Disable udev."):
cache, core = prepare()
Udev.disable()
with TestRun.step("Create and load IO class config."):
# direct IO class
ioclass_config.add_ioclass(
ioclass_id=ioclass_id,
eviction_priority=1,
allocation="1.00",
rule="direct",
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
with TestRun.step("Prepare fio command."):
fio = Fio().create_command() \
.io_engine(IoEngine.libaio) \
.size(io_size).offset(io_size) \
.read_write(ReadWrite.write) \
.target(f"{mountpoint}/tmp_file" if filesystem else core.path)
with TestRun.step("Prepare filesystem."):
if filesystem:
TestRun.LOGGER.info(
f"Preparing {filesystem.name} filesystem and mounting {core.path} at"
f" {mountpoint}"
)
core.create_filesystem(filesystem)
core.mount(mountpoint)
sync()
else:
TestRun.LOGGER.info("Testing on raw exported object.")
with TestRun.step(f"Run buffered writes to {'file' if filesystem else 'device'}"):
base_occupancy = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy
fio.run()
sync()
with TestRun.step("Check if buffered writes are not cached."):
new_occupancy = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy
if new_occupancy != base_occupancy:
TestRun.fail("Buffered writes were cached!\n"
f"Expected: {base_occupancy}, actual: {new_occupancy}")
with TestRun.step(f"Run direct writes to {'file' if filesystem else 'device'}"):
fio.direct()
fio.run()
sync()
with TestRun.step("Check if direct writes are cached."):
new_occupancy = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy
if new_occupancy != base_occupancy + io_size:
TestRun.fail("Wrong number of direct writes was cached!\n"
f"Expected: {base_occupancy + io_size}, actual: {new_occupancy}")
with TestRun.step(f"Run buffered reads from {'file' if filesystem else 'device'}"):
fio.remove_param("readwrite").remove_param("direct")
fio.read_write(ReadWrite.read)
fio.run()
sync()
with TestRun.step("Check if buffered reads caused reclassification."):
new_occupancy = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy
if new_occupancy != base_occupancy:
TestRun.fail("Buffered reads did not cause reclassification!"
f"Expected occupancy: {base_occupancy}, actual: {new_occupancy}")
with TestRun.step(f"Run direct reads from {'file' if filesystem else 'device'}"):
fio.direct()
fio.run()
sync()
with TestRun.step("Check if direct reads are cached."):
new_occupancy = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy
if new_occupancy != base_occupancy + io_size:
TestRun.fail("Wrong number of direct reads was cached!\n"
f"Expected: {base_occupancy + io_size}, actual: {new_occupancy}")
def test_one_core_release(cache_mode):
"""
title: Test if OpenCAS dynamically allocates space according to core devices needs.
description: |
When one or more core devices are unused in a single cache instance all blocks
previously occupied should be available to other core devices.
Test is without pass through mode.
pass_criteria:
- No system crash.
- The remaining core is able to use cache.
- OpenCAS frees blocks occupied by unused core and allocates it to the remaining core.
"""
with TestRun.step("Prepare two cache and one core devices."):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions([Size(512, Unit.MebiByte)])
cache_part = cache_dev.partitions[0]
core_dev = TestRun.disks['core']
core_dev.create_partitions([Size(1, Unit.GibiByte)] * 2)
core_part1 = core_dev.partitions[0]
core_part2 = core_dev.partitions[1]
Udev.disable()
with TestRun.step("Start cache"):
cache = casadm.start_cache(cache_part, cache_mode, force=True)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(
f"Expected caches count: 1; Actual caches count: {caches_count}."
)
with TestRun.step("Add both core devices to cache."):
core1 = cache.add_core(core_part1)
core2 = cache.add_core(core_part2)
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 2:
TestRun.fail(
f"Expected cores count: 2; Actual cores count: {cores_count}.")
with TestRun.step("Change sequential cutoff policy to 'never'."):
cache.set_seq_cutoff_policy(SeqCutOffPolicy.never)
with TestRun.step("Fill cache with pages from the first core."):
dd_builder(cache_mode, core1, cache.size).run()
core1_occupied_blocks_before = core1.get_occupancy()
with TestRun.step("Check if the remaining core is able to use cache."):
dd_builder(cache_mode, core2, Size(100, Unit.MebiByte)).run()
core1_occupied_blocks_after = core1.get_occupancy()
with TestRun.step(
"Check if occupancy from the first core is removed from cache."):
# The first core's occupancy should be lower than cache's occupancy
# by the value of the remaining core's occupancy because cache
# should reallocate blocks from unused core to used core.
if core1_occupied_blocks_after >= core1_occupied_blocks_before \
or cache.get_occupancy() <= core1_occupied_blocks_after \
or not float(core2.get_occupancy().get_value()) > 0:
TestRun.LOGGER.error(
"Blocks previously occupied by the first core aren't released."
)
with TestRun.step("Stop cache."):
casadm.stop_all_caches()
def test_ioclass_metadata(filesystem):
"""
title: Metadata IO classification.
description: |
Determine if every operation on files that cause metadata update results in increased
writes to cached metadata.
pass_criteria:
- No kernel bug.
- Metadata is classified properly.
"""
# Exact values may not be tested as each file system has different metadata structure.
test_dir_path = f"{mountpoint}/test_dir"
with TestRun.step("Prepare cache and core. Disable udev."):
cache, core = prepare()
Udev.disable()
with TestRun.step("Prepare and load IO class config file."):
ioclass_id = random.randint(1, ioclass_config.MAX_IO_CLASS_ID)
# metadata IO class
ioclass_config.add_ioclass(
ioclass_id=ioclass_id,
eviction_priority=1,
allocation="1.00",
rule="metadata&done",
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
with TestRun.step(f"Prepare {filesystem.name} filesystem and mount {core.path} "
f"at {mountpoint}."):
core.create_filesystem(filesystem)
core.mount(mountpoint)
sync()
with TestRun.step("Create 20 test files."):
requests_to_metadata_before = cache.get_io_class_statistics(
io_class_id=ioclass_id).request_stats.write
files = []
for i in range(1, 21):
file_path = f"{mountpoint}/test_file_{i}"
dd = (
Dd().input("/dev/urandom")
.output(file_path)
.count(random.randint(5, 50))
.block_size(Size(1, Unit.MebiByte))
.oflag("sync")
)
dd.run()
files.append(File(file_path))
with TestRun.step("Check requests to metadata."):
requests_to_metadata_after = cache.get_io_class_statistics(
io_class_id=ioclass_id).request_stats.write
if requests_to_metadata_after == requests_to_metadata_before:
TestRun.fail("No requests to metadata while creating files!")
with TestRun.step("Rename all test files."):
requests_to_metadata_before = requests_to_metadata_after
for file in files:
file.move(f"{file.full_path}_renamed")
sync()
with TestRun.step("Check requests to metadata."):
requests_to_metadata_after = cache.get_io_class_statistics(
io_class_id=ioclass_id).request_stats.write
if requests_to_metadata_after == requests_to_metadata_before:
TestRun.fail("No requests to metadata while renaming files!")
with TestRun.step(f"Create directory {test_dir_path}."):
requests_to_metadata_before = requests_to_metadata_after
fs_utils.create_directory(path=test_dir_path)
TestRun.LOGGER.info(f"Moving test files into {test_dir_path}")
for file in files:
file.move(test_dir_path)
sync()
with TestRun.step("Check requests to metadata."):
requests_to_metadata_after = cache.get_io_class_statistics(
io_class_id=ioclass_id).request_stats.write
if requests_to_metadata_after == requests_to_metadata_before:
TestRun.fail("No requests to metadata while moving files!")
with TestRun.step(f"Remove {test_dir_path}."):
fs_utils.remove(path=test_dir_path, force=True, recursive=True)
with TestRun.step("Check requests to metadata."):
requests_to_metadata_after = cache.get_io_class_statistics(
io_class_id=ioclass_id).request_stats.write
if requests_to_metadata_after == requests_to_metadata_before:
TestRun.fail("No requests to metadata while deleting directory with files!")
def test_one_core_fail(cache_mode):
"""
title: Test if OpenCAS correctly handles failure of one of multiple core devices.
description: |
When one core device fails in a single cache instance all blocks previously occupied
should be available to other core devices.
Test is without pass through mode.
pass_criteria:
- No system crash.
- Second core is able to use OpenCAS.
"""
with TestRun.step("Prepare one cache and two core devices."):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions([Size(1, Unit.GibiByte)] * 2)
cache_part = cache_dev.partitions[0]
core_dev1 = TestRun.disks['core1'] # This device would be unplugged.
core_dev1.create_partitions([Size(2, Unit.GibiByte)])
core_part1 = core_dev1.partitions[0]
core_dev2 = TestRun.disks['core2']
core_dev2.create_partitions([Size(2, Unit.GibiByte)])
core_part2 = core_dev2.partitions[0]
Udev.disable()
with TestRun.step("Start cache"):
cache = casadm.start_cache(cache_part, cache_mode, force=True)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(
f"Expected caches count: 1; Actual caches count: {caches_count}."
)
with TestRun.step("Add both core devices to cache."):
core1 = cache.add_core(core_part1)
core2 = cache.add_core(core_part2)
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 2:
TestRun.fail(
f"Expected cores count: 2; Actual cores count: {cores_count}.")
with TestRun.step("Change sequential cutoff policy."):
cache.set_seq_cutoff_policy(SeqCutOffPolicy.never)
with TestRun.step("Fill cache with pages from the first core."):
dd_builder(cache_mode, core1, cache.size).run()
cache_occupied_blocks_before = cache.get_occupancy()
with TestRun.step("Unplug the first core device."):
core_dev1.unplug()
with TestRun.step("Check if core device is really out of cache."):
output = str(casadm.list_caches().stdout.splitlines())
if core_part1.path in output:
TestRun.fail("The first core device should be unplugged!")
with TestRun.step("Check if the remaining core is able to use cache."):
dd_builder(cache_mode, core2, Size(100, Unit.MebiByte)).run()
if not float(core2.get_occupancy().get_value()) > 0:
TestRun.LOGGER.error(
"The remaining core is not able to use cache.")
with TestRun.step(
"Check if occupancy from the first core is removed from cache."):
# Cache occupancy cannot be lower than before the first core fails and after that
# should be equal to the sum of occupancy of the first and the remaining core
cache_occupied_blocks_after = cache.get_occupancy()
if cache_occupied_blocks_before > cache_occupied_blocks_after \
or cache_occupied_blocks_after != core2.get_occupancy() + core1.get_occupancy():
TestRun.fail("Blocks previously occupied by the first core "
"aren't released after this core failure.")
with TestRun.step("Stop cache."):
casadm.stop_all_caches()
with TestRun.step("Plug back the first core."):
core_dev1.plug()
def test_ioclass_effective_ioclass(filesystem):
"""
title: Effective IO class with multiple non-exclusive conditions
description: |
Test CAS ability to properly classify IO fulfilling multiple conditions based on
IO class ids and presence of '&done' annotation in IO class rules
pass_criteria:
- In every iteration first IO is classified to the last in order IO class
- In every iteration second IO is classified to the IO class with '&done' annotation
"""
with TestRun.LOGGER.step(f"Test prepare"):
cache, core = prepare()
Udev.disable()
file_size = Size(10, Unit.Blocks4096)
file_size_bytes = int(file_size.get_value(Unit.Byte))
test_dir = f"{mountpoint}/test"
rules = [
"direct", # rule contradicting other rules
f"directory:{test_dir}",
f"file_size:le:{2 * file_size_bytes}",
f"file_size:ge:{file_size_bytes // 2}"
]
with TestRun.LOGGER.step(
f"Preparing {filesystem.name} filesystem "
f"and mounting {core.system_path} at {mountpoint}"):
core.create_filesystem(filesystem)
core.mount(mountpoint)
fs_utils.create_directory(test_dir)
sync()
for i, permutation in TestRun.iteration(
enumerate(permutations(range(1, 5)), start=1)):
with TestRun.LOGGER.step(
"Load IO classes in order specified by permutation"):
load_io_classes_in_permutation_order(rules, permutation, cache)
io_class_id = 3 if rules[permutation.index(4)] == "direct" else 4
with TestRun.LOGGER.step(
"Perform IO fulfilling the non-contradicting conditions"):
base_occupancy = cache.get_io_class_statistics(
io_class_id=io_class_id).usage_stats.occupancy
fio = (Fio().create_command().io_engine(
IoEngine.libaio).size(file_size).read_write(
ReadWrite.write).target(f"{test_dir}/test_file{i}"))
fio.run()
sync()
with TestRun.LOGGER.step("Check if IO was properly classified "
"(to the last non-contradicting IO class)"):
new_occupancy = cache.get_io_class_statistics(
io_class_id=io_class_id).usage_stats.occupancy
if new_occupancy != base_occupancy + file_size:
TestRun.LOGGER.error(
"Wrong IO classification!\n"
f"Expected: {base_occupancy + file_size}, "
f"actual: {new_occupancy}")
with TestRun.LOGGER.step(
"Add '&done' to the second in order non-contradicting condition"
):
io_class_id = add_done_to_second_non_exclusive_condition(
rules, permutation, cache)
with TestRun.LOGGER.step("Repeat IO"):
base_occupancy = cache.get_io_class_statistics(
io_class_id=io_class_id).usage_stats.occupancy
fio.run()
sync()
with TestRun.LOGGER.step("Check if IO was properly classified "
"(to the IO class with '&done' annotation)"):
new_occupancy = cache.get_io_class_statistics(
io_class_id=io_class_id).usage_stats.occupancy
if new_occupancy != base_occupancy + file_size:
TestRun.LOGGER.error(
"Wrong IO classification!\n"
f"Expected: {base_occupancy + file_size}, "
f"actual: {new_occupancy}")
def test_one_core_remove(cache_mode):
"""
title: Test if OpenCAS correctly handles removal of one of multiple core devices.
description: |
When one core device is removed from a cache instance all blocks previously occupied
by the data from that core device should be removed. That means that the number of free
cache blocks should increase by the number of removed blocks.
Test is without pass through mode.
pass_criteria:
- No system crash.
- The remaining core is able to use cache.
- Removing core frees cache blocks occupied by this core.
"""
with TestRun.step("Prepare one device for cache and two for core."):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions([Size(512, Unit.MebiByte)])
cache_dev = cache_dev.partitions[0]
core_dev = TestRun.disks['core']
core_dev.create_partitions([Size(1, Unit.GibiByte)] * 2)
core_part1 = core_dev.partitions[0]
core_part2 = core_dev.partitions[1]
Udev.disable()
with TestRun.step("Start cache"):
cache = casadm.start_cache(cache_dev, cache_mode, force=True)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(
f"Expected caches count: 1; Actual caches count: {caches_count}."
)
with TestRun.step("Add both core devices to cache."):
core1 = cache.add_core(core_part1)
core2 = cache.add_core(core_part2)
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 2:
TestRun.fail(
f"Expected cores count: 2; Actual cores count: {cores_count}.")
with TestRun.step("Fill cache with pages from the first core."):
dd_builder(cache_mode, core1, cache.size).run()
core1_occupied_blocks = core1.get_occupancy()
occupied_blocks_before = cache.get_occupancy()
with TestRun.step("Remove the first core."):
cache.remove_core(core1.core_id, True)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(
f"Expected caches count: 1; Actual caches count: {caches_count}."
)
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 1:
TestRun.fail(
f"Expected cores count: 1; Actual cores count: {cores_count}.")
with TestRun.step(
"Check if occupancy from the first core is removed from cache."):
# Blocks occupied by the first core should be completely released.
if cache.get_occupancy(
) != occupied_blocks_before - core1_occupied_blocks:
TestRun.LOGGER.error(
"Blocks previously occupied by the first core "
"aren't released by removing this core.")
with TestRun.step("Check if the remaining core is able to use cache."):
dd_builder(cache_mode, core2, Size(100, Unit.MebiByte)).run()
if not float(core2.get_occupancy().get_value()) > 0:
TestRun.fail("The remaining core is not able to use cache.")
with TestRun.step("Stop cache."):
casadm.stop_all_caches()
def test_ioclass_directory_depth(filesystem):
"""
Test if directory classification works properly for deeply nested directories for read and
write operations.
"""
cache, core = prepare()
Udev.disable()
TestRun.LOGGER.info(f"Preparing {filesystem.name} filesystem "
f"and mounting {core.system_path} at {mountpoint}")
core.create_filesystem(filesystem)
core.mount(mountpoint)
sync()
base_dir_path = f"{mountpoint}/base_dir"
TestRun.LOGGER.info(f"Creating the base directory: {base_dir_path}")
fs_utils.create_directory(base_dir_path)
nested_dir_path = base_dir_path
random_depth = random.randint(40, 80)
for i in range(random_depth):
nested_dir_path += f"/dir_{i}"
TestRun.LOGGER.info(f"Creating a nested directory: {nested_dir_path}")
fs_utils.create_directory(path=nested_dir_path, parents=True)
# Test classification in nested dir by reading a previously unclassified file
TestRun.LOGGER.info("Creating the first file in the nested directory")
test_file_1 = File(f"{nested_dir_path}/test_file_1")
dd = (
Dd()
.input("/dev/urandom")
.output(test_file_1.full_path)
.count(random.randint(1, 200))
.block_size(Size(1, Unit.MebiByte))
)
dd.run()
sync()
drop_caches(DropCachesMode.ALL)
test_file_1.refresh_item()
ioclass_id = random.randint(1, ioclass_config.MAX_IO_CLASS_ID)
# directory IO class
ioclass_config.add_ioclass(
ioclass_id=ioclass_id,
eviction_priority=1,
allocation=True,
rule=f"directory:{base_dir_path}",
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
base_occupancy = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy
TestRun.LOGGER.info("Reading the file in the nested directory")
dd = (
Dd()
.input(test_file_1.full_path)
.output("/dev/null")
.block_size(Size(1, Unit.MebiByte))
)
dd.run()
new_occupancy = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy
assert new_occupancy == base_occupancy + test_file_1.size, \
"Wrong occupancy after reading file!\n" \
f"Expected: {base_occupancy + test_file_1.size}, actual: {new_occupancy}"
# Test classification in nested dir by creating a file
base_occupancy = new_occupancy
TestRun.LOGGER.info("Creating the second file in the nested directory")
test_file_2 = File(f"{nested_dir_path}/test_file_2")
dd = (
Dd()
.input("/dev/urandom")
.output(test_file_2.full_path)
.count(random.randint(1, 200))
.block_size(Size(1, Unit.MebiByte))
)
dd.run()
sync()
drop_caches(DropCachesMode.ALL)
test_file_2.refresh_item()
new_occupancy = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy
assert new_occupancy == base_occupancy + test_file_2.size, \
"Wrong occupancy after creating file!\n" \
f"Expected: {base_occupancy + test_file_2.size}, actual: {new_occupancy}"
def test_ioclass_repart(cache_mode, cache_line_size,
ioclass_size_multiplicatior):
"""
title: Check whether occupancy limit is respected during repart
description: |
Create ioclass for 3 different directories, each with different max
occupancy threshold. Create 3 files classified on default ioclass.
Move files to directories created earlier and force repart by reading
theirs contents.
pass_criteria:
- Partitions are evicted in specified order
"""
with TestRun.step("Prepare CAS device"):
cache, core = prepare(cache_mode=cache_mode,
cache_line_size=cache_line_size)
cache_size = cache.get_statistics().config_stats.cache_size
with TestRun.step("Disable udev"):
Udev.disable()
with TestRun.step(
f"Prepare filesystem and mount {core.path} at {mountpoint}"):
filesystem = Filesystem.xfs
core.create_filesystem(filesystem)
core.mount(mountpoint)
sync()
with TestRun.step("Prepare test dirs"):
IoclassConfig = namedtuple("IoclassConfig",
"id eviction_prio max_occupancy dir_path")
io_classes = [
IoclassConfig(1, 3, 0.40, f"{mountpoint}/A"),
IoclassConfig(2, 4, 0.30, f"{mountpoint}/B"),
IoclassConfig(3, 5, 0.30, f"{mountpoint}/C"),
]
for io_class in io_classes:
fs_utils.create_directory(io_class.dir_path, parents=True)
with TestRun.step("Remove old ioclass config"):
ioclass_config.remove_ioclass_config()
ioclass_config.create_ioclass_config(False)
with TestRun.step("Add default ioclasses"):
ioclass_config.add_ioclass(*str(IoClass.default(
allocation="1.00")).split(","))
with TestRun.step("Add ioclasses for all dirs"):
for io_class in io_classes:
ioclass_config.add_ioclass(
io_class.id,
f"directory:{io_class.dir_path}&done",
io_class.eviction_prio,
f"{io_class.max_occupancy*ioclass_size_multiplicatior:0.2f}",
)
casadm.load_io_classes(cache_id=cache.cache_id,
file=ioclass_config_path)
with TestRun.step("Reset cache stats"):
cache.purge_cache()
cache.reset_counters()
with TestRun.step(f"Create 3 files classified in default ioclass"):
for i, io_class in enumerate(io_classes[0:3]):
run_io_dir(
f"{mountpoint}/{i}",
int((io_class.max_occupancy * cache_size) / Unit.Blocks4096))
if not isclose(
get_io_class_occupancy(
cache, ioclass_config.DEFAULT_IO_CLASS_ID).value,
cache_size.value,
rel_tol=0.1,
):
TestRun.fail(f"Failed to populte default ioclass")
with TestRun.step("Check initial occupancy"):
for io_class in io_classes:
occupancy = get_io_class_occupancy(cache, io_class.id)
if occupancy.get_value() != 0:
TestRun.LOGGER.error(
f"Incorrect inital occupancy for ioclass id: {io_class.id}."
f" Expected 0, got: {occupancy}")
with TestRun.step(
"Force repart - move files to created directories and read theirs contents"
):
for i, io_class in enumerate(io_classes):
fs_utils.move(source=f"{mountpoint}/{i}",
destination=io_class.dir_path)
run_io_dir_read(f"{io_class.dir_path}/{i}")
with TestRun.step("Check if each ioclass reached it's occupancy limit"):
for io_class in io_classes[0:3]:
actuall_occupancy = get_io_class_occupancy(cache, io_class.id)
occupancy_limit = ((io_class.max_occupancy * cache_size *
ioclass_size_multiplicatior).align_down(
Unit.Blocks4096.get_value()).set_unit(
Unit.Blocks4096))
if not isclose(actuall_occupancy.value,
occupancy_limit.value,
rel_tol=0.1):
TestRun.LOGGER.error(
f"Occupancy for ioclass {io_class.id} does not match. "
f"Limit: {occupancy_limit}, actuall: {actuall_occupancy}")
def test_ioclass_direct(prepare_and_cleanup, filesystem):
"""
Perform buffered/direct IO to/from files or raw block device.
Data from buffered IO should be cached.
Data from buffered IO should not be cached and if performed to/from already cached data
should cause reclassification to unclassified IO class.
"""
cache, core = prepare()
Udev.disable()
ioclass_id = 1
io_size = Size(random.randint(1000, 2000), Unit.Blocks4096)
# direct IO class
ioclass_config.add_ioclass(
ioclass_id=ioclass_id,
eviction_priority=1,
allocation=True,
rule="direct",
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
fio = (Fio().create_command().io_engine(
IoEngine.libaio).size(io_size).offset(io_size).read_write(
ReadWrite.write).target(
f"{mountpoint}/tmp_file" if filesystem else core.system_path))
if filesystem:
TestRun.LOGGER.info(
f"Preparing {filesystem.name} filesystem and mounting {core.system_path} at"
f" {mountpoint}")
core.create_filesystem(filesystem)
core.mount(mountpoint)
sync()
else:
TestRun.LOGGER.info("Testing on raw exported object")
base_occupancy = cache.get_cache_statistics(
io_class_id=ioclass_id)["occupancy"]
TestRun.LOGGER.info(
f"Buffered writes to {'file' if filesystem else 'device'}")
fio.run()
sync()
new_occupancy = cache.get_cache_statistics(
io_class_id=ioclass_id)["occupancy"]
assert new_occupancy == base_occupancy, \
"Buffered writes were cached!\n" \
f"Expected: {base_occupancy}, actual: {new_occupancy}"
TestRun.LOGGER.info(
f"Direct writes to {'file' if filesystem else 'device'}")
fio.direct()
fio.run()
sync()
new_occupancy = cache.get_cache_statistics(
io_class_id=ioclass_id)["occupancy"]
assert new_occupancy == base_occupancy + io_size, \
"Wrong number of direct writes was cached!\n" \
f"Expected: {base_occupancy + io_size}, actual: {new_occupancy}"
TestRun.LOGGER.info(
f"Buffered reads from {'file' if filesystem else 'device'}")
fio.remove_param("readwrite").remove_param("direct")
fio.read_write(ReadWrite.read)
fio.run()
sync()
new_occupancy = cache.get_cache_statistics(
io_class_id=ioclass_id)["occupancy"]
assert new_occupancy == base_occupancy, \
"Buffered reads did not cause reclassification!" \
f"Expected occupancy: {base_occupancy}, actual: {new_occupancy}"
TestRun.LOGGER.info(
f"Direct reads from {'file' if filesystem else 'device'}")
fio.direct()
fio.run()
sync()
new_occupancy = cache.get_cache_statistics(
io_class_id=ioclass_id)["occupancy"]
assert new_occupancy == base_occupancy + io_size, \
"Wrong number of direct reads was cached!\n" \
f"Expected: {base_occupancy + io_size}, actual: {new_occupancy}"
def test_write_fetch_partial_misses(cache_mode, cache_line_size):
"""
title: No caching of partial write miss operations
description: |
Validate CAS ability to not cache entire cache line size for
partial write miss operations
pass_criteria:
- Appropriate number of write partial misses, write hits and writes to cache
in cache statistics
- Appropriate number of writes to cache in iostat
"""
pattern = f"0x{uuid.uuid4().hex}"
io_size = Size(600, Unit.MebiByte)
with TestRun.step("Prepare devices."):
cache_disk = TestRun.disks['cache']
core_disk = TestRun.disks['core']
core_disk.create_partitions([io_size + Size(1, Unit.MebiByte)])
core_part = core_disk.partitions[0]
with TestRun.step("Fill core partition with pattern."):
cache_mode_traits = CacheMode.get_traits(cache_mode)
if CacheModeTrait.InsertRead in cache_mode_traits:
run_fio(target=core_part.path,
operation_type=ReadWrite.write,
blocksize=Size(4, Unit.KibiByte),
io_size=io_size,
verify=True,
pattern=pattern)
else:
TestRun.LOGGER.info(f"Skipped for {cache_mode} cache mode.")
with TestRun.step("Start cache and add core."):
cache = casadm.start_cache(cache_disk, cache_mode, cache_line_size)
Udev.disable()
core = cache.add_core(core_part)
with TestRun.step("Cache half of file."):
operation_type = ReadWrite.read if CacheModeTrait.InsertRead in cache_mode_traits \
else ReadWrite.write
run_fio(target=core.path,
operation_type=operation_type,
skip=cache_line_size.value,
blocksize=cache_line_size.value,
io_size=io_size,
verify=True,
pattern=pattern)
if CacheModeTrait.InsertRead not in cache_mode_traits:
cache.flush_cache()
casadm.reset_counters(cache.cache_id, core.core_id)
with TestRun.step("Run writes to CAS device using fio."):
io_stats_before_io = cache_disk.get_io_stats()
blocksize = cache_line_size.value / 2 * 3
skip_size = cache_line_size.value / 2
run_fio(target=core.path,
operation_type=ReadWrite.write,
skip=skip_size,
blocksize=blocksize,
io_size=io_size)
with TestRun.step("Verify CAS statistics for partial misses, write hits and writes to cache."):
check_statistics(cache=cache,
blocksize=blocksize,
skip_size=skip_size,
io_size=io_size,
partial_misses=True)
with TestRun.step("Verify number of writes to cache device using iostat. Shall be 0.75 of "
f"io size ({str(io_size * 0.75)}) + metadata for cache mode with write "
f"insert feature."):
check_io_stats(cache_disk=cache_disk,
cache=cache,
io_stats_before=io_stats_before_io,
io_size=io_size,
blocksize=blocksize,
skip_size=skip_size)
def test_ioclass_id_as_condition(prepare_and_cleanup, filesystem):
"""
Load config in which IO class ids are used as conditions in other IO class definitions.
Check if performed IO is properly classified.
"""
cache, core = prepare()
Udev.disable()
base_dir_path = f"{mountpoint}/base_dir"
ioclass_file_size = Size(random.randint(25, 50), Unit.MebiByte)
ioclass_file_size_bytes = int(ioclass_file_size.get_value(Unit.Byte))
# directory condition
ioclass_config.add_ioclass(
ioclass_id=1,
eviction_priority=1,
allocation=True,
rule=f"directory:{base_dir_path}",
ioclass_config_path=ioclass_config_path,
)
# file size condition
ioclass_config.add_ioclass(
ioclass_id=2,
eviction_priority=1,
allocation=True,
rule=f"file_size:eq:{ioclass_file_size_bytes}",
ioclass_config_path=ioclass_config_path,
)
# direct condition
ioclass_config.add_ioclass(
ioclass_id=3,
eviction_priority=1,
allocation=True,
rule="direct",
ioclass_config_path=ioclass_config_path,
)
# IO class 1 OR 2 condition
ioclass_config.add_ioclass(
ioclass_id=4,
eviction_priority=1,
allocation=True,
rule="io_class:1|io_class:2",
ioclass_config_path=ioclass_config_path,
)
# IO class 4 AND file size condition (same as IO class 2)
ioclass_config.add_ioclass(
ioclass_id=5,
eviction_priority=1,
allocation=True,
rule=f"io_class:4&file_size:eq:{ioclass_file_size_bytes}",
ioclass_config_path=ioclass_config_path,
)
# IO class 3 condition
ioclass_config.add_ioclass(
ioclass_id=6,
eviction_priority=1,
allocation=True,
rule="io_class:3",
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
TestRun.LOGGER.info(f"Preparing {filesystem.name} filesystem "
f"and mounting {core.system_path} at {mountpoint}")
core.create_filesystem(filesystem)
core.mount(mountpoint)
fs_utils.create_directory(base_dir_path)
sync()
# IO fulfilling IO class 1 condition (and not IO class 2)
# Should be classified as IO class 4
base_occupancy = cache.get_cache_statistics(io_class_id=4)["occupancy"]
non_ioclass_file_size = Size(random.randrange(1, 25), Unit.MebiByte)
(Fio().create_command().io_engine(
IoEngine.libaio).size(non_ioclass_file_size).read_write(
ReadWrite.write).target(f"{base_dir_path}/test_file_1").run())
sync()
new_occupancy = cache.get_cache_statistics(io_class_id=4)["occupancy"]
assert new_occupancy == base_occupancy + non_ioclass_file_size, \
"Writes were not properly cached!\n" \
f"Expected: {base_occupancy + non_ioclass_file_size}, actual: {new_occupancy}"
# IO fulfilling IO class 2 condition (and not IO class 1)
# Should be classified as IO class 5
base_occupancy = cache.get_cache_statistics(io_class_id=5)["occupancy"]
(Fio().create_command().io_engine(
IoEngine.libaio).size(ioclass_file_size).read_write(
ReadWrite.write).target(f"{mountpoint}/test_file_2").run())
sync()
new_occupancy = cache.get_cache_statistics(io_class_id=5)["occupancy"]
assert new_occupancy == base_occupancy + ioclass_file_size, \
"Writes were not properly cached!\n" \
f"Expected: {base_occupancy + ioclass_file_size}, actual: {new_occupancy}"
# IO fulfilling IO class 1 and 2 conditions
# Should be classified as IO class 5
base_occupancy = new_occupancy
(Fio().create_command().io_engine(
IoEngine.libaio).size(ioclass_file_size).read_write(
ReadWrite.write).target(f"{base_dir_path}/test_file_3").run())
sync()
new_occupancy = cache.get_cache_statistics(io_class_id=5)["occupancy"]
assert new_occupancy == base_occupancy + ioclass_file_size, \
"Writes were not properly cached!\n" \
f"Expected: {base_occupancy + ioclass_file_size}, actual: {new_occupancy}"
# Same IO but direct
# Should be classified as IO class 6
base_occupancy = cache.get_cache_statistics(io_class_id=6)["occupancy"]
(Fio().create_command().io_engine(
IoEngine.libaio).size(ioclass_file_size).read_write(
ReadWrite.write).target(
f"{base_dir_path}/test_file_3").direct().run())
sync()
new_occupancy = cache.get_cache_statistics(io_class_id=6)["occupancy"]
assert new_occupancy == base_occupancy + ioclass_file_size, \
"Writes were not properly cached!\n" \
f"Expected: {base_occupancy + ioclass_file_size}, actual: {new_occupancy}"
def test_recovery_flush_reset_fs(cache_mode, fs):
"""
title: Recovery after reset during cache flushing - test on filesystem.
description: |
Verify that unflushed data can be safely recovered, when reset was pressed during
data flushing on filesystem.
pass_criteria:
- CAS recovers successfully after reboot
- No data corruption
"""
with TestRun.step("Prepare cache and core devices."):
cache_disk = TestRun.disks['cache']
core_disk = TestRun.disks['core']
cache_disk.create_partitions([Size(5, Unit.GibiByte)])
core_disk.create_partitions([Size(16, Unit.GibiByte)] * 2)
cache_device = cache_disk.partitions[0]
core_device = core_disk.partitions[0]
with TestRun.step(f"Create {fs} filesystem on core."):
core_device.create_filesystem(fs)
with TestRun.step("Create test files."):
source_file, target_file = create_test_files(test_file_size)
source_file_md5 = source_file.md5sum()
with TestRun.step("Setup cache and add core."):
cache = casadm.start_cache(cache_device, cache_mode)
Udev.disable()
core = cache.add_core(core_device)
cache.set_cleaning_policy(CleaningPolicy.nop)
cache.set_seq_cutoff_policy(SeqCutOffPolicy.never)
with TestRun.step("Mount CAS device."):
core.mount(mount_point)
with TestRun.step("Copy file to CAS."):
copy_file(source=source_file.full_path,
target=os.path.join(mount_point, "source_test_file"),
size=test_file_size, direct="oflag")
with TestRun.step("Unmount CAS device."):
core.unmount()
with TestRun.step("Trigger flush."):
os_utils.drop_caches(DropCachesMode.ALL)
TestRun.executor.run_in_background(cli.flush_cache_cmd(f"{cache.cache_id}"))
with TestRun.step("Hard reset DUT during data flushing."):
power_cycle_dut(True, core_device)
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache_device)
if cache.get_dirty_blocks() == Size.zero():
TestRun.LOGGER.error("There are no dirty blocks on cache device.")
with TestRun.step("Stop cache with dirty data flush."):
core_writes_before = core_device.get_io_stats().sectors_written
cache.stop()
if core_writes_before >= core_device.get_io_stats().sectors_written:
TestRun.fail("No data was flushed after stopping cache started with load option.")
with TestRun.step("Mount core device."):
core_device.mount(mount_point)
with TestRun.step("Copy test file from core device to temporary location. "
"Compare it with the first version – they should be the same."):
copy_file(source=os.path.join(mount_point, "source_test_file"),
target=target_file.full_path,
size=test_file_size, direct="iflag")
target_file_md5 = target_file.md5sum()
compare_files(source_file_md5, target_file_md5)
with TestRun.step("Unmount core device and remove test files."):
core_device.unmount()
try:
target_file.remove()
source_file.remove()
except Exception:
# On some OSes files at /tmp location are automatically removed after DUT hard reset
pass
Udev.enable()
def test_ioclass_eviction_priority(cache_line_size):
"""
title: Check whether eviction priorites are respected.
description: |
Create ioclass for 4 different directories, each with different
eviction priority configured. Saturate 3 of them and check if the
partitions are evicted in a good order during IO to the fourth
pass_criteria:
- Partitions are evicted in specified order
"""
with TestRun.step("Prepare CAS device"):
cache, core = prepare(cache_mode=CacheMode.WT,
cache_line_size=cache_line_size)
cache_size = cache.get_statistics().config_stats.cache_size
with TestRun.step("Disable udev"):
Udev.disable()
with TestRun.step(
f"Preparing filesystem and mounting {core.path} at {mountpoint}"):
filesystem = Filesystem.xfs
core.create_filesystem(filesystem)
core.mount(mountpoint)
sync()
with TestRun.step("Prepare test dirs"):
IoclassConfig = namedtuple("IoclassConfig",
"id eviction_prio max_occupancy dir_path")
io_classes = [
IoclassConfig(1, 3, 0.30, f"{mountpoint}/A"),
IoclassConfig(2, 4, 0.30, f"{mountpoint}/B"),
IoclassConfig(3, 5, 0.40, f"{mountpoint}/C"),
IoclassConfig(4, 1, 1.00, f"{mountpoint}/D"),
]
for io_class in io_classes:
fs_utils.create_directory(io_class.dir_path, parents=True)
with TestRun.step("Remove old ioclass config"):
ioclass_config.remove_ioclass_config()
ioclass_config.create_ioclass_config(False)
with TestRun.step("Adding default ioclasses"):
ioclass_config.add_ioclass(*str(IoClass.default(
allocation="0.00")).split(","))
with TestRun.step("Adding ioclasses for all dirs"):
for io_class in io_classes:
ioclass_config.add_ioclass(
io_class.id,
f"directory:{io_class.dir_path}&done",
io_class.eviction_prio,
f"{io_class.max_occupancy:0.2f}",
)
casadm.load_io_classes(cache_id=cache.cache_id,
file=ioclass_config_path)
with TestRun.step("Resetting cache stats"):
cache.purge_cache()
cache.reset_counters()
with TestRun.step("Checking initial occupancy"):
for io_class in io_classes:
occupancy = get_io_class_occupancy(cache, io_class.id)
if occupancy.get_value() != 0:
TestRun.LOGGER.error(
f"Incorrect inital occupancy for ioclass id: {io_class.id}."
f" Expected 0, got: {occupancy}")
with TestRun.step(
f"To A, B and C directories perform IO with size of max io_class occupancy"
):
for io_class in io_classes[0:3]:
run_io_dir(
f"{io_class.dir_path}/tmp_file",
int((io_class.max_occupancy * cache_size) / Unit.Blocks4096),
)
with TestRun.step("Check if each ioclass reached it's occupancy limit"):
for io_class in io_classes[0:3]:
actuall_occupancy = get_io_class_occupancy(cache, io_class.id)
occupancy_limit = ((io_class.max_occupancy *
cache_size).align_down(
Unit.Blocks4096.get_value()).set_unit(
Unit.Blocks4096))
if not isclose(actuall_occupancy.value,
occupancy_limit.value,
rel_tol=0.1):
TestRun.LOGGER.error(
f"Occupancy for ioclass {io_class.id} does not match. "
f"Limit: {occupancy_limit}, actuall: {actuall_occupancy}")
if get_io_class_occupancy(cache, io_classes[3].id).value != 0:
TestRun.LOGGER.error(
f"Occupancy for ioclass {io_classes[3].id} should be 0. "
f"Actuall: {actuall_occupancy}")
with TestRun.step("Perform IO to the fourth directory and check "
"if other partitions are evicted in a good order"):
target_io_class = io_classes[3]
io_classes_to_evict = io_classes[:
3][::
-1] # List is ordered by eviction priority
io_classes_evicted = []
io_offset = 0
for io_class in io_classes_to_evict:
io_size = int(
(io_class.max_occupancy * cache_size) / Unit.Blocks4096)
run_io_dir(f"{target_io_class.dir_path}/tmp_file_{io_class.id}",
io_size, io_offset)
io_offset += io_size
part_to_evict_end_occupancy = get_io_class_occupancy(cache,
io_class.id,
percent=True)
# Since number of evicted cachelines is always >= 128, occupancy is checked
# with approximation
if not isclose(part_to_evict_end_occupancy, 0, abs_tol=4):
TestRun.LOGGER.error(
f"Wrong percent of cache lines evicted from part {io_class.id}. "
f"Meant to be evicted {io_class.max_occupancy*100}%, actaully evicted "
f"{io_class.max_occupancy*100-part_to_evict_end_occupancy}%"
)
io_classes_evicted.append(io_class)
for i in io_classes_to_evict:
if i in io_classes_evicted:
continue
occupancy = get_io_class_occupancy(cache, i.id, percent=True)
if not isclose(occupancy, i.max_occupancy * 100, abs_tol=4):
TestRun.LOGGER.error(f"Ioclass {i.id} evicted incorrectly")
def test_ioclass_request_size():
"""
title: Test IO classification by request size.
description: Check if requests with size within defined range are cached.
pass_criteria:
- No kernel bug.
- IO is classified properly based on request size range defined in config.
"""
ioclass_id = 1
iterations = 100
with TestRun.step("Prepare cache and core."):
cache, core = prepare()
with TestRun.step("Create and load IO class config."):
ioclass_config.add_ioclass(
ioclass_id=ioclass_id,
eviction_priority=1,
allocation="1.00",
rule=f"request_size:ge:8192&request_size:le:16384&done",
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
with TestRun.step("Disable udev."):
Udev.disable()
with TestRun.step("Check if requests with size within defined range are cached."):
cached_req_sizes = [Size(2, Unit.Blocks4096), Size(4, Unit.Blocks4096)]
for i in range(iterations):
cache.flush_cache()
req_size = random.choice(cached_req_sizes)
dd = (
Dd().input("/dev/zero")
.output(core.path)
.count(1)
.block_size(req_size)
.oflag("direct")
)
dd.run()
dirty = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.dirty
if dirty.get_value(Unit.Blocks4096) != req_size.value / Unit.Blocks4096.value:
TestRun.fail("Incorrect number of dirty blocks!")
with TestRun.step("Flush cache."):
cache.flush_cache()
with TestRun.step("Check if requests with size outside of defined range are not cached"):
not_cached_req_sizes = [
Size(1, Unit.Blocks4096),
Size(8, Unit.Blocks4096),
Size(16, Unit.Blocks4096),
]
for i in range(iterations):
req_size = random.choice(not_cached_req_sizes)
dd = (
Dd().input("/dev/zero")
.output(core.path)
.count(1)
.block_size(req_size)
.oflag("direct")
)
dd.run()
dirty = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.dirty
if dirty.get_value(Unit.Blocks4096) != 0:
TestRun.fail("Dirty data present!")
def test_ioclass_directory_depth(filesystem):
"""
title: Test IO classification by directory.
description: |
Test if directory classification works properly for deeply nested directories for read and
write operations.
pass_criteria:
- No kernel bug.
- Read and write operations to directories are classified properly.
"""
base_dir_path = f"{mountpoint}/base_dir"
with TestRun.step("Prepare cache and core."):
cache, core = prepare()
Udev.disable()
with TestRun.step(f"Prepare {filesystem.name} filesystem and mount {core.path} "
f"at {mountpoint}."):
core.create_filesystem(filesystem)
core.mount(mountpoint)
sync()
with TestRun.step(f"Create the base directory: {base_dir_path}."):
fs_utils.create_directory(base_dir_path)
with TestRun.step(f"Create a nested directory."):
nested_dir_path = base_dir_path
random_depth = random.randint(40, 80)
for i in range(random_depth):
nested_dir_path += f"/dir_{i}"
fs_utils.create_directory(path=nested_dir_path, parents=True)
# Test classification in nested dir by reading a previously unclassified file
with TestRun.step("Create the first file in the nested directory."):
test_file_1 = File(f"{nested_dir_path}/test_file_1")
dd = (
Dd().input("/dev/urandom")
.output(test_file_1.full_path)
.count(random.randint(1, 200))
.block_size(Size(1, Unit.MebiByte))
)
dd.run()
sync()
drop_caches(DropCachesMode.ALL)
test_file_1.refresh_item()
with TestRun.step("Load IO class config."):
ioclass_id = random.randint(1, ioclass_config.MAX_IO_CLASS_ID)
# directory IO class
ioclass_config.add_ioclass(
ioclass_id=ioclass_id,
eviction_priority=1,
allocation="1.00",
rule=f"directory:{base_dir_path}",
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
with TestRun.step("Read the file in the nested directory"):
base_occupancy = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy
dd = (
Dd().input(test_file_1.full_path)
.output("/dev/null")
.block_size(Size(1, Unit.MebiByte))
)
dd.run()
with TestRun.step("Check occupancy after creating the file."):
new_occupancy = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy
if new_occupancy != base_occupancy + test_file_1.size:
TestRun.LOGGER.error("Wrong occupancy after reading file!\n"
f"Expected: {base_occupancy + test_file_1.size}, "
f"actual: {new_occupancy}")
# Test classification in nested dir by creating a file
with TestRun.step("Create the second file in the nested directory"):
base_occupancy = new_occupancy
test_file_2 = File(f"{nested_dir_path}/test_file_2")
dd = (
Dd().input("/dev/urandom")
.output(test_file_2.full_path)
.count(random.randint(25600, 51200)) # 100MB to 200MB
.block_size(Size(1, Unit.Blocks4096))
)
dd.run()
sync()
drop_caches(DropCachesMode.ALL)
test_file_2.refresh_item()
with TestRun.step("Check occupancy after creating the second file."):
new_occupancy = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.occupancy
expected_occpuancy = (base_occupancy + test_file_2.size).set_unit(Unit.Blocks4096)
if new_occupancy != base_occupancy + test_file_2.size:
TestRun.LOGGER.error("Wrong occupancy after creating file!\n"
f"Expected: {expected_occpuancy}, "
f"actual: {new_occupancy}")
def test_multistream_seq_cutoff_functional(threshold, streams_number):
"""
title: Functional test for multistream sequential cutoff
description: |
Testing if amount of data written to cache and core is correct after running sequential
writes from multiple streams with different sequential cut-off thresholds.
pass_criteria:
- Amount of data written to cache is equal to amount set with sequential cutoff threshold
- Amount of data written in pass-through is equal to io size run after reaching the
sequential cutoff threshold
"""
with TestRun.step("Start cache and add core device."):
cache_disk = TestRun.disks['cache']
core_disk = TestRun.disks['core']
cache = casadm.start_cache(cache_disk, CacheMode.WB, force=True)
Udev.disable()
core = cache.add_core(core_disk)
with TestRun.step(f"Set seq-cutoff policy to always, threshold to {threshold} "
f"and reset statistics counters."):
core.set_seq_cutoff_policy(SeqCutOffPolicy.always)
core.set_seq_cutoff_threshold(threshold)
core.set_seq_cutoff_promotion_count(1)
core.reset_counters()
with TestRun.step(f"Run {streams_number} I/O streams with amount of sequential writes equal to "
f"seq-cutoff threshold value minus one 4k block."):
kib_between_streams = 100
range_step = int(threshold.get_value(Unit.KibiByte)) + kib_between_streams
max_range_offset = streams_number * range_step
offsets = [o for o in range(0, max_range_offset, range_step)]
core_statistics_before = core.get_statistics()
for i in TestRun.iteration(range(0, len(offsets))):
TestRun.LOGGER.info(f"Statistics before I/O:\n{core_statistics_before}")
offset = Size(offsets[i], Unit.KibiByte)
run_dd(core.path, count=int(threshold.get_value(Unit.Blocks4096) - 1),
seek=int(offset.get_value(Unit.Blocks4096)))
core_statistics_after = core.get_statistics()
check_statistics(core_statistics_before,
core_statistics_after,
expected_pt=0,
expected_writes_to_cache=threshold - Size(1, Unit.Blocks4096))
core_statistics_before = core_statistics_after
with TestRun.step("Write random number of 4k block requests to each stream and check if all "
"writes were sent in pass-through mode."):
core_statistics_before = core.get_statistics()
random.shuffle(offsets)
for i in TestRun.iteration(range(0, len(offsets))):
TestRun.LOGGER.info(f"Statistics before second I/O:\n{core_statistics_before}")
additional_4k_blocks_writes = random.randint(1, kib_between_streams / 4)
offset = Size(offsets[i], Unit.KibiByte)
run_dd(
core.path, count=additional_4k_blocks_writes,
seek=int(offset.get_value(Unit.Blocks4096)
+ threshold.get_value(Unit.Blocks4096) - 1))
core_statistics_after = core.get_statistics()
check_statistics(core_statistics_before,
core_statistics_after,
expected_pt=additional_4k_blocks_writes,
expected_writes_to_cache=Size.zero())
core_statistics_before = core_statistics_after
