def remove_files_classification():
TestRun.LOGGER.info("Moving all files to 'unclassified' IO class")
ioclass_config.remove_ioclass_config(
ioclass_config_path=ioclass_config_path)
ioclass_config.create_ioclass_config(
add_default_rule=False, ioclass_config_path=ioclass_config_path)
ioclass_config.add_ioclass(
ioclass_id=0,
eviction_priority=22,
allocation=False,
rule="unclassified",
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id,
file=ioclass_config_path)
occupancy_before = cache.get_statistics_deprecated(
io_class_id=0)["occupancy"]
for file in test_files:
Dd().input(file.full_path).output("/dev/null").block_size(
file.size).run()
occupancy_after = cache.get_statistics_deprecated(
io_class_id=0)["occupancy"]
if occupancy_after != occupancy_before + file.size:
pytest.xfail("File not reclassified properly!\n"
f"Expected {occupancy_before + file.size}\n"
f"Actual {occupancy_after}")
occupancy_before = occupancy_after
sync()
drop_caches(DropCachesMode.ALL)
def start_monitoring(self,
buffer_size: Size = None,
number_of_subbuffers: int = None):
if self.blktrace_pid != -1:
raise Exception(
f"blktrace already running with PID: {self.blktrace_pid}")
self.__outputDirectoryPath = Directory.create_temp_directory(
).full_path
drop_caches(DropCachesMode.ALL)
number_of_subbuffers = ("" if number_of_subbuffers is None else
f" --num-sub-buffers={number_of_subbuffers}")
buffer_size = (
"" if buffer_size is None else
f" --buffer-size={buffer_size.get_value(Unit.KibiByte)}")
command = (
f"blktrace{number_of_subbuffers}{buffer_size} --dev={self.device.path}"
f"{self.masks} --output={PREFIX} --output-dir={self.__outputDirectoryPath}"
)
echo_output = TestRun.executor.run_expect_success(
f"nohup {command} </dev/null &>{self.__outputDirectoryPath}/out & echo $!"
)
self.blktrace_pid = int(echo_output.stdout)
TestRun.LOGGER.info(
f"blktrace monitoring for device {self.device.path} started"
f" (PID: {self.blktrace_pid}, output dir: {self.__outputDirectoryPath}"
)
def run_io_dir_read(path):
dd = Dd().output("/dev/null").input(f"{path}")
output = dd.run()
if output.exit_code != 0:
TestRun.fail(f"Failed to execute dd.\n {output.stdout}\n{output.stderr}")
sync()
drop_caches(DropCachesMode.ALL)
def run_io_dir(path, size_4k):
dd = (Dd().input("/dev/zero").output(f"{path}").count(size_4k).block_size(
Size(1, Unit.Blocks4096)))
TestRun.LOGGER.info(f"{dd}")
dd.run()
sync()
drop_caches(DropCachesMode.ALL)
def test_trim_eviction(cache_mode, cache_line_size, filesystem, cleaning):
"""
title: Test verifying if trim requests do not cause eviction on CAS device.
description: |
When trim requests enabled and files are being added and removed from CAS device,
there is no eviction (no reads from cache).
pass_criteria:
- Reads from cache device are the same before and after removing test file.
"""
mount_point = "/mnt"
test_file_path = os.path.join(mount_point, "test_file")
with TestRun.step("Prepare devices."):
cache_disk = TestRun.disks['cache']
cache_disk.create_partitions([Size(1, Unit.GibiByte)])
cache_dev = cache_disk.partitions[0]
core_disk = TestRun.disks['core']
core_disk.create_partitions([Size(1, Unit.GibiByte)])
core_dev = core_disk.partitions[0]
with TestRun.step("Start cache on device supporting trim and add core."):
cache = casadm.start_cache(cache_dev, cache_mode, cache_line_size)
cache.set_cleaning_policy(cleaning)
Udev.disable()
core = cache.add_core(core_dev)
with TestRun.step("Create filesystem on CAS device and mount it."):
core.create_filesystem(filesystem)
core.mount(mount_point, ["discard"])
with TestRun.step("Create random file using ddrescue."):
test_file = fs_utils.create_random_test_file(test_file_path,
core_dev.size * 0.9)
create_file_with_ddrescue(core_dev, test_file)
with TestRun.step("Remove file and create a new one."):
cache_iostats_before = cache_dev.get_io_stats()
test_file.remove()
os_utils.sync()
os_utils.drop_caches()
create_file_with_ddrescue(core_dev, test_file)
with TestRun.step(
"Check using iostat that reads from cache did not occur."):
cache_iostats_after = cache_dev.get_io_stats()
reads_before = cache_iostats_before.sectors_read
reads_after = cache_iostats_after.sectors_read
if reads_after != reads_before:
TestRun.fail(
f"Number of reads from cache before and after removing test file "
f"differs. Reads before: {reads_before}, reads after: {reads_after}."
)
else:
TestRun.LOGGER.info(
"Number of reads from cache before and after removing test file is the same."
)
def test_insufficient_memory_for_cas_module():
"""
title: Negative test of ability to load OpenCAS kernel module with insufficient memory.
description: |
Check that OpenCAS kernel module won’t be loaded in case not enough memory is available.
pass_criteria:
- Loading OpenCAS kernel module returns error.
"""
with TestRun.step("Disable caching and memory over-committing."):
disable_memory_affecting_functions()
drop_caches()
with TestRun.step("Measure memory usage without OpenCAS module."):
if is_kernel_module_loaded(CasModule.cache.value):
unload_kernel_module(CasModule.cache.value)
unload_kernel_module(CasModule.disk.value)
available_mem_before_cas = get_free_memory()
with TestRun.step("Load OpenCAS module"):
output = load_kernel_module(CasModule.cache.value)
if output.exit_code != 0:
TestRun.fail("Cannot load OpenCAS module!")
with TestRun.step("Measure memory usage with OpenCAS module."):
available_mem_with_cas = get_free_memory()
memory_used_by_cas = available_mem_before_cas - available_mem_with_cas
TestRun.LOGGER.info(
f"OpenCAS module uses {memory_used_by_cas.get_value(Unit.MiB):.2f} MiB of DRAM."
)
with TestRun.step("Unload OpenCAS module."):
unload_kernel_module(CasModule.cache.value)
unload_kernel_module(CasModule.disk.value)
with TestRun.step("Allocate memory leaving not enough memory for OpenCAS module."):
memory_to_leave = memory_used_by_cas * (3 / 4)
try:
allocate_memory(get_free_memory() - memory_to_leave)
except Exception as ex:
TestRun.LOGGER.error(f"{ex}")
with TestRun.step(
"Try to load OpenCAS module and check if error message is printed on failure."
):
output = load_kernel_module(CasModule.cache.value)
if output.stderr and output.exit_code != 0:
memory_left = get_free_memory()
TestRun.LOGGER.info(
f"Memory left for OpenCAS module: {memory_left.get_value(Unit.MiB):0.2f} MiB."
)
TestRun.LOGGER.info(f"Cannot load OpenCAS module as expected.\n{output.stderr}")
else:
TestRun.LOGGER.error("Loading OpenCAS module successfully finished, but should fail.")
with TestRun.step("Set memory options to default"):
unmount_ramfs()
defaultize_memory_affecting_functions()
def run_io_dir(path, size_4k):
dd = (Dd().input("/dev/zero").output(f"{path}").count(size_4k).block_size(
Size(1, Unit.Blocks4096)))
TestRun.LOGGER.info(f"{dd}")
output = dd.run()
if output.exit_code != 0:
TestRun.fail(
f"Failed to execute dd.\n {output.stdout}\n{output.stderr}")
sync()
drop_caches(DropCachesMode.ALL)
def read_files_with_reclassification_check(cache, target_ioclass_id: int,
source_ioclass_id: int,
directory: Directory,
with_delay: bool):
start_time = datetime.now()
target_occupancy_after = cache.get_io_class_statistics(
io_class_id=target_ioclass_id).usage_stats.occupancy
source_occupancy_after = cache.get_io_class_statistics(
io_class_id=source_ioclass_id).usage_stats.occupancy
files_to_reclassify = []
target_ioclass_is_enabled = ioclass_is_enabled(cache, target_ioclass_id)
for file in [item for item in directory.ls() if isinstance(item, File)]:
target_occupancy_before = target_occupancy_after
source_occupancy_before = source_occupancy_after
time_from_start = datetime.now() - start_time
dd = Dd().input(file.full_path).output("/dev/null").block_size(
Size(1, Unit.Blocks4096))
dd.run()
target_occupancy_after = cache.get_io_class_statistics(
io_class_id=target_ioclass_id).usage_stats.occupancy
source_occupancy_after = cache.get_io_class_statistics(
io_class_id=source_ioclass_id).usage_stats.occupancy
if target_ioclass_is_enabled:
if target_occupancy_after < target_occupancy_before:
TestRun.LOGGER.error("Target IO class occupancy lowered!")
elif target_occupancy_after - target_occupancy_before < file.size:
files_to_reclassify.append(file)
if with_delay and time_from_start <= ioclass_config.MAX_CLASSIFICATION_DELAY:
continue
TestRun.LOGGER.error(
"Target IO class occupancy not changed properly!\n"
f"Expected: {file.size + target_occupancy_before}\n"
f"Actual: {target_occupancy_after}")
elif target_occupancy_after > target_occupancy_before and with_delay:
files_to_reclassify.append(file)
if source_occupancy_after >= source_occupancy_before:
if file not in files_to_reclassify:
files_to_reclassify.append(file)
if with_delay and time_from_start <= ioclass_config.MAX_CLASSIFICATION_DELAY:
continue
TestRun.LOGGER.error(
"Source IO class occupancy not changed properly!\n"
f"Before: {source_occupancy_before}\n"
f"After: {source_occupancy_after}")
if len(files_to_reclassify):
TestRun.LOGGER.info("Rereading unclassified test files...")
sync()
drop_caches(DropCachesMode.ALL)
for file in files_to_reclassify:
(Dd().input(file.full_path).output("/dev/null").block_size(
Size(1, Unit.Blocks4096)).run())
def stop_monitoring_and_check_discards(blktraces, discard_support):
time.sleep(60)
os_utils.sync()
os_utils.drop_caches()
time.sleep(5)
discard_flag = RwbsKind.D # Discard
for key in blktraces.keys():
output = blktraces[key].stop_monitoring()
discard_messages = [h for h in output if discard_flag in h.rwbs]
check_discards(len(discard_messages), blktraces[key].device, discard_support[key])
def run_io_dir(path, num_ios):
dd = (
Dd()
.input("/dev/zero")
.output(f"{path}/tmp_file")
.count(num_ios)
.block_size(Size(1, Unit.Blocks4096))
)
dd.run()
sync()
drop_caches(DropCachesMode.ALL)
def test_load_after_clean_shutdown(reboot_type, cache_mode, filesystem):
"""
title: Planned system shutdown test.
description: Test for data consistency after clean system shutdown.
pass_criteria:
- DUT should reboot successfully.
- Checksum of file on core device should be the same before and after reboot.
"""
with TestRun.step("Prepare CAS device."):
cache_disk = TestRun.disks['cache']
cache_disk.create_partitions([Size(1, Unit.GibiByte)])
cache_dev = cache_disk.partitions[0]
core_dev = TestRun.disks['core']
cache = casadm.start_cache(cache_dev, cache_mode, force=True)
core = cache.add_core(core_dev)
core.create_filesystem(filesystem,
blocksize=int(Size(1, Unit.Blocks4096)))
core.mount(mount_point)
with TestRun.step("Create file on cache and count its checksum."):
test_file = File(os.path.join(mount_point, "test_file"))
Dd()\
.input("/dev/zero")\
.output(test_file.full_path)\
.block_size(Size(1, Unit.KibiByte))\
.count(1024)\
.run()
test_file.refresh_item()
test_file_md5 = test_file.md5sum()
sync()
drop_caches(DropCachesMode.ALL)
with TestRun.step("Reset platform."):
if reboot_type == "soft":
TestRun.executor.reboot()
else:
power_control = TestRun.plugin_manager.get_plugin('power_control')
power_control.power_cycle()
with TestRun.step("Load cache."):
casadm.load_cache(cache_dev)
core.mount(mount_point)
with TestRun.step("Check file md5sum."):
test_file.refresh_item()
if test_file_md5 != test_file.md5sum():
TestRun.LOGGER.error(
"Checksums does not match - file is corrupted.")
else:
TestRun.LOGGER.info("File checksum is correct.")
with TestRun.step("Remove test file."):
test_file.remove()
def stop_monitoring(self):
if self.blktrace_pid == -1:
raise Exception("PID for blktrace is not set - has monitoring been started?")
drop_caches(DropCachesMode.ALL)
TestRun.executor.run_expect_success(f"kill -s SIGINT {self.blktrace_pid}")
self.blktrace_pid = -1
# dummy command for swallowing output of killed command
TestRun.executor.run("sleep 2 && echo dummy")
TestRun.LOGGER.info(f"blktrace monitoring for device {self.device.system_path} stopped")
return self.__parse_blktrace_output()
def reclassify_files():
TestRun.LOGGER.info("Reading files belonging to different IO classes "
"(classification by reads).")
for file in test_files:
ioclass_id = size_to_class[file.size]
occupancy_before = cache.get_cache_statistics(io_class_id=ioclass_id)["occupancy"]
Dd().input(file.full_path).output("/dev/null").block_size(file.size).run()
occupancy_after = cache.get_cache_statistics(io_class_id=ioclass_id)["occupancy"]
if occupancy_after != occupancy_before + file.size:
pytest.xfail("File not reclassified properly!\n"
f"Expected {occupancy_before + file.size}\n"
f"Actual {occupancy_after}")
sync()
drop_caches(DropCachesMode.ALL)
def create_files_and_check_classification():
TestRun.LOGGER.info("Creating files belonging to different IO classes "
"(classification by writes).")
for size, ioclass_id in size_to_class.items():
occupancy_before = cache.get_cache_statistics(io_class_id=ioclass_id)["occupancy"]
file_path = f"{mountpoint}/test_file_{size.get_value()}"
Dd().input("/dev/zero").output(file_path).oflag("sync").block_size(size).count(1).run()
occupancy_after = cache.get_cache_statistics(io_class_id=ioclass_id)["occupancy"]
if occupancy_after != occupancy_before + size:
pytest.xfail("File not cached properly!\n"
f"Expected {occupancy_before + size}\n"
f"Actual {occupancy_after}")
test_files.append(File(file_path).refresh_item())
sync()
drop_caches(DropCachesMode.ALL)
def read_files_with_reclassification_check(target_ioclass_id: int,
source_ioclass_id: int,
directory: Directory,
with_delay: bool):
start_time = datetime.now()
target_occupancy_after = cache.get_statistics_deprecated(
io_class_id=target_ioclass_id)["occupancy"]
source_occupancy_after = cache.get_statistics_deprecated(
io_class_id=source_ioclass_id)["occupancy"]
unclassified_files = []
for file in [
item for item in directory.ls() if isinstance(item, File)
]:
target_occupancy_before = target_occupancy_after
source_occupancy_before = source_occupancy_after
time_from_start = datetime.now() - start_time
(Dd().input(file.full_path).output("/dev/null").block_size(
Size(1, Unit.Blocks4096)).run())
target_occupancy_after = cache.get_statistics_deprecated(
io_class_id=target_ioclass_id)["occupancy"]
source_occupancy_after = cache.get_statistics_deprecated(
io_class_id=source_ioclass_id)["occupancy"]
if target_occupancy_after < target_occupancy_before:
pytest.xfail("Target IO class occupancy lowered!")
elif target_occupancy_after - target_occupancy_before < file.size:
unclassified_files.append(file)
if with_delay and time_from_start <= ioclass_config.MAX_CLASSIFICATION_DELAY:
continue
pytest.xfail("Target IO class occupancy not changed properly!")
if source_occupancy_after >= source_occupancy_before:
if file not in unclassified_files:
unclassified_files.append(file)
if with_delay and time_from_start <= ioclass_config.MAX_CLASSIFICATION_DELAY:
continue
pytest.xfail("Source IO class occupancy not changed properly!")
if len(unclassified_files):
TestRun.LOGGER.info("Rereading unclassified test files...")
sync()
drop_caches(DropCachesMode.ALL)
for file in unclassified_files:
(Dd().input(file.full_path).output("/dev/null").block_size(
Size(1, Unit.Blocks4096)).run())
def reclassify_files():
TestRun.LOGGER.info("Reading files belonging to different IO classes "
"(classification by reads).")
for file in test_files:
ioclass_id = size_to_class[file.size]
occupancy_before = cache.get_io_class_statistics(
io_class_id=ioclass_id).usage_stats.occupancy
Dd().input(file.full_path).output("/dev/null").block_size(
file.size).run()
occupancy_after = cache.get_io_class_statistics(
io_class_id=ioclass_id).usage_stats.occupancy
actual_blocks = occupancy_after.get_value(Unit.Blocks4096)
expected_blocks = (occupancy_before + file.size).get_value(
Unit.Blocks4096)
if actual_blocks != expected_blocks:
TestRun.fail("File not reclassified properly!\n"
f"Expected {occupancy_before + file.size}\n"
f"Actual {occupancy_after}")
sync()
drop_caches(DropCachesMode.ALL)
def test_ioclass_directory_depth(filesystem):
"""
Test if directory classification works properly for deeply nested directories for read and
write operations.
"""
cache, core = prepare()
Udev.disable()
TestRun.LOGGER.info(f"Preparing {filesystem.name} filesystem "
f"and mounting {core.system_path} at {mountpoint}")
core.create_filesystem(filesystem)
core.mount(mountpoint)
sync()
base_dir_path = f"{mountpoint}/base_dir"
TestRun.LOGGER.info(f"Creating the base directory: {base_dir_path}")
fs_utils.create_directory(base_dir_path)
nested_dir_path = base_dir_path
random_depth = random.randint(40, 80)
for i in range(random_depth):
nested_dir_path += f"/dir_{i}"
TestRun.LOGGER.info(f"Creating a nested directory: {nested_dir_path}")
fs_utils.create_directory(path=nested_dir_path, parents=True)
# Test classification in nested dir by reading a previously unclassified file
TestRun.LOGGER.info("Creating the first file in the nested directory")
test_file_1 = File(f"{nested_dir_path}/test_file_1")
dd = (Dd().input("/dev/urandom").output(test_file_1.full_path).count(
random.randint(1, 200)).block_size(Size(1, Unit.MebiByte)))
dd.run()
sync()
drop_caches(DropCachesMode.ALL)
test_file_1.refresh_item()
ioclass_id = random.randint(1, ioclass_config.MAX_IO_CLASS_ID)
# directory IO class
ioclass_config.add_ioclass(
ioclass_id=ioclass_id,
eviction_priority=1,
allocation=True,
rule=f"directory:{base_dir_path}",
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
base_occupancy = cache.get_statistics_deprecated(
io_class_id=ioclass_id)["occupancy"]
TestRun.LOGGER.info("Reading the file in the nested directory")
dd = (Dd().input(test_file_1.full_path).output("/dev/null").block_size(
Size(1, Unit.MebiByte)))
dd.run()
new_occupancy = cache.get_statistics_deprecated(
io_class_id=ioclass_id)["occupancy"]
assert new_occupancy == base_occupancy + test_file_1.size, \
"Wrong occupancy after reading file!\n" \
f"Expected: {base_occupancy + test_file_1.size}, actual: {new_occupancy}"
# Test classification in nested dir by creating a file
base_occupancy = new_occupancy
TestRun.LOGGER.info("Creating the second file in the nested directory")
test_file_2 = File(f"{nested_dir_path}/test_file_2")
dd = (Dd().input("/dev/urandom").output(test_file_2.full_path).count(
random.randint(1, 200)).block_size(Size(1, Unit.MebiByte)))
dd.run()
sync()
drop_caches(DropCachesMode.ALL)
test_file_2.refresh_item()
new_occupancy = cache.get_statistics_deprecated(
io_class_id=ioclass_id)["occupancy"]
assert new_occupancy == base_occupancy + test_file_2.size, \
"Wrong occupancy after creating file!\n" \
f"Expected: {base_occupancy + test_file_2.size}, actual: {new_occupancy}"
def test_ioclass_directory_file_operations(filesystem):
"""
title: Test IO classification by file operations.
description: |
Test if directory classification works properly after file operations like move or rename.
pass_criteria:
- No kernel bug.
- The operations themselves should not cause reclassification but IO after those
operations should be reclassified to proper IO class.
"""
test_dir_path = f"{mountpoint}/test_dir"
nested_dir_path = f"{test_dir_path}/nested_dir"
dd_blocks = random.randint(5, 50)
with TestRun.step("Prepare cache and core."):
cache, core = prepare()
Udev.disable()
with TestRun.step("Create and load IO class config file."):
ioclass_id = random.randint(1, ioclass_config.MAX_IO_CLASS_ID)
# directory IO class
ioclass_config.add_ioclass(
ioclass_id=ioclass_id,
eviction_priority=1,
allocation=True,
rule=f"directory:{test_dir_path}",
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id,
file=ioclass_config_path)
with TestRun.step(f"Prepare {filesystem.name} filesystem "
f"and mounting {core.system_path} at {mountpoint}."):
core.create_filesystem(fs_type=filesystem)
core.mount(mount_point=mountpoint)
sync()
with TestRun.step(f"Create directory {nested_dir_path}."):
Directory.create_directory(path=nested_dir_path, parents=True)
sync()
drop_caches(DropCachesMode.ALL)
with TestRun.step("Create test file."):
classified_before = cache.get_io_class_statistics(
io_class_id=ioclass_id).usage_stats.occupancy
file_path = f"{test_dir_path}/test_file"
(Dd().input("/dev/urandom").output(file_path).oflag("sync").block_size(
Size(1, Unit.MebiByte)).count(dd_blocks).run())
sync()
drop_caches(DropCachesMode.ALL)
test_file = File(file_path).refresh_item()
with TestRun.step("Check classified occupancy."):
classified_after = cache.get_io_class_statistics(
io_class_id=ioclass_id).usage_stats.occupancy
check_occupancy(classified_before + test_file.size, classified_after)
with TestRun.step("Move test file out of classified directory."):
classified_before = classified_after
non_classified_before = cache.get_io_class_statistics(
io_class_id=0).usage_stats.occupancy
test_file.move(destination=mountpoint)
sync()
drop_caches(DropCachesMode.ALL)
with TestRun.step("Check classified occupancy."):
classified_after = cache.get_io_class_statistics(
io_class_id=ioclass_id).usage_stats.occupancy
check_occupancy(classified_before, classified_after)
TestRun.LOGGER.info("Checking non-classified occupancy")
non_classified_after = cache.get_io_class_statistics(
io_class_id=0).usage_stats.occupancy
check_occupancy(non_classified_before, non_classified_after)
with TestRun.step("Read test file."):
classified_before = classified_after
non_classified_before = non_classified_after
(Dd().input(test_file.full_path).output("/dev/null").block_size(
Size(1, Unit.MebiByte)).run())
with TestRun.step("Check classified occupancy."):
classified_after = cache.get_io_class_statistics(
io_class_id=ioclass_id).usage_stats.occupancy
check_occupancy(classified_before - test_file.size, classified_after)
TestRun.LOGGER.info("Checking non-classified occupancy")
non_classified_after = cache.get_io_class_statistics(
io_class_id=0).usage_stats.occupancy
check_occupancy(non_classified_before + test_file.size,
non_classified_after)
with TestRun.step(f"Move test file to {nested_dir_path}."):
classified_before = classified_after
non_classified_before = non_classified_after
test_file.move(destination=nested_dir_path)
sync()
drop_caches(DropCachesMode.ALL)
with TestRun.step("Check classified occupancy."):
classified_after = cache.get_io_class_statistics(
io_class_id=ioclass_id).usage_stats.occupancy
check_occupancy(classified_before, classified_after)
TestRun.LOGGER.info("Checking non-classified occupancy")
non_classified_after = cache.get_io_class_statistics(
io_class_id=0).usage_stats.occupancy
check_occupancy(non_classified_before, non_classified_after)
with TestRun.step("Read test file."):
classified_before = classified_after
non_classified_before = non_classified_after
(Dd().input(test_file.full_path).output("/dev/null").block_size(
Size(1, Unit.MebiByte)).run())
with TestRun.step("Check classified occupancy."):
classified_after = cache.get_io_class_statistics(
io_class_id=ioclass_id).usage_stats.occupancy
check_occupancy(classified_before + test_file.size, classified_after)
with TestRun.step("Check non-classified occupancy."):
non_classified_after = cache.get_io_class_statistics(
io_class_id=0).usage_stats.occupancy
check_occupancy(non_classified_before - test_file.size,
non_classified_after)
def run_io_dir_read(path):
dd = Dd().output("/dev/null").input(f"{path}")
dd.run()
sync()
drop_caches(DropCachesMode.ALL)
def test_ioclass_directory_dir_operations(filesystem):
"""
title: Test IO classification by directory operations.
description: |
Test if directory classification works properly after directory operations like move or
rename.
pass_criteria:
- No kernel bug.
- The operations themselves should not cause reclassification but IO after those
operations should be reclassified to proper IO class.
- Directory classification may work with a delay after loading IO class configuration or
move/rename operations. Test checks if maximum delay is not exceeded.
"""
non_classified_dir_path = f"{mountpoint}/non_classified"
with TestRun.step("Prepare cache and core."):
cache, core = prepare()
Udev.disable()
with TestRun.step("Create and load IO class config file."):
proper_ids = random.sample(
range(1, ioclass_config.MAX_IO_CLASS_ID + 1), 2)
ioclass_id_1 = proper_ids[0]
classified_dir_path_1 = f"{mountpoint}/dir_{ioclass_id_1}"
ioclass_id_2 = proper_ids[1]
classified_dir_path_2 = f"{mountpoint}/dir_{ioclass_id_2}"
# directory IO classes
ioclass_config.add_ioclass(
ioclass_id=ioclass_id_1,
eviction_priority=1,
allocation=True,
rule=f"directory:{classified_dir_path_1}",
ioclass_config_path=ioclass_config_path,
)
ioclass_config.add_ioclass(
ioclass_id=ioclass_id_2,
eviction_priority=1,
allocation=True,
rule=f"directory:{classified_dir_path_2}",
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id,
file=ioclass_config_path)
with TestRun.step(f"Prepare {filesystem.name} filesystem "
f"and mount {core.system_path} at {mountpoint}."):
core.create_filesystem(fs_type=filesystem)
core.mount(mount_point=mountpoint)
sync()
with TestRun.step(
f"Create a non-classified directory: {non_classified_dir_path}."):
dir_1 = Directory.create_directory(path=non_classified_dir_path)
with TestRun.step(
f"Rename {non_classified_dir_path} to {classified_dir_path_1}."):
dir_1.move(destination=classified_dir_path_1)
with TestRun.step("Create files with delay check."):
create_files_with_classification_delay_check(cache,
directory=dir_1,
ioclass_id=ioclass_id_1)
with TestRun.step(f"Create {classified_dir_path_2}/subdir."):
dir_2 = Directory.create_directory(
path=f"{classified_dir_path_2}/subdir", parents=True)
with TestRun.step("Create files with delay check."):
create_files_with_classification_delay_check(cache,
directory=dir_2,
ioclass_id=ioclass_id_2)
sync()
drop_caches(DropCachesMode.ALL)
with TestRun.step(f"Move {dir_2.full_path} to {classified_dir_path_1}."):
dir_2.move(destination=classified_dir_path_1)
with TestRun.step("Read files with reclassification check."):
read_files_with_reclassification_check(cache,
target_ioclass_id=ioclass_id_1,
source_ioclass_id=ioclass_id_2,
directory=dir_2,
with_delay=False)
sync()
drop_caches(DropCachesMode.ALL)
with TestRun.step(f"Move {dir_2.full_path} to {mountpoint}."):
dir_2.move(destination=mountpoint)
with TestRun.step("Read files with reclassification check."):
read_files_with_reclassification_check(cache,
target_ioclass_id=0,
source_ioclass_id=ioclass_id_1,
directory=dir_2,
with_delay=False)
with TestRun.step(f"Remove {classified_dir_path_2}."):
fs_utils.remove(path=classified_dir_path_2, force=True, recursive=True)
sync()
drop_caches(DropCachesMode.ALL)
with TestRun.step(
f"Rename {classified_dir_path_1} to {classified_dir_path_2}."):
dir_1.move(destination=classified_dir_path_2)
with TestRun.step("Read files with reclassification check."):
read_files_with_reclassification_check(cache,
target_ioclass_id=ioclass_id_2,
source_ioclass_id=ioclass_id_1,
directory=dir_1,
with_delay=True)
with TestRun.step(
f"Rename {classified_dir_path_2} to {non_classified_dir_path}."):
dir_1.move(destination=non_classified_dir_path)
with TestRun.step("Read files with reclassification check."):
read_files_with_reclassification_check(cache,
target_ioclass_id=0,
source_ioclass_id=ioclass_id_2,
directory=dir_1,
with_delay=True)
def test_trim_device_discard_support(trim_support_cache_core, cache_mode,
filesystem, cleaning_policy):
"""
title: Trim requests supported on various cache and core devices.
description: |
Handling trim requests support when various combination of SSD and HDD are used as
cache and core.
pass_criteria:
- No system crash.
- Discards detected on CAS.
- Discards detected on SSD device when it is used as core.
- Discards not detected on HDD device used as cache or core.
- Discards not detected on cache device.
"""
mount_point = "/mnt"
with TestRun.step(f"Create partitions on SSD and HDD devices."):
TestRun.disks["ssd1"].create_partitions([Size(1, Unit.GibiByte)])
TestRun.disks["ssd2"].create_partitions([Size(1, Unit.GibiByte)])
TestRun.disks["hdd"].create_partitions([Size(1, Unit.GibiByte)])
ssd1_dev = TestRun.disks["ssd1"].partitions[0]
ssd2_dev = TestRun.disks["ssd2"].partitions[0]
hdd_dev = TestRun.disks["hdd"].partitions[0]
with TestRun.step(f"Start cache and add core."):
cache_dev = ssd1_dev if trim_support_cache_core[0] else hdd_dev
core_dev = ssd2_dev if trim_support_cache_core[1] else hdd_dev
cache = casadm.start_cache(cache_dev, cache_mode, force=True)
cache.set_cleaning_policy(cleaning_policy)
core = cache.add_core(core_dev)
with TestRun.step("Make filesystem and mount it with discard option."):
core.create_filesystem(filesystem)
core.mount(mount_point, ["discard"])
with TestRun.step("Create random file."):
test_file = fs_utils.create_random_test_file(
os.path.join(mount_point, "test_file"), core_dev.size * 0.9)
occupancy_before = core.get_occupancy()
TestRun.LOGGER.info(str(core.get_statistics()))
with TestRun.step("Start blktrace monitoring on all devices."):
blktraces = start_monitoring(core_dev, cache_dev, core)
with TestRun.step("Remove file."):
os_utils.sync()
os_utils.drop_caches()
test_file.remove()
with TestRun.step(
"Ensure that discards were detected by blktrace on proper devices."
):
discard_expected = {
"core": trim_support_cache_core[1],
"cache": False,
"cas": True
}
stop_monitoring_and_check_discards(blktraces, discard_expected)
with TestRun.step("Ensure occupancy reduced."):
occupancy_after = core.get_occupancy()
TestRun.LOGGER.info(str(core.get_statistics()))
if occupancy_after >= occupancy_before:
TestRun.LOGGER.error(
"Occupancy on core after removing test file greater than before."
)
else:
TestRun.LOGGER.info(
"Occupancy on core after removing test file smaller than before."
)
with TestRun.step("Check CAS sysfs properties values."):
check_sysfs_properties(
cache,
cache_dev,
core,
core_dev.parent_device,
core_supporting_discards=trim_support_cache_core[1])
def test_ioclass_file_size(filesystem):
"""
title: Test IO classification by file size.
description: Test if file size classification works properly.
pass_criteria:
- No kernel bug.
- IO is classified properly based on IO class rule with file size.
"""
# File size IO class rules are configured in a way that each tested file size is unambiguously
# classified.
# Firstly write operations are tested (creation of files), secondly read operations.
base_size = Size(random.randint(50, 1000) * 2, Unit.Blocks4096)
size_to_class = {
base_size: 1,
base_size - Unit.Blocks4096: 2,
base_size + Unit.Blocks4096: 3,
base_size / 2: 4,
base_size / 2 - Unit.Blocks4096: 4,
base_size / 2 + Unit.Blocks4096: 2,
base_size * 2: 5,
base_size * 2 - Unit.Blocks4096: 3,
base_size * 2 + Unit.Blocks4096: 5,
}
with TestRun.step("Prepare cache and core."):
cache, core = prepare(default_allocation="1.00")
with TestRun.step("Prepare and load IO class config."):
load_file_size_io_classes(cache, base_size)
with TestRun.step(
f"Prepare {filesystem.name} filesystem and mount {core.path} "
f"at {mountpoint}."):
core.create_filesystem(filesystem)
core.mount(mountpoint)
sync()
with TestRun.step(
"Create files belonging to different IO classes (classification by writes)."
):
test_files = []
for size, ioclass_id in size_to_class.items():
occupancy_before = cache.get_io_class_statistics(
io_class_id=ioclass_id).usage_stats.occupancy
file_path = f"{mountpoint}/test_file_{size.get_value()}"
Dd().input("/dev/zero").output(file_path).oflag("sync").block_size(
size).count(1).run()
sync()
drop_caches(DropCachesMode.ALL)
occupancy_after = cache.get_io_class_statistics(
io_class_id=ioclass_id).usage_stats.occupancy
if occupancy_after != occupancy_before + size:
TestRun.fail("File not cached properly!\n"
f"Expected {occupancy_before + size}\n"
f"Actual {occupancy_after}")
test_files.append(File(file_path).refresh_item())
sync()
drop_caches(DropCachesMode.ALL)
with TestRun.step("Move all files to 'unclassified' IO class."):
ioclass_config.remove_ioclass_config(
ioclass_config_path=ioclass_config_path)
ioclass_config.create_ioclass_config(
add_default_rule=False, ioclass_config_path=ioclass_config_path)
ioclass_config.add_ioclass(
ioclass_id=0,
eviction_priority=22,
allocation="1.00",
rule="unclassified",
ioclass_config_path=ioclass_config_path,
)
ioclass_config.add_ioclass(
ioclass_id=6,
eviction_priority=1,
allocation="0.00",
rule=f"metadata",
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id,
file=ioclass_config_path)
occupancy_before = cache.get_io_class_statistics(
io_class_id=0).usage_stats.occupancy
for file in test_files:
Dd().input(file.full_path).output("/dev/null").block_size(
file.size).run()
sync()
drop_caches(DropCachesMode.ALL)
occupancy_after = cache.get_io_class_statistics(
io_class_id=0).usage_stats.occupancy
occupancy_expected = occupancy_before + file.size
if occupancy_after != occupancy_expected:
TestRun.fail("File not reclassified properly!\n"
f"Expected {occupancy_expected}\n"
f"Actual {occupancy_after}")
occupancy_before = occupancy_after
sync()
drop_caches(DropCachesMode.ALL)
with TestRun.step("Restore IO class configuration."):
ioclass_config.remove_ioclass_config(
ioclass_config_path=ioclass_config_path)
ioclass_config.create_ioclass_config(
add_default_rule=False, ioclass_config_path=ioclass_config_path)
ioclass_config.add_ioclass(
ioclass_id=0,
eviction_priority=22,
allocation="1.00",
rule="unclassified",
ioclass_config_path=ioclass_config_path,
)
load_file_size_io_classes(cache, base_size)
with TestRun.step(
"Read files belonging to different IO classes (classification by reads)."
):
# CAS device should be unmounted and mounted because data can be sometimes still cached by
# OS cache so occupancy statistics will not match
core.unmount()
core.mount(mountpoint)
for file in test_files:
ioclass_id = size_to_class[file.size]
occupancy_before = cache.get_io_class_statistics(
io_class_id=ioclass_id).usage_stats.occupancy
Dd().input(file.full_path).output("/dev/null").block_size(
file.size).run()
sync()
drop_caches(DropCachesMode.ALL)
occupancy_after = cache.get_io_class_statistics(
io_class_id=ioclass_id).usage_stats.occupancy
actual_blocks = occupancy_after.get_value(Unit.Blocks4096)
expected_blocks = (occupancy_before + file.size).get_value(
Unit.Blocks4096)
if actual_blocks != expected_blocks:
TestRun.fail("File not reclassified properly!\n"
f"Expected {occupancy_before + file.size}\n"
f"Actual {occupancy_after}")
sync()
drop_caches(DropCachesMode.ALL)
def test_trim_eviction(cache_mode, cache_line_size, filesystem, cleaning):
"""
title: Test verifying if trim requests do not cause eviction on CAS device.
description: |
When trim requests enabled and files are being added and removed from CAS device,
there is no eviction (no reads from cache).
pass_criteria:
- Reads from cache device are the same before and after removing test file.
"""
mount_point = "/mnt"
test_file_path = os.path.join(mount_point, "test_file")
with TestRun.step("Prepare devices."):
cache_disk = TestRun.disks['cache']
cache_disk.create_partitions([Size(1, Unit.GibiByte)])
cache_dev = cache_disk.partitions[0]
core_disk = TestRun.disks['core']
core_disk.create_partitions([Size(1, Unit.GibiByte)])
core_dev = core_disk.partitions[0]
cache_block_size = disk_utils.get_block_size(cache_disk)
with TestRun.step("Start cache on device supporting trim and add core."):
cache = casadm.start_cache(cache_dev,
cache_mode,
cache_line_size,
force=True)
cache.set_cleaning_policy(cleaning)
Udev.disable()
core = cache.add_core(core_dev)
with TestRun.step("Create filesystem on CAS device and mount it."):
core.create_filesystem(filesystem)
core.mount(mount_point, ["discard"])
with TestRun.step("Create ioclass config."):
ioclass_config.create_ioclass_config()
ioclass_config.add_ioclass(ioclass_id=1,
eviction_priority=1,
allocation="0.00",
rule=f"metadata")
casadm.load_io_classes(cache_id=cache.cache_id,
file=ioclass_config.default_config_file_path)
with TestRun.step("Create random file using ddrescue."):
test_file = create_file_with_ddrescue(core_dev, test_file_path)
os_utils.sync()
os_utils.drop_caches()
time.sleep(ioclass_config.MAX_CLASSIFICATION_DELAY.seconds)
with TestRun.step("Remove file and create a new one."):
cache_iostats_before = cache_dev.get_io_stats()
data_reads_before = cache.get_io_class_statistics(
io_class_id=0).block_stats.cache.reads
metadata_reads_before = cache.get_io_class_statistics(
io_class_id=1).block_stats.cache.reads
test_file.remove()
os_utils.sync()
os_utils.drop_caches()
create_file_with_ddrescue(core_dev, test_file_path)
os_utils.sync()
os_utils.drop_caches()
time.sleep(ioclass_config.MAX_CLASSIFICATION_DELAY.seconds)
with TestRun.step(
"Check using iostat that reads from cache did not occur."):
cache_iostats_after = cache_dev.get_io_stats()
data_reads_after = cache.get_io_class_statistics(
io_class_id=0).block_stats.cache.reads
metadata_reads_after = cache.get_io_class_statistics(
io_class_id=1).block_stats.cache.reads
reads_before = cache_iostats_before.sectors_read
reads_after = cache_iostats_after.sectors_read
metadata_reads_diff = metadata_reads_after - metadata_reads_before
data_reads_diff = data_reads_after - data_reads_before
iostat_diff = (reads_after - reads_before) * cache_block_size
if iostat_diff > int(metadata_reads_diff) or int(data_reads_diff) > 0:
TestRun.fail(
f"Number of reads from cache before and after removing test file "
f"differs. Sectors read before: {reads_before}, sectors read after: {reads_after}."
f"Data read from cache before {data_reads_before}, after {data_reads_after}."
f"Metadata read from cache before {metadata_reads_before}, "
f"after {metadata_reads_after}.")
else:
TestRun.LOGGER.info(
"Number of reads from cache before and after removing test file is the same."
)
def test_recovery_all_options(cache_mode, cache_line_size, cleaning_policy,
filesystem):
"""
title: Test for recovery after reset with various cache options.
description: Verify that unflushed data can be safely recovered after reset.
pass_criteria:
- CAS recovers successfully after reboot
- No data corruption
"""
with TestRun.step("Prepare cache and core devices."):
cache_disk = TestRun.disks['cache']
core_disk = TestRun.disks['core']
cache_disk.create_partitions([Size(200, Unit.MebiByte)])
core_disk.create_partitions([Size(2000, Unit.MebiByte)] * 2)
cache_device = cache_disk.partitions[0]
core_device = core_disk.partitions[0]
test_file = File(os.path.join(mount_point, filename))
file_operation(test_file.full_path, pattern, ReadWrite.write)
file_md5 = test_file.md5sum()
with TestRun.step(f"Make {filesystem} on core device."):
core_device.create_filesystem(filesystem)
with TestRun.step("Mount core device."):
core_device.mount(mount_point)
file_operation(test_file.full_path, other_pattern, ReadWrite.write)
os_utils.drop_caches(DropCachesMode.ALL)
with TestRun.step("Unmount core device."):
core_device.unmount()
with TestRun.step(
f"Start cache in {cache_mode.name} with given configuration."):
cache = casadm.start_cache(cache_device,
cache_mode,
cache_line_size,
force=True)
cache.set_cleaning_policy(cleaning_policy)
if cleaning_policy == CleaningPolicy.acp:
cache.set_params_acp(
FlushParametersAcp(wake_up_time=Time(seconds=1)))
with TestRun.step("Add core."):
core = cache.add_core(core_device)
with TestRun.step("Mount CAS device."):
core.mount(mount_point)
file_operation(test_file.full_path, pattern, ReadWrite.write)
with TestRun.step(
"Change cache mode to Write-Through without flush option."):
cache.set_cache_mode(CacheMode.WT, flush=False)
with TestRun.step("Reset platform."):
os_utils.sync()
core.unmount()
TestRun.LOGGER.info(
f"Number of dirty blocks in cache: {cache.get_dirty_blocks()}")
power_cycle_dut()
with TestRun.step("Try to start cache without load and force option."):
try:
casadm.start_cache(cache_device, cache_mode, cache_line_size)
TestRun.fail("Cache started without load or force option.")
except Exception:
TestRun.LOGGER.info(
"Cache did not start without load and force option.")
with TestRun.step("Load cache and stop it with flush."):
cache = casadm.load_cache(cache_device)
cache.stop()
with TestRun.step("Check md5sum of tested file on core device."):
core_device.mount(mount_point)
cas_md5 = test_file.md5sum()
core_device.unmount()
if cas_md5 == file_md5:
TestRun.LOGGER.info(
"Source and target file checksums are identical.")
else:
TestRun.fail("Source and target file checksums are different.")
def test_ioclass_directory_depth(filesystem):
"""
title: Test IO classification by directory.
description: |
Test if directory classification works properly for deeply nested directories for read and
write operations.
pass_criteria:
- No kernel bug.
- Read and write operations to directories are classified properly.
"""
base_dir_path = f"{mountpoint}/base_dir"
with TestRun.step("Prepare cache and core."):
cache, core = prepare()
Udev.disable()
with TestRun.step(
f"Prepare {filesystem.name} filesystem and mount {core.system_path} "
f"at {mountpoint}."):
core.create_filesystem(filesystem)
core.mount(mountpoint)
sync()
with TestRun.step(f"Create the base directory: {base_dir_path}."):
fs_utils.create_directory(base_dir_path)
with TestRun.step(f"Create a nested directory."):
nested_dir_path = base_dir_path
random_depth = random.randint(40, 80)
for i in range(random_depth):
nested_dir_path += f"/dir_{i}"
fs_utils.create_directory(path=nested_dir_path, parents=True)
# Test classification in nested dir by reading a previously unclassified file
with TestRun.step("Create the first file in the nested directory."):
test_file_1 = File(f"{nested_dir_path}/test_file_1")
dd = (Dd().input("/dev/urandom").output(test_file_1.full_path).count(
random.randint(1, 200)).block_size(Size(1, Unit.MebiByte)))
dd.run()
sync()
drop_caches(DropCachesMode.ALL)
test_file_1.refresh_item()
with TestRun.step("Load IO class config."):
ioclass_id = random.randint(1, ioclass_config.MAX_IO_CLASS_ID)
# directory IO class
ioclass_config.add_ioclass(
ioclass_id=ioclass_id,
eviction_priority=1,
allocation=True,
rule=f"directory:{base_dir_path}",
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id,
file=ioclass_config_path)
with TestRun.step("Read the file in the nested directory"):
base_occupancy = cache.get_io_class_statistics(
io_class_id=ioclass_id).usage_stats.occupancy
dd = (Dd().input(test_file_1.full_path).output("/dev/null").block_size(
Size(1, Unit.MebiByte)))
dd.run()
with TestRun.step("Check occupancy after creating the file."):
new_occupancy = cache.get_io_class_statistics(
io_class_id=ioclass_id).usage_stats.occupancy
if new_occupancy != base_occupancy + test_file_1.size:
TestRun.LOGGER.error(
"Wrong occupancy after reading file!\n"
"Expected: {base_occupancy + test_file_1.size}, "
f"actual: {new_occupancy}")
# Test classification in nested dir by creating a file
with TestRun.step("Create the second file in the nested directory"):
base_occupancy = new_occupancy
test_file_2 = File(f"{nested_dir_path}/test_file_2")
dd = (Dd().input("/dev/urandom").output(test_file_2.full_path).count(
random.randint(1, 200)).block_size(Size(1, Unit.MebiByte)))
dd.run()
sync()
drop_caches(DropCachesMode.ALL)
test_file_2.refresh_item()
with TestRun.step("Check occupancy after creating the second file."):
new_occupancy = cache.get_io_class_statistics(
io_class_id=ioclass_id).usage_stats.occupancy
if new_occupancy != base_occupancy + test_file_2.size:
TestRun.LOGGER.error(
"Wrong occupancy after creating file!\n"
f"Expected: {base_occupancy + test_file_2.size}, "
f"actual: {new_occupancy}")
def test_init_reboot_runlevels(runlevel, cache_mode):
"""
title: Initialize CAS devices after reboot
description: |
Verify that CAS init script starts cache properly after reboot in different runlevels.
pass_criteria:
- Cache should be loaded successfully after reboot.
"""
with TestRun.step(f"Set runlevel to {runlevel.value}."):
os_utils.change_runlevel(runlevel)
with TestRun.step("Prepare CAS device."):
cache_disk = TestRun.disks['cache']
cache_disk.create_partitions([Size(2, Unit.GibiByte)])
cache_dev = cache_disk.partitions[0]
core_disk = TestRun.disks['core']
core_disk.create_partitions([Size(1, Unit.GibiByte)])
core_dev = core_disk.partitions[0]
cache = casadm.start_cache(cache_dev, cache_mode, force=True)
core = cache.add_core(core_dev)
with TestRun.step("Create CAS init config based on running configuration."):
InitConfig.create_init_config_from_running_configuration()
with TestRun.step("Make filesystem on CAS device and mount it."):
core.create_filesystem(Filesystem.xfs)
core.mount(mount_point)
with TestRun.step("Start writing file to CAS."):
fio = Fio().create_command()\
.file_name(os.path.join(mount_point, "test_file"))\
.read_write(ReadWrite.randwrite)\
.io_engine(IoEngine.sync)\
.num_jobs(1).direct()\
.file_size(Size(30, Unit.GibiByte))
fio.run_in_background()
os_utils.sync()
os_utils.drop_caches()
time.sleep(10)
TestRun.executor.run_expect_success("pgrep fio")
with TestRun.step("Reboot machine during writing a file."):
TestRun.executor.reboot()
with TestRun.step("Check if cache was properly started at boot time"):
# Wait for CAS to load after boot
time.sleep(60)
caches = casadm_parser.get_caches()
if len(caches) == 1:
TestRun.LOGGER.info("Cache started properly at boot time.")
else:
TestRun.LOGGER.error("Cache did not start properly at boot time.")
with TestRun.step("Stop cache and set default runlevel."):
if len(caches) != 0:
casadm.stop_all_caches()
os_utils.change_runlevel(Runlevel.runlevel3)
TestRun.executor.reboot()
def test_interrupt_cache_stop(cache_mode, filesystem):
"""
title: Test if OpenCAS works correctly after cache stopping interruption.
description: |
Negative test of the ability of OpenCAS to handle cache's stop interruption.
pass_criteria:
- No system crash.
- Flushing would be stopped after interruption.
- Md5sum are correct during all test steps.
- Dirty blocks quantity after interruption is lower but non-zero.
"""
with TestRun.step("Prepare cache and core."):
cache_part, core_part = prepare()
for _ in TestRun.iteration(
range(iterations_per_config),
f"Reload cache configuration {iterations_per_config} times."):
with TestRun.step("Start cache."):
cache = casadm.start_cache(cache_part, cache_mode, force=True)
with TestRun.step("Set cleaning policy to NOP."):
cache.set_cleaning_policy(CleaningPolicy.nop)
with TestRun.step(
f"Add core device with {filesystem} filesystem and mount it."):
core_part.create_filesystem(filesystem)
core = cache.add_core(core_part)
core.mount(mount_point)
with TestRun.step(
f"Create test file in mount point of exported object."):
test_file = create_test_file()
with TestRun.step("Check md5 sum of test file."):
test_file_md5sum_before = test_file.md5sum()
with TestRun.step(
"Get number of dirty data on exported object before interruption."
):
os_utils.sync()
os_utils.drop_caches(DropCachesMode.ALL)
cache_dirty_blocks_before = cache.get_dirty_blocks()
with TestRun.step("Unmount core."):
core.unmount()
with TestRun.step("Start stopping cache."):
flush_pid = TestRun.executor.run_in_background(
cli.stop_cmd(str(cache.cache_id)))
sleep(2)
with TestRun.step("Interrupt cache stopping."):
percentage = casadm_parser.get_flushing_progress(
cache.cache_id, core.core_id)
while percentage < 50:
percentage = casadm_parser.get_flushing_progress(
cache.cache_id, core.core_id)
TestRun.executor.run(f"kill -s SIGINT {flush_pid}")
with TestRun.step(
"Check number of dirty data on exported object after interruption."
):
cache_dirty_blocks_after = cache.get_dirty_blocks()
if cache_dirty_blocks_after >= cache_dirty_blocks_before:
TestRun.LOGGER.error(
"Quantity of dirty lines after cache stop interruption "
"should be lower.")
if int(cache_dirty_blocks_after) == 0:
TestRun.LOGGER.error(
"Quantity of dirty lines after cache stop interruption "
"should not be zero.")
with TestRun.step("Stop cache."):
cache.stop()
with TestRun.step("Mount core device."):
core_part.mount(mount_point)
with TestRun.step("Check md5 sum of test file again."):
if test_file_md5sum_before != test_file.md5sum():
TestRun.LOGGER.error("Md5 sums before and after interrupting"
" cache stop are different.")
with TestRun.step("Unmount core device."):
core_part.unmount()
def test_ioclass_directory_file_operations(filesystem):
"""
Test if directory classification works properly after file operations like move or rename.
The operations themselves should not cause reclassification but IO after those operations
should be reclassified to proper IO class.
"""
def check_occupancy(expected: Size, actual: Size):
if expected != actual:
pytest.xfail("Occupancy check failed!\n"
f"Expected: {expected}, actual: {actual}")
cache, core = prepare()
Udev.disable()
test_dir_path = f"{mountpoint}/test_dir"
nested_dir_path = f"{test_dir_path}/nested_dir"
dd_blocks = random.randint(5, 50)
ioclass_id = random.randint(1, ioclass_config.MAX_IO_CLASS_ID)
# directory IO class
ioclass_config.add_ioclass(
ioclass_id=ioclass_id,
eviction_priority=1,
allocation=True,
rule=f"directory:{test_dir_path}",
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
TestRun.LOGGER.info(f"Preparing {filesystem.name} filesystem "
f"and mounting {core.system_path} at {mountpoint}")
core.create_filesystem(fs_type=filesystem)
core.mount(mount_point=mountpoint)
sync()
TestRun.LOGGER.info(f"Creating directory {nested_dir_path}")
Directory.create_directory(path=nested_dir_path, parents=True)
sync()
drop_caches(DropCachesMode.ALL)
TestRun.LOGGER.info("Creating test file")
classified_before = cache.get_statistics_deprecated(
io_class_id=ioclass_id)["occupancy"]
file_path = f"{test_dir_path}/test_file"
(Dd().input("/dev/urandom").output(file_path).oflag("sync").block_size(
Size(1, Unit.MebiByte)).count(dd_blocks).run())
sync()
drop_caches(DropCachesMode.ALL)
test_file = File(file_path).refresh_item()
TestRun.LOGGER.info("Checking classified occupancy")
classified_after = cache.get_statistics_deprecated(
io_class_id=ioclass_id)["occupancy"]
check_occupancy(classified_before + test_file.size, classified_after)
TestRun.LOGGER.info("Moving test file out of classified directory")
classified_before = classified_after
non_classified_before = cache.get_statistics_deprecated(
io_class_id=0)["occupancy"]
test_file.move(destination=mountpoint)
sync()
drop_caches(DropCachesMode.ALL)
TestRun.LOGGER.info("Checking classified occupancy")
classified_after = cache.get_statistics_deprecated(
io_class_id=ioclass_id)["occupancy"]
check_occupancy(classified_before, classified_after)
TestRun.LOGGER.info("Checking non-classified occupancy")
non_classified_after = cache.get_statistics_deprecated(
io_class_id=0)["occupancy"]
check_occupancy(non_classified_before, non_classified_after)
TestRun.LOGGER.info("Reading test file")
classified_before = classified_after
non_classified_before = non_classified_after
(Dd().input(test_file.full_path).output("/dev/null").block_size(
Size(1, Unit.MebiByte)).run())
TestRun.LOGGER.info("Checking classified occupancy")
classified_after = cache.get_statistics_deprecated(
io_class_id=ioclass_id)["occupancy"]
check_occupancy(classified_before - test_file.size, classified_after)
TestRun.LOGGER.info("Checking non-classified occupancy")
non_classified_after = cache.get_statistics_deprecated(
io_class_id=0)["occupancy"]
check_occupancy(non_classified_before + test_file.size,
non_classified_after)
TestRun.LOGGER.info(f"Moving test file to {nested_dir_path}")
classified_before = classified_after
non_classified_before = non_classified_after
test_file.move(destination=nested_dir_path)
sync()
drop_caches(DropCachesMode.ALL)
TestRun.LOGGER.info("Checking classified occupancy")
classified_after = cache.get_statistics_deprecated(
io_class_id=ioclass_id)["occupancy"]
check_occupancy(classified_before, classified_after)
TestRun.LOGGER.info("Checking non-classified occupancy")
non_classified_after = cache.get_statistics_deprecated(
io_class_id=0)["occupancy"]
check_occupancy(non_classified_before, non_classified_after)
TestRun.LOGGER.info("Reading test file")
classified_before = classified_after
non_classified_before = non_classified_after
(Dd().input(test_file.full_path).output("/dev/null").block_size(
Size(1, Unit.MebiByte)).run())
TestRun.LOGGER.info("Checking classified occupancy")
classified_after = cache.get_statistics_deprecated(
io_class_id=ioclass_id)["occupancy"]
check_occupancy(classified_before + test_file.size, classified_after)
TestRun.LOGGER.info("Checking non-classified occupancy")
non_classified_after = cache.get_statistics_deprecated(
io_class_id=0)["occupancy"]
check_occupancy(non_classified_before - test_file.size,
non_classified_after)
def test_ioclass_core_id(filesystem):
"""
title: Test for `core_id` classification rule
description: |
Test if IO to core with selective allocation enabled is cached and IO to core with
selective allocation disabled is redirected to pass-through mode
pass_criteria:
- IO to core with enabled selective allocation is cached
- IO to core with disabled selective allocation is not cached
"""
fs_info = f"with {filesystem}" if filesystem else ""
with TestRun.step(
f"Start cache with two cores on created partitions {fs_info}, "
"with NOP, disabled seq cutoff"):
cache, cores = prepare(filesystem, 2)
core_1, core_2 = cores[0], cores[1]
with TestRun.step(f"Add core_id based classification rules"):
cached_ioclass_id = 11
not_cached_ioclass_id = 12
ioclass_config.add_ioclass(
ioclass_id=cached_ioclass_id,
eviction_priority=22,
allocation="1.00",
rule=f"core_id:eq:{core_1.core_id}&done",
ioclass_config_path=ioclass_config.default_config_file_path,
)
ioclass_config.add_ioclass(
ioclass_id=not_cached_ioclass_id,
eviction_priority=22,
allocation="0.00",
rule=f"core_id:eq:{core_2.core_id}&done",
ioclass_config_path=ioclass_config.default_config_file_path,
)
with TestRun.step(f"Load ioclass config file"):
casadm.load_io_classes(cache_id=cache.cache_id,
file=ioclass_config.default_config_file_path)
if filesystem:
with TestRun.step(f"Mount cores"):
core_1.mount(cached_mountpoint)
core_2.mount(not_cached_mountpoint)
with TestRun.step(f"Reset counters"):
sync()
drop_caches()
cache.purge_cache()
cache.reset_counters()
with TestRun.step(f"Trigger IO to both cores"):
if filesystem:
dd_dst_paths = [
cached_mountpoint + "/test_file",
not_cached_mountpoint + "/test_file"
]
else:
dd_dst_paths = [core_1.path, core_2.path]
for path in dd_dst_paths:
dd = (Dd().input("/dev/zero").output(path).count(
dd_count).block_size(dd_bs).oflag("sync"))
dd.run()
sync()
drop_caches()
with TestRun.step(f"Check cores occupancy"):
dd_size = (dd_bs * dd_count).set_unit(Unit.Blocks4096)
core_1_occupancy = core_1.get_statistics().usage_stats.occupancy
core_2_occupancy = core_2.get_statistics().usage_stats.occupancy
if core_1_occupancy < dd_size:
TestRun.LOGGER.error(
f"First core's occupancy is {core_1_occupancy} "
f"- it is less than {dd_size} - triggerd IO size!")
if core_2_occupancy.get_value() != 0:
TestRun.LOGGER.error(
f"First core's occupancy is {core_2_occupancy} instead of 0!")
with TestRun.step(f"Check ioclasses occupancy"):
cached_ioclass_occupancy = cache.get_io_class_statistics(
io_class_id=cached_ioclass_id).usage_stats.occupancy
not_cached_ioclass_occupancy = cache.get_io_class_statistics(
io_class_id=not_cached_ioclass_id).usage_stats.occupancy
if cached_ioclass_occupancy < dd_size:
TestRun.LOGGER.error(
f"Cached ioclass occupancy is {cached_ioclass_occupancy} "
f"- it is less than {dd_size} - triggerd IO size!")
if not_cached_ioclass_occupancy.get_value() != 0:
TestRun.LOGGER.error(
f"Not cached ioclass occupancy is {not_cached_ioclass_occupancy} instead of 0!"
)
with TestRun.step(f"Check number of serviced requests to not cached core"):
core_2_serviced_requests = core_2.get_statistics(
).request_stats.requests_serviced
if core_2_serviced_requests != 0:
TestRun.LOGGER.error(
f"Second core should have 0 serviced requests "
f"instead of {core_2_serviced_requests}")
def test_ioclass_directory_dir_operations(filesystem):
"""
Test if directory classification works properly after directory operations like move or rename.
The operations themselves should not cause reclassification but IO after those operations
should be reclassified to proper IO class.
Directory classification may work with a delay after loading IO class configuration or
move/rename operations. Test checks if maximum delay is not exceeded.
"""
def create_files_with_classification_delay_check(directory: Directory,
ioclass_id: int):
start_time = datetime.now()
occupancy_after = cache.get_statistics_deprecated(
io_class_id=ioclass_id)["occupancy"]
dd_blocks = 10
dd_size = Size(dd_blocks, Unit.Blocks4096)
file_counter = 0
unclassified_files = []
time_from_start = datetime.now() - start_time
while time_from_start < ioclass_config.MAX_CLASSIFICATION_DELAY:
occupancy_before = occupancy_after
file_path = f"{directory.full_path}/test_file_{file_counter}"
file_counter += 1
time_from_start = datetime.now() - start_time
(Dd().input(
"/dev/zero").output(file_path).oflag("sync").block_size(
Size(1, Unit.Blocks4096)).count(dd_blocks).run())
occupancy_after = cache.get_statistics_deprecated(
io_class_id=ioclass_id)["occupancy"]
if occupancy_after - occupancy_before < dd_size:
unclassified_files.append(file_path)
if len(unclassified_files) == file_counter:
pytest.xfail(
"No files were properly classified within max delay time!")
if len(unclassified_files):
TestRun.LOGGER.info("Rewriting unclassified test files...")
for file_path in unclassified_files:
(Dd().input("/dev/zero").output(
file_path).oflag("sync").block_size(
Size(1, Unit.Blocks4096)).count(dd_blocks).run())
def read_files_with_reclassification_check(target_ioclass_id: int,
source_ioclass_id: int,
directory: Directory,
with_delay: bool):
start_time = datetime.now()
target_occupancy_after = cache.get_statistics_deprecated(
io_class_id=target_ioclass_id)["occupancy"]
source_occupancy_after = cache.get_statistics_deprecated(
io_class_id=source_ioclass_id)["occupancy"]
unclassified_files = []
for file in [
item for item in directory.ls() if isinstance(item, File)
]:
target_occupancy_before = target_occupancy_after
source_occupancy_before = source_occupancy_after
time_from_start = datetime.now() - start_time
(Dd().input(file.full_path).output("/dev/null").block_size(
Size(1, Unit.Blocks4096)).run())
target_occupancy_after = cache.get_statistics_deprecated(
io_class_id=target_ioclass_id)["occupancy"]
source_occupancy_after = cache.get_statistics_deprecated(
io_class_id=source_ioclass_id)["occupancy"]
if target_occupancy_after < target_occupancy_before:
pytest.xfail("Target IO class occupancy lowered!")
elif target_occupancy_after - target_occupancy_before < file.size:
unclassified_files.append(file)
if with_delay and time_from_start <= ioclass_config.MAX_CLASSIFICATION_DELAY:
continue
pytest.xfail("Target IO class occupancy not changed properly!")
if source_occupancy_after >= source_occupancy_before:
if file not in unclassified_files:
unclassified_files.append(file)
if with_delay and time_from_start <= ioclass_config.MAX_CLASSIFICATION_DELAY:
continue
pytest.xfail("Source IO class occupancy not changed properly!")
if len(unclassified_files):
TestRun.LOGGER.info("Rereading unclassified test files...")
sync()
drop_caches(DropCachesMode.ALL)
for file in unclassified_files:
(Dd().input(file.full_path).output("/dev/null").block_size(
Size(1, Unit.Blocks4096)).run())
cache, core = prepare()
Udev.disable()
proper_ids = random.sample(range(1, ioclass_config.MAX_IO_CLASS_ID + 1), 2)
ioclass_id_1 = proper_ids[0]
classified_dir_path_1 = f"{mountpoint}/dir_{ioclass_id_1}"
ioclass_id_2 = proper_ids[1]
classified_dir_path_2 = f"{mountpoint}/dir_{ioclass_id_2}"
# directory IO classes
ioclass_config.add_ioclass(
ioclass_id=ioclass_id_1,
eviction_priority=1,
allocation=True,
rule=f"directory:{classified_dir_path_1}",
ioclass_config_path=ioclass_config_path,
)
ioclass_config.add_ioclass(
ioclass_id=ioclass_id_2,
eviction_priority=1,
allocation=True,
rule=f"directory:{classified_dir_path_2}",
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
TestRun.LOGGER.info(f"Preparing {filesystem.name} filesystem "
f"and mounting {core.system_path} at {mountpoint}")
core.create_filesystem(fs_type=filesystem)
core.mount(mount_point=mountpoint)
sync()
non_classified_dir_path = f"{mountpoint}/non_classified"
TestRun.LOGGER.info(
f"Creating a non-classified directory: {non_classified_dir_path}")
dir_1 = Directory.create_directory(path=non_classified_dir_path)
TestRun.LOGGER.info(
f"Renaming {non_classified_dir_path} to {classified_dir_path_1}")
dir_1.move(destination=classified_dir_path_1)
TestRun.LOGGER.info("Creating files with delay check")
create_files_with_classification_delay_check(directory=dir_1,
ioclass_id=ioclass_id_1)
TestRun.LOGGER.info(f"Creating {classified_dir_path_2}/subdir")
dir_2 = Directory.create_directory(path=f"{classified_dir_path_2}/subdir",
parents=True)
TestRun.LOGGER.info("Creating files with delay check")
create_files_with_classification_delay_check(directory=dir_2,
ioclass_id=ioclass_id_2)
sync()
drop_caches(DropCachesMode.ALL)
TestRun.LOGGER.info(f"Moving {dir_2.full_path} to {classified_dir_path_1}")
dir_2.move(destination=classified_dir_path_1)
TestRun.LOGGER.info("Reading files with reclassification check")
read_files_with_reclassification_check(target_ioclass_id=ioclass_id_1,
source_ioclass_id=ioclass_id_2,
directory=dir_2,
with_delay=False)
sync()
drop_caches(DropCachesMode.ALL)
TestRun.LOGGER.info(f"Moving {dir_2.full_path} to {mountpoint}")
dir_2.move(destination=mountpoint)
TestRun.LOGGER.info("Reading files with reclassification check")
read_files_with_reclassification_check(target_ioclass_id=0,
source_ioclass_id=ioclass_id_1,
directory=dir_2,
with_delay=False)
TestRun.LOGGER.info(f"Removing {classified_dir_path_2}")
fs_utils.remove(path=classified_dir_path_2, force=True, recursive=True)
sync()
drop_caches(DropCachesMode.ALL)
TestRun.LOGGER.info(
f"Renaming {classified_dir_path_1} to {classified_dir_path_2}")
dir_1.move(destination=classified_dir_path_2)
TestRun.LOGGER.info("Reading files with reclassification check")
read_files_with_reclassification_check(target_ioclass_id=ioclass_id_2,
source_ioclass_id=ioclass_id_1,
directory=dir_1,
with_delay=True)
TestRun.LOGGER.info(
f"Renaming {classified_dir_path_2} to {non_classified_dir_path}")
dir_1.move(destination=non_classified_dir_path)
TestRun.LOGGER.info("Reading files with reclassification check")
read_files_with_reclassification_check(target_ioclass_id=0,
source_ioclass_id=ioclass_id_2,
directory=dir_1,
with_delay=True)
