def prepare():
base_prepare()
ioclass_config.remove_ioclass_config()
cache_device = next(
disk
for disk in TestProperties.dut.disks
if disk.disk_type in [DiskType.optane, DiskType.nand]
)
core_device = next(
disk
for disk in TestProperties.dut.disks
if (disk.disk_type.value > cache_device.disk_type.value and disk != cache_device)
)
cache_device.create_partitions([Size(500, Unit.MebiByte)])
core_device.create_partitions([Size(2, Unit.GibiByte)])
cache_device = cache_device.partitions[0]
core_device_1 = core_device.partitions[0]
Udev.disable()
TestProperties.LOGGER.info(f"Staring cache")
cache = casadm.start_cache(cache_device, force=True)
TestProperties.LOGGER.info(f"Setting cleaning policy to NOP")
cache.set_cleaning_policy(CleaningPolicy.nop)
TestProperties.LOGGER.info(f"Adding core devices")
core = cache.add_core(core_dev=core_device_1)
output = TestProperties.executor.execute(f"mkdir -p {mountpoint}")
if output.exit_code != 0:
raise Exception(f"Failed to create mountpoint")
return cache, core
def prepare(cache_mode: CacheMode):
ioclass_config.remove_ioclass_config()
cache_device = TestRun.disks['cache']
core_device = TestRun.disks['core']
cache_device.create_partitions([Size(500, Unit.MebiByte)])
core_device.create_partitions([
Size(1, Unit.GibiByte),
Size(1, Unit.GibiByte),
Size(1, Unit.GibiByte)
])
cache_device = cache_device.partitions[0]
core_device_1 = core_device.partitions[0]
core_device_2 = core_device.partitions[1]
core_device_3 = core_device.partitions[2]
Udev.disable()
TestRun.LOGGER.info(f"Starting cache")
cache = casadm.start_cache(cache_device, cache_mode=cache_mode, force=True)
TestRun.LOGGER.info(f"Setting cleaning policy to NOP")
casadm.set_param_cleaning(cache_id=cache_id, policy=CleaningPolicy.nop)
TestRun.LOGGER.info(f"Adding core devices")
core_1 = cache.add_core(core_dev=core_device_1)
core_2 = cache.add_core(core_dev=core_device_2)
core_3 = cache.add_core(core_dev=core_device_3)
output = TestRun.executor.run(f"mkdir -p {mountpoint}")
if output.exit_code != 0:
raise Exception(f"Failed to create mountpoint")
return cache, [core_1, core_2, core_3]
def test_ioclass_process_name(prepare_and_cleanup):
"""Check if data generated by process with particular name is cached"""
cache, core = prepare()
ioclass_id = 1
dd_size = Size(4, Unit.KibiByte)
dd_count = 1
iterations = 100
ioclass_config.add_ioclass(
ioclass_id=ioclass_id,
eviction_priority=1,
allocation=True,
rule=f"process_name:dd&done",
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
cache.flush_cache()
Udev.disable()
TestProperties.LOGGER.info(
f"Check if all data generated by dd process is cached.")
for i in range(iterations):
dd = (Dd().input("/dev/zero").output(
core.system_path).count(dd_count).block_size(dd_size).seek(i))
dd.run()
sync()
time.sleep(0.1)
stats = cache.get_cache_statistics(io_class_id=ioclass_id)
assert stats["dirty"].get_value(Unit.Blocks4096) == (i + 1) * dd_count
def test_cleaning_policies_in_write_back(cleaning_policy):
"""
title: Test for cleaning policy operation in Write-Back cache mode.
description: |
Check if ALRU, NOP and ACP cleaning policies preserve their
parameters when changed and if they flush dirty data properly
in Write-Back cache mode.
pass_criteria:
- Flush parameters preserve their values when changed.
- Dirty data is flushed or not according to the policy used.
"""
with TestRun.step("Partition cache and core devices"):
cache_dev, core_dev = storage_prepare()
Udev.disable()
with TestRun.step(
f"Start cache in Write-Back mode with {cleaning_policy} cleaning policy"
):
cache = casadm.start_cache(cache_dev.partitions[0],
CacheMode.WB,
force=True)
set_cleaning_policy_and_params(cache, cleaning_policy)
with TestRun.step("Check for running CAS cleaner"):
if TestRun.executor.run(
f"pgrep {cas_cleaner_process_name}").exit_code != 0:
TestRun.fail("CAS cleaner process is not running!")
with TestRun.step(f"Add {cores_count} cores to the cache"):
core = []
for i in range(cores_count):
core.append(cache.add_core(core_dev.partitions[i]))
with TestRun.step("Run 'fio'"):
fio = fio_prepare()
for i in range(cores_count):
fio.add_job().target(core[i].path)
fio.run()
time.sleep(3)
core_writes_before_wait_for_cleaning = (
cache.get_statistics().block_stats.core.writes)
with TestRun.step(f"Wait {time_to_wait} seconds"):
time.sleep(time_to_wait)
with TestRun.step("Check write statistics for core device"):
core_writes_after_wait_for_cleaning = (
cache.get_statistics().block_stats.core.writes)
check_cleaning_policy_operation(
cleaning_policy,
core_writes_before_wait_for_cleaning,
core_writes_after_wait_for_cleaning,
)
with TestRun.step("Stop all caches"):
casadm.stop_all_caches()
Udev.enable()
def test_trim_eviction(cache_mode, cache_line_size, filesystem, cleaning):
"""
title: Test verifying if trim requests do not cause eviction on CAS device.
description: |
When trim requests enabled and files are being added and removed from CAS device,
there is no eviction (no reads from cache).
pass_criteria:
- Reads from cache device are the same before and after removing test file.
"""
mount_point = "/mnt"
test_file_path = os.path.join(mount_point, "test_file")
with TestRun.step("Prepare devices."):
cache_disk = TestRun.disks['cache']
cache_disk.create_partitions([Size(1, Unit.GibiByte)])
cache_dev = cache_disk.partitions[0]
core_disk = TestRun.disks['core']
core_disk.create_partitions([Size(1, Unit.GibiByte)])
core_dev = core_disk.partitions[0]
with TestRun.step("Start cache on device supporting trim and add core."):
cache = casadm.start_cache(cache_dev, cache_mode, cache_line_size)
cache.set_cleaning_policy(cleaning)
Udev.disable()
core = cache.add_core(core_dev)
with TestRun.step("Create filesystem on CAS device and mount it."):
core.create_filesystem(filesystem)
core.mount(mount_point, ["discard"])
with TestRun.step("Create random file using ddrescue."):
test_file = fs_utils.create_random_test_file(test_file_path,
core_dev.size * 0.9)
create_file_with_ddrescue(core_dev, test_file)
with TestRun.step("Remove file and create a new one."):
cache_iostats_before = cache_dev.get_io_stats()
test_file.remove()
os_utils.sync()
os_utils.drop_caches()
create_file_with_ddrescue(core_dev, test_file)
with TestRun.step(
"Check using iostat that reads from cache did not occur."):
cache_iostats_after = cache_dev.get_io_stats()
reads_before = cache_iostats_before.sectors_read
reads_after = cache_iostats_after.sectors_read
if reads_after != reads_before:
TestRun.fail(
f"Number of reads from cache before and after removing test file "
f"differs. Reads before: {reads_before}, reads after: {reads_after}."
)
else:
TestRun.LOGGER.info(
"Number of reads from cache before and after removing test file is the same."
)
def test_one_core_release(cache_mode):
"""
title: Test if OpenCAS dynamically allocates space according to core devices needs.
description: |
When one or more core devices are unused in a single cache instance all blocks
previously occupied should be available to other core devices.
Test is without pass through mode.
pass_criteria:
- No system crash.
- The remaining core is able to use cache.
- OpenCAS frees blocks occupied by unused core and allocates it to the remaining core.
"""
with TestRun.step("Prepare two cache and one core devices."):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions([Size(512, Unit.MebiByte)])
cache_part = cache_dev.partitions[0]
core_dev = TestRun.disks['core']
core_dev.create_partitions([Size(1, Unit.GibiByte)] * 2)
core_part1 = core_dev.partitions[0]
core_part2 = core_dev.partitions[1]
Udev.disable()
with TestRun.step("Start cache"):
cache = casadm.start_cache(cache_part, cache_mode, force=True)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(f"Expected caches count: 1; Actual caches count: {caches_count}.")
with TestRun.step("Add both core devices to cache."):
core1 = cache.add_core(core_part1)
core2 = cache.add_core(core_part2)
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 2:
TestRun.fail(f"Expected cores count: 2; Actual cores count: {cores_count}.")
with TestRun.step("Change sequential cutoff policy to 'never'."):
cache.set_seq_cutoff_policy(SeqCutOffPolicy.never)
with TestRun.step("Fill cache with pages from the first core."):
dd_builder(cache_mode, core1, cache.size).run()
core1_occupied_blocks_before = core1.get_occupancy()
with TestRun.step("Check if the remaining core is able to use cache."):
dd_builder(cache_mode, core2, Size(100, Unit.MebiByte)).run()
core1_occupied_blocks_after = core1.get_occupancy()
with TestRun.step("Check if occupancy from the first core is removed from cache."):
# The first core's occupancy should be lower than cache's occupancy
# by the value of the remaining core's occupancy because cache
# should reallocate blocks from unused core to used core.
if core1_occupied_blocks_after >= core1_occupied_blocks_before \
or cache.get_occupancy() <= core1_occupied_blocks_after \
or not float(core2.get_occupancy().get_value()) > 0:
TestRun.LOGGER.error("Blocks previously occupied by the first core aren't released.")
with TestRun.step("Stop cache."):
casadm.stop_all_caches()
def prepare():
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions([cache_size])
cache_part = cache_dev.partitions[0]
core_dev = TestRun.disks['core']
core_dev.create_partitions([cache_size * 2])
core_part = core_dev.partitions[0]
Udev.disable()
return cache_part, core_part
def test_ioclass_pid():
cache, core = prepare()
ioclass_id = 1
iterations = 20
dd_count = 100
dd_size = Size(4, Unit.KibiByte)
Udev.disable()
# Since 'dd' has to be executed right after writing pid to 'ns_last_pid',
# 'dd' command is created and is appended to 'echo' command instead of running it
dd_command = str(
Dd()
.input("/dev/zero")
.output(core.system_path)
.count(dd_count)
.block_size(dd_size)
)
for i in range(iterations):
cache.flush_cache()
output = TestRun.executor.run("cat /proc/sys/kernel/ns_last_pid")
if output.exit_code != 0:
raise Exception(
f"Failed to retrieve pid. stdout: {output.stdout} \n stderr :{output.stderr}"
)
# Few pids might be used by system during test preparation
pid = int(output.stdout) + 50
ioclass_config.add_ioclass(
ioclass_id=ioclass_id,
eviction_priority=1,
allocation=True,
rule=f"pid:eq:{pid}&done",
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
TestRun.LOGGER.info(f"Running dd with pid {pid}")
# pid saved in 'ns_last_pid' has to be smaller by one than target dd pid
dd_and_pid_command = (
f"echo {pid-1} > /proc/sys/kernel/ns_last_pid && {dd_command}"
)
output = TestRun.executor.run(dd_and_pid_command)
if output.exit_code != 0:
raise Exception(
f"Failed to run dd with target pid. "
f"stdout: {output.stdout} \n stderr :{output.stderr}"
)
sync()
dirty = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.dirty
if dirty.get_value(Unit.Blocks4096) != dd_count:
TestRun.LOGGER.error(f"Wrong amount of dirty data ({dirty}).")
ioclass_config.remove_ioclass(ioclass_id)
def test_ioclass_conditions_and(filesystem):
"""
title: IO class condition 'and'.
description: |
Load config with IO class combining 5 conditions contradicting
at least one other condition.
pass_criteria:
- No kernel bug.
- Every IO fulfilling one of the conditions is not classified.
"""
file_size = Size(random.randint(25, 50), Unit.MebiByte)
file_size_bytes = int(file_size.get_value(Unit.Byte))
with TestRun.step("Prepare cache and core. Disable udev."):
cache, core = prepare()
Udev.disable()
with TestRun.step("Create and load IO class config file."):
# directories OR condition
ioclass_config.add_ioclass(
ioclass_id=1,
eviction_priority=1,
allocation=True,
rule=
f"file_size:gt:{file_size_bytes}&file_size:lt:{file_size_bytes}&"
f"file_size:ge:{file_size_bytes}&file_size:le:{file_size_bytes}&"
f"file_size:eq:{file_size_bytes}",
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id,
file=ioclass_config_path)
TestRun.LOGGER.info(f"Preparing {filesystem.name} filesystem "
f"and mounting {core.system_path} at {mountpoint}")
core.create_filesystem(filesystem)
core.mount(mountpoint)
sync()
base_occupancy = cache.get_io_class_statistics(
io_class_id=1).usage_stats.occupancy
# Perform IO
for size in [
file_size, file_size + Size(1, Unit.MebiByte),
file_size - Size(1, Unit.MebiByte)
]:
(Fio().create_command().io_engine(
IoEngine.libaio).size(size).read_write(
ReadWrite.write).target(f"{mountpoint}/test_file").run())
sync()
new_occupancy = cache.get_io_class_statistics(
io_class_id=1).usage_stats.occupancy
if new_occupancy != base_occupancy:
TestRun.fail(
"Unexpected occupancy increase!\n"
f"Expected: {base_occupancy}, actual: {new_occupancy}")
def test_ioclass_conditions_or(filesystem):
"""
title: IO class condition 'or'.
description: |
Load config with IO class combining 5 contradicting conditions connected by OR operator.
pass_criteria:
- No kernel bug.
- Every IO fulfilling one condition is classified properly.
"""
with TestRun.step("Prepare cache and core. Disable udev."):
cache, core = prepare()
Udev.disable()
with TestRun.step("Create and load IO class config file."):
# directories OR condition
ioclass_config.add_ioclass(
ioclass_id=1,
eviction_priority=1,
allocation=True,
rule=
f"directory:{mountpoint}/dir1|directory:{mountpoint}/dir2|directory:"
f"{mountpoint}/dir3|directory:{mountpoint}/dir4|directory:{mountpoint}/dir5",
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id,
file=ioclass_config_path)
with TestRun.step(f"Prepare {filesystem.name} filesystem "
f"and mount {core.system_path} at {mountpoint}."):
core.create_filesystem(filesystem)
core.mount(mountpoint)
for i in range(1, 6):
fs_utils.create_directory(f"{mountpoint}/dir{i}")
sync()
with TestRun.step(
"Perform IO fulfilling each condition and check if occupancy raises."
):
for i in range(1, 6):
file_size = Size(random.randint(25, 50), Unit.MebiByte)
base_occupancy = cache.get_io_class_statistics(
io_class_id=1).usage_stats.occupancy
(Fio().create_command().io_engine(
IoEngine.libaio).size(file_size).read_write(
ReadWrite.write).target(
f"{mountpoint}/dir{i}/test_file").run())
sync()
new_occupancy = cache.get_io_class_statistics(
io_class_id=1).usage_stats.occupancy
if new_occupancy != base_occupancy + file_size:
TestRun.fail(
"Occupancy has not increased correctly!\n"
f"Expected: {base_occupancy + file_size}, actual: {new_occupancy}"
)
def test_multistream_seq_cutoff_stress_fs(streams_seq_rand, filesystem, cache_mode):
"""
title: Stress test for multistream sequential cutoff on the device with a filesystem
description: |
Testing the stability of a system when there are multiple sequential and random I/O streams
running against the exported object with a filesystem when the sequential cutoff policy is
set to always and the sequential cutoff threshold is set to a value which is able
to be reached by sequential I/O streams.
pass_criteria:
- No system crash
"""
mount_point = "/mnt"
with TestRun.step("Prepare devices. Create filesystem on core device."):
cache_disk = TestRun.disks['cache']
core_disk = TestRun.disks['core']
core_disk.create_filesystem(filesystem)
with TestRun.step("Start cache and add core."):
cache = casadm.start_cache(cache_disk, cache_mode, force=True)
Udev.disable()
core = cache.add_core(core_disk)
with TestRun.step("Mount core."):
core.mount(mount_point)
with TestRun.step(f"Set seq-cutoff policy to always and threshold to 20MiB."):
core.set_seq_cutoff_policy(SeqCutOffPolicy.always)
core.set_seq_cutoff_threshold(Size(20, Unit.MebiByte))
with TestRun.step("Reset core statistics counters."):
core.reset_counters()
with TestRun.step("Run I/O"):
sequential_streams = streams_seq_rand[0]
random_streams = streams_seq_rand[1]
stream_size = core_disk.size / 256
fio = (Fio().create_command()
.io_engine(IoEngine.libaio)
.block_size(Size(1, Unit.Blocks4096))
.direct()
.offset_increment(stream_size))
for i in range(0, sequential_streams + random_streams):
fio_job = fio.add_job(job_name=f"stream_{i}")
fio_job.size(stream_size)
fio_job.target(os.path.join(mount_point, f"file_{i}"))
if i < sequential_streams:
fio_job.read_write(ReadWrite.write)
else:
fio_job.read_write(ReadWrite.randwrite)
pid = fio.run_in_background()
while TestRun.executor.check_if_process_exists(pid):
sleep(5)
TestRun.LOGGER.info(f"{core.get_statistics()}")
def prepare(filesystem, cores_number):
ioclass_config.remove_ioclass_config()
cache_device = TestRun.disks["cache"]
core_device = TestRun.disks["core"]
cache_device.create_partitions([Size(10, Unit.GibiByte)])
core_device.create_partitions([Size(5, Unit.GibiByte)] * cores_number)
cache_device = cache_device.partitions[0]
cache = casadm.start_cache(cache_device,
cache_mode=CacheMode.WT,
force=True)
Udev.disable()
casadm.set_param_cleaning(cache_id=cache.cache_id,
policy=CleaningPolicy.nop)
cores = []
for part in core_device.partitions:
if filesystem:
part.create_filesystem(filesystem)
cores.append(casadm.add_core(cache, core_dev=part))
cache.set_seq_cutoff_policy(SeqCutOffPolicy.never)
ioclass_config.create_ioclass_config(
add_default_rule=False,
ioclass_config_path=ioclass_config.default_config_file_path)
# To make test more precise all workload except of tested ioclass should be
# put in pass-through mode
ioclass_config.add_ioclass(
ioclass_id=0,
eviction_priority=22,
allocation="1.00",
rule="unclassified",
ioclass_config_path=ioclass_config.default_config_file_path,
)
ioclass_config.add_ioclass(
ioclass_id=1,
eviction_priority=22,
allocation="0.00",
rule="metadata",
ioclass_config_path=ioclass_config.default_config_file_path,
)
ioclass_config.add_ioclass(
ioclass_id=2,
eviction_priority=22,
allocation="0.00",
rule="direct",
ioclass_config_path=ioclass_config.default_config_file_path,
)
return cache, cores
def test_ioclass_request_size():
cache, core = prepare()
ioclass_id = 1
iterations = 100
ioclass_config.add_ioclass(
ioclass_id=ioclass_id,
eviction_priority=1,
allocation=True,
rule=f"request_size:ge:8192&request_size:le:16384&done",
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
Udev.disable()
# Check if requests with appropriate size are cached
TestRun.LOGGER.info(
f"Check if requests with size within defined range are cached")
cached_req_sizes = [Size(2, Unit.Blocks4096), Size(4, Unit.Blocks4096)]
for i in range(iterations):
cache.flush_cache()
req_size = random.choice(cached_req_sizes)
dd = (Dd().input("/dev/zero").output(
core.system_path).count(1).block_size(req_size).oflag("direct"))
dd.run()
dirty = cache.get_io_class_statistics(
io_class_id=ioclass_id).usage_stats.dirty
if dirty.get_value(
Unit.Blocks4096) != req_size.value / Unit.Blocks4096.value:
TestRun.fail("Incorrect number of dirty blocks!")
cache.flush_cache()
# Check if requests with inappropriate size are not cached
TestRun.LOGGER.info(
f"Check if requests with size outside defined range are not cached")
not_cached_req_sizes = [
Size(1, Unit.Blocks4096),
Size(8, Unit.Blocks4096),
Size(16, Unit.Blocks4096),
]
for i in range(iterations):
req_size = random.choice(not_cached_req_sizes)
dd = (Dd().input("/dev/zero").output(
core.system_path).count(1).block_size(req_size).oflag("direct"))
dd.run()
dirty = cache.get_io_class_statistics(
io_class_id=ioclass_id).usage_stats.dirty
if dirty.get_value(Unit.Blocks4096) != 0:
TestRun.fail("Dirty data present!")
def test_ioclass_request_size(prepare_and_cleanup):
cache, core = prepare()
ioclass_id = 1
iterations = 100
ioclass_config.add_ioclass(
ioclass_id=ioclass_id,
eviction_priority=1,
allocation=True,
rule=f"request_size:ge:8192&request_size:le:16384&done",
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
Udev.disable()
# Check if requests with appropriate size are cached
TestProperties.LOGGER.info(
f"Check if requests with size within defined range are cached")
cached_req_sizes = [Size(2, Unit.Blocks4096), Size(4, Unit.Blocks4096)]
for i in range(iterations):
cache.flush_cache()
req_size = random.choice(cached_req_sizes)
dd = (Dd().input("/dev/zero").output(
core.system_path).count(1).block_size(req_size).oflag("direct"))
dd.run()
stats = cache.get_cache_statistics(per_io_class=True,
io_class_id=ioclass_id)
assert (stats["dirty"].get_value(Unit.Blocks4096) == req_size.value /
Unit.Blocks4096.value)
cache.flush_cache()
# Check if requests with inappropriate size are not cached
TestProperties.LOGGER.info(
f"Check if requests with size outside defined range are not cached")
not_cached_req_sizes = [
Size(1, Unit.Blocks4096),
Size(8, Unit.Blocks4096),
Size(16, Unit.Blocks4096),
]
for i in range(iterations):
req_size = random.choice(not_cached_req_sizes)
dd = (Dd().input("/dev/zero").output(
core.system_path).count(1).block_size(req_size).oflag("direct"))
dd.run()
stats = cache.get_cache_statistics(per_io_class=True,
io_class_id=ioclass_id)
assert stats["dirty"].get_value(Unit.Blocks4096) == 0
def test_multistream_seq_cutoff_stress_raw(streams_seq_rand):
"""
title: Stress test for multistream sequential cutoff on raw device
description: |
Testing the stability of a system when there are multiple sequential and random I/O streams
running against the raw exported object with the sequential cutoff policy set to always and
the sequential cutoff threshold set to a value which is able to be reached by
sequential I/O streams.
pass_criteria:
- No system crash
"""
with TestRun.step("Start cache and add core device."):
cache_disk = TestRun.disks['cache']
core_disk = TestRun.disks['core']
cache_disk.create_partitions([Size(1.5, Unit.GibiByte)])
cache_dev = cache_disk.partitions[0]
cache = casadm.start_cache(cache_dev, CacheMode.WB, force=True)
Udev.disable()
core = cache.add_core(core_disk)
with TestRun.step(f"Set seq-cutoff policy to always and threshold to 512KiB."):
core.set_seq_cutoff_policy(SeqCutOffPolicy.always)
core.set_seq_cutoff_threshold(Size(512, Unit.KibiByte))
with TestRun.step("Reset core statistics counters."):
core.reset_counters()
with TestRun.step("Run I/O"):
stream_size = min(core_disk.size / 256, Size(256, Unit.MebiByte))
sequential_streams = streams_seq_rand[0]
random_streams = streams_seq_rand[1]
fio = (Fio().create_command()
.io_engine(IoEngine.libaio)
.block_size(Size(1, Unit.Blocks4096))
.direct()
.offset_increment(stream_size))
for i in range(0, sequential_streams + random_streams):
fio_job = fio.add_job(job_name=f"stream_{i}")
fio_job.size(stream_size)
fio_job.target(core.path)
if i < sequential_streams:
fio_job.read_write(ReadWrite.write)
else:
fio_job.read_write(ReadWrite.randwrite)
pid = fio.run_in_background()
while TestRun.executor.check_if_process_exists(pid):
sleep(5)
TestRun.LOGGER.info(f"{core.get_statistics()}")
def prepare_configuration(cache_mode, cache_line_size):
cache_device = TestRun.disks["cache"]
core_device = TestRun.disks["core"]
with TestRun.step("Creating cache partition"):
cache_device.create_partitions([Size(25, Unit.MebiByte)])
with TestRun.step("Creating cache error device"):
error_device = ErrorDevice("error", cache_device.partitions[0])
with TestRun.step("Staring cache to check metadata offset"):
cache = casadm.start_cache(error_device,
cache_line_size=cache_line_size,
force=True)
cache_size = cache.size
cache.stop()
with TestRun.step("Setting errors on non-metadata area"):
error_device.change_table(
DmTable.error_table(
offset=(cache_device.partitions[0].size -
cache_size).get_value(Unit.Blocks512),
size=cache_size,
).fill_gaps(cache_device.partitions[0]))
with TestRun.step("Create core partition with size of usable cache space"):
core_device.create_partitions([cache_size])
with TestRun.step("Starting and configuring cache"):
cache = casadm.start_cache(error_device,
cache_mode=cache_mode,
cache_line_size=cache_line_size,
force=True)
result = cache.set_seq_cutoff_policy(SeqCutOffPolicy.never)
if result.exit_code:
TestRun.LOGGER.exception("Couldn't set seq cutoff policy")
result = cache.set_cleaning_policy(CleaningPolicy.nop)
if result.exit_code:
TestRun.LOGGER.exception("Couldn't set cleaning policy")
with TestRun.step("Stopping udev"):
Udev.disable()
with TestRun.step("Adding core device"):
core = cache.add_core(core_dev=core_device.partitions[0])
return cache, core, core_device.partitions[0]
def test_flush_over_640_gibibytes_raw_device(cache_mode):
"""
title: Test of the ability to flush huge amount of dirty data on raw device.
description: |
Flush cache when amount of dirty data in cache exceeds 640 GiB.
pass_criteria:
- Flushing completes successfully without any errors.
"""
with TestRun.step("Prepare devices for cache and core."):
cache_dev = TestRun.disks['cache']
check_disk_size(cache_dev)
cache_dev.create_partitions([required_disk_size])
cache_part = cache_dev.partitions[0]
core_dev = TestRun.disks['core']
check_disk_size(core_dev)
Udev.disable()
with TestRun.step(f"Start cache in {cache_mode} mode."):
cache = casadm.start_cache(cache_part, cache_mode)
with TestRun.step(f"Add core to cache."):
core = cache.add_core(core_dev)
with TestRun.step("Disable cleaning and sequential cutoff."):
cache.set_cleaning_policy(CleaningPolicy.nop)
cache.set_seq_cutoff_policy(SeqCutOffPolicy.never)
with TestRun.step("Create test file"):
fio = (Fio().create_command().io_engine(IoEngine.libaio).read_write(
ReadWrite.write).block_size(bs).direct().io_depth(256).target(
core).size(file_size))
fio.default_run_time = timedelta(
hours=4) # timeout for non-time-based fio
fio.run()
with TestRun.step(f"Check if dirty data exceeded {file_size * 0.98} GiB."):
minimum_4KiB_blocks = int(
(file_size * 0.98).get_value(Unit.Blocks4096))
if int(cache.get_statistics().usage_stats.dirty) < minimum_4KiB_blocks:
TestRun.fail("There is not enough dirty data in the cache!")
with TestRun.step("Stop cache with flush."):
# this operation could take few hours, depending on core disk
output = TestRun.executor.run(stop_cmd(str(cache.cache_id)),
timedelta(hours=12))
if output.exit_code != 0:
TestRun.fail(f"Stopping cache with flush failed!\n{output.stderr}")
def prepare(
cache_size=Size(500, Unit.MebiByte),
core_size=Size(10, Unit.GibiByte),
cache_mode=CacheMode.WB,
cache_line_size=CacheLineSize.LINE_4KiB,
):
ioclass_config.remove_ioclass_config()
cache_device = TestRun.disks["cache"]
core_device = TestRun.disks["core"]
cache_device.create_partitions([cache_size])
core_device.create_partitions([core_size])
cache_device = cache_device.partitions[0]
core_device = core_device.partitions[0]
TestRun.LOGGER.info(f"Starting cache")
cache = casadm.start_cache(cache_device,
cache_mode=cache_mode,
cache_line_size=cache_line_size,
force=True)
Udev.disable()
TestRun.LOGGER.info(f"Setting cleaning policy to NOP")
casadm.set_param_cleaning(cache_id=cache.cache_id,
policy=CleaningPolicy.nop)
TestRun.LOGGER.info(f"Adding core device")
core = casadm.add_core(cache, core_dev=core_device)
TestRun.LOGGER.info(f"Setting seq cutoff policy to never")
core.set_seq_cutoff_policy(SeqCutOffPolicy.never)
ioclass_config.create_ioclass_config(
add_default_rule=False, ioclass_config_path=ioclass_config_path)
# To make test more precise all workload except of tested ioclass should be
# put in pass-through mode
ioclass_config.add_ioclass(
ioclass_id=ioclass_config.DEFAULT_IO_CLASS_ID,
eviction_priority=ioclass_config.DEFAULT_IO_CLASS_PRIORITY,
allocation="0.00",
rule=ioclass_config.DEFAULT_IO_CLASS_RULE,
ioclass_config_path=ioclass_config_path,
)
output = TestRun.executor.run(f"mkdir -p {mountpoint}")
if output.exit_code != 0:
raise Exception(f"Failed to create mountpoint")
return cache, core
def test_ioclass_process_name():
"""
title: Test IO classification by process name.
description: Check if data generated by process with particular name is cached.
pass_criteria:
- No kernel bug.
- IO is classified properly based on process generating IO name.
"""
ioclass_id = 1
dd_size = Size(4, Unit.KibiByte)
dd_count = 1
iterations = 100
with TestRun.step("Prepare cache and core."):
cache, core = prepare()
with TestRun.step("Create and load IO class config file."):
ioclass_config.add_ioclass(
ioclass_id=ioclass_id,
eviction_priority=1,
allocation="1.00",
rule=f"process_name:dd&done",
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
with TestRun.step("Flush cache and disable udev."):
cache.flush_cache()
Udev.disable()
with TestRun.step("Check if all data generated by dd process is cached."):
for i in range(iterations):
dd = (
Dd()
.input("/dev/zero")
.output(core.path)
.count(dd_count)
.block_size(dd_size)
.seek(i)
)
dd.run()
sync()
time.sleep(0.1)
dirty = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.dirty
if dirty.get_value(Unit.Blocks4096) != (i + 1) * dd_count:
TestRun.LOGGER.error(f"Wrong amount of dirty data ({dirty}).")
def test_stat_max_core(cache_mode):
"""
title: CAS statistics values for maximum core devices.
description: |
Check CAS ability to display correct values in statistics
for 62 core devices.
pass_criteria:
- Core's statistics matches cache's statistics.
"""
cores_per_cache = 62
with TestRun.step(f"Create 1 cache and {cores_per_cache} core partitions"):
cache_dev = TestRun.disks["cache"]
cache_dev.create_partitions([cache_size])
core_dev = TestRun.disks["core"]
core_parts = [core_size] * cores_per_cache
core_dev.create_partitions(core_parts)
Udev.disable()
with TestRun.step(
f"Start cache in {cache_mode} cache mode and add {cores_per_cache} cores"
):
cache = casadm.start_cache(cache_dev.partitions[0],
cache_mode=cache_mode,
force=True)
cache.set_cleaning_policy(CleaningPolicy.nop)
cores = []
for j in range(cores_per_cache):
cores.append(cache.add_core(core_dev.partitions[j]))
with TestRun.step("Run 'fio'"):
fio = fio_prepare()
for j in range(cores_per_cache):
fio.add_job().target(cores[j].system_path)
fio.run()
sleep(3)
with TestRun.step("Check if cache's statistics matches core's statistics"):
cache_stats = cache.get_statistics_flat(stat_filter=stat_filter)
cores_stats = [
cores[j].get_statistics_flat(stat_filter=stat_filter)
for j in range(cores_per_cache)
]
fail_message = f"In {cache_mode} cache mode "
stats_compare(cache_stats, cores_stats, cores_per_cache, fail_message)
def test_ioclass_conditions_and(filesystem):
"""
Load config with IO class combining 5 conditions contradicting at least one other condition
connected by AND operator.
Check if every IO fulfilling one of the conditions is not classified.
"""
cache, core = prepare()
Udev.disable()
file_size = Size(random.randint(25, 50), Unit.MebiByte)
file_size_bytes = int(file_size.get_value(Unit.Byte))
# directories OR condition
ioclass_config.add_ioclass(
ioclass_id=1,
eviction_priority=1,
allocation=True,
rule=f"file_size:gt:{file_size_bytes}&file_size:lt:{file_size_bytes}&"
f"file_size:ge:{file_size_bytes}&file_size:le:{file_size_bytes}&"
f"file_size:eq:{file_size_bytes}",
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
TestRun.LOGGER.info(f"Preparing {filesystem.name} filesystem "
f"and mounting {core.system_path} at {mountpoint}")
core.create_filesystem(filesystem)
core.mount(mountpoint)
sync()
base_occupancy = cache.get_statistics_deprecated(
io_class_id=1)["occupancy"]
# Perform IO
for size in [
file_size, file_size + Size(1, Unit.MebiByte),
file_size - Size(1, Unit.MebiByte)
]:
(Fio().create_command().io_engine(
IoEngine.libaio).size(size).read_write(
ReadWrite.write).target(f"{mountpoint}/test_file").run())
sync()
new_occupancy = cache.get_statistics_deprecated(
io_class_id=1)["occupancy"]
assert new_occupancy == base_occupancy, \
"Unexpected occupancy increase!\n" \
f"Expected: {base_occupancy}, actual: {new_occupancy}"
def test_stats_values():
"""
title: Check for proper statistics values.
description: |
Check if CAS displays proper usage, request, block and error statistics values
for core devices in every cache mode - at the start, after IO and after cache
reload. Also check if core's statistics match cache's statistics.
pass_criteria:
- Usage, request, block and error statistics have proper values.
- Core's statistics match cache's statistics.
"""
with TestRun.step("Partition cache and core devices"):
cache_dev, core_dev = storage_prepare()
Udev.disable()
with TestRun.step(
f"Start {caches_count} caches (one for every cache mode) "
f"and add {cores_per_cache} cores per cache"
):
caches, cores = cache_prepare(cache_dev, core_dev)
with TestRun.step("Check initial statistics values for each core"):
check_stats_initial(caches, cores)
with TestRun.step("Run 'fio'"):
fio = fio_prepare()
for i in range(caches_count):
for j in range(cores_per_cache):
fio.add_job().target(cores[i][j].path)
fio.run()
sleep(3)
with TestRun.step("Check statistics values after IO"):
check_stats_after_io(caches, cores)
with TestRun.step("Check if cache's statistics match core's statistics"):
check_stats_sum(caches, cores)
with TestRun.step("Stop and load caches back"):
casadm.stop_all_caches()
caches = cache_load(cache_dev)
with TestRun.step("Check statistics values after reload"):
check_stats_after_io(caches, cores, after_reload=True)
def test_ioclass_conditions_or(filesystem):
"""
Load config with IO class combining 5 contradicting conditions connected by OR operator.
Check if every IO fulfilling one condition is classified properly.
"""
cache, core = prepare()
Udev.disable()
# directories OR condition
ioclass_config.add_ioclass(
ioclass_id=1,
eviction_priority=1,
allocation=True,
rule=
f"directory:{mountpoint}/dir1|directory:{mountpoint}/dir2|directory:"
f"{mountpoint}/dir3|directory:{mountpoint}/dir4|directory:{mountpoint}/dir5",
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
TestRun.LOGGER.info(f"Preparing {filesystem.name} filesystem "
f"and mounting {core.system_path} at {mountpoint}")
core.create_filesystem(filesystem)
core.mount(mountpoint)
for i in range(1, 6):
fs_utils.create_directory(f"{mountpoint}/dir{i}")
sync()
# Perform IO fulfilling each condition and check if occupancy raises
for i in range(1, 6):
file_size = Size(random.randint(25, 50), Unit.MebiByte)
base_occupancy = cache.get_statistics_deprecated(
io_class_id=1)["occupancy"]
(Fio().create_command().io_engine(
IoEngine.libaio).size(file_size).read_write(
ReadWrite.write).target(
f"{mountpoint}/dir{i}/test_file").run())
sync()
new_occupancy = cache.get_statistics_deprecated(
io_class_id=1)["occupancy"]
assert new_occupancy == base_occupancy + file_size, \
"Occupancy has not increased correctly!\n" \
f"Expected: {base_occupancy + file_size}, actual: {new_occupancy}"
def test_write_fetch_full_misses(cache_mode, cache_line_size):
"""
title: No caching of full write miss operations with block size smaller than cache line size
description: |
Validate CAS ability to not cache entire cache line size for full write miss operations
when block size is smaller than cache line size – no fetch for writes
pass_criteria:
- Appropriate number of write full misses and writes to cache in cache statistics
- Appropriate number of writes to cache in iostat
"""
io_size = Size(300, Unit.MebiByte)
with TestRun.step("Start cache and add core."):
cache_disk = TestRun.disks['cache']
core_disk = TestRun.disks['core']
cache = casadm.start_cache(cache_disk, cache_mode, cache_line_size)
Udev.disable()
core = cache.add_core(core_disk)
with TestRun.step("Run writes to CAS device using fio."):
io_stats_before_io = cache_disk.get_io_stats()
blocksize = cache_line_size.value / 2
skip_size = cache_line_size.value / 2
run_fio(target=core.path,
operation_type=ReadWrite.write,
skip=skip_size,
blocksize=blocksize,
io_size=io_size)
with TestRun.step(
"Verify CAS statistics for write full misses and writes to cache."
):
check_statistics(cache=cache,
blocksize=blocksize,
skip_size=skip_size,
io_size=io_size)
with TestRun.step(
"Verify number of writes to cache device using iostat. Shall be half of "
f"io size ({str(io_size / 2)}) + metadata for WB."):
check_io_stats(cache_disk=cache_disk,
cache=cache,
io_stats_before=io_stats_before_io,
io_size=io_size,
blocksize=blocksize,
skip_size=skip_size)
def prepare(cores_count=1, cache_line_size: CacheLineSize = None):
cache_device = TestRun.disks['cache']
core_device = TestRun.disks['core']
cache_device.create_partitions(
[(SEQ_CUTOFF_THRESHOLD_MAX * cores_count + Size(5, Unit.GibiByte)).align_down(0x1000)])
partitions = \
[(SEQ_CUTOFF_THRESHOLD_MAX + Size(10, Unit.GibiByte)).align_down(0x1000)] * cores_count
core_device.create_partitions(partitions)
cache_part = cache_device.partitions[0]
core_parts = core_device.partitions
TestRun.LOGGER.info("Starting cache")
cache = casadm.start_cache(cache_part, force=True, cache_line_size=cache_line_size)
Udev.disable()
TestRun.LOGGER.info("Adding core devices")
core_list = []
for core_part in core_parts:
core_list.append(cache.add_core(core_dev=core_part))
return cache, core_list
def prepare():
cache_dev = TestRun.disks["cache"]
core_dev = TestRun.disks["core"]
cache_dev.create_partitions([Size(100, Unit.MiB)])
core_dev.create_partitions([Size(200, Unit.MiB)])
Udev.disable()
cache = casadm.start_cache(cache_dev.partitions[0], force=True, cache_mode=CacheMode.WB)
core = cache.add_core(core_dev.partitions[0])
cache.set_seq_cutoff_policy(SeqCutOffPolicy.never)
cache.set_cleaning_policy(CleaningPolicy.alru)
cache.set_params_alru(
FlushParametersAlru(
activity_threshold=Time(seconds=100),
staleness_time=Time(seconds=1),
)
)
return cache, core
def prepare_config(cache_line_size, cache_mode):
cache_device = TestRun.disks["cache"]
core_device = TestRun.disks["core"]
core_device.create_partitions([Size(3, Unit.GiB)])
cache = casadm.start_cache(
cache_device,
cache_mode=cache_mode,
cache_line_size=cache_line_size,
force=True,
)
cache.set_seq_cutoff_policy(SeqCutOffPolicy.never)
cache.set_cleaning_policy(CleaningPolicy.nop)
Udev.disable()
core = cache.add_core(core_device.partitions[0])
return cache, core
def prepare(random_cls, cache_count=1, cores_per_cache=1):
cache_modes = [CacheMode.WT, CacheMode.WB, CacheMode.WA, CacheMode.WO]
ioclass_config.remove_ioclass_config()
cache_device = TestRun.disks['cache']
core_device = TestRun.disks['core']
cache_device.create_partitions([Size(500, Unit.MebiByte)] * cache_count)
core_device.create_partitions([Size(2, Unit.GibiByte)] * cache_count *
cores_per_cache)
cache_devices = cache_device.partitions
core_devices = core_device.partitions
for core_device in core_devices:
core_device.create_filesystem(Filesystem.ext4)
Udev.disable()
caches, cores = [], []
for i, cache_device in enumerate(cache_devices):
TestRun.LOGGER.info(f"Starting cache on {cache_device.path}")
cache = casadm.start_cache(cache_device,
force=True,
cache_mode=cache_modes[i],
cache_line_size=random_cls)
caches.append(cache)
TestRun.LOGGER.info("Setting cleaning policy to NOP")
cache.set_cleaning_policy(CleaningPolicy.nop)
for core_device in core_devices[i * cores_per_cache:(i + 1) *
cores_per_cache]:
TestRun.LOGGER.info(
f"Adding core device {core_device.path} to cache {cache.cache_id}"
)
core = cache.add_core(core_dev=core_device)
core.reset_counters()
cores.append(core)
TestRun.executor.run_expect_success(f"mkdir -p {mountpoint}")
return caches, cores
def prepare():
ioclass_config.remove_ioclass_config()
cache_device = TestRun.disks['cache']
core_device = TestRun.disks['core']
cache_device.create_partitions([Size(500, Unit.MebiByte)])
core_device.create_partitions([Size(1, Unit.GibiByte)])
cache_device = cache_device.partitions[0]
core_device = core_device.partitions[0]
TestRun.LOGGER.info(f"Starting cache")
cache = casadm.start_cache(cache_device, cache_mode=CacheMode.WB, force=True)
Udev.disable()
TestRun.LOGGER.info(f"Setting cleaning policy to NOP")
casadm.set_param_cleaning(cache_id=cache.cache_id, policy=CleaningPolicy.nop)
TestRun.LOGGER.info(f"Adding core device")
core = casadm.add_core(cache, core_dev=core_device)
core.set_seq_cutoff_policy(SeqCutOffPolicy.never)
ioclass_config.create_ioclass_config(
add_default_rule=False, ioclass_config_path=ioclass_config_path
)
# To make test more precise all workload except of tested ioclass should be
# put in pass-through mode
ioclass_config.add_ioclass(
ioclass_id=0,
eviction_priority=22,
allocation=False,
rule="unclassified",
ioclass_config_path=ioclass_config_path,
)
output = TestRun.executor.run(f"mkdir -p {mountpoint}")
if output.exit_code != 0:
raise Exception(f"Failed to create mountpoint")
return cache, core
def prepare_configuration(cache_mode, cache_line_size):
cache_device = TestRun.disks["cache"]
core_device = TestRun.disks["core"]
cache_device.create_partitions([Size(70, Unit.MebiByte)])
core_device.create_partitions(
[Size(70, Unit.MebiByte), Size(70, Unit.MebiByte)]
)
core1 = core_device.partitions[0]
core2 = core_device.partitions[1]
error_device = ErrorDevice(
"error",
core1,
DmTable.uniform_error_table(
start_lba=0,
stop_lba=int(core1.size.get_value(Unit.Blocks512)),
num_error_zones=100,
error_zone_size=Size(5, Unit.Blocks512),
).fill_gaps(core1),
)
cache = casadm.start_cache(
cache_device.partitions[0],
cache_mode=cache_mode,
cache_line_size=cache_line_size,
force=True,
)
cache.set_seq_cutoff_policy(SeqCutOffPolicy.never)
cache.set_cleaning_policy(CleaningPolicy.nop)
Udev.disable()
error_core = cache.add_core(core_dev=error_device)
good_core = cache.add_core(core_dev=core2)
return cache, error_core, good_core
