def create_partitions(device,
sizes: [],
partition_table_type=PartitionTable.gpt):
from storage_devices.partition import Partition
if create_partition_table(device, partition_table_type):
device.partition_table = partition_table_type
partition_type = PartitionType.primary
partition_number_offset = 0
for s in sizes:
size = Size(
s.get_value(device.block_size) - device.block_size.value,
device.block_size)
if partition_table_type == PartitionTable.msdos and \
len(sizes) > 4 and len(device.partitions) == 3:
create_partition(device, Size.zero(), 4,
PartitionType.extended, Unit.MebiByte, True)
partition_type = PartitionType.logical
partition_number_offset = 1
partition_number = len(
device.partitions) + 1 + partition_number_offset
if create_partition(device, size, partition_number, partition_type,
Unit.MebiByte, True):
new_part = Partition(device, partition_type, partition_number)
dd = Dd().input("/dev/zero") \
.output(new_part.system_path) \
.count(1) \
.block_size(Size(1, Unit.Blocks4096)) \
.oflag("direct")
dd.run()
device.partitions.append(new_part)
def run_io_dir_read(path):
dd = Dd().output("/dev/null").input(f"{path}")
output = dd.run()
if output.exit_code != 0:
TestRun.fail(f"Failed to execute dd.\n {output.stdout}\n{output.stderr}")
sync()
drop_caches(DropCachesMode.ALL)
def test_ioclass_file_offset(prepare_and_cleanup):
cache, core = prepare()
ioclass_id = 1
iterations = 100
dd_size = Size(4, Unit.KibiByte)
dd_count = 1
min_cached_offset = 16384
max_cached_offset = 65536
ioclass_config.add_ioclass(
ioclass_id=ioclass_id,
eviction_priority=1,
allocation=True,
rule=
f"file_offset:gt:{min_cached_offset}&file_offset:lt:{max_cached_offset}&done",
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
TestProperties.LOGGER.info(
f"Preparing filesystem and mounting {core.system_path} at {mountpoint}"
)
core.create_filesystem(Filesystem.ext3)
core.mount(mountpoint)
cache.flush_cache()
# Since ioclass rule consists of strict inequalities, 'seek' can't be set to first
# nor last sector
min_seek = int(
(min_cached_offset + Unit.Blocks4096.value) / Unit.Blocks4096.value)
max_seek = int(
(max_cached_offset - min_cached_offset - Unit.Blocks4096.value) /
Unit.Blocks4096.value)
TestProperties.LOGGER.info(f"Writing to file within cached offset range")
for i in range(iterations):
file_offset = random.choice(range(min_seek, max_seek))
dd = (Dd().input("/dev/zero").output(f"{mountpoint}/tmp_file").count(
dd_count).block_size(dd_size).seek(file_offset))
dd.run()
sync()
stats = cache.get_cache_statistics(io_class_id=ioclass_id)
assert (stats["dirty"].get_value(
Unit.Blocks4096) == 1), f"Offset not cached: {file_offset}"
cache.flush_cache()
min_seek = 0
max_seek = int(min_cached_offset / Unit.Blocks4096.value)
TestProperties.LOGGER.info(
f"Writing to file outside of cached offset range")
for i in range(iterations):
file_offset = random.choice(range(min_seek, max_seek))
dd = (Dd().input("/dev/zero").output(f"{mountpoint}/tmp_file").count(
dd_count).block_size(dd_size).seek(file_offset))
dd.run()
sync()
stats = cache.get_cache_statistics(io_class_id=ioclass_id)
assert (stats["dirty"].get_value(Unit.Blocks4096) == 0
), f"Inappropriately cached offset: {file_offset}"
def create_files_with_classification_delay_check(directory: Directory,
ioclass_id: int):
start_time = datetime.now()
occupancy_after = cache.get_statistics_deprecated(
io_class_id=ioclass_id)["occupancy"]
dd_blocks = 10
dd_size = Size(dd_blocks, Unit.Blocks4096)
file_counter = 0
unclassified_files = []
time_from_start = datetime.now() - start_time
while time_from_start < ioclass_config.MAX_CLASSIFICATION_DELAY:
occupancy_before = occupancy_after
file_path = f"{directory.full_path}/test_file_{file_counter}"
file_counter += 1
time_from_start = datetime.now() - start_time
(Dd().input(
"/dev/zero").output(file_path).oflag("sync").block_size(
Size(1, Unit.Blocks4096)).count(dd_blocks).run())
occupancy_after = cache.get_statistics_deprecated(
io_class_id=ioclass_id)["occupancy"]
if occupancy_after - occupancy_before < dd_size:
unclassified_files.append(file_path)
if len(unclassified_files) == file_counter:
pytest.xfail(
"No files were properly classified within max delay time!")
if len(unclassified_files):
TestRun.LOGGER.info("Rewriting unclassified test files...")
for file_path in unclassified_files:
(Dd().input("/dev/zero").output(
file_path).oflag("sync").block_size(
Size(1, Unit.Blocks4096)).count(dd_blocks).run())
def test_ioclass_lba(prepare_and_cleanup):
"""Write data to random lba and check if it is cached according to range
defined in ioclass rule"""
cache, core = prepare()
ioclass_id = 1
min_cached_lba = 56
max_cached_lba = 200
iterations = 100
dd_size = Size(1, Unit.Blocks512)
dd_count = 1
# Prepare ioclass config
ioclass_config.add_ioclass(
ioclass_id=ioclass_id,
eviction_priority=1,
allocation=True,
rule=f"lba:ge:{min_cached_lba}&lba:le:{max_cached_lba}&done",
ioclass_config_path=ioclass_config_path,
)
# Prepare cache for test
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
cache.flush_cache()
# Check if lbas from defined range are cached
dirty_count = 0
# '8' step is set to prevent writting cache line more than once
TestProperties.LOGGER.info(
f"Writing to one sector in each cache line from range.")
for lba in range(min_cached_lba, max_cached_lba, 8):
dd = (Dd().input("/dev/zero").output(f"{core.system_path}").count(
dd_count).block_size(dd_size).seek(lba))
dd.run()
sync()
dirty_count += 1
stats = cache.get_cache_statistics(per_io_class=True,
io_class_id=ioclass_id)
assert (stats["dirty"].get_value(
Unit.Blocks4096) == dirty_count), f"LBA {lba} not cached"
cache.flush_cache()
# Check if lba outside of defined range are not cached
TestProperties.LOGGER.info(
f"Writing to random sectors outside of cached range.")
for i in range(iterations):
rand_lba = random.randrange(2000)
if min_cached_lba <= rand_lba <= max_cached_lba:
continue
dd = (Dd().input("/dev/zero").output(f"{core.system_path}").count(
dd_count).block_size(dd_size).seek(rand_lba))
dd.run()
sync()
stats = cache.get_cache_statistics(per_io_class=True,
io_class_id=ioclass_id)
assert (stats["dirty"].get_value(
Unit.Blocks4096) == 0), f"Inappropriately cached lba: {rand_lba}"
def test_activate_corrupted_after_dump():
"""
title: Activate cache instance on metadata corrupted after the detach
description: |
Initialize standby cache, populate it with metadata, detach cache, corrupt metadata
on the cache-to-be device and try to activate.
pass_criteria:
- Kernel panic doesn't occur
"""
with TestRun.step("Prepare devices for the cache and core."):
cache_device = TestRun.disks["cache"]
cache_device.create_partitions([Size(200, Unit.MebiByte)])
cache_device = cache_device.partitions[0]
core_device = TestRun.disks["core"]
core_device.create_partitions([Size(500, Unit.MebiByte)])
core_device = core_device.partitions[0]
with TestRun.step("Prepare metadata dump"):
cache_id = 1
cls = CacheLineSize.LINE_32KiB
md_dump = prepare_md_dump(cache_device, core_device, cls, cache_id)
for offset in get_offsets_to_corrupt(md_dump.size, block_size):
with TestRun.step("Prepare standby instance"):
cache = casadm.standby_init(
cache_dev=cache_device,
cache_line_size=int(cls.value.value / Unit.KibiByte.value),
cache_id=cache_id,
force=True,
)
with TestRun.step(
f"Populate the passive instance with valid metadata"):
Dd().input(
md_dump.full_path).output(f"/dev/cas-cache-{cache_id}").run()
sync()
with TestRun.step(f"Standby detach"):
cache.standby_detach()
with TestRun.step(
f"Corrupt {block_size} on the offset {offset*block_size}"):
corrupted_md = prepare_corrupted_md(md_dump, offset, block_size)
with TestRun.step(f"Copy corrupted metadata to the passive instance"):
Dd().input(corrupted_md.full_path).output(cache_device.path).run()
sync()
with TestRun.step("Try to activate cache instance"):
output = TestRun.executor.run(
standby_activate_cmd(cache_dev=cache_device.path,
cache_id=str(cache_id)))
with TestRun.step("Per iteration cleanup"):
cache.stop()
corrupted_md.remove(force=True, ignore_errors=True)
with TestRun.step("Test cleanup"):
md_dump.remove()
def test_ioclass_file_extension(prepare_and_cleanup):
cache, core = prepare()
iterations = 50
ioclass_id = 1
tested_extension = "tmp"
wrong_extensions = ["tm", "tmpx", "txt", "t", "", "123", "tmp.xx"]
dd_size = Size(4, Unit.KibiByte)
dd_count = 10
ioclass_config.add_ioclass(
ioclass_id=ioclass_id,
eviction_priority=1,
allocation=True,
rule=f"extension:{tested_extension}&done",
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
TestRun.LOGGER.info(
f"Preparing filesystem and mounting {core.system_path} at {mountpoint}"
)
core.create_filesystem(Filesystem.ext3)
core.mount(mountpoint)
cache.flush_cache()
# Check if file with proper extension is cached
dd = (
Dd()
.input("/dev/zero")
.output(f"{mountpoint}/test_file.{tested_extension}")
.count(dd_count)
.block_size(dd_size)
)
TestRun.LOGGER.info(f"Writing to file with cached extension.")
for i in range(iterations):
dd.run()
sync()
stats = cache.get_cache_statistics(io_class_id=ioclass_id)
assert stats["dirty"].get_value(Unit.Blocks4096) == (i + 1) * dd_count
cache.flush_cache()
# Check if file with improper extension is not cached
TestRun.LOGGER.info(f"Writing to file with no cached extension.")
for ext in wrong_extensions:
dd = (
Dd()
.input("/dev/zero")
.output(f"{mountpoint}/test_file.{ext}")
.count(dd_count)
.block_size(dd_size)
)
dd.run()
sync()
stats = cache.get_cache_statistics(io_class_id=ioclass_id)
assert stats["dirty"].get_value(Unit.Blocks4096) == 0
def run_dd(target_path, count, seek):
dd = Dd() \
.input("/dev/zero") \
.output(target_path) \
.block_size(Size(1, Unit.Blocks4096)) \
.count(count) \
.oflag("direct") \
.seek(seek)
dd.run()
TestRun.LOGGER.info(f"dd command:\n{dd}")
def read_files_with_reclassification_check(cache, target_ioclass_id: int,
source_ioclass_id: int,
directory: Directory,
with_delay: bool):
start_time = datetime.now()
target_occupancy_after = cache.get_io_class_statistics(
io_class_id=target_ioclass_id).usage_stats.occupancy
source_occupancy_after = cache.get_io_class_statistics(
io_class_id=source_ioclass_id).usage_stats.occupancy
files_to_reclassify = []
target_ioclass_is_enabled = ioclass_is_enabled(cache, target_ioclass_id)
for file in [item for item in directory.ls() if isinstance(item, File)]:
target_occupancy_before = target_occupancy_after
source_occupancy_before = source_occupancy_after
time_from_start = datetime.now() - start_time
dd = Dd().input(file.full_path).output("/dev/null").block_size(
Size(1, Unit.Blocks4096))
dd.run()
target_occupancy_after = cache.get_io_class_statistics(
io_class_id=target_ioclass_id).usage_stats.occupancy
source_occupancy_after = cache.get_io_class_statistics(
io_class_id=source_ioclass_id).usage_stats.occupancy
if target_ioclass_is_enabled:
if target_occupancy_after < target_occupancy_before:
TestRun.LOGGER.error("Target IO class occupancy lowered!")
elif target_occupancy_after - target_occupancy_before < file.size:
files_to_reclassify.append(file)
if with_delay and time_from_start <= ioclass_config.MAX_CLASSIFICATION_DELAY:
continue
TestRun.LOGGER.error(
"Target IO class occupancy not changed properly!\n"
f"Expected: {file.size + target_occupancy_before}\n"
f"Actual: {target_occupancy_after}")
elif target_occupancy_after > target_occupancy_before and with_delay:
files_to_reclassify.append(file)
if source_occupancy_after >= source_occupancy_before:
if file not in files_to_reclassify:
files_to_reclassify.append(file)
if with_delay and time_from_start <= ioclass_config.MAX_CLASSIFICATION_DELAY:
continue
TestRun.LOGGER.error(
"Source IO class occupancy not changed properly!\n"
f"Before: {source_occupancy_before}\n"
f"After: {source_occupancy_after}")
if len(files_to_reclassify):
TestRun.LOGGER.info("Rereading unclassified test files...")
sync()
drop_caches(DropCachesMode.ALL)
for file in files_to_reclassify:
(Dd().input(file.full_path).output("/dev/null").block_size(
Size(1, Unit.Blocks4096)).run())
def create_partition(
device,
part_size,
part_number,
part_type: PartitionType = PartitionType.primary,
unit=Unit.MebiByte,
aligned: bool = True):
TestRun.LOGGER.info(
f"Creating {part_type.name} partition on device: {device.path}")
begin = get_first_partition_offset(device, aligned)
for part in device.partitions:
begin += part.size
if part.type == PartitionType.logical:
begin += Size(1, Unit.MebiByte if not aligned else device.block_size)
if part_type == PartitionType.logical:
begin += Size(1, Unit.MebiByte if not aligned else device.block_size)
end = (begin + part_size) if part_size != Size.zero() else '100%'
cmd = f'parted --script {device.path} mkpart ' \
f'{part_type.name} ' \
f'{begin.get_value(unit)}{unit_to_string(unit)} ' \
f'{end.get_value(unit)}{unit_to_string(unit)}'
output = TestRun.executor.run(cmd)
if output.exit_code != 0:
TestRun.executor.run_expect_success("partprobe")
TestRun.executor.run_expect_success("udevadm settle")
if not check_partition_after_create(
part_size,
part_number,
device.path,
part_type,
aligned):
raise Exception("Could not create partition!")
if part_type != PartitionType.extended:
from storage_devices.partition import Partition
new_part = Partition(device,
part_type,
part_number,
begin,
end if type(end) is Size else device.size)
dd = Dd().input("/dev/zero") \
.output(new_part.path) \
.count(1) \
.block_size(Size(1, Unit.Blocks4096)) \
.oflag("direct")
dd.run()
device.partitions.append(new_part)
TestRun.LOGGER.info(f"Successfully created {part_type.name} partition on {device.path}")
def test_ioclass_request_size():
cache, core = prepare()
ioclass_id = 1
iterations = 100
ioclass_config.add_ioclass(
ioclass_id=ioclass_id,
eviction_priority=1,
allocation=True,
rule=f"request_size:ge:8192&request_size:le:16384&done",
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
Udev.disable()
# Check if requests with appropriate size are cached
TestRun.LOGGER.info(
f"Check if requests with size within defined range are cached")
cached_req_sizes = [Size(2, Unit.Blocks4096), Size(4, Unit.Blocks4096)]
for i in range(iterations):
cache.flush_cache()
req_size = random.choice(cached_req_sizes)
dd = (Dd().input("/dev/zero").output(
core.system_path).count(1).block_size(req_size).oflag("direct"))
dd.run()
dirty = cache.get_io_class_statistics(
io_class_id=ioclass_id).usage_stats.dirty
if dirty.get_value(
Unit.Blocks4096) != req_size.value / Unit.Blocks4096.value:
TestRun.fail("Incorrect number of dirty blocks!")
cache.flush_cache()
# Check if requests with inappropriate size are not cached
TestRun.LOGGER.info(
f"Check if requests with size outside defined range are not cached")
not_cached_req_sizes = [
Size(1, Unit.Blocks4096),
Size(8, Unit.Blocks4096),
Size(16, Unit.Blocks4096),
]
for i in range(iterations):
req_size = random.choice(not_cached_req_sizes)
dd = (Dd().input("/dev/zero").output(
core.system_path).count(1).block_size(req_size).oflag("direct"))
dd.run()
dirty = cache.get_io_class_statistics(
io_class_id=ioclass_id).usage_stats.dirty
if dirty.get_value(Unit.Blocks4096) != 0:
TestRun.fail("Dirty data present!")
def create_test_file():
from test_utils.filesystem.file import File
from test_tools.dd import Dd
bs = Size(512, Unit.KibiByte)
cnt = int(cache_size.value / bs.value)
test_file = File.create_file(test_file_path)
dd = Dd().output(test_file_path) \
.input("/dev/zero") \
.block_size(bs) \
.count(cnt)
dd.run()
test_file.refresh_item()
return test_file
def test_ioclass_file_extension_preexisting_filesystem():
"""Create files on filesystem, add device with filesystem as a core,
write data to files and check if they are cached properly"""
cache, core = prepare()
ioclass_id = 1
extensions = ["tmp", "tm", "out", "txt", "log", "123"]
dd_size = Size(4, Unit.KibiByte)
dd_count = 10
TestRun.LOGGER.info(f"Preparing files on raw block device")
casadm.remove_core(cache.cache_id, core_id=core.core_id)
core.core_device.create_filesystem(Filesystem.ext3)
core.core_device.mount(mountpoint)
# Prepare files
for ext in extensions:
dd = (Dd().input("/dev/zero").output(f"{mountpoint}/test_file.{ext}").
count(dd_count).block_size(dd_size))
dd.run()
core.core_device.unmount()
# Prepare ioclass config
rule = "|".join([f"extension:{ext}" for ext in extensions])
ioclass_config.add_ioclass(
ioclass_id=ioclass_id,
eviction_priority=1,
allocation=True,
rule=f"{rule}&done",
ioclass_config_path=ioclass_config_path,
)
# Prepare cache for test
TestRun.LOGGER.info(f"Adding device with preexisting data as a core")
core = casadm.add_core(cache, core_dev=core.core_device)
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
core.mount(mountpoint)
cache.flush_cache()
# Check if files with proper extensions are cached
TestRun.LOGGER.info(f"Writing to file with cached extension.")
for ext in extensions:
dd = (Dd().input("/dev/zero").output(f"{mountpoint}/test_file.{ext}").
count(dd_count).block_size(dd_size))
dd.run()
sync()
dirty = cache.get_io_class_statistics(
io_class_id=ioclass_id).usage_stats.dirty
if dirty.get_value(
Unit.Blocks4096) != (extensions.index(ext) + 1) * dd_count:
TestRun.LOGGER.error(f"Wrong amount of dirty data ({dirty}).")
def test_ioclass_request_size(prepare_and_cleanup):
cache, core = prepare()
ioclass_id = 1
iterations = 100
ioclass_config.add_ioclass(
ioclass_id=ioclass_id,
eviction_priority=1,
allocation=True,
rule=f"request_size:ge:8192&request_size:le:16384&done",
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
Udev.disable()
# Check if requests with appropriate size are cached
TestProperties.LOGGER.info(
f"Check if requests with size within defined range are cached")
cached_req_sizes = [Size(2, Unit.Blocks4096), Size(4, Unit.Blocks4096)]
for i in range(iterations):
cache.flush_cache()
req_size = random.choice(cached_req_sizes)
dd = (Dd().input("/dev/zero").output(
core.system_path).count(1).block_size(req_size).oflag("direct"))
dd.run()
stats = cache.get_cache_statistics(per_io_class=True,
io_class_id=ioclass_id)
assert (stats["dirty"].get_value(Unit.Blocks4096) == req_size.value /
Unit.Blocks4096.value)
cache.flush_cache()
# Check if requests with inappropriate size are not cached
TestProperties.LOGGER.info(
f"Check if requests with size outside defined range are not cached")
not_cached_req_sizes = [
Size(1, Unit.Blocks4096),
Size(8, Unit.Blocks4096),
Size(16, Unit.Blocks4096),
]
for i in range(iterations):
req_size = random.choice(not_cached_req_sizes)
dd = (Dd().input("/dev/zero").output(
core.system_path).count(1).block_size(req_size).oflag("direct"))
dd.run()
stats = cache.get_cache_statistics(per_io_class=True,
io_class_id=ioclass_id)
assert stats["dirty"].get_value(Unit.Blocks4096) == 0
def create_random_test_file(target_file_path: str,
file_size: Size = Size(1, Unit.MebiByte),
random: bool = True):
from test_utils.filesystem.file import File
bs = Size(512, Unit.KibiByte)
cnt = math.ceil(file_size.value / bs.value)
file = File.create_file(target_file_path)
dd = Dd().output(target_file_path) \
.input("/dev/urandom" if random else "/dev/zero") \
.block_size(bs) \
.count(cnt) \
.oflag("direct")
dd.run()
file.refresh_item()
return file
def run_io_dir(path, size_4k):
dd = (Dd().input("/dev/zero").output(f"{path}").count(size_4k).block_size(
Size(1, Unit.Blocks4096)))
TestRun.LOGGER.info(f"{dd}")
dd.run()
sync()
drop_caches(DropCachesMode.ALL)
def remove_files_classification():
TestRun.LOGGER.info("Moving all files to 'unclassified' IO class")
ioclass_config.remove_ioclass_config(
ioclass_config_path=ioclass_config_path)
ioclass_config.create_ioclass_config(
add_default_rule=False, ioclass_config_path=ioclass_config_path)
ioclass_config.add_ioclass(
ioclass_id=0,
eviction_priority=22,
allocation=False,
rule="unclassified",
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id,
file=ioclass_config_path)
occupancy_before = cache.get_statistics_deprecated(
io_class_id=0)["occupancy"]
for file in test_files:
Dd().input(file.full_path).output("/dev/null").block_size(
file.size).run()
occupancy_after = cache.get_statistics_deprecated(
io_class_id=0)["occupancy"]
if occupancy_after != occupancy_before + file.size:
pytest.xfail("File not reclassified properly!\n"
f"Expected {occupancy_before + file.size}\n"
f"Actual {occupancy_after}")
occupancy_before = occupancy_after
sync()
drop_caches(DropCachesMode.ALL)
def test_ioclass_process_name(prepare_and_cleanup):
"""Check if data generated by process with particular name is cached"""
cache, core = prepare()
ioclass_id = 1
dd_size = Size(4, Unit.KibiByte)
dd_count = 1
iterations = 100
ioclass_config.add_ioclass(
ioclass_id=ioclass_id,
eviction_priority=1,
allocation=True,
rule=f"process_name:dd&done",
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
cache.flush_cache()
Udev.disable()
TestProperties.LOGGER.info(
f"Check if all data generated by dd process is cached.")
for i in range(iterations):
dd = (Dd().input("/dev/zero").output(
core.system_path).count(dd_count).block_size(dd_size).seek(i))
dd.run()
sync()
time.sleep(0.1)
stats = cache.get_cache_statistics(io_class_id=ioclass_id)
assert stats["dirty"].get_value(Unit.Blocks4096) == (i + 1) * dd_count
def prepare_md_dump(cache_device, core_device, cls, cache_id):
with TestRun.step("Setup WB cache instance with one core"):
cache = casadm.start_cache(
cache_dev=cache_device,
cache_line_size=cls,
cache_mode=CacheMode.WB,
cache_id=cache_id,
force=True,
)
cache.add_core(core_device)
with TestRun.step("Get metadata size"):
dmesg_out = TestRun.executor.run_expect_success("dmesg").stdout
md_size = dmesg.get_metadata_size(dmesg_out)
with TestRun.step("Dump the metadata of the cache"):
dump_file_path = "/tmp/test_activate_corrupted.dump"
md_dump = File(dump_file_path)
md_dump.remove(force=True, ignore_errors=True)
dd_count = int(md_size / Size(1, Unit.MebiByte)) + 1
(Dd().input(cache_device.path).output(md_dump.full_path).block_size(
Size(1, Unit.MebiByte)).count(dd_count).run())
md_dump.refresh_item()
with TestRun.step("Stop cache device"):
cache.stop()
return md_dump
def dd_builder(cache_mode: CacheMode, dev: Core, size: Size):
blocks = int(size.value / block_size.value)
dd = (Dd().block_size(block_size).count(blocks))
if CacheModeTrait.InsertRead in CacheMode.get_traits(cache_mode):
dd.input(dev.path).output("/dev/null")
else:
dd.input("/dev/urandom").output(dev.path)
return dd
def test_ioclass_pid():
cache, core = prepare()
ioclass_id = 1
iterations = 20
dd_count = 100
dd_size = Size(4, Unit.KibiByte)
Udev.disable()
# Since 'dd' has to be executed right after writing pid to 'ns_last_pid',
# 'dd' command is created and is appended to 'echo' command instead of running it
dd_command = str(
Dd()
.input("/dev/zero")
.output(core.system_path)
.count(dd_count)
.block_size(dd_size)
)
for i in range(iterations):
cache.flush_cache()
output = TestRun.executor.run("cat /proc/sys/kernel/ns_last_pid")
if output.exit_code != 0:
raise Exception(
f"Failed to retrieve pid. stdout: {output.stdout} \n stderr :{output.stderr}"
)
# Few pids might be used by system during test preparation
pid = int(output.stdout) + 50
ioclass_config.add_ioclass(
ioclass_id=ioclass_id,
eviction_priority=1,
allocation=True,
rule=f"pid:eq:{pid}&done",
ioclass_config_path=ioclass_config_path,
)
casadm.load_io_classes(cache_id=cache.cache_id, file=ioclass_config_path)
TestRun.LOGGER.info(f"Running dd with pid {pid}")
# pid saved in 'ns_last_pid' has to be smaller by one than target dd pid
dd_and_pid_command = (
f"echo {pid-1} > /proc/sys/kernel/ns_last_pid && {dd_command}"
)
output = TestRun.executor.run(dd_and_pid_command)
if output.exit_code != 0:
raise Exception(
f"Failed to run dd with target pid. "
f"stdout: {output.stdout} \n stderr :{output.stderr}"
)
sync()
dirty = cache.get_io_class_statistics(io_class_id=ioclass_id).usage_stats.dirty
if dirty.get_value(Unit.Blocks4096) != dd_count:
TestRun.LOGGER.error(f"Wrong amount of dirty data ({dirty}).")
ioclass_config.remove_ioclass(ioclass_id)
def remove_partitions(device):
if device.is_mounted():
device.unmount()
for partition in device.partitions:
unmount(partition)
TestProperties.LOGGER.info(f"Removing partitions from device: {device.system_path}.")
dd = Dd().input("/dev/zero") \
.output(device.system_path) \
.count(1) \
.block_size(Size(device.block_size.value, Unit.Byte))
dd.run()
output = TestProperties.executor.execute(f"ls {device.system_path}* -1")
if len(output.stdout.split('\n')) > 1:
TestProperties.LOGGER.error(f"Could not remove partitions from device {device.system_path}")
return False
return True
def dd_builder(cache_mode, cache_line_size, count, device):
dd = (Dd().block_size(cache_line_size.value).count(count))
if CacheModeTrait.InsertRead in CacheMode.get_traits(cache_mode):
dd.input(device.path).output("/dev/null").iflag("direct")
else:
dd.input("/dev/urandom").output(device.path).oflag("direct")
return dd
def read_files_with_reclassification_check(target_ioclass_id: int,
source_ioclass_id: int,
directory: Directory,
with_delay: bool):
start_time = datetime.now()
target_occupancy_after = cache.get_statistics_deprecated(
io_class_id=target_ioclass_id)["occupancy"]
source_occupancy_after = cache.get_statistics_deprecated(
io_class_id=source_ioclass_id)["occupancy"]
unclassified_files = []
for file in [
item for item in directory.ls() if isinstance(item, File)
]:
target_occupancy_before = target_occupancy_after
source_occupancy_before = source_occupancy_after
time_from_start = datetime.now() - start_time
(Dd().input(file.full_path).output("/dev/null").block_size(
Size(1, Unit.Blocks4096)).run())
target_occupancy_after = cache.get_statistics_deprecated(
io_class_id=target_ioclass_id)["occupancy"]
source_occupancy_after = cache.get_statistics_deprecated(
io_class_id=source_ioclass_id)["occupancy"]
if target_occupancy_after < target_occupancy_before:
pytest.xfail("Target IO class occupancy lowered!")
elif target_occupancy_after - target_occupancy_before < file.size:
unclassified_files.append(file)
if with_delay and time_from_start <= ioclass_config.MAX_CLASSIFICATION_DELAY:
continue
pytest.xfail("Target IO class occupancy not changed properly!")
if source_occupancy_after >= source_occupancy_before:
if file not in unclassified_files:
unclassified_files.append(file)
if with_delay and time_from_start <= ioclass_config.MAX_CLASSIFICATION_DELAY:
continue
pytest.xfail("Source IO class occupancy not changed properly!")
if len(unclassified_files):
TestRun.LOGGER.info("Rereading unclassified test files...")
sync()
drop_caches(DropCachesMode.ALL)
for file in unclassified_files:
(Dd().input(file.full_path).output("/dev/null").block_size(
Size(1, Unit.Blocks4096)).run())
def test_purge(purge_target):
"""
title: Call purge without and with `--script` switch
description: |
Check if purge is called only when `--script` switch is used.
pass_criteria:
- casadm returns an error when `--script` is missing
- cache is wiped when purge command is used properly
"""
with TestRun.step("Prepare devices"):
cache_device = TestRun.disks["cache"]
core_device = TestRun.disks["core"]
cache_device.create_partitions([Size(500, Unit.MebiByte)])
core_device.create_partitions([Size(500, Unit.MebiByte)])
cache_device = cache_device.partitions[0]
core_device = core_device.partitions[0]
with TestRun.step("Prepare cache instance"):
cache = casadm.start_cache(cache_device, force=True)
core = casadm.add_core(cache, core_device)
with TestRun.step("Trigger IO to prepared cache instance"):
dd = (Dd().input("/dev/zero").output(core.path).count(100).block_size(
Size(1, Unit.Blocks512)).oflag("direct"))
dd.run()
sync()
with TestRun.step(
f"Try to call purge-{purge_target} without `--script` switch"):
original_occupancy = cache.get_statistics().usage_stats.occupancy
purge_params = f"--cache-id {cache.cache_id} "
if purge_target == "core":
purge_params += f"--core-id {core.core_id}"
TestRun.executor.run_expect_fail(
f"casadm --purge-{purge_target} {purge_params}")
if cache.get_statistics().usage_stats.occupancy != original_occupancy:
TestRun.fail(
f"Purge {purge_target} should not be possible to use without `--script` switch!"
)
with TestRun.step(
f"Try to call purge-{purge_target} with `--script` switch"):
TestRun.executor.run_expect_success(
f"casadm --script --purge-{purge_target} {purge_params}")
if cache.get_statistics().usage_stats.occupancy.get_value() != 0:
TestRun.fail(
f"{cache.get_statistics().usage_stats.occupancy.get_value()}")
TestRun.fail(
f"Purge {purge_target} should invalidate all cache lines!")
with TestRun.step(f"Stop cache"):
casadm.stop_all_caches()
def copy_file(source, target, size, direct=None):
dd = Dd() \
.input(source) \
.output(target) \
.block_size(Size(1, Unit.Blocks4096)) \
.count(int(size.get_value(Unit.Blocks4096)))
if direct == "oflag":
dd.oflag("direct")
elif direct == "iflag":
dd.iflag("direct")
dd.run()
def run_io_dir(path, size_4k):
dd = (Dd().input("/dev/zero").output(f"{path}").count(size_4k).block_size(
Size(1, Unit.Blocks4096)))
TestRun.LOGGER.info(f"{dd}")
output = dd.run()
if output.exit_code != 0:
TestRun.fail(
f"Failed to execute dd.\n {output.stdout}\n{output.stderr}")
sync()
drop_caches(DropCachesMode.ALL)
def prepare_corrupted_md(md_dump, offset_to_corrupt, bs):
invalid_dump_path = "/tmp/test_activate_corrupted.invalid_dump"
dd_count = offset_to_corrupt + 1
md_dump.copy(destination=invalid_dump_path, force=True)
corrupted_md = File(invalid_dump_path)
(Dd().input("/dev/urandom").output(corrupted_md.full_path).block_size(
bs).count(dd_count).seek(offset_to_corrupt).conv("notrunc").run())
corrupted_md.refresh_item()
return corrupted_md
def run_io_dir(path, num_ios):
dd = (
Dd()
.input("/dev/zero")
.output(f"{path}/tmp_file")
.count(num_ios)
.block_size(Size(1, Unit.Blocks4096))
)
dd.run()
sync()
drop_caches(DropCachesMode.ALL)
def allocate_memory(size: Size):
"""Allocates given amount of memory"""
mount_ramfs()
TestRun.LOGGER.info(
f"Allocating {size.get_value(Unit.MiB):0.2f} MiB of memory.")
bs = Size(1, Unit.Blocks512)
dd = (Dd().block_size(bs).count(math.ceil(
size / bs)).input("/dev/zero").output(f"{MEMORY_MOUNT_POINT}/data"))
output = dd.run()
if output.exit_code != 0:
raise CmdException("Allocating memory failed.", output)
