def pytest_runtest_teardown():
"""
This method is executed always in the end of each test, even if it fails or raises exception in
prepare stage.
"""
TestRun.LOGGER.end_all_groups()
with TestRun.LOGGER.step("Cleanup after test"):
try:
if TestRun.executor:
if not TestRun.executor.is_active():
TestRun.executor.wait_for_connection()
Udev.enable()
kill_all_io()
unmount_cas_devices()
if installer.check_if_installed():
casadm.remove_all_detached_cores()
casadm.stop_all_caches()
from api.cas.init_config import InitConfig
InitConfig.create_default_init_config()
DeviceMapper.remove_all()
except Exception as ex:
TestRun.LOGGER.warning(f"Exception occurred during platform cleanup.\n"
f"{str(ex)}\n{traceback.format_exc()}")
TestRun.LOGGER.end()
for dut in TestRun.duts:
with TestRun.use_dut(dut):
if TestRun.executor:
os.makedirs(os.path.join(TestRun.LOGGER.base_dir, "dut_info", dut.ip),
exist_ok=True)
TestRun.LOGGER.get_additional_logs()
Log.destroy()
TestRun.teardown()
def base_prepare(item):
with TestRun.LOGGER.step("Cleanup before test"):
Udev.enable()
kill_all_io()
DeviceMapper.remove_all()
if installer.check_if_installed():
try:
from api.cas.init_config import InitConfig
InitConfig.create_default_init_config()
unmount_cas_devices()
casadm.stop_all_caches()
casadm.remove_all_detached_cores()
except Exception:
pass # TODO: Reboot DUT if test is executed remotely
for disk in TestRun.dut.disks:
disk.umount_all_partitions()
if not create_partition_table(disk, PartitionTable.gpt):
raise Exception(f"Failed to remove partitions from {disk}")
if get_force_param(item) and not TestRun.usr.already_updated:
installer.reinstall_opencas()
elif not installer.check_if_installed():
installer.install_opencas()
TestRun.usr.already_updated = True
TestRun.LOGGER.add_build_info(f'Commit hash:')
TestRun.LOGGER.add_build_info(f"{git.get_current_commit_hash()}")
TestRun.LOGGER.add_build_info(f'Commit message:')
TestRun.LOGGER.add_build_info(f'{git.get_current_commit_message()}')
def base_prepare(item):
with TestRun.LOGGER.step("Cleanup before test"):
TestRun.executor.run("pkill --signal=SIGKILL fsck")
Udev.enable()
kill_all_io()
DeviceMapper.remove_all()
if installer.check_if_installed():
try:
from api.cas.init_config import InitConfig
InitConfig.create_default_init_config()
unmount_cas_devices()
casadm.stop_all_caches()
casadm.remove_all_detached_cores()
except Exception:
pass # TODO: Reboot DUT if test is executed remotely
raids = Raid.discover()
for raid in raids:
# stop only those RAIDs, which are comprised of test disks
if all(map(lambda device:
any(map(lambda disk_path:
disk_path in device.get_device_id(),
[bd.get_device_id() for bd in TestRun.dut.disks])),
raid.array_devices)):
raid.umount_all_partitions()
raid.remove_partitions()
raid.stop()
for device in raid.array_devices:
Mdadm.zero_superblock(os.path.join('/dev', device.get_device_id()))
Udev.settle()
for disk in TestRun.dut.disks:
disk_serial = get_disk_serial_number(disk.path)
if disk.serial_number != disk_serial:
raise Exception(
f"Serial for {disk.path} doesn't match the one from the config."
f"Serial from config {disk.serial_number}, actual serial {disk_serial}"
)
disk.umount_all_partitions()
Mdadm.zero_superblock(os.path.join('/dev', disk.get_device_id()))
TestRun.executor.run_expect_success("udevadm settle")
disk.remove_partitions()
create_partition_table(disk, PartitionTable.gpt)
if get_force_param(item) and not TestRun.usr.already_updated:
installer.rsync_opencas_sources()
installer.reinstall_opencas()
elif not installer.check_if_installed():
installer.rsync_opencas_sources()
installer.set_up_opencas()
TestRun.usr.already_updated = True
TestRun.LOGGER.add_build_info(f'Commit hash:')
TestRun.LOGGER.add_build_info(f"{git.get_current_commit_hash()}")
TestRun.LOGGER.add_build_info(f'Commit message:')
TestRun.LOGGER.add_build_info(f'{git.get_current_commit_message()}')
def test_acp_param_wake_up_time(cache_line_size, cache_mode):
"""
title: Functional test for ACP wake-up parameter.
description: |
Verify if interval between ACP cleaning iterations is not longer than
wake-up time parameter value.
pass_criteria:
- ACP flush iterations are triggered with defined frequency.
"""
with TestRun.step("Test prepare."):
error_threshold_ms = 50
generated_vals = get_random_list(
min_val=FlushParametersAcp.acp_params_range().wake_up_time[0],
max_val=FlushParametersAcp.acp_params_range().wake_up_time[1],
n=10,
)
acp_configs = []
for config in generated_vals:
acp_configs.append(
FlushParametersAcp(wake_up_time=Time(milliseconds=config)))
acp_configs.append(FlushParametersAcp.default_acp_params())
with TestRun.step("Prepare partitions."):
core_size = Size(5, Unit.GibiByte)
cache_device = TestRun.disks["cache"]
core_device = TestRun.disks["core"]
cache_device.create_partitions([Size(10, Unit.GibiByte)])
core_device.create_partitions([core_size])
with TestRun.step(
f"Start cache in {cache_mode} with {cache_line_size} and add core."
):
cache = casadm.start_cache(cache_device.partitions[0], cache_mode,
cache_line_size)
core = cache.add_core(core_device.partitions[0])
with TestRun.step("Set cleaning policy to NOP."):
cache.set_cleaning_policy(CleaningPolicy.nop)
with TestRun.step("Start IO in background."):
fio = get_fio_cmd(core, core_size)
fio.run_in_background()
time.sleep(10)
with TestRun.step("Set cleaning policy to ACP."):
cache.set_cleaning_policy(CleaningPolicy.acp)
with TestRun.group("Verify IO number for different wake_up_time values."):
for acp_config in acp_configs:
with TestRun.step(f"Setting {acp_config}"):
cache.set_params_acp(acp_config)
accepted_interval_threshold = (
acp_config.wake_up_time.total_milliseconds() +
error_threshold_ms)
with TestRun.step(
"Using blktrace verify if interval between ACP cleaning iterations "
f"is shorter or equal than wake-up parameter value "
f"(including {error_threshold_ms}ms error threshold)"):
blktrace = BlkTrace(core.core_device, BlkTraceMask.write)
blktrace.start_monitoring()
time.sleep(15)
blktrace_output = blktrace.stop_monitoring()
for (prev, curr) in zip(blktrace_output, blktrace_output[1:]):
if not new_acp_iteration(prev, curr):
continue
interval_ms = (curr.timestamp - prev.timestamp) / 10**6
if interval_ms > accepted_interval_threshold:
TestRun.LOGGER.error(
f"{interval_ms} is not within accepted range for "
f"{acp_config.wake_up_time.total_milliseconds()} "
f"wake_up_time param value.")
with TestRun.step("Stop all caches"):
kill_all_io()
casadm.stop_all_caches()
def test_acp_param_flush_max_buffers(cache_line_size, cache_mode):
"""
title: Functional test for ACP flush-max-buffers parameter.
description: |
Verify if there is appropriate number of I/O requests between wake-up time intervals,
which depends on flush-max-buffer parameter.
pass_criteria:
- ACP triggered dirty data flush
- Number of writes to core is lower or equal than flush_max_buffers
"""
with TestRun.step("Test prepare."):
buffer_values = get_random_list(
min_val=FlushParametersAcp.acp_params_range().flush_max_buffers[0],
max_val=FlushParametersAcp.acp_params_range().flush_max_buffers[1],
n=10,
)
default_config = FlushParametersAcp.default_acp_params()
acp_configs = [
FlushParametersAcp(flush_max_buffers=buf,
wake_up_time=Time(seconds=1))
for buf in buffer_values
]
acp_configs.append(default_config)
with TestRun.step("Prepare partitions."):
core_size = Size(5, Unit.GibiByte)
cache_device = TestRun.disks["cache"]
core_device = TestRun.disks["core"]
cache_device.create_partitions([Size(10, Unit.GibiByte)])
core_device.create_partitions([core_size])
with TestRun.step(
f"Start cache in {cache_mode} with {cache_line_size} and add core."
):
cache = casadm.start_cache(cache_device.partitions[0], cache_mode,
cache_line_size)
core = cache.add_core(core_device.partitions[0])
with TestRun.step("Set cleaning policy to NOP."):
cache.set_cleaning_policy(CleaningPolicy.nop)
with TestRun.step("Start IO in background."):
fio = get_fio_cmd(core, core_size)
fio.run_in_background()
time.sleep(10)
with TestRun.step("Set cleaning policy to ACP."):
cache.set_cleaning_policy(CleaningPolicy.acp)
with TestRun.group(
"Verify IO number for different max_flush_buffers values."):
for acp_config in acp_configs:
with TestRun.step(f"Setting {acp_config}"):
cache.set_params_acp(acp_config)
with TestRun.step(
"Using blktrace verify if there is appropriate number of I/O requests, "
"which depends on flush-max-buffer parameter."):
blktrace = BlkTrace(core.core_device, BlkTraceMask.write)
blktrace.start_monitoring()
time.sleep(20)
blktrace_output = blktrace.stop_monitoring()
cleaning_started = False
flush_writes = 0
for (prev, curr) in zip(blktrace_output, blktrace_output[1:]):
if cleaning_started and write_to_core(prev):
flush_writes += 1
if new_acp_iteration(prev, curr):
if cleaning_started:
if flush_writes <= acp_config.flush_max_buffers:
flush_writes = 0
else:
TestRun.LOGGER.error(
f"Incorrect number of handled io requests. "
f"Expected {acp_config.flush_max_buffers} - "
f"actual {flush_writes}")
flush_writes = 0
cleaning_started = True
if not cleaning_started:
TestRun.fail(f"ACP flush not triggered for {acp_config}")
with TestRun.step("Stop all caches"):
kill_all_io()
casadm.stop_all_caches()
def test_wb_throttling():
"""
title: Test CAS with write back throttling enabled on exported object
description: |
Fill cache with data, run intensive IO (rwmix=74) with occasional trims.
pass_criteria:
- Hang task did not occurred
- System did not crashed
"""
with TestRun.step("Prepare devices."):
cache_device = TestRun.disks["cache"]
core_device = TestRun.disks["core"]
set_wbt_lat(cache_device, 0)
set_wbt_lat(core_device, 0)
if core_device.size < cache_device.size:
TestRun.LOGGER.info("Starting cache on partition")
cache_device.create_partitions([core_device.size])
cache_device = cache_device.partitions[0]
with TestRun.step(
f"Start cache with one core in write back with 64k cache line, NOP and disabled seq cutoff"
):
cache = casadm.start_cache(cache_device, CacheMode.WB, CacheLineSize.LINE_64KiB)
cache.set_cleaning_policy(CleaningPolicy.nop)
core = cache.add_core(core_device)
cache.set_seq_cutoff_policy(SeqCutOffPolicy.never)
with TestRun.step("Check wbt_lat value on exported object"):
wbt_lat_val = get_wbt_lat(core)
TestRun.LOGGER.info(f"wbt_lat for exported object is {wbt_lat_val}")
if wbt_lat_val == 0:
TestRun.LOGGER.info(f"Setting wbt_lat for exported object to 75000us")
set_wbt_lat(core, 75000)
with TestRun.step("Fill cache with dirty data"):
fio = get_fio_rw_cmd(core, write_percentage=100)
fio_pid = fio.run_in_background()
wait(
lambda: core.get_statistics(percentage_val=True).usage_stats.dirty == 100,
timeout=timedelta(hours=1),
interval=timedelta(seconds=1),
)
kill_all_io()
with TestRun.step("Run fio with rwmix=74% and occasional trims"):
get_fio_rw_cmd(core, write_percentage=74).run_in_background()
get_fio_trim(core).run_in_background()
with TestRun.step("Change cleaning policy to ACP"):
cache.set_cleaning_policy(CleaningPolicy.acp)
with TestRun.step("Wait for IO processes to finish and print debug informations"):
sleep_interval = timedelta(seconds=5)
eta = runtime
while eta.total_seconds() > 0:
# Instead of explicit sleeping with `time.sleep()` iostat is used for waiting
iostat = IOstat.get_iostat_list(
[core, cache_device, core_device],
since_boot=False,
interval=sleep_interval.total_seconds(),
)
TestRun.LOGGER.debug(f"{iostat}")
eta -= sleep_interval
TestRun.LOGGER.debug(f"ETA: {str(eta)}")
with TestRun.step("Stop all caches"):
kill_all_io()
casadm.stop_all_caches()
def test_alru_no_idle():
"""
title: Test ALRU with activity threshold set to 0
description: |
Verify that ALRU is able to perform cleaning if cache is under constant load and
activity threshold is set to 0. Constant load is performed by using fio instance running
in background.
pass_criteria:
- Dirty cache lines are cleaned successfuly.
"""
with TestRun.step("Prepare configuration"):
cache, core = prepare()
with TestRun.step("Prepare dirty data to be cleaned"):
bg_size = Size(2, Unit.MiB)
(
Fio()
.create_command()
.io_engine(IoEngine.libaio)
.offset(bg_size)
.size(Size(10, Unit.MiB))
.block_size(Size(4, Unit.KiB))
.target(core)
.direct()
.read_write(ReadWrite.randwrite)
.run()
)
with TestRun.step("Run background fio"):
(
Fio()
.create_command()
.io_engine(IoEngine.libaio)
.size(bg_size)
.block_size(Size(4, Unit.KiB))
.target(core)
.direct()
.time_based(True)
.run_time(timedelta(hours=1))
.read_write(ReadWrite.randwrite)
.run_in_background()
)
with TestRun.step("Verify that cache is dirty"):
# Wait for bg fio to dirty whole workset
time.sleep(5)
dirty_before = cache.get_statistics().usage_stats.dirty
if dirty_before == Size(0):
TestRun.fail("Cache should be dirty")
with TestRun.step("Check that cleaning doesn't occur under constant load"):
time.sleep(5)
dirty_now = cache.get_statistics().usage_stats.dirty
if dirty_before > dirty_now:
TestRun.fail(
f"Cleaning has run, while it shouldn't"
" (dirty down from {dirty_before} to {dirty_now}"
)
with TestRun.step("Set 0 idle time and wake up time for ALRU"):
cache.set_params_alru(FlushParametersAlru(activity_threshold=Time(0), wake_up_time=Time(0)))
with TestRun.step("Check that cleaning is progressing"):
time.sleep(5)
if dirty_before <= cache.get_statistics().usage_stats.dirty:
TestRun.fail("Cleaning didn't run")
kill_all_io()
