def base_prepare(item):
with TestRun.LOGGER.step("Cleanup before test"):
Udev.enable()
kill_all_io()
if installer.check_if_installed():
try:
unmount_cas_devices()
casadm.stop_all_caches()
except Exception:
pass # TODO: Reboot DUT if test is executed remotely
for disk in TestRun.dut.disks:
disk.umount_all_partitions()
if not create_partition_table(disk, PartitionTable.gpt):
TestRun.exception(f"Failed to remove partitions from {disk}")
if get_force_param(
item) and not TestRun.plugins['opencas'].already_updated:
installer.reinstall_opencas()
elif not installer.check_if_installed():
installer.install_opencas()
TestRun.plugins['opencas'].already_updated = True
from api.cas import init_config
init_config.create_default_init_config()
TestRun.LOGGER.add_build_info(f'Commit hash:')
TestRun.LOGGER.add_build_info(f"{git.get_current_commit_hash()}")
TestRun.LOGGER.add_build_info(f'Commit message:')
TestRun.LOGGER.add_build_info(f'{git.get_current_commit_message()}')
def base_prepare(item):
with TestRun.LOGGER.step("Cleanup before test"):
TestRun.executor.run("pkill --signal=SIGKILL fsck")
Udev.enable()
kill_all_io()
DeviceMapper.remove_all()
if installer.check_if_installed():
try:
from api.cas.init_config import InitConfig
InitConfig.create_default_init_config()
unmount_cas_devices()
casadm.stop_all_caches()
casadm.remove_all_detached_cores()
except Exception:
pass # TODO: Reboot DUT if test is executed remotely
for disk in TestRun.dut.disks:
disk.umount_all_partitions()
create_partition_table(disk, PartitionTable.gpt)
if get_force_param(item) and not TestRun.usr.already_updated:
installer.rsync_opencas_sources()
installer.reinstall_opencas()
elif not installer.check_if_installed():
installer.rsync_opencas_sources()
installer.set_up_opencas()
TestRun.usr.already_updated = True
TestRun.LOGGER.add_build_info(f'Commit hash:')
TestRun.LOGGER.add_build_info(f"{git.get_current_commit_hash()}")
TestRun.LOGGER.add_build_info(f'Commit message:')
TestRun.LOGGER.add_build_info(f'{git.get_current_commit_message()}')
def pytest_runtest_teardown():
"""
This method is executed always in the end of each test, even if it fails or raises exception in
prepare stage.
"""
if TestRun.outcome == "skipped":
return
TestRun.LOGGER.end_all_groups()
with TestRun.LOGGER.step("Cleanup after test"):
try:
if TestRun.executor:
if TestRun.executor.is_active():
TestRun.executor.wait_for_connection()
Udev.enable()
unmount_cas_devices()
casadm.stop_all_caches()
except Exception as ex:
TestRun.LOGGER.warning(
f"Exception occured during platform cleanup.\n"
f"{str(ex)}\n{traceback.format_exc()}")
if 'test_wrapper' in sys.modules:
try:
test_wrapper.cleanup()
except Exception as ex:
TestRun.LOGGER.warning(
f"Exception occured during test wrapper cleanup.\n{str(ex)}"
f"\n{traceback.format_exc()}")
TestRun.LOGGER.end()
if TestRun.executor:
TestRun.LOGGER.get_additional_logs()
Log.destroy()
def pytest_runtest_teardown():
"""
This method is executed always in the end of each test, even if it fails or raises exception in
prepare stage.
"""
TestRun.LOGGER.end_all_groups()
with TestRun.LOGGER.step("Cleanup after test"):
try:
if TestRun.executor:
if not TestRun.executor.is_active():
TestRun.executor.wait_for_connection()
Udev.enable()
kill_all_io()
unmount_cas_devices()
if installer.check_if_installed():
casadm.remove_all_detached_cores()
casadm.stop_all_caches()
from api.cas.init_config import InitConfig
InitConfig.create_default_init_config()
DeviceMapper.remove_all()
except Exception as ex:
TestRun.LOGGER.warning(f"Exception occurred during platform cleanup.\n"
f"{str(ex)}\n{traceback.format_exc()}")
TestRun.LOGGER.end()
for dut in TestRun.duts:
with TestRun.use_dut(dut):
if TestRun.executor:
os.makedirs(os.path.join(TestRun.LOGGER.base_dir, "dut_info", dut.ip),
exist_ok=True)
TestRun.LOGGER.get_additional_logs()
Log.destroy()
TestRun.teardown()
def test_cleaning_policies_in_write_back(cleaning_policy):
"""
title: Test for cleaning policy operation in Write-Back cache mode.
description: |
Check if ALRU, NOP and ACP cleaning policies preserve their
parameters when changed and if they flush dirty data properly
in Write-Back cache mode.
pass_criteria:
- Flush parameters preserve their values when changed.
- Dirty data is flushed or not according to the policy used.
"""
with TestRun.step("Partition cache and core devices"):
cache_dev, core_dev = storage_prepare()
Udev.disable()
with TestRun.step(
f"Start cache in Write-Back mode with {cleaning_policy} cleaning policy"
):
cache = casadm.start_cache(cache_dev.partitions[0],
CacheMode.WB,
force=True)
set_cleaning_policy_and_params(cache, cleaning_policy)
with TestRun.step("Check for running CAS cleaner"):
if TestRun.executor.run(
f"pgrep {cas_cleaner_process_name}").exit_code != 0:
TestRun.fail("CAS cleaner process is not running!")
with TestRun.step(f"Add {cores_count} cores to the cache"):
core = []
for i in range(cores_count):
core.append(cache.add_core(core_dev.partitions[i]))
with TestRun.step("Run 'fio'"):
fio = fio_prepare()
for i in range(cores_count):
fio.add_job().target(core[i].path)
fio.run()
time.sleep(3)
core_writes_before_wait_for_cleaning = (
cache.get_statistics().block_stats.core.writes)
with TestRun.step(f"Wait {time_to_wait} seconds"):
time.sleep(time_to_wait)
with TestRun.step("Check write statistics for core device"):
core_writes_after_wait_for_cleaning = (
cache.get_statistics().block_stats.core.writes)
check_cleaning_policy_operation(
cleaning_policy,
core_writes_before_wait_for_cleaning,
core_writes_after_wait_for_cleaning,
)
with TestRun.step("Stop all caches"):
casadm.stop_all_caches()
Udev.enable()
def base_prepare(item):
with TestRun.LOGGER.step("Cleanup before test"):
TestRun.executor.run("pkill --signal=SIGKILL fsck")
Udev.enable()
kill_all_io()
DeviceMapper.remove_all()
if installer.check_if_installed():
try:
from api.cas.init_config import InitConfig
InitConfig.create_default_init_config()
unmount_cas_devices()
casadm.stop_all_caches()
casadm.remove_all_detached_cores()
except Exception:
pass # TODO: Reboot DUT if test is executed remotely
raids = Raid.discover()
for raid in raids:
# stop only those RAIDs, which are comprised of test disks
if all(map(lambda device:
any(map(lambda disk_path:
disk_path in device.get_device_id(),
[bd.get_device_id() for bd in TestRun.dut.disks])),
raid.array_devices)):
raid.umount_all_partitions()
raid.remove_partitions()
raid.stop()
for device in raid.array_devices:
Mdadm.zero_superblock(os.path.join('/dev', device.get_device_id()))
Udev.settle()
for disk in TestRun.dut.disks:
disk_serial = get_disk_serial_number(disk.path)
if disk.serial_number != disk_serial:
raise Exception(
f"Serial for {disk.path} doesn't match the one from the config."
f"Serial from config {disk.serial_number}, actual serial {disk_serial}"
)
disk.umount_all_partitions()
Mdadm.zero_superblock(os.path.join('/dev', disk.get_device_id()))
TestRun.executor.run_expect_success("udevadm settle")
disk.remove_partitions()
create_partition_table(disk, PartitionTable.gpt)
if get_force_param(item) and not TestRun.usr.already_updated:
installer.rsync_opencas_sources()
installer.reinstall_opencas()
elif not installer.check_if_installed():
installer.rsync_opencas_sources()
installer.set_up_opencas()
TestRun.usr.already_updated = True
TestRun.LOGGER.add_build_info(f'Commit hash:')
TestRun.LOGGER.add_build_info(f"{git.get_current_commit_hash()}")
TestRun.LOGGER.add_build_info(f'Commit message:')
TestRun.LOGGER.add_build_info(f'{git.get_current_commit_message()}')
def prepare_and_cleanup(request):
"""
This fixture returns the dictionary, which contains DUT ip, IPMI, spider, list of disks.
This fixture also returns the executor of commands
"""
# There should be dut config file added to config package and
# pytest should be executed with option --dut-config=conf_name'.
#
# 'ip' field should be filled with valid IP string to use remote ssh executor
# or it should be commented out when user want to execute tests on local machine
#
# User can also have own test wrapper, which runs test prepare, cleanup, etc.
# Then in the config/configuration.py file there should be added path to it:
# test_wrapper_dir = 'wrapper_path'
try:
with open(request.config.getoption('--dut-config')) as cfg:
dut_config = yaml.safe_load(cfg)
except Exception:
dut_config = {}
if 'test_wrapper' in sys.modules:
if 'ip' in dut_config:
try:
IP(dut_config['ip'])
except ValueError:
raise Exception(
"IP address from configuration file is in invalid format.")
dut_config = test_wrapper.prepare(request.param, dut_config)
TestRun.prepare(dut_config)
TestRun.plugins['opencas'] = {'already_updated': False}
TestRun.LOGGER.info(
f"**********Test {request.node.name} started!**********")
yield
TestRun.LOGGER.info("Test cleanup")
Udev.enable()
unmount_cas_devices()
casadm.stop_all_caches()
if 'test_wrapper' in sys.modules:
test_wrapper.cleanup()
def base_prepare():
TestRun.LOGGER.info("Base test prepare")
TestRun.LOGGER.info(f"DUT info: {TestRun.dut}")
Udev.enable()
kill_all_io()
if installer.check_if_installed():
try:
unmount_cas_devices()
casadm.stop_all_caches()
except Exception:
pass # TODO: Reboot DUT if test is executed remotely
if get_force_param(
) is not "False" and not TestRun.plugins['opencas']['already_updated']:
installer.reinstall_opencas()
elif not installer.check_if_installed():
installer.install_opencas()
TestRun.plugins['opencas']['already_updated'] = True
def base_prepare():
LOGGER.info("Base test prepare")
LOGGER.info(f"DUT info: {TestProperties.dut}")
Udev.enable()
if installer.check_if_installed():
try:
unmount_cas_devices()
casadm.stop_all_caches()
except Exception:
pass # TODO: Reboot DUT if test is executed remotely
for disk in TestProperties.dut.disks:
if disk.is_mounted():
disk.unmount()
disk.remove_partitions()
if get_force_param() is not "False" and not hasattr(c, "already_updated"):
installer.reinstall_opencas()
elif not installer.check_if_installed():
installer.install_opencas()
c.already_updated = True # to skip reinstall every test
def test_cache_stop_and_load(cache_mode):
"""
title: Test for stopping and loading cache back with dynamic cache mode switching.
description: |
Validate the ability of the CAS to switch cache modes at runtime and
check if all of them are working properly after switching and
after stopping and reloading cache back.
Check also other parameters consistency after reload.
pass_criteria:
- In all cache modes data reads and writes are handled properly before and after reload.
- All cache parameters preserve their values after reload.
"""
with TestRun.step("Partition cache and core devices"):
cache_dev, core_dev = storage_prepare()
with TestRun.step(f"Start cache in {cache_mode[0]} mode"):
cache = casadm.start_cache(cache_dev, cache_mode[0], force=True)
Udev.disable()
with TestRun.step("Add core to the cache"):
core = cache.add_core(core_dev)
with TestRun.step(f"Change cache mode to {cache_mode[1]}"):
cache.set_cache_mode(cache_mode[1], flush=True)
check_cache_config = cache.get_cache_config()
with TestRun.step(f"Check if {cache_mode[1]} cache mode works properly"):
check_cache_mode_operation(cache, core, cache_mode[1])
with TestRun.step("Stop and load cache back"):
cache.stop()
cache = casadm.load_cache(cache_dev)
with TestRun.step("Check parameters consistency"):
if check_cache_config != cache.get_cache_config():
failed_params = ""
if check_cache_config.cache_mode != cache.get_cache_mode():
failed_params += (
f"Cache mode is: {check_cache_config.cache_mode}, "
f"should be: {cache.get_cache_mode()}\n")
if check_cache_config.cleaning_policy != cache.get_cleaning_policy(
):
failed_params += (
f"Cleaning policy is: {check_cache_config.cleaning_policy}, "
f"should be: {cache.get_cleaning_policy()}\n")
if check_cache_config.cache_line_size != cache.get_cache_line_size(
):
failed_params += (
f"Cache line size is: {check_cache_config.cache_line_size}, "
f"should be: {cache.get_cache_line_size()}\n")
TestRun.fail(
f"Parameters do not match after reload:\n{failed_params}")
with TestRun.step(
f"Check if {cache_mode[1]} cache mode works properly after reload"
):
if cache_mode[1] == CacheMode.WA or cache_mode[1] == CacheMode.WO:
check_separated_read_write_after_reload(cache, core, cache_mode[1],
io_size)
else:
check_cache_mode_operation(cache, core, cache_mode[1])
with TestRun.step("Stop all caches"):
casadm.stop_all_caches()
Udev.enable()
def test_recovery_flush_reset_fs(cache_mode, fs):
"""
title: Recovery after reset during cache flushing - test on filesystem.
description: |
Verify that unflushed data can be safely recovered, when reset was pressed during
data flushing on filesystem.
pass_criteria:
- CAS recovers successfully after reboot
- No data corruption
"""
with TestRun.step("Prepare cache and core devices."):
cache_disk = TestRun.disks['cache']
core_disk = TestRun.disks['core']
cache_disk.create_partitions([Size(2, Unit.GibiByte)])
core_disk.create_partitions([Size(16, Unit.GibiByte)] * 2)
cache_device = cache_disk.partitions[0]
core_device = core_disk.partitions[0]
core_device_link = core_device.get_device_link("/dev/disk/by-id")
cache_device_link = cache_device.get_device_link("/dev/disk/by-id")
with TestRun.step(f"Create {fs} filesystem on core."):
core_device.create_filesystem(fs)
with TestRun.step("Create test files."):
source_file, target_file = create_test_files(test_file_size)
with TestRun.step("Setup cache and add core."):
cache = casadm.start_cache(cache_device, cache_mode)
Udev.disable()
core = cache.add_core(core_device)
cache.set_cleaning_policy(CleaningPolicy.nop)
cache.set_seq_cutoff_policy(SeqCutOffPolicy.never)
with TestRun.step("Mount CAS device."):
core.mount(mount_point)
with TestRun.step("Copy file to CAS."):
copy_file(source=source_file.full_path,
target=os.path.join(mount_point, "source_test_file"),
size=test_file_size,
direct="oflag")
with TestRun.step("Unmount CAS device."):
core.unmount()
with TestRun.step("Trigger flush."):
TestRun.executor.run_in_background(
cli.flush_cache_cmd(f"{cache.cache_id}"))
with TestRun.step("Hard reset DUT during data flushing."):
power_cycle_dut(True, core_device)
cache_device.path = cache_device_link.get_target()
core_device.path = core_device_link.get_target()
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache_device)
if cache.get_dirty_blocks() == Size.zero():
TestRun.fail("There are no dirty blocks on cache device.")
with TestRun.step("Stop cache with dirty data flush."):
core_writes_before = core_device.get_io_stats().sectors_written
cache.stop()
if core_writes_before >= core_device.get_io_stats().sectors_written:
TestRun.fail(
"No data was flushed after stopping cache started with load option."
)
with TestRun.step("Mount core device."):
core_device.mount(mount_point)
with TestRun.step(
"Copy test file from core device to temporary location. "
"Compare it with the first version – they should be the same."):
copy_file(source=os.path.join(mount_point, "source_test_file"),
target=target_file.full_path,
size=test_file_size,
direct="iflag")
compare_files(source_file, target_file)
with TestRun.step("Unmount core device and remove test files."):
core_device.unmount()
target_file.remove()
source_file.remove()
Udev.enable()
def test_trace_stats():
"""
title: Test for generated statistics correctness
description: |
Generate workload usign fio and compare fio output with iotracer output.
pass_criteria:
- Output generated by iotracer matches with workload triggered with fio.
"""
Udev.disable()
iotrace = TestRun.plugins["iotrace"]
dev = TestRun.disks["traced_dev"]
with TestRun.group("Compare read stats"):
fio_cmd = get_fio_cmd(dev)
fio_cmd = fio_cmd.read_write(ReadWrite.randread)
with TestRun.step("Start tracing"):
iotrace.start_tracing([dev.system_path])
fio_output = fio_cmd.run()[0]
with TestRun.step("Stop tracing"):
iotrace.stop_tracing()
trace_path = iotrace.get_latest_trace_path()
stats = DeviceTraceStatistics(
iotrace.get_trace_statistics(trace_path=trace_path,
dev_path=dev.system_path))
read_stats = stats.read
with TestRun.step("Compare names of tested device"):
if len(fio_output.disks_name()) != 1:
TestRun.LOGGER.error("Number of traced devices doesn't match!")
if stats.device.name != fio_output.disks_name()[0]:
TestRun.LOGGER.error("Names of devices doesn't match!")
with TestRun.step("Compare read average bandwidth"):
fio_bandwidth = fio_output.read_bandwidth_average().get_value()
iotrace_bandwidth = read_stats.metrics.bandwidth.get_value()
if not isclose(iotrace_bandwidth,
fio_bandwidth,
rel_tol=tolerance_threshold):
TestRun.LOGGER.error(
"Difference between fio bandwidth and iotrace read bandwidth is out of "
f"acceptable threshold. Iotrace: {iotrace_bandwidth}, "
f"fio: {fio_bandwidth}")
with TestRun.step("Compare read workset"):
fio_workset = fio_output.read_io().get_value()
iotrace_workset = read_stats.metrics.workset.get_value()
if not isclose(
iotrace_workset, fio_workset, rel_tol=tolerance_threshold):
TestRun.LOGGER.error(
"Difference between fio and iotrace read workset is out of "
f"acceptable threshold. Iotrace: {iotrace_workset}, "
f"fio: {fio_workset}")
with TestRun.step("Compare read IOPS"):
fio_iops = fio_output.read_iops()
iotrace_iops = read_stats.metrics.iops
if not isclose(iotrace_iops, fio_iops,
rel_tol=tolerance_threshold):
TestRun.LOGGER.error(
"Difference between fio and iotrace read iops is out of "
f"acceptable threshold. Iotrace: {iotrace_iops}, "
f"fio: {fio_iops}")
with TestRun.step("Compare read latency"):
fio_latency_avg = (fio_output.read_completion_latency_average().
total_nanoseconds())
iotrace_latency_avg = read_stats.latency.average.total_nanoseconds(
)
if not isclose(
iotrace_latency_avg,
fio_latency_avg,
abs_tol=latency_tolerance_threshold,
):
TestRun.LOGGER.error(
"Difference between fio and iotrace read avarage latency is out of "
f"acceptable threshold. Iotrace: {iotrace_latency_avg}, "
f"fio: {fio_latency_avg}")
fio_latency_min = (
fio_output.read_completion_latency_min().total_nanoseconds())
iotrace_latency_min = read_stats.latency.min.total_nanoseconds()
if not isclose(
iotrace_latency_min,
fio_latency_min,
abs_tol=latency_tolerance_threshold,
):
TestRun.LOGGER.error(
"Difference between fio and iotrace read min latency is out of "
f"acceptable threshold. Iotrace: {iotrace_latency_min}, "
f"fio: {fio_latency_min}")
fio_latency_max = (
fio_output.read_completion_latency_max().total_nanoseconds())
iotrace_latency_max = read_stats.latency.max.total_nanoseconds()
if not isclose(
iotrace_latency_max,
fio_latency_max,
abs_tol=latency_tolerance_threshold,
):
TestRun.LOGGER.error(
"Difference between fio and iotrace read max latency is out of "
f"acceptable threshold. Iotrace: {iotrace_latency_max}, "
f"fio: {fio_latency_max}")
with TestRun.step("Compare read size"):
fio_size = fio_output.read_io().get_value()
iotrace_size = read_stats.size.total.get_value()
if not isclose(iotrace_size, fio_size,
rel_tol=tolerance_threshold):
TestRun.LOGGER.error(
"Difference between fio and iotrace read size is out of "
f"acceptable threshold. Iotrace: {iotrace_size}, "
f"fio: {fio_size}")
with TestRun.step("Compare number of read requests"):
fio_reqs = fio_output.read_requests_number()
iotrace_reqs = read_stats.count
if not isclose(iotrace_reqs, fio_reqs,
rel_tol=tolerance_threshold):
TestRun.LOGGER.error(
"Difference between fio and iotrace number of read reqs is out of "
f"acceptable threshold. Iotrace: {iotrace_reqs}, "
f"fio: {fio_reqs}")
with TestRun.step("Compare read latency percentiles"):
fio_percentiles = fio_output.read_completion_latency_percentile()
iotrace_percentiles = read_stats.latency.percentiles
fio_90 = fio_percentiles["90.000000"]
fio_99 = fio_percentiles["99.000000"]
fio_99_9 = fio_percentiles["99.900000"]
fio_99_99 = fio_percentiles["99.990000"]
if not isclose(iotrace_percentiles.le90,
fio_90,
rel_tol=tolerance_threshold):
TestRun.LOGGER.error(
"Difference between fio and iotrace read 90 percentile is out of "
f"acceptable threshold. Iotrace: {iotrace_percentiles.le90}, "
f"fio: {fio_90}")
if not isclose(iotrace_percentiles.le99,
fio_99,
rel_tol=tolerance_threshold):
TestRun.LOGGER.error(
"Difference between fio and iotrace read 99 percentile is out of "
f"acceptable threshold. Iotrace: {iotrace_percentiles.le99}, "
f"fio: {fio_99}")
if not isclose(iotrace_percentiles.le99_90,
fio_99_9,
rel_tol=tolerance_threshold):
TestRun.LOGGER.error(
"Difference between fio and iotrace read 99.9 percentile is out of "
f"acceptable threshold. Iotrace: {iotrace_percentiles.le99_90}, "
f"fio: {fio_99_9}")
if not isclose(iotrace_percentiles.le99_99,
fio_99_99,
rel_tol=tolerance_threshold):
TestRun.LOGGER.error(
"Difference between fio and iotrace read 99.99 percentile is out of "
f"acceptable threshold. Iotrace: {iotrace_percentiles.le99_99}, "
f"fio: {fio_99_99}")
with TestRun.step("Compare number of errors"):
fio_errors = fio_output.total_errors()
iotrace_errors = read_stats.errors
if not isclose(
iotrace_errors, fio_errors, rel_tol=tolerance_threshold):
TestRun.LOGGER.error(
"Difference between fio and iotrace errors is out of "
f"acceptable threshold. Iotrace: {iotrace_errors}, "
f"fio: {fio_errors}")
with TestRun.group("Compare write stats"):
fio_cmd = get_fio_cmd(dev)
fio_cmd = fio_cmd.read_write(ReadWrite.randwrite)
with TestRun.step("Start tracing"):
iotrace.start_tracing([dev.system_path])
fio_output = fio_cmd.run()[0]
with TestRun.step("Stop tracing"):
iotrace.stop_tracing()
trace_path = iotrace.get_latest_trace_path()
stats = DeviceTraceStatistics(
iotrace.get_trace_statistics(trace_path=trace_path,
dev_path=dev.system_path))
write_stats = stats.write
with TestRun.step("Compare write average bandwidth"):
fio_bandwidth = fio_output.write_bandwidth_average().get_value()
iotrace_bandwidth = write_stats.metrics.bandwidth.get_value()
if not isclose(iotrace_bandwidth,
fio_bandwidth,
rel_tol=tolerance_threshold):
TestRun.LOGGER.error(
"Difference between fio bandwidth and iotrace write bandwidth is out of "
f"acceptable threshold. Iotrace: {iotrace_bandwidth}, "
f"fio: {fio_bandwidth}")
with TestRun.step("Compare number of requests"):
fio_requests = fio_output.write_requests_number()
iotrace_requests = write_stats.count
if not isclose(iotrace_requests,
fio_requests,
rel_tol=tolerance_threshold):
TestRun.LOGGER.error(
"Difference between fio and iotrace number of write reqs is out of "
f"acceptable threshold. Iotrace: {iotrace_bandwidth}, "
f"fio: {fio_bandwidth}")
with TestRun.step("Compare write workset"):
fio_workset = fio_output.write_io().get_value()
iotrace_workset = write_stats.metrics.workset.get_value()
if not isclose(
iotrace_workset, fio_workset, rel_tol=tolerance_threshold):
TestRun.LOGGER.error(
"Difference between fio and iotrace write workset is out of "
f"acceptable threshold. Iotrace: {iotrace_workset}, "
f"fio: {fio_workset}")
with TestRun.step("Compare write IOPS"):
fio_iops = fio_output.write_iops()
iotrace_iops = write_stats.metrics.iops
if not isclose(iotrace_iops, fio_iops,
rel_tol=tolerance_threshold):
TestRun.LOGGER.error(
"Difference between fio and iotrace write iops is out of "
f"acceptable threshold. Iotrace: {iotrace_bandwidth}, "
f"fio: {fio_bandwidth}")
with TestRun.step("Compare write latency"):
fio_latency_avg = (fio_output.write_completion_latency_average().
total_nanoseconds())
iotrace_latency_avg = write_stats.latency.average.total_nanoseconds(
)
if not isclose(
iotrace_latency_avg,
fio_latency_avg,
abs_tol=latency_tolerance_threshold,
):
TestRun.LOGGER.error(
"Difference between fio and iotrace write avarage latency is out of "
f"acceptable threshold. Iotrace: {iotrace_latency_avg}, "
f"fio: {fio_latency_avg}")
fio_latency_min = (
fio_output.write_completion_latency_min().total_nanoseconds())
iotrace_latency_min = write_stats.latency.min.total_nanoseconds()
if not isclose(
iotrace_latency_min,
fio_latency_min,
abs_tol=latency_tolerance_threshold,
):
TestRun.LOGGER.error(
"Difference between fio and iotrace write min latency is out of "
f"acceptable threshold. Iotrace: {iotrace_latency_min}, "
f"fio: {fio_latency_min}")
fio_latency_max = (
fio_output.write_completion_latency_max().total_nanoseconds())
iotrace_latency_max = write_stats.latency.max.total_nanoseconds()
if not isclose(
iotrace_latency_max,
fio_latency_max,
abs_tol=latency_tolerance_threshold,
):
TestRun.LOGGER.error(
"Difference between fio and iotrace write max latency is out of "
f"acceptable threshold. Iotrace: {iotrace_latency_max}, "
f"fio: {fio_latency_max}")
with TestRun.step("Compare write latency percentiles"):
fio_percentiles = fio_output.write_completion_latency_percentile()
iotrace_percentiles = write_stats.latency.percentiles
fio_90 = fio_percentiles["90.000000"]
fio_99 = fio_percentiles["99.000000"]
fio_99_9 = fio_percentiles["99.900000"]
fio_99_99 = fio_percentiles["99.990000"]
if not isclose(iotrace_percentiles.le90,
fio_90,
rel_tol=tolerance_threshold):
TestRun.LOGGER.error(
"Difference between fio and iotrace write 90 percentile is out of "
f"acceptable threshold. Iotrace: {iotrace_percentiles.le90}, "
f"fio: {fio_90}")
if not isclose(iotrace_percentiles.le99,
fio_99,
rel_tol=tolerance_threshold):
TestRun.LOGGER.error(
"Difference between fio and iotrace write 99 percentile is out of "
f"acceptable threshold. Iotrace: {iotrace_percentiles.le99}, "
f"fio: {fio_99}")
if not isclose(iotrace_percentiles.le99_90,
fio_99_9,
rel_tol=tolerance_threshold):
TestRun.LOGGER.error(
"Difference between fio and iotrace write 99.9 percentile is out of "
f"acceptable threshold. Iotrace: {iotrace_percentiles.le99_90}, "
f"fio: {fio_99_9}")
if not isclose(iotrace_percentiles.le99_99,
fio_99_99,
rel_tol=tolerance_threshold):
TestRun.LOGGER.error(
"Difference between fio and iotrace write 99.99 percentile is out of "
f"acceptable threshold. Iotrace: {iotrace_percentiles.le99_99}, "
f"fio: {fio_99_99}")
with TestRun.step("Compare number of errors"):
fio_errors = fio_output.total_errors()
iotrace_errors = write_stats.errors
if not isclose(
iotrace_errors, fio_errors, rel_tol=tolerance_threshold):
TestRun.LOGGER.error(
"Difference between fio and iotrace errors is out of "
f"acceptable threshold. Iotrace: {iotrace_errors}, "
f"fio: {fio_errors}")
with TestRun.group("Compare discard stats"):
fio_cmd = get_fio_cmd(dev)
fio_cmd = fio_cmd.read_write(ReadWrite.randtrim)
with TestRun.step("Start tracing"):
iotrace.start_tracing([dev.system_path])
fio_output = fio_cmd.run()[0]
with TestRun.step("Stop tracing"):
iotrace.stop_tracing()
trace_path = iotrace.get_latest_trace_path()
stats = DeviceTraceStatistics(
iotrace.get_trace_statistics(trace_path=trace_path,
dev_path=dev.system_path))
discard_stats = stats.discard
with TestRun.step("Compare discard average bandwidth"):
fio_bandwidth = fio_output.trim_bandwidth_average().get_value()
iotrace_bandwidth = discard_stats.metrics.bandwidth.get_value()
if not isclose(iotrace_bandwidth,
fio_bandwidth,
rel_tol=tolerance_threshold):
TestRun.LOGGER.error(
"Difference between fio bandwidth and iotrace discard bandwidth is out of "
f"acceptable threshold. Iotrace: {iotrace_bandwidth}, "
f"fio: {fio_bandwidth}")
with TestRun.step("Compare discard workset"):
fio_workset = fio_output.trim_io().get_value()
iotrace_workset = discard_stats.metrics.workset.get_value()
if not isclose(
iotrace_workset, fio_workset, rel_tol=tolerance_threshold):
TestRun.LOGGER.error(
"Difference between fio and iotrace discard workset is out of "
f"acceptable threshold. Iotrace: {iotrace_workset}, "
f"fio: {fio_workset}")
with TestRun.step("Compare discard IOPS"):
fio_iops = fio_output.trim_iops()
iotrace_iops = discard_stats.metrics.iops
if not isclose(iotrace_iops, fio_iops,
rel_tol=tolerance_threshold):
TestRun.LOGGER.error(
"Difference between fio and iotrace discard iops is out of "
f"acceptable threshold. Iotrace: {iotrace_bandwidth}, "
f"fio: {fio_bandwidth}")
with TestRun.step("Compare discard latency"):
fio_latency_avg = (fio_output.trim_completion_latency_average().
total_nanoseconds())
iotrace_latency_avg = discard_stats.latency.average.total_nanoseconds(
)
if not isclose(
iotrace_latency_avg,
fio_latency_avg,
abs_tol=latency_tolerance_threshold,
):
TestRun.LOGGER.error(
"Difference between fio and iotrace discard avarage latency is out of "
f"acceptable threshold. Iotrace: {iotrace_latency_avg}, "
f"fio: {fio_latency_avg}")
fio_latency_min = (
fio_output.trim_completion_latency_min().total_nanoseconds())
iotrace_latency_min = discard_stats.latency.min.total_nanoseconds()
if not isclose(
iotrace_latency_min,
fio_latency_min,
abs_tol=latency_tolerance_threshold,
):
TestRun.LOGGER.error(
"Difference between fio and iotrace discard min latency is out of "
f"acceptable threshold. Iotrace: {iotrace_latency_min}, "
f"fio: {fio_latency_min}")
fio_latency_max = (
fio_output.trim_completion_latency_max().total_nanoseconds())
iotrace_latency_max = discard_stats.latency.max.total_nanoseconds()
if not isclose(
iotrace_latency_max,
fio_latency_max,
abs_tol=latency_tolerance_threshold,
):
TestRun.LOGGER.error(
"Difference between fio and iotrace discard max latency is out of "
f"acceptable threshold. Iotrace: {iotrace_latency_max}, "
f"fio: {fio_latency_max}")
with TestRun.step("Compare discard latency percentiles"):
fio_percentiles = fio_output.trim_completion_latency_percentile()
iotrace_percentiles = discard_stats.latency.percentiles
fio_90 = fio_percentiles["90.000000"]
fio_99 = fio_percentiles["99.000000"]
fio_99_9 = fio_percentiles["99.900000"]
fio_99_99 = fio_percentiles["99.990000"]
if not isclose(iotrace_percentiles.le90,
fio_90,
rel_tol=tolerance_threshold):
TestRun.LOGGER.error(
"Difference between fio and iotrace discard 90 percentile is out of "
f"acceptable threshold. Iotrace: {iotrace_percentiles.le90}, "
f"fio: {fio_90}")
if not isclose(iotrace_percentiles.le99,
fio_99,
rel_tol=tolerance_threshold):
TestRun.LOGGER.error(
"Difference between fio and iotrace discard 99 percentile is out of "
f"acceptable threshold. Iotrace: {iotrace_percentiles.le99}, "
f"fio: {fio_99}")
if not isclose(iotrace_percentiles.le99_90,
fio_99_9,
rel_tol=tolerance_threshold):
TestRun.LOGGER.error(
"Difference between fio and iotrace discard 99.9 percentile is out of "
f"acceptable threshold. Iotrace: {iotrace_percentiles.le99_90}, "
f"fio: {fio_99_9}")
if not isclose(iotrace_percentiles.le99_99,
fio_99_99,
rel_tol=tolerance_threshold):
TestRun.LOGGER.error(
"Difference between fio and iotrace discard 99.99 percentile is out of "
f"acceptable threshold. Iotrace: {iotrace_percentiles.le99_99}, "
f"fio: {fio_99_99}")
with TestRun.step("Compare number of errors"):
fio_errors = fio_output.total_errors()
iotrace_errors = discard_stats.errors
if not isclose(
iotrace_errors, fio_errors, rel_tol=tolerance_threshold):
TestRun.LOGGER.error(
"Difference between fio and iotrace errors is out of "
f"acceptable threshold. Iotrace: {iotrace_errors}, "
f"fio: {fio_errors}")
Udev.enable()
def test_print_statistics_inactive(cache_mode):
"""
title: Print statistics for cache with inactive cache volumes.
description: |
Check if statistics are displayed properly when there is one or more
inactive cache volumes added to cache.
pass_criteria:
- No kernel error
- All statistics should contain appropriate information depending on situation of
cache and core devices (as described in test steps)
"""
with TestRun.step("Prepare devices."):
devices = prepare_devices([("cache", 1), ("core1", 1), ("core2", 1)])
cache_dev = devices["cache"].partitions[0]
first_core_dev = devices["core1"].partitions[0]
second_core_dev = devices["core2"].partitions[0]
first_plug_device = devices["core1"]
second_plug_device = devices["core2"]
Udev.disable(
) # disabling udev for a while prevents creating clean data on cores
with TestRun.step("Start cache and add cores."):
cache = casadm.start_cache(cache_dev,
cache_mode=cache_mode,
force=True)
first_core = cache.add_core(first_core_dev)
second_core = cache.add_core(second_core_dev)
cache_mode_traits = CacheMode.get_traits(cache.get_cache_mode())
with TestRun.step("Disable cleaning and sequential cutoff policies."):
cache.set_cleaning_policy(CleaningPolicy.nop)
cache.set_seq_cutoff_policy(SeqCutOffPolicy.never)
with TestRun.step(
"Create init config file using current CAS configuration."):
InitConfig.create_init_config_from_running_configuration()
with TestRun.step("Run IO."):
run_fio([first_core.path, second_core.path])
with TestRun.step(
"Print statistics and check if there is no inactive usage section."
):
active_stats = cache.get_statistics()
check_if_inactive_section_exists(active_stats, False)
with TestRun.step("Stop cache."):
if CacheModeTrait.LazyWrites in cache_mode_traits:
cache.stop(no_data_flush=True)
else:
cache.stop()
with TestRun.step("Remove both core devices from OS."):
Udev.enable() # enable udev back because it's necessary now
first_plug_device.unplug()
second_plug_device.unplug()
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache_dev)
with TestRun.step(
"Check if inactive devices section appeared and contains appropriate "
"information."):
inactive_stats_before = cache.get_statistics()
check_if_inactive_section_exists(inactive_stats_before)
check_number_of_inactive_devices(inactive_stats_before, 2)
with TestRun.step(
"Attach one of detached core devices and add it to cache."):
first_plug_device.plug()
time.sleep(1)
first_core_status = first_core.get_status()
if first_core_status != CoreStatus.active:
TestRun.fail(
f"Core {first_core.path} should be in active state but it is not. "
f"Actual state: {first_core_status}.")
with TestRun.step("Check cache statistics section of inactive devices."):
inactive_stats_after = cache.get_statistics()
check_if_inactive_section_exists(inactive_stats_after)
check_number_of_inactive_devices(inactive_stats_after, 1)
# criteria for checks below
insert_write_traits = CacheModeTrait.InsertWrite in cache_mode_traits
lazy_write_traits = CacheModeTrait.LazyWrites in cache_mode_traits
lazy_writes_or_no_insert_write_traits = (not insert_write_traits
or lazy_write_traits)
check_inactive_usage_stats(
inactive_stats_before.inactive_usage_stats.inactive_occupancy,
inactive_stats_after.inactive_usage_stats.inactive_occupancy,
"inactive occupancy", not insert_write_traits)
check_inactive_usage_stats(
inactive_stats_before.inactive_usage_stats.inactive_clean,
inactive_stats_after.inactive_usage_stats.inactive_clean,
"inactive clean", lazy_writes_or_no_insert_write_traits)
check_inactive_usage_stats(
inactive_stats_before.inactive_usage_stats.inactive_dirty,
inactive_stats_after.inactive_usage_stats.inactive_dirty,
"inactive dirty", not lazy_write_traits)
with TestRun.step("Check statistics per inactive core."):
inactive_core_stats = second_core.get_statistics()
if inactive_stats_after.inactive_usage_stats.inactive_occupancy == \
inactive_core_stats.usage_stats.occupancy:
TestRun.LOGGER.info(
"Inactive occupancy in cache statistics is equal to inactive core "
"occupancy.")
else:
TestRun.fail(
f"Inactive core occupancy ({inactive_core_stats.usage_stats.occupancy}) "
f"should be the same as cache inactive occupancy "
f"({inactive_stats_after.inactive_usage_stats.inactive_occupancy})."
)
with TestRun.step(
"Remove inactive core from cache and check if cache is in running state."
):
cache.remove_core(second_core.core_id, force=True)
cache_status = cache.get_status()
if cache_status != CacheStatus.running:
TestRun.fail(
f"Cache did not change status to 'running' after plugging core device. "
f"Actual status: {cache_status}.")
with TestRun.step(
"Check if there is no inactive devices statistics section and if cache has "
"Running status."):
cache_stats = cache.get_statistics()
check_if_inactive_section_exists(cache_stats, False)
check_number_of_inactive_devices(cache_stats, 0)
with TestRun.step("Plug missing disk and stop cache."):
second_plug_device.plug()
time.sleep(1)
cache.stop()
def prepare_and_cleanup(request):
"""
This fixture returns the dictionary, which contains DUT ip, IPMI, spider, list of disks.
This fixture also returns the executor of commands
"""
# There should be dut config file added to config package and
# pytest should be executed with option --dut-config=conf_name'.
#
# 'ip' field should be filled with valid IP string to use remote ssh executor
# or it should be commented out when user want to execute tests on local machine
#
# User can also have own test wrapper, which runs test prepare, cleanup, etc.
# Then in the config/configuration.py file there should be added path to it:
# test_wrapper_dir = 'wrapper_path'
LOGGER.info(f"**********Test {request.node.name} started!**********")
try:
dut_config = importlib.import_module(
f"config.{request.config.getoption('--dut-config')}")
except:
dut_config = None
if os.path.exists(c.test_wrapper_dir):
if hasattr(dut_config, 'ip'):
try:
IP(dut_config.ip)
except ValueError:
raise Exception(
"IP address from configuration file is in invalid format.")
TestProperties.dut = Dut(test_wrapper.prepare(request, dut_config))
elif dut_config is not None:
if hasattr(dut_config, 'ip'):
try:
IP(dut_config.ip)
if hasattr(dut_config, 'user') and hasattr(
dut_config, 'password'):
executor = SshExecutor(dut_config.ip, dut_config.user,
dut_config.password)
TestProperties.executor = executor
else:
raise Exception("There is no credentials in config file.")
if hasattr(dut_config, 'disks'):
TestProperties.dut = Dut({
'ip': dut_config.ip,
'disks': dut_config.disks
})
else:
TestProperties.dut = Dut({
'ip': dut_config.ip,
'disks': disk_finder.find_disks()
})
except ValueError:
raise Exception(
"IP address from configuration file is in invalid format.")
elif hasattr(dut_config, 'disks'):
TestProperties.executor = LocalExecutor()
TestProperties.dut = Dut({'disks': dut_config.disks})
else:
TestProperties.executor = LocalExecutor()
TestProperties.dut = Dut({'disks': disk_finder.find_disks()})
else:
raise Exception(
"There is neither configuration file nor test wrapper attached to tests execution."
)
yield
TestProperties.LOGGER.info("Test cleanup")
Udev.enable()
unmount_cas_devices()
casadm.stop_all_caches()
if os.path.exists(c.test_wrapper_dir):
test_wrapper.cleanup(TestProperties.dut)
