def wait(iotracer: IotracePlugin):
time = 0
while iotracer.check_if_tracing_active():
sleep(2)
time += 2
if time >= test_timeout:
iotracer.stop_tracing()
TestRun.fail("Tracing reached test time limit.")
def test_source_package_installation():
TestRun.LOGGER.info("Testing source package installation")
iotrace: IotracePlugin = TestRun.plugins['iotrace']
work_path: str = f"{iotrace.working_dir}/standalone-linux-io-tracer"
disk = TestRun.dut.disks[0]
with TestRun.step("Building iotrace source package"):
TestRun.executor.run_expect_success(
f"cd {work_path} && "
"make package_source -j`nproc --all`")
iotrace_installed = check_if_installed()
if iotrace_installed:
with TestRun.step("Uninstall existing iotrace if needed"):
uninstall_iotrace()
with TestRun.step("Remove old source package"):
TestRun.executor.run_expect_success(f"cd {work_path} && "
f"rm -rf iotrace-*-Source")
with TestRun.step("Unpack and install source package"):
TestRun.executor.run_expect_success(
f"tar -xvf {work_path}/build/release/iotrace-*.tar.gz -C {work_path} &&"
f"cd {work_path}/iotrace-*-Source &&"
"./setup_dependencies.sh &&"
"make install -j`nproc --all`")
with TestRun.step("Check if iotrace is installed"):
iotrace.version()
iotrace.start_tracing([disk.system_path])
stopped = iotrace.stop_tracing()
if not stopped:
raise Exception("Could not stop active tracing.")
trace_path = IotracePlugin.get_latest_trace_path()
summary_parsed = IotracePlugin.get_trace_summary(trace_path)
if summary_parsed['state'] != "COMPLETE":
TestRun.LOGGER.error("Trace state is not complete")
with TestRun.step("Uninstall iotrace"):
uninstall_iotrace()
with TestRun.step("Check if iotrace was uninstalled"):
TestRun.executor.run_expect_fail("iotrace -V")
if iotrace_installed:
with TestRun.step("Reinstall iotrace"):
install_iotrace()
def test_fuzz_trace_file():
fuzzing_time_seconds = 20 * 60
iotrace: IotracePlugin = TestRun.plugins['iotrace']
repo_path: str = f"{iotrace.working_dir}/slit-afl"
# Create trace files
iotrace.start_tracing()
iotrace.stop_tracing()
trace_repo_path = IotracePlugin.get_trace_repository_path()
trace_path = trace_repo_path + "/" + IotracePlugin.get_latest_trace_path()
tracefile_path = f'{trace_path}/octf.trace.0'
copied_tracefile_path = f'{repo_path}/rootfs/var/lib/octf/trace/' + \
f'{IotracePlugin.get_latest_trace_path()}' \
f'/octf.trace.0'
# Create patch file for redirecting fuzzed stdin to trace file
new_patch_path: str = f'{iotrace.working_dir}/redirect_to_tracefile.patch'
create_patch_redirect_fuzz_to_file(f'{copied_tracefile_path}',
new_patch_path)
# Install iotrace locally with AFL support and redirect patch
install_iotrace_with_afl_support(new_patch_path, [
'modules/open-cas-telemetry-framework', 'modules/test-framework',
'source/kernel'
])
# Copy trace files to local instalation of iotrace
TestRun.executor.run_expect_success(
f'cp -r {trace_repo_path}/kernel '
f'{repo_path}/rootfs/var/lib/octf/trace/')
TestRun.executor.run_expect_success(
f'cd {repo_path} && mkdir -p afl-i afl-o')
# Add input seed which shall be mutated
TestRun.executor.run_expect_success(
f'cd {repo_path} && echo "0" > afl-i/case0')
TestRun.LOGGER.info(
f'Starting fuzzing {tracefile_path} This should take ' +
str(fuzzing_time_seconds / 60) + ' minutes')
TestRun.executor.run(f'cd {repo_path} && ./tests/security/fuzzy/fuzz.sh '
'"rootfs/bin/iotrace --get-trace-statistics -p '
f'{IotracePlugin.get_latest_trace_path()}" --one-job')
output = wait_for_completion(fuzzing_time_seconds, repo_path)
TestRun.executor.run(
f'cd {repo_path} && ./tests/security/fuzzy/fuzz.sh clean')
TestRun.executor.run_expect_success(f'rm -rf {repo_path}/afl-i')
detect_crashes(output, "trace-file")
def test_trace_start_stop():
TestRun.LOGGER.info("Testing starting and stopping of tracing")
iotrace: IotracePlugin = TestRun.plugins['iotrace']
iotrace.start_tracing()
stopped = iotrace.stop_tracing()
if not stopped:
raise Exception("Could not stop active tracing.")
trace_path = IotracePlugin.get_latest_trace_path()
summary_parsed = IotracePlugin.get_trace_summary(trace_path)
if summary_parsed['state'] != "COMPLETE":
raise Exception("Trace state is not complete")
def test_compare_summary():
"""
title: Compare trace summaries.
description: |
Verify the summary returned at the end of tracing
(output of 'iotrace –S') is the same as the summary read
from trace files (output of 'iotrace -M -G').
pass_criteria:
- Trace summaries are the same.
"""
with TestRun.step("Run tracing for a while"):
disk = TestRun.dut.disks[0]
output = parse_json(TestRun.executor.run(
f'iotrace -S -d {disk.system_path} -t {int(runtime.total_seconds())}').stdout)[-1]
with TestRun.step("Compare tracing summaries."):
summary = IotracePlugin.get_trace_summary(output["tracePath"])
if output != summary:
err_output = "The summary read from trace file differs from summary " \
"returned at the end of tracing.\n"
fault = False
if summary["tracePath"] != output["tracePath"]:
fault = True
err_output += (f'Trace paths are different:'
f'\ntrace path from file: "{summary["tracePath"]}"'
f'\ntrace path from output: "{output["tracePath"]}"\n')
if summary["state"] != output["state"]:
fault = True
err_output += (f'Trace states are different:'
f'\ntrace state from file: "{summary["state"]}"'
f'\ntrace state from output: "{output["state"]}"\n')
if summary["sourceNode"]["node"][0]["id"] != output["sourceNode"]["node"][0]["id"]:
fault = True
err_output += (f'Node\'s IDs are different:'
f'\nnode\'s ID from file: {summary["sourceNode"]["node"][0]["id"]}'
f'\nnode\'s ID from output: '
f'{output["sourceNode"]["node"][0]["id"]}\n')
if summary["traceStartDateTime"] != output["traceStartDateTime"]:
fault = True
err_output += (f'Trace start times are different:'
f'\ntrace start time from file: {summary["traceStartDateTime"]}'
f'\ntrace start time from output: {output["traceStartDateTime"]}\n')
if summary["traceDuration"] != output["traceDuration"]:
fault = True
err_output += (f'Trace duration times are different:'
f'\ntrace duration time from file: {summary["traceDuration"]}'
f'\ntrace duration time from output: {output["traceDuration"]}\n')
if summary["label"] != output["label"]:
fault = True
err_output += (f'Trace labels are different:'
f'\ntrace label from output: "{summary["label"]}"'
f'\nexpected trace label: "{output["label"]}"\n')
if fault:
TestRun.fail(err_output)
def test_verify_flushes():
TestRun.LOGGER.info("Testing io events during tracing")
iotrace = TestRun.plugins['iotrace']
for disk in TestRun.dut.disks:
with TestRun.step("Start tracing"):
iotrace.start_tracing([disk.system_path])
time.sleep(5)
with TestRun.step("Run test workload with sync"):
fio = (Fio().create_command().io_engine(IoEngine.sync).block_size(
Size(4, Unit.KibiByte)).time_based().read_write(
ReadWrite.randwrite).target(
disk.system_path).direct(value=False).run_time(
datetime.timedelta(seconds=10)).fsync(value=1))
fio.run()
with TestRun.step("Stop tracing"):
iotrace.stop_tracing()
with TestRun.step("Verify trace correctness"):
trace_path = IotracePlugin.get_latest_trace_path()
events_parsed = IotracePlugin.get_trace_events(trace_path)
result = any('io' in event and 'fua' in event['io'] and bool(
event['io']['fua'] is True) for event in events_parsed)
if not result:
TestRun.fail("Could not find event with fua")
result = any('io' in event and 'flush' in event['io'] and bool(
event['io']['flush'] is True) for event in events_parsed)
if not result:
TestRun.fail("Could not find event with flush")
result = all(
'flush' in event['io'] and bool(event['io']['flush']) is True
for event in filter(
lambda event: 'io' in event and 'lba' in event['io'] and
int(event['io']['lba']) == 0 and int(event['io'][
'len']) == 0, events_parsed))
if not result:
TestRun.fail(
"All events with lba 0 and len 0 should have flush")
def pytest_runtest_setup(item):
TestRun.prepare(item)
test_name = item.name.split('[')[0]
TestRun.LOGGER = create_log(item.config.getoption('--log-path'), test_name)
with TestRun.LOGGER.step("Test initialization"):
try:
# Open and parse yaml config file
try:
with open(item.config.getoption('--dut-config')) as cfg:
dut_config = yaml.safe_load(cfg)
except Exception as e:
print(e)
exit(1)
# Setup from dut config
TestRun.setup(dut_config)
TestRun.plugins['iotrace'] = IotracePlugin(
repo_dir=os.path.join(os.path.dirname(__file__), "../"),
working_dir=dut_config['working_dir'])
except Exception as e:
TestRun.LOGGER.exception(f"{str(e)}\n{traceback.format_exc()}")
TestRun.LOGGER.info(f"DUT info: {TestRun.dut}")
# Prepare DUT for test
with TestRun.LOGGER.step("DUT prepare"):
TestRun.LOGGER.add_build_info(f'Commit hash:')
TestRun.LOGGER.add_build_info(f"{get_current_commit_hash()}")
TestRun.LOGGER.add_build_info(f'Commit message:')
TestRun.LOGGER.add_build_info(f'{get_current_commit_message()}')
TestRun.LOGGER.add_build_info(f'OCTF commit hash:')
TestRun.LOGGER.add_build_info(f'{get_current_octf_hash()}')
dut_prepare(item)
TestRun.LOGGER.start_group("Test body")
def pytest_runtest_setup(item):
try:
with open(item.config.getoption('--dut-config')) as cfg:
dut_config = yaml.safe_load(cfg)
except Exception:
raise Exception("You need to specify DUT config. "
"See the example_dut_config.py file.")
try:
TestRun.prepare(item, dut_config)
test_name = item.name.split('[')[0]
TestRun.LOGGER = create_log(item.config.getoption('--log-path'),
test_name)
TestRun.setup()
TestRun.plugins['iotrace'] = IotracePlugin(
repo_dir=os.path.join(os.path.dirname(__file__), "../"),
working_dir=dut_config['working_dir']
)
except Exception as ex:
raise Exception(f"Exception occurred during test setup:\n"
f"{str(ex)}\n{traceback.format_exc()}")
TestRun.LOGGER.info(f"DUT info: {TestRun.dut}")
# Prepare DUT for test
TestRun.LOGGER.add_build_info(f'Commit hash:')
TestRun.LOGGER.add_build_info(f"{get_current_commit_hash()}")
TestRun.LOGGER.add_build_info(f'Commit message:')
TestRun.LOGGER.add_build_info(f'{get_current_commit_message()}')
TestRun.LOGGER.add_build_info(f'OCTF commit hash:')
TestRun.LOGGER.add_build_info(f'{get_current_octf_hash()}')
dut_prepare(item.config.getoption('--force-reinstall'))
TestRun.LOGGER.info(f"DUT info: {TestRun.dut}")
TestRun.LOGGER.write_to_command_log("Test body")
TestRun.LOGGER.start_group("Test body")
def test_package_installation():
TestRun.LOGGER.info("Testing package installation")
iotrace: IotracePlugin = TestRun.plugins['iotrace']
work_path: str = f"{iotrace.working_dir}/iotrace_package"
disk = TestRun.dut.disks[0]
with TestRun.step("Copying iotrace repository to DUT"):
TestRun.executor.rsync_to(f"{iotrace.repo_dir}/",
work_path,
delete=True,
symlinks=True,
exclude_list=['build'] + ['*.pyc'],
timeout=timedelta(minutes=2))
with TestRun.step("Setup dependencies"):
TestRun.executor.run_expect_success(f"cd {work_path} && "
"./setup_dependencies.sh")
with TestRun.step("Remove old packages"):
TestRun.executor.run_expect_success(
f"rm -rf {work_path}/build/release/iotrace-*.deb && "
f"rm -rf {work_path}/build/release/iotrace-*.rpm")
with TestRun.step("Building iotrace package"):
TestRun.executor.run_expect_success(f"cd {work_path} && "
"make package -j`nproc --all`")
iotrace_installed = check_if_installed()
if iotrace_installed:
with TestRun.step("Uninstall existing iotrace if needed"):
uninstall_iotrace()
with TestRun.step("Install from iotrace package"):
if check_if_ubuntu():
TestRun.executor.run_expect_success(
f"cd {work_path}/build/release && "
"dpkg -i iotrace-*.deb")
else:
TestRun.executor.run_expect_success(
f"cd {work_path}/build/release && "
"rpm -i iotrace-*.rpm")
with TestRun.step("Check if iotrace is installed"):
iotrace.version()
iotrace.start_tracing([disk.system_path])
time.sleep(1)
stopped = iotrace.stop_tracing()
if not stopped:
raise Exception("Could not stop active tracing.")
trace_path = IotracePlugin.get_latest_trace_path()
summary_parsed = IotracePlugin.get_trace_summary(trace_path)
if summary_parsed['state'] != "COMPLETE":
TestRun.LOGGER.error("Trace state is not complete")
with TestRun.step("Uninstall rpm package"):
if check_if_ubuntu():
TestRun.executor.run_expect_success("apt remove -y iotrace")
else:
TestRun.executor.run_expect_success("rpm -e iotrace")
with TestRun.step("Check if iotrace was uninstalled"):
TestRun.executor.run_expect_fail("iotrace -V")
if iotrace_installed:
with TestRun.step("Reinstall iotrace"):
install_iotrace()
def test_lba_histogram():
TestRun.LOGGER.info("Testing lba histogram")
iotrace = TestRun.plugins['iotrace']
number_buckets = 32
bucket_size = Size(256)
start_lba = 10240
end_lba = start_lba + number_buckets * bucket_size.value
for disk in TestRun.dut.disks:
io_len = Size(1, disk.block_size)
with TestRun.step("Start tracing"):
iotrace.start_tracing([disk.system_path])
time.sleep(5)
with TestRun.step("Send write commands"):
for bucket_index in range(number_buckets):
for i in range(bucket_index + 1):
# Generate IO for a given bucket. The LBA may need to be
# translated from 512B (which iotrace always uses) to 4KiB
# (if that's what the underlying disk surfaces)
seek = ((start_lba + bucket_index * bucket_size.value +
randrange(bucket_size.value)) * iotrace_lba_len /
disk.block_size.get_value())
dd = (Dd().input("/dev/urandom").output(
disk.system_path).count(1).block_size(io_len).oflag(
'direct,sync').seek(int(seek)))
dd.run()
with TestRun.step("Send read commands"):
for bucket_index in range(number_buckets):
for i in range(bucket_index + 1):
seek = ((start_lba + bucket_index * bucket_size.value +
randrange(bucket_size.value)) * iotrace_lba_len /
disk.block_size.get_value())
dd = (Dd().input(
disk.system_path).output("/dev/null").count(
1).block_size(io_len).iflag('direct,sync').skip(
int(seek)))
dd.run()
with TestRun.step("Send discard commands"):
for bucket_index in range(number_buckets):
for i in range(bucket_index + 1):
seek = (start_lba + bucket_index * bucket_size.value +
randrange(bucket_size.value))
seek = round_down(seek * iotrace_lba_len,
disk.block_size.get_value())
TestRun.executor.run_expect_success(
f"blkdiscard -o {int(seek)} "
f"-l {disk.block_size.get_value()} {disk.system_path}")
with TestRun.step("Stop tracing"):
iotrace.stop_tracing()
with TestRun.step("Verify histogram correctness"):
trace_path = IotracePlugin.get_latest_trace_path()
json = IotracePlugin.get_lba_histogram(trace_path, bucket_size,
start_lba, end_lba)
TestRun.LOGGER.info(str(json[0]['histogram'][0]))
TestRun.LOGGER.info(
str(json[0]['histogram'][0]['total']['range'][4]))
for bucket_index in range(number_buckets):
read = json[0]['histogram'][0]['read']['range'][bucket_index]
write = json[0]['histogram'][0]['write']['range'][bucket_index]
discard = json[0]['histogram'][0]['discard']['range'][
bucket_index]
total = json[0]['histogram'][0]['total']['range'][bucket_index]
if int(read['begin']
) != start_lba + bucket_index * bucket_size.value:
TestRun.fail(
f"Invalid read begin range: {read['begin']} for index {bucket_index}"
)
if int(read['end']) != start_lba + (bucket_index +
1) * bucket_size.value - 1:
TestRun.fail(
f"Invalid read end range: {read['end']} for index {bucket_index}"
)
if int(read['count']) != bucket_index + 1:
TestRun.fail(
f"Invalid read count: {read['count']} for index {bucket_index}"
)
if int(write['begin']
) != start_lba + bucket_index * bucket_size.value:
TestRun.fail(
f"Invalid write begin range: {write['begin']} for index {bucket_index}"
)
if int(
write['end']
) != start_lba + (bucket_index + 1) * bucket_size.value - 1:
TestRun.fail(
f"Invalid write end range: {write['end']} for index {bucket_index}"
)
if int(write['count']) != bucket_index + 1:
TestRun.fail(
f"Invalid write count: {write['count']} for index {bucket_index}"
)
if int(discard['begin']
) != start_lba + bucket_index * bucket_size.value:
TestRun.fail(
f"Invalid discard begin range: {discard['begin']} "
f"for index {bucket_index}")
if int(
discard['end']
) != start_lba + (bucket_index + 1) * bucket_size.value - 1:
TestRun.fail(
f"Invalid discard end range: {discard['end']} for index {bucket_index}"
)
if int(discard['count']) != bucket_index + 1:
TestRun.fail(
f"Invalid discard count: {discard['count']} for index {bucket_index}"
)
if int(total['begin']
) != start_lba + bucket_index * bucket_size.value:
TestRun.fail(
f"Invalid total begin range: {total['begin']} for index {bucket_index}"
)
if int(
total['end']
) != start_lba + (bucket_index + 1) * bucket_size.value - 1:
TestRun.fail(
f"Invalid total end range: {total['end']} for index {bucket_index}"
)
if int(total['count']) != 3 * (bucket_index + 1):
TestRun.fail(
f"Invalid total count: {total['count']} for index {bucket_index}"
)
def test_io_events():
TestRun.LOGGER.info("Testing io events during tracing")
iotrace = TestRun.plugins['iotrace']
for disk in TestRun.dut.disks:
with TestRun.step("Start tracing"):
iotrace.start_tracing([disk.system_path])
time.sleep(5)
with TestRun.step("Send write command"):
write_length = Size(17, disk.block_size)
write_offset = 2 * write_length.get_value()
dd = (Dd().input("/dev/urandom").output(disk.system_path).count(
1).block_size(write_length).oflag('direct,sync').seek(
int(write_offset / write_length.get_value())))
dd.run()
with TestRun.step("Send read command"):
read_length = Size(19, disk.block_size)
read_offset = 2 * read_length.get_value()
dd = (Dd().input(disk.system_path).output("/dev/null").count(
1).block_size(read_length).iflag('direct,sync').skip(
int(read_offset / read_length.get_value())))
dd.run()
with TestRun.step("Send discard command"):
discard_length = Size(21, disk.block_size).get_value()
discard_offset = int(2 * discard_length)
TestRun.executor.run_expect_success(
f"blkdiscard -o {discard_offset}"
f" -l {int(discard_length)} {disk.system_path}")
with TestRun.step("Stop tracing"):
iotrace.stop_tracing()
with TestRun.step("Verify trace correctness"):
trace_path = IotracePlugin.get_latest_trace_path()
events_parsed = IotracePlugin.get_trace_events(trace_path)
result = any(
'io' in event and 'operation' in event['io']
and event['io']['operation'] == 'Write'
# LBA 0 events don't have a lba field, so skip them
and 'lba' in event['io'] and int(event['io']['lba']) == int(
write_offset / iotrace_lba_len) and int(event['io']['len'])
== int(write_length.get_value() / iotrace_lba_len) and
f"/dev/{event['device']['name']}" == disk.system_path
for event in events_parsed)
if not result:
TestRun.fail("Could not find write event")
result = any(
'io' in event and 'operation' in event['io']
and event['io']['operation'] == 'Read'
# LBA 0 events don't have a lba field, so skip them
and 'lba' in event['io'] and int(event['io']['lba']) == int(
read_offset / iotrace_lba_len) and int(event['io']['len'])
== int(read_length.get_value() / iotrace_lba_len) and
f"/dev/{event['device']['name']}" == disk.system_path
for event in events_parsed)
if not result:
TestRun.fail("Could not find read event")
result = any('io' in event and 'operation' in event['io']
and event['io']['operation'] == 'Discard'
# LBA 0 events don't have a lba field, so skip them
and 'lba' in event['io'] and int(event['io']['lba'])
== int(discard_offset / iotrace_lba_len) and
int(event['io']['len']) == int(discard_length /
iotrace_lba_len) and
f"/dev/{event['device']['name']}" == disk.system_path
for event in events_parsed)
if not result:
TestRun.fail("Could not find discard event")
def test_iotracer_limits():
"""
title: Check if io-tracer respects its own tracing limits .
description: |
Check if io-tracer respects the time limit and the trace file size limit
set with start command and finishes tracing when one of two limits is reached.
pass_criteria:
- No system crash.
- Tracing stops when one of the limits is reached.
"""
with TestRun.step("Generate workload on device."):
disk = TestRun.dut.disks[0]
fio_pid = fio_workload(disk.system_path, fio_runtime).run_in_background()
with TestRun.step("Prepare io-tracer."):
iotracer: IotracePlugin = TestRun.plugins['iotrace']
with TestRun.step("Run io-tracer with duration limit set."):
iotracer.start_tracing([disk.system_path], timeout=runtime_short)
with TestRun.step("Wait for tracing to finish."):
wait(iotracer)
output = IotracePlugin.get_trace_summary(IotracePlugin.get_latest_trace_path())
with TestRun.step("Check tracing duration."):
if not is_time_almost_equal(runtime_short, output['traceDuration']):
TestRun.LOGGER.error("Tracing duration is different than set.")
with TestRun.step("Run io-tracer with file size limit set."):
iotracer.start_tracing([disk.system_path], trace_file_size=size_limit_low)
with TestRun.step("Wait for tracing to finish."):
wait(iotracer)
output = IotracePlugin.get_trace_summary(IotracePlugin.get_latest_trace_path())
with TestRun.step("Check trace file size."):
if not is_size_almost_equal(size_limit_low, output['traceSize']):
TestRun.LOGGER.error("Tracing file size is different than set.")
with TestRun.step("Run io-tracer with low file size limit and high duration limit set."):
iotracer.start_tracing([disk.system_path], timeout=runtime_long,
trace_file_size=size_limit_low)
with TestRun.step("Wait for tracing to finish."):
wait(iotracer)
output = IotracePlugin.get_trace_summary(IotracePlugin.get_latest_trace_path())
with TestRun.step("Check tracing duration and trace file size."):
if not ((is_time_almost_equal(runtime_long, output['traceDuration'])
and is_size_lower_or_equal(output['traceSize'], size_limit_low))
or (is_time_lower_or_equal(output['traceDuration'], runtime_long)
and is_size_almost_equal(size_limit_low, output['traceSize']))):
TestRun.LOGGER.error("Tracing did not stop after reaching size nor time limit.")
with TestRun.step("Run io-tracer with high file size limit and low duration limit set."):
iotracer.start_tracing([disk.system_path], timeout=runtime_short,
trace_file_size=size_limit_high)
with TestRun.step("Wait for tracing to finish."):
wait(iotracer)
output = IotracePlugin.get_trace_summary(IotracePlugin.get_latest_trace_path())
with TestRun.step("Check tracing duration and trace file size."):
if not ((is_time_almost_equal(runtime_short, output['traceDuration'])
and is_size_lower_or_equal(output['traceSize'], size_limit_high))
or (is_time_lower_or_equal(output['traceDuration'], runtime_short)
and is_size_almost_equal(size_limit_high, output['traceSize']))):
TestRun.LOGGER.error("Tracing did not stop after reaching size nor time limit.")
with TestRun.step("Stop fio workload."):
TestRun.executor.kill_process(fio_pid)
def pytest_runtest_setup(item):
iotrace: IotracePlugin = IotracePlugin()
TestRun.plugins['iotrace'] = iotrace
iotrace.runtest_setup(item)
dut_prepare(item.config.getoption('--force-reinstall'))
def test_fs_operations():
TestRun.LOGGER.info("Testing file system events during tracing")
iotrace = TestRun.plugins['iotrace']
for disk in TestRun.dut.disks:
try:
with TestRun.step("Create file system"):
disk.create_filesystem(Filesystem.ext4)
with TestRun.step("Mount device"):
disk.mount(mountpoint)
with TestRun.step("Start tracing"):
iotrace.start_tracing([disk.system_path])
time.sleep(5)
with TestRun.step("Create test directory and file"):
write_file(f"{mountpoint}/test_file", content="foo")
sync()
test_file_inode = get_inode(f"{mountpoint}/test_file")
create_directory(f"{mountpoint}/test_dir")
sync()
with TestRun.step("Write to test file"):
write_file(f"{mountpoint}/test_file",
overwrite=False,
content="bar")
sync()
with TestRun.step("Create new test file"):
create_file(f"{mountpoint}/test_file2")
test_file2_inode = get_inode(f"{mountpoint}/test_file2")
sync()
with TestRun.step("Move test file"):
move(f"{mountpoint}/test_file", f"{mountpoint}/test_dir")
sync()
with TestRun.step("Delete test file"):
remove(f"{mountpoint}/test_dir/test_file")
sync()
with TestRun.step("Stop tracing"):
sync()
iotrace.stop_tracing()
with TestRun.step("Verify trace correctness"):
trace_path = IotracePlugin.get_latest_trace_path()
events_parsed = IotracePlugin.get_trace_events(trace_path)
result = any(
'file' in event and event['file']['eventType'] == 'Create'
and event['file']['id'] == test_file2_inode
for event in events_parsed)
if not result:
raise Exception("Could not find Create event")
result = any(
'file' in event and event['file']['eventType'] == 'Delete'
and event['file']['id'] == test_file_inode
for event in events_parsed)
if not result:
raise Exception("Could not find Delete event")
result = any(
'file' in event and event['file']['eventType'] == 'MoveTo'
and event['file']['id'] == test_file_inode
for event in events_parsed)
if not result:
raise Exception("Could not find MoveTo event")
result = any(
'file' in event and event['file']['eventType'] ==
'MoveFrom' and event['file']['id'] == test_file_inode
for event in events_parsed)
if not result:
raise Exception("Could not find MoveFrom event")
result = any(
'file' in event and event['file']['eventType'] == 'Access'
and event['file']['id'] == test_file_inode
for event in events_parsed)
if not result:
raise Exception("Could not find Access event")
finally:
with TestRun.step("Unmount device"):
disk.unmount()
def test_fs_statistics(fs):
"""
title: Test if FS statistics are properly calculated by iotrace
description: |
Create files on filesystem (relative to fs root):
* test_dir/A.x
* test_dir/B.y
* A.y
* B.x
and execute known workload on them. Each file has different size and is written with
different block size, also varying number of times. This way number of loops on given job
is equal to WiF, and size of given file is equal to workset. Only workset and WiF stats
are verified.
pass_criteria:
- Statistics for file prefixes A and B match with expected values
- Statistics for file extensions x and y match with expected values
- Statistics for root fs directory and test_dir directory match with expected values
"""
iotrace = TestRun.plugins["iotrace"]
(disk, filesystem) = fs
with TestRun.step("Prepare fio configuration"):
test_dir = f"{filesystem}/test_dir"
fio_cfg = (Fio().create_command().io_engine(
IoEngine.libaio).sync(True).read_write(ReadWrite.write))
dir_Ax_size = Size(3, Unit.MebiByte)
dir_Ax_WiF = 2
dir_Ax_written = dir_Ax_size * dir_Ax_WiF
(fio_cfg.add_job("test_dir/A.x").target(f"{test_dir}/A.x").file_size(
dir_Ax_size).block_size(Size(4, Unit.KibiByte)).loops(dir_Ax_WiF))
dir_By_size = Size(16, Unit.KibiByte)
dir_By_WiF = 2
dir_By_written = dir_By_size * dir_By_WiF
(fio_cfg.add_job("test_dir/B.y").target(f"{test_dir}/B.y").file_size(
dir_By_size).block_size(Size(16, Unit.KibiByte)).loops(dir_By_WiF))
Ay_size = Size(2, Unit.MebiByte)
Ay_WiF = 5
Ay_written = Ay_size * Ay_WiF
(fio_cfg.add_job("A.y").target(f"{filesystem}/A.y").file_size(
Ay_size).block_size(Size(64, Unit.KibiByte)).loops(Ay_WiF))
Bx_size = Size(5, Unit.MebiByte)
Bx_WiF = 1
Bx_written = Bx_size * Bx_WiF
(fio_cfg.add_job("B.x").target(f"{filesystem}/B.x").file_size(
Bx_size).block_size(Size(128, Unit.KibiByte)))
with TestRun.step("Prepare directory and files for test"):
create_directory(test_dir)
# In this run FIO will only create all the files needed by jobs and quit.
# If we didn't do it WiF would be +1 for each of the files (one file write on creation).
# For simplicity of calculations we create files first and after that start tracing.
fio_cfg.edit_global().create_only(True)
fio_cfg.run()
with TestRun.step("Start tracing"):
iotrace.start_tracing([disk.system_path], Size(1, Unit.GibiByte))
time.sleep(3)
with TestRun.step("Run workload"):
fio_cfg.edit_global().create_only(False)
fio_cfg.run()
with TestRun.step("Stop tracing"):
iotrace.stop_tracing()
with TestRun.step("Verify trace correctness"):
A_prefix_workset = dir_Ax_size + Ay_size
B_prefix_workset = dir_By_size + Bx_size
A_prefix_WiF = (dir_Ax_written + Ay_written) / A_prefix_workset
B_prefix_WiF = (dir_By_written + Bx_written) / B_prefix_workset
x_extension_workset = dir_Ax_size + Bx_size
y_extension_workset = dir_By_size + Ay_size
x_extension_WiF = (dir_Ax_written + Bx_written) / x_extension_workset
y_extension_WiF = (Ay_written + dir_By_written) / y_extension_workset
test_dir_workset = dir_Ax_size + dir_By_size
root_dir_workset = Ay_size + Bx_size
test_dir_WiF = (dir_Ax_written + dir_By_written) / test_dir_workset
root_dir_WiF = (Ay_written + Bx_written) / root_dir_workset
trace_path = IotracePlugin.get_latest_trace_path()
stats = parse_fs_stats(
IotracePlugin.get_fs_statistics(trace_path)[0]['entries'])
prefix_stats = {
stat.file_name_prefix: stat
for stat in stats if type(stat) == FileTraceStatistics
}
extension_stats = {
stat.extension: stat
for stat in stats if type(stat) == ExtensionTraceStatistics
}
dir_stats = {
stat.directory: stat
for stat in stats if type(stat) == DirectoryTraceStatistics
}
for (desc, (expect_workset, expect_WiF), got) in [
("A file prefix", (A_prefix_workset, A_prefix_WiF),
prefix_stats["A"]),
("B file prefix", (B_prefix_workset, B_prefix_WiF),
prefix_stats["B"]),
("x file extension", (x_extension_workset, x_extension_WiF),
extension_stats["x"]),
("y file extension", (y_extension_workset, y_extension_WiF),
extension_stats["y"]),
("test_dir directory", (test_dir_workset, test_dir_WiF),
dir_stats["/test_dir"]),
("root directory", (root_dir_workset, root_dir_WiF),
dir_stats["/"]),
]:
expect_equal(f"{desc} workset", expect_workset,
got.statistics.total.metrics.workset)
expect_equal(f"{desc} write invalidation factor", expect_WiF,
got.statistics.write.metrics.wif)
