def test_ior_intercept_multi_client(self):
"""Jira ID: DAOS-3499.
Test Description:
Purpose of this test is to run ior through dfuse in multiple
clients for 5 minutes and capture the metrics and use the
intercepiton library by exporting LD_PRELOAD to the libioil.so
path and rerun the above ior and capture the metrics and
compare the performance difference and check using interception
library make significant performance improvement.
Use case:
Run ior with read, write for 5 minutes
Run ior with read, write for 5 minutes with interception library
Compare the results and check whether using interception
library provides better performance.
:avocado: tags=all,full_regression,hw,large
:avocado: tags=daosio,iorinterceptmulticlient
"""
suffix = self.ior_cmd.transfer_size.value
out = self.run_ior_with_pool(test_file_suffix=suffix)
without_intercept = IorCommand.get_ior_metrics(out)
intercept = os.path.join(self.prefix, 'lib64', 'libioil.so')
suffix = suffix + "intercept"
out = self.run_ior_with_pool(intercept, test_file_suffix=suffix)
with_intercept = IorCommand.get_ior_metrics(out)
max_mib = int(IorMetrics.Max_MiB)
min_mib = int(IorMetrics.Min_MiB)
mean_mib = int(IorMetrics.Mean_MiB)
write_x = self.params.get("write_x", "/run/ior/iorflags/ssf/*", 1)
# Verifying write performance
self.assertTrue(
float(with_intercept[0][max_mib]) > write_x *
float(without_intercept[0][max_mib]))
self.assertTrue(
float(with_intercept[0][min_mib]) > write_x *
float(without_intercept[0][min_mib]))
self.assertTrue(
float(with_intercept[0][mean_mib]) > write_x *
float(without_intercept[0][mean_mib]))
# Verifying read performance
# The read performance is almost same with or without intercept
# library. But arbitarily the read performance with interception
# library can be bit lower than without it. Verifying that it is
# not drastically lower by checking it is at least 60% or above.
read_x = 0.6
self.assertTrue(
float(with_intercept[1][max_mib]) > read_x *
float(without_intercept[1][max_mib]))
self.assertTrue(
float(with_intercept[1][min_mib]) > read_x *
float(without_intercept[1][min_mib]))
self.assertTrue(
float(with_intercept[1][mean_mib]) > read_x *
float(without_intercept[1][mean_mib]))
def test_ior_intercept(self):
"""Jira ID: DAOS-3498.
Test Description:
Purpose of this test is to run ior using dfuse for 5 minutes
and capture the metrics and use the intercepiton library by
exporting LD_PRELOAD to the libioil.so path and rerun the
above ior and capture the metrics and compare the
performance difference and check using interception
library make significant performance improvement.
Use case:
Run ior with read, write, CheckWrite, CheckRead
for 5 minutes
Run ior with read, write, CheckWrite, CheckRead
for 5 minutes with interception library
Compare the results and check whether using interception
library provides better performance.
:avocado: tags=all,full_regression,hw,small,daosio,iorinterceptbasic
"""
apis = self.params.get("ior_api", '/run/ior/iorflags/ssf/*')
for api in apis:
self.ior_cmd.api.update(api)
out = self.run_ior_with_pool(fail_on_warning=False)
without_intercept = IorCommand.get_ior_metrics(out)
if api == "POSIX":
intercept = os.path.join(self.prefix, 'lib64', 'libioil.so')
out = self.run_ior_with_pool(intercept, fail_on_warning=False)
with_intercept = IorCommand.get_ior_metrics(out)
max_mib = int(IorMetrics.Max_MiB)
min_mib = int(IorMetrics.Min_MiB)
mean_mib = int(IorMetrics.Mean_MiB)
write_x = self.params.get("write_x", "/run/ior/iorflags/ssf/*",
1)
read_x = self.params.get("read_x", "/run/ior/iorflags/ssf/*",
1)
# Verifying write performance
self.assertTrue(
float(with_intercept[0][max_mib]) > write_x *
float(without_intercept[0][max_mib]))
self.assertTrue(
float(with_intercept[0][min_mib]) > write_x *
float(without_intercept[0][min_mib]))
self.assertTrue(
float(with_intercept[0][mean_mib]) > write_x *
float(without_intercept[0][mean_mib]))
# Verifying read performance
self.assertTrue(
float(with_intercept[1][max_mib]) > read_x *
float(without_intercept[1][max_mib]))
self.assertTrue(
float(with_intercept[1][min_mib]) > read_x *
float(without_intercept[1][min_mib]))
self.assertTrue(
float(with_intercept[1][mean_mib]) > read_x *
float(without_intercept[1][mean_mib]))
def run_il_perf_check(self):
"""Verify IOR performance with DFUSE + IL is similar to DFS.
Steps:
Run IOR with DFS.
Run IOR with DFUSE + IL.
Verify performance with DFUSE + IL is similar to DFS.
"""
# Write and read performance thresholds
write_x = self.params.get("write_x", self.ior_cmd.namespace, None)
read_x = self.params.get("read_x", self.ior_cmd.namespace, None)
if write_x is None or read_x is None:
self.fail("Failed to get write_x and read_x from config")
# Run IOR with DFS
self.ior_cmd.api.update("DFS")
dfs_out = self.run_ior_with_pool(fail_on_warning=self.log.info)
dfs_perf = IorCommand.get_ior_metrics(dfs_out)
# Destroy and use a new pool and container
self.container.destroy()
self.container = None
self.pool.destroy()
self.pool = None
# Run IOR with dfuse + IL
self.ior_cmd.api.update("POSIX")
dfuse_out = self.run_ior_with_pool(intercept=os.path.join(
self.prefix, 'lib64', 'libioil.so'),
fail_on_warning=self.log.info)
dfuse_perf = IorCommand.get_ior_metrics(dfuse_out)
# Verify write and read performance are within the thresholds.
# Since Min can have a lot of variance, don't check Min or Mean.
# Ideally, we might want to look at the Std Dev to ensure the results are admissible.
dfs_max_write = float(dfs_perf[0][IorMetrics.Max_MiB])
dfuse_max_write = float(dfuse_perf[0][IorMetrics.Max_MiB])
actual_write_x = percent_change(dfs_max_write, dfuse_max_write)
self.log.info("DFS Max Write: %.2f", dfs_max_write)
self.log.info("DFUSE IL Max Write: %.2f", dfuse_max_write)
self.log.info("Percent Diff: %.2f%%", actual_write_x * 100)
self.assertLessEqual(abs(actual_write_x), write_x,
"Max Write Diff too large")
dfs_max_read = float(dfs_perf[1][IorMetrics.Max_MiB])
dfuse_max_read = float(dfuse_perf[1][IorMetrics.Max_MiB])
actual_read_x = percent_change(dfs_max_read, dfuse_max_read)
self.log.info("DFS Max Read: %.2f", dfs_max_read)
self.log.info("DFUSE IL Max Read: %.2f", dfuse_max_read)
self.log.info("Percent Diff: %.2f%%", actual_read_x * 100)
self.assertLessEqual(abs(actual_read_x), read_x,
"Max Read Diff too large")
def start_ior_thread(self, results, create_cont, operation='WriteRead'):
"""Start IOR write/read threads and wait until all threads are finished.
Args:
results (queue): queue for returning thread results
operation (str): IOR operation for read/write.
Default it will do whatever mention in ior_flags
set.
"""
self.ior_cmd.flags.value = self.ior_default_flags
#For IOR Other operation, calculate the block size based on server %
#to fill up. Store the container UUID for future reading operation.
if operation == 'Write':
block_size = self.calculate_ior_block_size()
self.ior_cmd.block_size.update('{}'.format(block_size))
#For IOR Read only operation, retrieve the stored container UUID
elif operation == 'Read':
create_cont = False
self.ior_cmd.flags.value = self.ior_read_flags
# run IOR Command
try:
out = self.run_ior_with_pool(create_cont=create_cont,
fail_on_warning=self.fail_on_warning)
self.ior_matrix = IorCommand.get_ior_metrics(out)
results.put("PASS")
except (CommandFailure, TestFail) as _error:
results.put("FAIL")
def start_ior_thread(self, results, create_cont, operation):
"""Start IOR write/read threads and wait until all threads are finished.
Args:
results (queue): queue for returning thread results
create_cont (Bool): To create the new container or not.
operation (str):
Write/WriteRead: It will Write or Write/Read base on IOR parameter in yaml file.
Auto_Write/Auto_Read: It will calculate the IOR block size based on requested
storage % to be fill.
"""
# IOR flag can be Write only or Write/Read based on test yaml
self.ior_cmd.flags.value = self.ior_default_flags
# Calculate the block size based on server % to fill up.
if 'Auto' in operation:
block_size = self.calculate_ior_block_size()
self.ior_cmd.block_size.update('{}'.format(block_size))
# For IOR Read operation update the read flax from yaml file.
if 'Auto_Read' in operation or operation == "Read":
create_cont = False
self.ior_cmd.flags.value = self.ior_read_flags
# run IOR Command
try:
out = self.run_ior_with_pool(create_cont=create_cont,
fail_on_warning=self.fail_on_warning)
self.ior_matrix = IorCommand.get_ior_metrics(out)
results.put("PASS")
except (CommandFailure, TestFail) as _error:
results.put("FAIL")
def run_ior_collect_error(self, results, job_num, file_name, clients):
"""Run IOR command and store error in results.
Args:
results (dict): A dictionary object to store the ior metrics.
job_num (int): Assigned job number.
file_name (str): File name used for self.ior_cmd.test_file.
clients (list): Client hostnames to run IOR from.
"""
ior_cmd = IorCommand()
ior_cmd.get_params(self)
ior_cmd.set_daos_params(
group=self.server_group, pool=self.pool, cont_uuid=self.container.uuid)
testfile = os.path.join("/", file_name)
ior_cmd.test_file.update(testfile)
manager = get_job_manager(
test=self, class_name="Mpirun", job=ior_cmd, subprocess=self.subprocess,
mpi_type="mpich")
manager.assign_hosts(clients, self.workdir, self.hostfile_clients_slots)
ppn = self.params.get("ppn", '/run/ior/client_processes/*')
manager.ppn.update(ppn, 'mpirun.ppn')
manager.processes.update(None, 'mpirun.np')
try:
ior_output = manager.run()
results[job_num] = [True]
# For debugging.
results[job_num].extend(IorCommand.get_ior_metrics(ior_output))
# We'll verify the error message.
results[job_num].append(ior_output.stderr_text)
except CommandFailure as error:
results[job_num] = [False, "IOR failed: {}".format(error)]
def run_custom_ior_cmd(self,
ior_command,
clients,
results,
job_num,
intercept=None):
"""Run customized IOR command, not self.ior_cmd.
Expected to be used with a threaded code where multiple IOR commands are
executed in parallel.
Display pool space before running it for a reference.
Args:
ior_command (IorCommand): Custom IOR command instance.
clients (list): hosts on which to run ior
results (dict): A dictionary object to store the ior metrics
job_num (int): Assigned job number
intercept (str, optional): path to interception library. Defaults to
None.
"""
self.log.info("--- IOR Thread %d: Start ---", job_num)
tsize = ior_command.transfer_size.value
testfile = os.path.join(self.dfuse.mount_dir.value,
"testfile{}{}".format(tsize, job_num))
if intercept:
testfile += "intercept"
ior_command.test_file.update(testfile)
# Get the custom job manager that's associated with this thread.
manager = get_job_manager(self, "Mpirun", ior_command, self.subprocess,
"mpich")
procs = (self.processes // len(self.hostlist_clients)) * len(clients)
env = ior_command.get_default_env(str(manager), self.client_log)
if intercept:
env["LD_PRELOAD"] = intercept
manager.assign_hosts(clients, self.workdir,
self.hostfile_clients_slots)
manager.assign_processes(procs)
manager.assign_environment(env)
self.log.info("--- IOR Thread %d: Starting IOR ---", job_num)
self.display_pool_space()
try:
ior_output = manager.run()
results[job_num] = [True]
results[job_num].extend(IorCommand.get_ior_metrics(ior_output))
except CommandFailure as error:
results[job_num] = [False, "IOR failed: {}".format(error)]
finally:
self.display_pool_space()
self.log.info("--- IOR Thread %d: End ---", job_num)
def start_ior_thread(self, create_cont, operation):
"""Start IOR write/read threads and wait until all threads are finished.
Args:
create_cont (Bool): To create the new container or not.
operation (str):
Write/WriteRead: It will Write or Write/Read base on IOR parameter in yaml file.
Auto_Write/Auto_Read: It will calculate the IOR block size based on requested
storage % to be fill.
"""
# IOR flag can Write/Read based on test yaml
self.ior_local_cmd.flags.value = self.ior_default_flags
# Calculate the block size based on server % to fill up.
if 'Auto' in operation:
block_size = self.calculate_ior_block_size()
self.ior_local_cmd.block_size.update('{}'.format(block_size))
# For IOR Read operation update the read only flag from yaml file.
if 'Auto_Read' in operation or operation == "Read":
create_cont = False
self.ior_local_cmd.flags.value = self.ior_read_flags
self.ior_local_cmd.set_daos_params(self.server_group, self.pool)
self.ior_local_cmd.test_file.update('/testfile')
# Created new container or use the existing container for reading
if create_cont:
self.create_container()
self.ior_local_cmd.dfs_cont.update(self.nvme_local_cont.uuid)
# Define the job manager for the IOR command
job_manager_main = get_job_manager(self, "Mpirun", self.ior_local_cmd, mpi_type="mpich")
env = self.ior_local_cmd.get_default_env(str(job_manager_main))
job_manager_main.assign_hosts(self.hostlist_clients, self.workdir, None)
job_manager_main.assign_environment(env, True)
job_manager_main.assign_processes(self.params.get("np", '/run/ior/client_processes/*'))
# run IOR Command
try:
output = job_manager_main.run()
self.ior_matrix = IorCommand.get_ior_metrics(output)
for line in output.stdout_text.splitlines():
if 'WARNING' in line and self.fail_on_warning:
self.result.append("FAIL-IOR command issued warnings.")
except (CommandFailure, TestFail) as error:
self.result.append("FAIL - {}".format(error))
def run_ior_report_error(self, results, job_num, file_name, pool,
container, namespace):
"""Run IOR command and store the results to results dictionary.
Create a new IorCommand object instead of using the one in IorTestBase because
we'll run a test that runs multiple IOR processes at the same time.
Args:
results (dict): A dictionary object to store the ior metrics
job_num (int): Assigned job number
file_name (str): File name used for self.ior_cmd.test_file.
oclass (str): Value for dfs_oclass and dfs_dir_oclass.
pool (TestPool): Pool to run IOR.
container (TestContainer): Container to run IOR.
"""
# Update the object class depending on the test case.
ior_cmd = IorCommand(namespace=namespace)
ior_cmd.get_params(self)
# Standard IOR prep sequence.
ior_cmd.set_daos_params(self.server_group, pool, container.uuid)
testfile = os.path.join("/", file_name)
ior_cmd.test_file.update(testfile)
manager = get_job_manager(test=self,
class_name="Mpirun",
job=ior_cmd,
subprocess=self.subprocess,
mpi_type="mpich")
manager.assign_hosts(self.hostlist_clients, self.workdir,
self.hostfile_clients_slots)
ppn = self.params.get("ppn", '/run/ior/client_processes/*')
manager.ppn.update(ppn, 'mpirun.ppn')
manager.processes.update(None, 'mpirun.np')
# Run the command.
try:
self.log.info("--- IOR command %d start ---", job_num)
ior_output = manager.run()
results[job_num] = [True]
# For debugging.
results[job_num].extend(IorCommand.get_ior_metrics(ior_output))
# Command worked, but append the error message if any.
results[job_num].append(ior_output.stderr_text)
self.log.info("--- IOR command %d end ---", job_num)
except CommandFailure as error:
self.log.info("--- IOR command %d failed ---", job_num)
results[job_num] = [False, "IOR failed: {}".format(error)]
def run_multiple_ior(self,
hostfile,
num_clients,
results,
job_num,
intercept=None):
# pylint: disable=too-many-arguments
"""Run the IOR command.
Args:
manager (str): mpi job manager command
processes (int): number of host processes
intercept (str): path to interception library.
"""
self.lock.acquire(True)
tsize = self.ior_cmd.transfer_size.value
testfile = os.path.join(self.dfuse.mount_dir.value,
"testfile{}{}".format(tsize, job_num))
if intercept:
testfile += "intercept"
self.ior_cmd.test_file.update(testfile)
manager = self.get_ior_job_manager_command()
procs = (self.processes // len(self.hostlist_clients)) * num_clients
env = self.ior_cmd.get_default_env(str(manager), self.client_log)
if intercept:
env["LD_PRELOAD"] = intercept
manager.setup_command(env, hostfile, procs)
self.lock.release()
try:
self.pool.display_pool_daos_space()
out = manager.run()
self.lock.acquire(True)
results[job_num] = IorCommand.get_ior_metrics(out)
self.lock.release()
except CommandFailure as error:
self.log.error("IOR Failed: %s", str(error))
self.fail("Test was expected to pass but it failed.\n")
finally:
self.pool.display_pool_daos_space()
def run_multiple_ior(self, clients, results, job_num, intercept=None):
"""Run the IOR command.
Args:
clients (list): hosts on which to run ior
results (dict): A dictionary object to store the ior metrics
job_num (int): Assigned job number
intercept (str, optional): path to interception library. Defaults to
None.
"""
self.lock.acquire(True)
tsize = self.ior_cmd.transfer_size.value
testfile = os.path.join(self.dfuse.mount_dir.value,
"testfile{}{}".format(tsize, job_num))
if intercept:
testfile += "intercept"
self.ior_cmd.test_file.update(testfile)
manager = self.get_ior_job_manager_command()
procs = (self.processes // len(self.hostlist_clients)) * len(clients)
env = self.ior_cmd.get_default_env(str(manager), self.client_log)
if intercept:
env["LD_PRELOAD"] = intercept
manager.assign_hosts(clients, self.workdir,
self.hostfile_clients_slots)
manager.assign_processes(procs)
manager.assign_environment(env)
self.lock.release()
try:
self.pool.display_pool_daos_space()
out = manager.run()
self.lock.acquire(True)
results[job_num] = IorCommand.get_ior_metrics(out)
self.lock.release()
except CommandFailure as error:
self.log.error("IOR Failed: %s", str(error))
self.fail("Test was expected to pass but it failed.\n")
finally:
self.pool.display_pool_daos_space()
def test_aggregation_throttling(self):
"""Jira ID: DAOS-3749
Test Description:
Verify the ior throttling during aggregation
in the background is affecting the ior
performance only by +/- 30%
Use case:
Create a pool and container
Disable the aggregation
Run ior with same file option
Capture the initial ior performance.
Run ior the second time with same file option, so the
logical partitioning can be overwritten.
Enable the aggregation and wait for 90 seconds
Now as the aggregation is running in the background, run
ior again so both aggregation and ior runs in parallel
Capture the ior performance now and verify that it is
+/- 30% of the initial performance.
Also, verify the aggregation reclaimed the space used by
second ior.
:avocado: tags=all,hw,large,full_regression,aggregate,daosio
:avocado: tags=aggregatethrottling
"""
# Create pool and container
self.update_ior_cmd_with_pool()
# Disable the aggregation
self.pool.set_property("reclaim", "disabled")
# Run ior with -k option to retain the file and not delete it
out = self.run_ior_with_pool()
metric_before_aggregate = IorCommand.get_ior_metrics(out)
# Run ior the second time on the same pool and container, so another
# copy of the file is inserted in DAOS.
out = self.run_ior_with_pool(create_pool=False)
# wait 90 seconds for files to get old enough for aggregation
self.log.info("Waiting for 90 seconds for aggregation to start")
time.sleep(90)
# Enable the aggregation
self.pool.set_property("reclaim", "time")
# Run ior the third time while the aggregation of first two runs
# are running in the background.
out = self.run_ior_with_pool(create_pool=False)
metric_after_aggregate = IorCommand.get_ior_metrics(out)
# When DAOS-5057 is fixed, adjust the percentage. For now,
# keep it at 30 %
expected_perf_diff = 30.0
self.verify_performance(
metric_before_aggregate,
metric_after_aggregate,
0, # write_perf
expected_perf_diff) # 30% perf difference
self.verify_performance(
metric_before_aggregate,
metric_after_aggregate,
1, # read_perf
expected_perf_diff)
def test_caching_check(self):
"""Jira ID: DAOS-4874.
Test Description:
Purpose of this test is to check if dfuse caching is working.
Use case:
Write using ior over dfuse with caching disabled.
Perform ior read twice to get base read performance.
Unmount dfuse and mount it again with caching enabled.
Perform ior read after fresh mount to get read performance.
Run ior again to get second read performance numbers with caching enabled.
Compared cached read performance numbers after refresh to the baseline
read performance and confirm cached read performance is multiple folds
higher than with caching disabled.
:avocado: tags=all,full_regression
:avocado: tags=hw,medium,ib2
:avocado: tags=daosio,dfuse
:avocado: tags=dfusecachingcheck
"""
# get params
flags = self.params.get("iorflags", '/run/ior/*')
read_x = self.params.get("read_x", "/run/ior/*", 1)
# update flag
self.ior_cmd.flags.update(flags[0])
# run ior to write to the dfuse mount point
self.run_ior_with_pool(fail_on_warning=False, stop_dfuse=False)
# update ior flag to read
self.ior_cmd.flags.update(flags[1])
# run ior to read and store the read performance
base_read_arr = []
out = self.run_ior_with_pool(fail_on_warning=False, stop_dfuse=False)
base_read_arr.append(IorCommand.get_ior_metrics(out))
# run ior again to read with caching disabled and store performance
out = self.run_ior_with_pool(fail_on_warning=False, stop_dfuse=False)
base_read_arr.append(IorCommand.get_ior_metrics(out))
# the index of max_mib
max_mib = int(IorMetrics.Max_MiB)
# unmount dfuse and mount again with caching enabled
pcmd(self.hostlist_clients,
self.dfuse.get_umount_command(),
expect_rc=None)
self.dfuse.disable_caching.update(False)
self.dfuse.run()
# run ior to obtain first read performance after mount
out = self.run_ior_with_pool(fail_on_warning=False, stop_dfuse=False)
base_read_arr.append(IorCommand.get_ior_metrics(out))
# run ior again to obtain second read performance with caching enabled
# second read should be multiple times greater than first read
out = self.run_ior_with_pool(fail_on_warning=False)
with_caching = IorCommand.get_ior_metrics(out)
# verify cached read performance is multiple times greater than without caching
for base_read in base_read_arr:
actual_change = percent_change(base_read[0][max_mib],
with_caching[0][max_mib])
self.log.info('assert actual_change > min_change: %f > %f',
actual_change, read_x)
self.assertTrue(actual_change > read_x)
def test_ior_intercept_verify_data(self):
"""Jira ID: DAOS-3502.
Test Description:
Purpose of this test is to run ior through dfuse with
interception library on 5 clients and without interception
library on 1 client for at least 30 minutes and verify the
data integrity using ior's Read Verify and Write Verify
options.
Use case:
Run ior with read, write, fpp, read verify
write verify for 30 minutes
Run ior with read, write, read verify
write verify for 30 minutes
:avocado: tags=all,full_regression
:avocado: tags=hw,large
:avocado: tags=daosio,dfuse,il,ior_intercept
:avocado: tags=ior_intercept_verify_data
"""
self.add_pool()
self.add_container(self.pool)
# Start dfuse for POSIX api. This is specific to interception library test requirements.
self.start_dfuse(self.hostlist_clients, self.pool, self.container)
# Setup the thread manager
thread_manager = ThreadManager(run_ior, self.timeout - 30)
index_clients_intercept_file = [
(0, self.hostlist_clients[0:-1],
os.path.join(self.prefix, 'lib64', 'libioil.so'),
os.path.join(self.dfuse.mount_dir.value, "testfile_0_intercept")),
(1, self.hostlist_clients[-1:], None,
os.path.join(self.dfuse.mount_dir.value, "testfile_1")),
]
self.job_manager = []
for index, clients, intercept, test_file in index_clients_intercept_file:
# Add a job manager for each ior command. Use a timeout for the ior command that leaves
# enough time to report the summary of all the threads
job_manager = get_job_manager(self, "Mpirun", None, False, "mpich",
self.get_remaining_time() - 30)
# Define the parameters that will be used to run an ior command in this thread
thread_manager.add(
test=self,
manager=job_manager,
log=self.client_log,
hosts=clients,
path=self.workdir,
slots=None,
group=self.server_group,
pool=self.pool,
container=self.container,
processes=(self.processes // len(self.hostlist_clients)) *
len(clients),
intercept=intercept,
ior_params={"test_file": test_file})
self.log.info("Created thread %s for %s with intercept: %s", index,
clients, str(intercept))
# Launch the IOR threads
self.log.info("Launching %d IOR threads", thread_manager.qty)
results = thread_manager.run()
# Stop dfuse
self.stop_dfuse()
# Check the ior thread results
failed_thread_count = thread_manager.check(results)
if failed_thread_count > 0:
msg = "{} FAILED IOR Thread(s)".format(failed_thread_count)
self.d_log.error(msg)
self.fail(msg)
for index, clients, intercept, _ in index_clients_intercept_file:
with_intercept = "without" if intercept is None else "with"
IorCommand.log_metrics(
self.log, "{} clients {} interception library".format(
len(clients), with_intercept),
IorCommand.get_ior_metrics(results[index].result))
def test_dfuse_caching_check(self):
"""Jira ID: DAOS-4874.
Test Description:
Purpose of this test is to check if dfuse caching is working.
Use case:
Write using ior over dfuse with caching disabled.
Perform ior read to get base read performance.
Run ior read to get second read performance with caching disabled.
Compare first and second read performance numbers and they should
be similar.
Unmount dfuse and mount it again with caching enabled.
Perform ior read after fresh mount to get read performance.
Run ior again to get second read performance numbers with caching
enabled.
Compare first and second read performance numbers after dfuse
refresh and the second read should be multiple folds higher than
the first one.
:avocado: tags=all,full_regression
:avocado: tags=hw,small
:avocado: tags=daosio,dfuse
:avocado: tags=dfusecachingcheck
"""
# get params
flags = self.params.get("iorflags", '/run/ior/*')
read_x = self.params.get("read_x", "/run/ior/*", 1)
# update flag
self.ior_cmd.flags.update(flags[0])
# run ior to write to the dfuse mount point
self.run_ior_with_pool(fail_on_warning=False, stop_dfuse=False)
# update ior flag to read
self.ior_cmd.flags.update(flags[1])
# run ior to read and store the read performance
out = self.run_ior_with_pool(fail_on_warning=False, stop_dfuse=False)
base_read = IorCommand.get_ior_metrics(out)
# run ior again to read with caching disabled and store performance
out = self.run_ior_with_pool(fail_on_warning=False, stop_dfuse=False)
without_caching = IorCommand.get_ior_metrics(out)
max_mib = int(IorMetrics.Max_MiB)
# Compare read performance with caching disabled
# it should be similar to last read
lower_bound = (float(base_read[0][max_mib]) -
(float(base_read[0][max_mib]) * read_x[0]))
upper_bound = (float(base_read[0][max_mib]) +
(float(base_read[0][max_mib]) * read_x[0]))
# verify read performance is similar to last read and within
# the range of 1% up or down the first read performance
self.assertTrue(
lower_bound <= float(without_caching[0][max_mib]) <= upper_bound)
# unmount dfuse and mount again with caching enabled
pcmd(self.hostlist_clients,
self.dfuse.get_umount_command(),
expect_rc=None)
self.dfuse.disable_caching.update(False)
self.dfuse.run()
# run ior to obtain first read performance after mount
out = self.run_ior_with_pool(fail_on_warning=False, stop_dfuse=False)
base_read = IorCommand.get_ior_metrics(out)
# run ior again to obtain second read performance with caching enabled
# second read should be multiple times greater than first read
out = self.run_ior_with_pool(fail_on_warning=False)
with_caching = IorCommand.get_ior_metrics(out)
# verifying read performance
self.assertTrue(
float(with_caching[0][max_mib]) > read_x[1] *
float(base_read[0][max_mib]))
