def workload_fio_storageutilization(
fixture_name,
target_percentage,
project,
fio_pvc_dict,
fio_job_dict,
fio_configmap_dict,
measurement_dir,
tmp_path,
):
"""
This function implements core functionality of fio storage utilization
workload fixture. This is necessary because we can't parametrize single
general fixture over multiple parameters (it would mess with test case id
and polarion test case tracking).
"""
if fixture_name.endswith("rbd"):
storage_class_name = "ocs-storagecluster-ceph-rbd"
ceph_pool_name = "ocs-storagecluster-cephblockpool"
elif fixture_name.endswith("cephfs"):
storage_class_name = "ocs-storagecluster-cephfs"
ceph_pool_name = "ocs-storagecluster-cephfilesystem-data0"
else:
raise UnexpectedVolumeType(
"unexpected volume type, ocs-ci code is wrong")
# make sure we communicate what is going to happen
logger.info((
f"starting {fixture_name} fixture, "
f"using {storage_class_name} storage class "
f"backed by {ceph_pool_name} ceph pool"))
pvc_size = get_storageutilization_size(target_percentage, ceph_pool_name)
# For cephfs we can't use fill_fs because of BZ 1763808 (the process
# will get *Disk quota exceeded* error instead of *No space left on
# device* error).
# On the other hand, we can't use size={pvc_size} for rbd, as we can't
# write pvc_size bytes to a filesystem on a block device of {pvc_size}
# size (obviously, some space is used by filesystem metadata).
if fixture_name.endswith("rbd"):
fio_conf = textwrap.dedent("""
[simple-write]
readwrite=write
buffered=1
blocksize=4k
ioengine=libaio
directory=/mnt/target
fill_fs=1
""")
else:
fio_conf = textwrap.dedent(f"""
[simple-write]
readwrite=write
buffered=1
blocksize=4k
ioengine=libaio
directory=/mnt/target
size={pvc_size}G
""")
# put the dicts together into yaml file of the Job
fio_configmap_dict["data"]["workload.fio"] = fio_conf
fio_pvc_dict["spec"]["storageClassName"] = storage_class_name
fio_pvc_dict["spec"]["resources"]["requests"]["storage"] = f"{pvc_size}Gi"
fio_objs = [fio_pvc_dict, fio_configmap_dict, fio_job_dict]
fio_job_file = ObjectConfFile(fixture_name, fio_objs, project, tmp_path)
# How long do we let the job running while writing data to the volume?
# Based on min. fio write speed of the enviroment ...
fio_min_mbps = config.ENV_DATA['fio_storageutilization_min_mbps']
logger.info(
"Assuming %.2f MB/s is a minimal write speed of fio.", fio_min_mbps)
# ... we compute max. time we are going to wait for fio to write all data
min_time_to_write_gb = 1 / (fio_min_mbps / 2**10)
write_timeout = pvc_size * min_time_to_write_gb # seconds
logger.info((
f"fixture will wait {write_timeout} seconds for the Job "
f"to write {pvc_size} Gi data on OCS backed volume"))
def write_data():
"""
Write data via fio Job (specified in ``tf`` tmp file) to reach desired
utilization level, and keep this level for ``minimal_time`` seconds.
"""
# deploy the fio Job to the cluster
fio_job_file.create()
# This is a WORKAROUND of particular ocsci design choices: I just wait
# for one pod in the namespace, and then ask for the pod again to get
# it's name (but it would be much better to just wait for the job to
# finish instead, then ask for a name of the successful pod and use it
# to get logs ...)
ocp_pod = ocp.OCP(kind="Pod", namespace=project.namespace)
try:
ocp_pod.wait_for_resource(
resource_count=1,
condition=constants.STATUS_COMPLETED,
timeout=write_timeout,
sleep=30)
except TimeoutExpiredError as ex:
# report some high level error as well
msg = (
f"Job fio failed to write {pvc_size} Gi data on OCS backed "
f"volume in expected time {write_timeout} seconds.")
logger.error(msg)
# TODO: if the job is still running, report more specific error
# message instead of the generic one which is pushed to ex. below
ex.message = msg + (
" If the fio pod were still runing"
" (see 'last actual status was' in some previous log message),"
" this is caused either by"
" severe product performance regression"
" or by a misconfiguration of the clusterr, ping infra team.")
raise(ex)
pod_data = ocp_pod.get()
# explicit list of assumptions, if these assumptions are not met, the
# code won't work and it either means that something went terrible
# wrong or that the code needs to be changed
assert pod_data['kind'] == "List"
pod_dict = pod_data['items'][0]
assert pod_dict['kind'] == "Pod"
pod_name = pod_dict['metadata']['name']
logger.info(f"Identified pod name of the finished fio Job: {pod_name}")
fio_output = ocp_pod.exec_oc_cmd(
f"logs {pod_name}", out_yaml_format=False)
# parse fio output
fio_report = fio_to_dict(fio_output)
logger.info(fio_report)
# data which will be available to the test via:
# fixture_name['result']
result = {
'fio': fio_report,
'pvc_size': pvc_size,
'target_p': target_percentage,
'namespace': project.namespace}
return result
test_file = os.path.join(measurement_dir, f"{fixture_name}.json")
measured_op = measure_operation(
write_data, test_file, measure_after=True, minimal_time=480)
# we don't need to delete anything if this fixture has been already
# executed
if measured_op['first_run']:
# make sure we communicate what is going to happen
logger.info(f"going to delete {fixture_name} Job")
fio_job_file.delete()
logger.info(
f"going to wait a bit to make sure that "
f"data written by {fixture_name} Job are really deleted")
def check_pvc_size():
"""
Check whether data created by the Job were actually deleted.
"""
# By asking again for pvc_size necessary to reach the target
# cluster utilization, we can see how much data were already
# deleted. Negative or small value of current pvc_size means that
# the data were not yet deleted.
pvc_size_tmp = get_storageutilization_size(
target_percentage, ceph_pool_name)
# If no other components were utilizing OCS storage, the space
# would be considered reclaimed when current pvc_size reaches
# it's original value again. But since this is not the case (eg.
# constantly growing monitoring or log data are stored there),
# we are ok with just 90% of the original value.
result = pvc_size_tmp >= pvc_size * 0.90
if result:
logger.info("storage space was reclaimed")
else:
logger.info(
"storage space was not yet fully reclaimed, "
f"current pvc size {pvc_size_tmp} value "
f"should be close to {pvc_size}")
return result
check_timeout = 660 # seconds
check_sampler = TimeoutSampler(
timeout=check_timeout, sleep=30, func=check_pvc_size)
finished_in_time = check_sampler.wait_for_func_status(result=True)
if not finished_in_time:
error_msg = (
"it seems that the storage space was not reclaimed "
f"within {check_timeout} seconds, "
"this is most likely a product bug or misconfiguration")
logger.error(error_msg)
raise Exception(error_msg)
return measured_op
def test_bulk_clone_performance(self, namespace, tmp_path):
"""
Creates number of PVCs in a bulk using kube job
Write 60% of PVC capacity to each one of the created PVCs
Creates 1 clone per each PVC altogether in a bulk
Measuring total and csi creation times for bulk of clones
"""
pvc_count = 50
vol_size = "5Gi"
job_pod_file, job_pvc_file, job_clone_file = [None, None, None]
log.info(f"Start creating {self.interface} {pvc_count} PVC")
if self.interface == constants.CEPHBLOCKPOOL:
sc_name = constants.DEFAULT_STORAGECLASS_RBD
clone_yaml = constants.CSI_RBD_PVC_CLONE_YAML
elif self.interface == constants.CEPHFILESYSTEM:
sc_name = constants.DEFAULT_STORAGECLASS_CEPHFS
clone_yaml = constants.CSI_CEPHFS_PVC_CLONE_YAML
try:
pvc_dict_list = scale_lib.construct_pvc_creation_yaml_bulk_for_kube_job(
no_of_pvc=pvc_count,
access_mode=constants.ACCESS_MODE_RWO,
sc_name=sc_name,
pvc_size=vol_size,
)
job_pvc_file = ObjectConfFile(
name="job_profile_pvc",
obj_dict_list=pvc_dict_list,
project=self.namespace,
tmp_path=tmp_path,
)
# Create kube_job
job_pvc_file.create(namespace=self.namespace)
# Check all the PVC reached Bound state
pvc_bound_list = scale_lib.check_all_pvc_reached_bound_state_in_kube_job(
kube_job_obj=job_pvc_file,
namespace=self.namespace,
no_of_pvc=pvc_count,
)
log.info(f"Number of PVCs in Bound state {len(pvc_bound_list)}")
# Kube_job to Create pod
pod_dict_list = scale_lib.attach_multiple_pvc_to_pod_dict(
pvc_list=pvc_bound_list,
namespace=self.namespace,
pvcs_per_pod=1,
start_io=False,
pod_yaml=constants.NGINX_POD_YAML,
)
job_pod_file = ObjectConfFile(
name="job_profile_pod",
obj_dict_list=pod_dict_list,
project=self.namespace,
tmp_path=tmp_path,
)
job_pod_file.create(namespace=self.namespace)
# Check all PODs in Running state
scale_lib.check_all_pod_reached_running_state_in_kube_job(
kube_job_obj=job_pod_file,
namespace=self.namespace,
no_of_pod=len(pod_dict_list),
timeout=90,
)
log.info(f"Number of PODs in Running state {len(pod_dict_list)}")
total_files_size = self.run_fio_on_pvcs(vol_size)
clone_dict_list = scale_lib.construct_pvc_clone_yaml_bulk_for_kube_job(
pvc_dict_list, clone_yaml, sc_name)
log.info("Created clone dict list")
csi_bulk_start_time = self.get_time(time_format="csi")
job_clone_file = ObjectConfFile(
name="job_profile_clone",
obj_dict_list=clone_dict_list,
project=self.namespace,
tmp_path=tmp_path,
)
# Create kube_job that creates clones
job_clone_file.create(namespace=self.namespace)
log.info("Going to check bound status for clones")
# Check all the clones reached Bound state
clone_bound_list = scale_lib.check_all_pvc_reached_bound_state_in_kube_job(
kube_job_obj=job_clone_file,
namespace=self.namespace,
no_of_pvc=pvc_count,
timeout=180,
)
log.info(
f"Number of clones in Bound state {len(clone_bound_list)}")
clone_objs = []
all_pvc_objs = pvc.get_all_pvc_objs(namespace=self.namespace)
for clone_yaml in clone_dict_list:
name = clone_yaml["metadata"]["name"]
size = clone_yaml["spec"]["resources"]["requests"]["storage"]
log.info(f"Clone {name} of size {size} created")
for pvc_obj in all_pvc_objs:
if pvc_obj.name == name:
clone_objs.append(pvc_obj)
assert len(clone_bound_list) == len(
clone_objs
), "Not all clones reached BOUND state, cannot measure time"
start_time = helpers.get_provision_time(self.interface,
clone_objs,
status="start")
end_time = helpers.get_provision_time(self.interface,
clone_objs,
status="end")
total_time = (end_time - start_time).total_seconds()
speed = round(total_files_size / total_time, 2)
csi_creation_time = performance_lib.csi_bulk_pvc_time_measure(
self.interface, clone_objs, "create", csi_bulk_start_time)
log.info(
f"Total creation time = {total_time} secs, csi creation time = {csi_creation_time},"
f" data size = {total_files_size} MB, speed = {speed} MB/sec "
f"for {self.interface} clone in bulk of {pvc_count} clones.")
# Produce ES report
# Collecting environment information
self.get_env_info()
# Initialize the results doc file.
full_results = self.init_full_results(
ResultsAnalyse(
self.uuid,
self.crd_data,
self.full_log_path,
"bulk_clone_perf_fullres",
))
full_results.add_key("interface", self.interface)
full_results.add_key("bulk_size", pvc_count)
full_results.add_key("clone_size", vol_size)
full_results.add_key("bulk_creation_time", total_time)
full_results.add_key("bulk_csi_creation_time", csi_creation_time)
full_results.add_key("data_size(MB)", total_files_size)
full_results.add_key("speed", speed)
full_results.add_key("es_results_link",
full_results.results_link())
# Write the test results into the ES server
full_results.es_write()
self.results_path = get_full_test_logs_path(cname=self)
res_link = full_results.results_link()
# write the ES link to the test results in the test log.
log.info(f"The result can be found at : {res_link}")
# Create text file with results of all subtest (3 - according to the parameters)
self.write_result_to_file(res_link)
# Finally is used to clean-up the resources created
# Irrespective of try block pass/fail finally will be executed.
finally:
# Cleanup activities
log.info(
"Cleanup of all the resources created during test execution")
if job_pod_file:
job_pod_file.delete(namespace=self.namespace)
job_pod_file.wait_for_delete(resource_name=job_pod_file.name,
namespace=self.namespace)
if job_clone_file:
job_clone_file.delete(namespace=self.namespace)
job_clone_file.wait_for_delete(
resource_name=job_clone_file.name,
namespace=self.namespace)
if job_pvc_file:
job_pvc_file.delete(namespace=self.namespace)
job_pvc_file.wait_for_delete(resource_name=job_pvc_file.name,
namespace=self.namespace)
# Check ceph health status
utils.ceph_health_check(tries=20)
def test_all_4_type_pvc_creation_deletion_scale(self, namespace, tmp_path):
"""
Measuring PVC creation time while scaling PVC of all 4 types,
A total of 500 times the number of worker nodes
will be created, i.e. 375 each pvc type
Measure PVC deletion time in scale env
"""
scale_pvc_count = scale_lib.get_max_pvc_count()
log.info(f"Start creating {scale_pvc_count} PVC of all 4 types")
cephfs_sc_obj = constants.DEFAULT_STORAGECLASS_CEPHFS
rbd_sc_obj = constants.DEFAULT_STORAGECLASS_RBD
# Get pvc_dict_list, append all the pvc.yaml dict to pvc_dict_list
rbd_pvc_dict_list, cephfs_pvc_dict_list = ([] for i in range(2))
for mode in [constants.ACCESS_MODE_RWO, constants.ACCESS_MODE_RWX]:
rbd_pvc_dict_list.extend(
scale_lib.construct_pvc_creation_yaml_bulk_for_kube_job(
no_of_pvc=int(scale_pvc_count / 4),
access_mode=mode,
sc_name=rbd_sc_obj,
))
cephfs_pvc_dict_list.extend(
scale_lib.construct_pvc_creation_yaml_bulk_for_kube_job(
no_of_pvc=int(scale_pvc_count / 4),
access_mode=mode,
sc_name=cephfs_sc_obj,
))
# There is 2 kube_job for cephfs and rbd PVCs
job_file_rbd = ObjectConfFile(
name="rbd_pvc_job",
obj_dict_list=rbd_pvc_dict_list,
project=self.namespace,
tmp_path=tmp_path,
)
job_file_cephfs = ObjectConfFile(
name="cephfs_pvc_job",
obj_dict_list=cephfs_pvc_dict_list,
project=self.namespace,
tmp_path=tmp_path,
)
# Create kube_job
job_file_rbd.create(namespace=self.namespace)
job_file_cephfs.create(namespace=self.namespace)
# Check all the PVC reached Bound state
rbd_pvc_name = scale_lib.check_all_pvc_reached_bound_state_in_kube_job(
kube_job_obj=job_file_rbd,
namespace=self.namespace,
no_of_pvc=int(scale_pvc_count / 2),
)
fs_pvc_name = scale_lib.check_all_pvc_reached_bound_state_in_kube_job(
kube_job_obj=job_file_cephfs,
namespace=self.namespace,
no_of_pvc=int(scale_pvc_count / 2),
)
# Get pvc objs from namespace, which is used to identify backend pv
rbd_pvc_obj, cephfs_pvc_obj = ([] for i in range(2))
pvc_objs = pvc.get_all_pvc_objs(namespace=self.namespace)
for pvc_obj in pvc_objs:
if pvc_obj.backed_sc == constants.DEFAULT_STORAGECLASS_RBD:
rbd_pvc_obj.append(pvc_obj)
elif pvc_obj.backed_sc == constants.DEFAULT_STORAGECLASS_CEPHFS:
cephfs_pvc_obj.append(pvc_obj)
# Get PVC creation time
fs_pvc_create_time = helpers.measure_pvc_creation_time_bulk(
interface=constants.CEPHFS_INTERFACE, pvc_name_list=fs_pvc_name)
rbd_pvc_create_time = helpers.measure_pvc_creation_time_bulk(
interface=constants.CEPHBLOCKPOOL, pvc_name_list=rbd_pvc_name)
fs_pvc_create_time.update(rbd_pvc_create_time)
# TODO: Update below code with google API, to record value in spreadsheet
# TODO: For now observing Google API limit to write more than 100 writes
log_path = f"{ocsci_log_path()}/All-type-PVC"
with open(f"{log_path}-creation-time.csv", "w") as fd:
csv_obj = csv.writer(fd)
for k, v in fs_pvc_create_time.items():
csv_obj.writerow([k, v])
log.info(f"Create data present in {log_path}-creation-time.csv file")
# Get pv_name, require pv_name to fetch deletion time data from log
rbd_pv_list, fs_pv_list = ([] for i in range(2))
get_rbd_kube_job = job_file_rbd.get(namespace=self.namespace)
for i in range(int(scale_pvc_count / 2)):
rbd_pv_list.append(
get_rbd_kube_job["items"][i]["spec"]["volumeName"])
get_fs_kube_job = job_file_cephfs.get(namespace=self.namespace)
for i in range(int(scale_pvc_count / 2)):
fs_pv_list.append(
get_fs_kube_job["items"][i]["spec"]["volumeName"])
# Delete kube_job
job_file_rbd.delete(namespace=self.namespace)
job_file_cephfs.delete(namespace=self.namespace)
# Adding 1min wait time for PVC deletion logs to be updated
# Observed failure when we immediately check the logs for pvc delete time
# https://github.com/red-hat-storage/ocs-ci/issues/3371
time.sleep(60)
# Get PV deletion time
fs_pvc_deletion_time = helpers.measure_pv_deletion_time_bulk(
interface=constants.CEPHFS_INTERFACE, pv_name_list=fs_pv_list)
rbd_pvc_deletion_time = helpers.measure_pv_deletion_time_bulk(
interface=constants.CEPHBLOCKPOOL, pv_name_list=rbd_pv_list)
fs_pvc_deletion_time.update(rbd_pvc_deletion_time)
# TODO: Update below code with google API, to record value in spreadsheet
# TODO: For now observing Google API limit to write more than 100 writes
with open(f"{log_path}-deletion-time.csv", "w") as fd:
csv_obj = csv.writer(fd)
for k, v in fs_pvc_deletion_time.items():
csv_obj.writerow([k, v])
log.info(f"Delete data present in {log_path}-deletion-time.csv file")
end_time = default_timer()
log.info(f"Elapsed time -- {end_time - self.start_time} seconds")
def test_multiple_pvc_creation_deletion_scale(self, namespace, tmp_path,
access_mode, interface):
"""
Measuring PVC creation time while scaling PVC
Measure PVC deletion time after creation test
"""
scale_pvc_count = scale_lib.get_max_pvc_count()
log.info(
f"Start creating {access_mode}-{interface} {scale_pvc_count} PVC")
if interface == constants.CEPHBLOCKPOOL:
sc_name = constants.DEFAULT_STORAGECLASS_RBD
elif interface == constants.CEPHFS_INTERFACE:
sc_name = constants.DEFAULT_STORAGECLASS_CEPHFS
# Get pvc_dict_list, append all the pvc.yaml dict to pvc_dict_list
pvc_dict_list1 = scale_lib.construct_pvc_creation_yaml_bulk_for_kube_job(
no_of_pvc=int(scale_pvc_count / 2),
access_mode=access_mode,
sc_name=sc_name)
pvc_dict_list2 = scale_lib.construct_pvc_creation_yaml_bulk_for_kube_job(
no_of_pvc=int(scale_pvc_count / 2),
access_mode=access_mode,
sc_name=sc_name)
# There is 2 kube_job to reduce the load, observed time_out problems
# during delete process of single kube_job and heavy load.
job_file1 = ObjectConfFile(
name="job_profile_1",
obj_dict_list=pvc_dict_list1,
project=self.namespace,
tmp_path=tmp_path,
)
job_file2 = ObjectConfFile(
name="job_profile_2",
obj_dict_list=pvc_dict_list2,
project=self.namespace,
tmp_path=tmp_path,
)
# Create kube_job
job_file1.create(namespace=self.namespace)
job_file2.create(namespace=self.namespace)
# Check all the PVC reached Bound state
pvc_bound_list = scale_lib.check_all_pvc_reached_bound_state_in_kube_job(
kube_job_obj=job_file1,
namespace=self.namespace,
no_of_pvc=int(scale_pvc_count / 2),
)
pvc_bound_list.extend(
scale_lib.check_all_pvc_reached_bound_state_in_kube_job(
kube_job_obj=job_file2,
namespace=self.namespace,
no_of_pvc=int(scale_pvc_count / 2),
))
log.info(f"Number of PVCs in Bound state {len(pvc_bound_list)}")
# Get PVC creation time
pvc_create_time = helpers.measure_pvc_creation_time_bulk(
interface=interface,
pvc_name_list=pvc_bound_list,
wait_time=300,
)
# TODO: Update below code with google API, to record value in spreadsheet
# TODO: For now observing Google API limit to write more than 100 writes
log_path = f"{ocsci_log_path()}/{interface}-{access_mode}"
with open(f"{log_path}-creation-time.csv", "w") as fd:
csv_obj = csv.writer(fd)
for k, v in pvc_create_time.items():
csv_obj.writerow([k, v])
log.info(f"Create data present in {log_path}-creation-time.csv file")
# Get pv_name, require pv_name to fetch deletion time data from log
pv_name_list = list()
get_kube_job_1 = job_file1.get(namespace=self.namespace)
for i in range(int(scale_pvc_count / 2)):
pv_name_list.append(
get_kube_job_1["items"][i]["spec"]["volumeName"])
get_kube_job_2 = job_file2.get(namespace=self.namespace)
for i in range(int(scale_pvc_count / 2)):
pv_name_list.append(
get_kube_job_2["items"][i]["spec"]["volumeName"])
# Delete kube_job
job_file1.delete(namespace=self.namespace)
job_file2.delete(namespace=self.namespace)
# Adding 1min wait time for PVC deletion logs to be updated
# Observed failure when we immediately check the logs for pvc delete time
# https://github.com/red-hat-storage/ocs-ci/issues/3371
time.sleep(60)
# Get PVC deletion time
pvc_deletion_time = helpers.measure_pv_deletion_time_bulk(
interface=interface, pv_name_list=pv_name_list)
# Update result to csv file.
# TODO: Update below code with google API, to record value in spreadsheet
# TODO: For now observing Google API limit to write more than 100 writes
with open(f"{log_path}-deletion-time.csv", "w") as fd:
csv_obj = csv.writer(fd)
for k, v in pvc_deletion_time.items():
csv_obj.writerow([k, v])
log.info(f"Delete data present in {log_path}-deletion-time.csv file")
end_time = default_timer()
log.info(f"Elapsed time -- {end_time - self.start_time} seconds")
def workload_fio_storageutilization(
fixture_name,
target_percentage,
project,
fio_pvc_dict,
fio_job_dict,
fio_configmap_dict,
measurement_dir,
tmp_path,
):
"""
This function implements core functionality of fio storage utilization
workload fixture. This is necessary because we can't parametrize single
general fixture over multiple parameters (it would mess with test case id
and polarion test case tracking).
"""
if fixture_name.endswith("rbd"):
storage_class_name = "ocs-storagecluster-ceph-rbd"
ceph_pool_name = "ocs-storagecluster-cephblockpool"
elif fixture_name.endswith("cephfs"):
storage_class_name = "ocs-storagecluster-cephfs"
ceph_pool_name = "ocs-storagecluster-cephfilesystem-data0"
else:
raise UnexpectedVolumeType(
"unexpected volume type, ocs-ci code is wrong")
# make sure we communicate what is going to happen
logger.info((f"starting {fixture_name} fixture, "
f"using {storage_class_name} storage class "
f"backed by {ceph_pool_name} ceph pool"))
pvc_size = get_storageutilization_size(target_percentage, ceph_pool_name)
# For cephfs we can't use fill_fs because of BZ 1763808 (the process
# will get *Disk quota exceeded* error instead of *No space left on
# device* error).
# On the other hand, we can't use size={pvc_size} for rbd, as we can't
# write pvc_size bytes to a filesystem on a block device of {pvc_size}
# size (obviously, some space is used by filesystem metadata).
if fixture_name.endswith("rbd"):
fio_conf = textwrap.dedent("""
[simple-write]
readwrite=write
buffered=1
blocksize=4k
ioengine=libaio
directory=/mnt/target
fill_fs=1
""")
else:
fio_conf = textwrap.dedent(f"""
[simple-write]
readwrite=write
buffered=1
blocksize=4k
ioengine=libaio
directory=/mnt/target
size={pvc_size}G
""")
# put the dicts together into yaml file of the Job
fio_configmap_dict["data"]["workload.fio"] = fio_conf
fio_pvc_dict["spec"]["storageClassName"] = storage_class_name
fio_pvc_dict["spec"]["resources"]["requests"]["storage"] = f"{pvc_size}Gi"
fio_objs = [fio_pvc_dict, fio_configmap_dict, fio_job_dict]
fio_job_file = ObjectConfFile(fixture_name, fio_objs, project, tmp_path)
# how long do we let the job running while writing data to the volume
# TODO: increase this value or make it configurable
write_timeout = pvc_size * 30 # seconds
logger.info((f"fixture will wait {write_timeout} seconds for the Job "
f"to write {pvc_size} Gi data on OCS backed volume"))
def write_data():
"""
Write data via fio Job (specified in ``tf`` tmp file) to reach desired
utilization level, and keep this level for ``minimal_time`` seconds.
"""
# deploy the fio Job to the cluster
fio_job_file.create()
# This is a WORKAROUND of particular ocsci design choices: I just wait
# for one pod in the namespace, and then ask for the pod again to get
# it's name (but it would be much better to just wait for the job to
# finish instead, then ask for a name of the successful pod and use it
# to get logs ...)
ocp_pod = ocp.OCP(kind="Pod", namespace=project.namespace)
ocp_pod.wait_for_resource(resource_count=1,
condition=constants.STATUS_COMPLETED,
timeout=write_timeout,
sleep=30)
pod_data = ocp_pod.get()
# explicit list of assumptions, if these assumptions are not met, the
# code won't work and it either means that something went terrible
# wrong or that the code needs to be changed
assert pod_data['kind'] == "List"
pod_dict = pod_data['items'][0]
assert pod_dict['kind'] == "Pod"
pod_name = pod_dict['metadata']['name']
logger.info(f"Identified pod name of the finished fio Job: {pod_name}")
fio_output = ocp_pod.exec_oc_cmd(f"logs {pod_name}",
out_yaml_format=False)
# parse fio output
fio_report = fio_to_dict(fio_output)
logger.info(fio_report)
# data which will be available to the test via:
# fixture_name['result']
result = {
'fio': fio_report,
'pvc_size': pvc_size,
'target_p': target_percentage,
'namespace': project.namespace
}
return result
test_file = os.path.join(measurement_dir, f"{fixture_name}.json")
measured_op = measure_operation(write_data,
test_file,
measure_after=True,
minimal_time=480)
# we don't need to delete anything if this fixture has been already
# executed
if measured_op['first_run']:
# make sure we communicate what is going to happen
logger.info(f"going to delete {fixture_name} Job")
fio_job_file.delete()
return measured_op
