def workload_fio_storageutilization( fixture_name, target_percentage, project, fio_pvc_dict, fio_job_dict, fio_configmap_dict, measurement_dir, tmp_path, ): """ This function implements core functionality of fio storage utilization workload fixture. This is necessary because we can't parametrize single general fixture over multiple parameters (it would mess with test case id and polarion test case tracking). """ if fixture_name.endswith("rbd"): storage_class_name = "ocs-storagecluster-ceph-rbd" ceph_pool_name = "ocs-storagecluster-cephblockpool" elif fixture_name.endswith("cephfs"): storage_class_name = "ocs-storagecluster-cephfs" ceph_pool_name = "ocs-storagecluster-cephfilesystem-data0" else: raise UnexpectedVolumeType( "unexpected volume type, ocs-ci code is wrong") # make sure we communicate what is going to happen logger.info(( f"starting {fixture_name} fixture, " f"using {storage_class_name} storage class " f"backed by {ceph_pool_name} ceph pool")) pvc_size = get_storageutilization_size(target_percentage, ceph_pool_name) # For cephfs we can't use fill_fs because of BZ 1763808 (the process # will get *Disk quota exceeded* error instead of *No space left on # device* error). # On the other hand, we can't use size={pvc_size} for rbd, as we can't # write pvc_size bytes to a filesystem on a block device of {pvc_size} # size (obviously, some space is used by filesystem metadata). if fixture_name.endswith("rbd"): fio_conf = textwrap.dedent(""" [simple-write] readwrite=write buffered=1 blocksize=4k ioengine=libaio directory=/mnt/target fill_fs=1 """) else: fio_conf = textwrap.dedent(f""" [simple-write] readwrite=write buffered=1 blocksize=4k ioengine=libaio directory=/mnt/target size={pvc_size}G """) # put the dicts together into yaml file of the Job fio_configmap_dict["data"]["workload.fio"] = fio_conf fio_pvc_dict["spec"]["storageClassName"] = storage_class_name fio_pvc_dict["spec"]["resources"]["requests"]["storage"] = f"{pvc_size}Gi" fio_objs = [fio_pvc_dict, fio_configmap_dict, fio_job_dict] fio_job_file = ObjectConfFile(fixture_name, fio_objs, project, tmp_path) # How long do we let the job running while writing data to the volume? # Based on min. fio write speed of the enviroment ... fio_min_mbps = config.ENV_DATA['fio_storageutilization_min_mbps'] logger.info( "Assuming %.2f MB/s is a minimal write speed of fio.", fio_min_mbps) # ... we compute max. time we are going to wait for fio to write all data min_time_to_write_gb = 1 / (fio_min_mbps / 2**10) write_timeout = pvc_size * min_time_to_write_gb # seconds logger.info(( f"fixture will wait {write_timeout} seconds for the Job " f"to write {pvc_size} Gi data on OCS backed volume")) def write_data(): """ Write data via fio Job (specified in ``tf`` tmp file) to reach desired utilization level, and keep this level for ``minimal_time`` seconds. """ # deploy the fio Job to the cluster fio_job_file.create() # This is a WORKAROUND of particular ocsci design choices: I just wait # for one pod in the namespace, and then ask for the pod again to get # it's name (but it would be much better to just wait for the job to # finish instead, then ask for a name of the successful pod and use it # to get logs ...) ocp_pod = ocp.OCP(kind="Pod", namespace=project.namespace) try: ocp_pod.wait_for_resource( resource_count=1, condition=constants.STATUS_COMPLETED, timeout=write_timeout, sleep=30) except TimeoutExpiredError as ex: # report some high level error as well msg = ( f"Job fio failed to write {pvc_size} Gi data on OCS backed " f"volume in expected time {write_timeout} seconds.") logger.error(msg) # TODO: if the job is still running, report more specific error # message instead of the generic one which is pushed to ex. below ex.message = msg + ( " If the fio pod were still runing" " (see 'last actual status was' in some previous log message)," " this is caused either by" " severe product performance regression" " or by a misconfiguration of the clusterr, ping infra team.") raise(ex) pod_data = ocp_pod.get() # explicit list of assumptions, if these assumptions are not met, the # code won't work and it either means that something went terrible # wrong or that the code needs to be changed assert pod_data['kind'] == "List" pod_dict = pod_data['items'][0] assert pod_dict['kind'] == "Pod" pod_name = pod_dict['metadata']['name'] logger.info(f"Identified pod name of the finished fio Job: {pod_name}") fio_output = ocp_pod.exec_oc_cmd( f"logs {pod_name}", out_yaml_format=False) # parse fio output fio_report = fio_to_dict(fio_output) logger.info(fio_report) # data which will be available to the test via: # fixture_name['result'] result = { 'fio': fio_report, 'pvc_size': pvc_size, 'target_p': target_percentage, 'namespace': project.namespace} return result test_file = os.path.join(measurement_dir, f"{fixture_name}.json") measured_op = measure_operation( write_data, test_file, measure_after=True, minimal_time=480) # we don't need to delete anything if this fixture has been already # executed if measured_op['first_run']: # make sure we communicate what is going to happen logger.info(f"going to delete {fixture_name} Job") fio_job_file.delete() logger.info( f"going to wait a bit to make sure that " f"data written by {fixture_name} Job are really deleted") def check_pvc_size(): """ Check whether data created by the Job were actually deleted. """ # By asking again for pvc_size necessary to reach the target # cluster utilization, we can see how much data were already # deleted. Negative or small value of current pvc_size means that # the data were not yet deleted. pvc_size_tmp = get_storageutilization_size( target_percentage, ceph_pool_name) # If no other components were utilizing OCS storage, the space # would be considered reclaimed when current pvc_size reaches # it's original value again. But since this is not the case (eg. # constantly growing monitoring or log data are stored there), # we are ok with just 90% of the original value. result = pvc_size_tmp >= pvc_size * 0.90 if result: logger.info("storage space was reclaimed") else: logger.info( "storage space was not yet fully reclaimed, " f"current pvc size {pvc_size_tmp} value " f"should be close to {pvc_size}") return result check_timeout = 660 # seconds check_sampler = TimeoutSampler( timeout=check_timeout, sleep=30, func=check_pvc_size) finished_in_time = check_sampler.wait_for_func_status(result=True) if not finished_in_time: error_msg = ( "it seems that the storage space was not reclaimed " f"within {check_timeout} seconds, " "this is most likely a product bug or misconfiguration") logger.error(error_msg) raise Exception(error_msg) return measured_op
def test_bulk_clone_performance(self, namespace, tmp_path): """ Creates number of PVCs in a bulk using kube job Write 60% of PVC capacity to each one of the created PVCs Creates 1 clone per each PVC altogether in a bulk Measuring total and csi creation times for bulk of clones """ pvc_count = 50 vol_size = "5Gi" job_pod_file, job_pvc_file, job_clone_file = [None, None, None] log.info(f"Start creating {self.interface} {pvc_count} PVC") if self.interface == constants.CEPHBLOCKPOOL: sc_name = constants.DEFAULT_STORAGECLASS_RBD clone_yaml = constants.CSI_RBD_PVC_CLONE_YAML elif self.interface == constants.CEPHFILESYSTEM: sc_name = constants.DEFAULT_STORAGECLASS_CEPHFS clone_yaml = constants.CSI_CEPHFS_PVC_CLONE_YAML try: pvc_dict_list = scale_lib.construct_pvc_creation_yaml_bulk_for_kube_job( no_of_pvc=pvc_count, access_mode=constants.ACCESS_MODE_RWO, sc_name=sc_name, pvc_size=vol_size, ) job_pvc_file = ObjectConfFile( name="job_profile_pvc", obj_dict_list=pvc_dict_list, project=self.namespace, tmp_path=tmp_path, ) # Create kube_job job_pvc_file.create(namespace=self.namespace) # Check all the PVC reached Bound state pvc_bound_list = scale_lib.check_all_pvc_reached_bound_state_in_kube_job( kube_job_obj=job_pvc_file, namespace=self.namespace, no_of_pvc=pvc_count, ) log.info(f"Number of PVCs in Bound state {len(pvc_bound_list)}") # Kube_job to Create pod pod_dict_list = scale_lib.attach_multiple_pvc_to_pod_dict( pvc_list=pvc_bound_list, namespace=self.namespace, pvcs_per_pod=1, start_io=False, pod_yaml=constants.NGINX_POD_YAML, ) job_pod_file = ObjectConfFile( name="job_profile_pod", obj_dict_list=pod_dict_list, project=self.namespace, tmp_path=tmp_path, ) job_pod_file.create(namespace=self.namespace) # Check all PODs in Running state scale_lib.check_all_pod_reached_running_state_in_kube_job( kube_job_obj=job_pod_file, namespace=self.namespace, no_of_pod=len(pod_dict_list), timeout=90, ) log.info(f"Number of PODs in Running state {len(pod_dict_list)}") total_files_size = self.run_fio_on_pvcs(vol_size) clone_dict_list = scale_lib.construct_pvc_clone_yaml_bulk_for_kube_job( pvc_dict_list, clone_yaml, sc_name) log.info("Created clone dict list") csi_bulk_start_time = self.get_time(time_format="csi") job_clone_file = ObjectConfFile( name="job_profile_clone", obj_dict_list=clone_dict_list, project=self.namespace, tmp_path=tmp_path, ) # Create kube_job that creates clones job_clone_file.create(namespace=self.namespace) log.info("Going to check bound status for clones") # Check all the clones reached Bound state clone_bound_list = scale_lib.check_all_pvc_reached_bound_state_in_kube_job( kube_job_obj=job_clone_file, namespace=self.namespace, no_of_pvc=pvc_count, timeout=180, ) log.info( f"Number of clones in Bound state {len(clone_bound_list)}") clone_objs = [] all_pvc_objs = pvc.get_all_pvc_objs(namespace=self.namespace) for clone_yaml in clone_dict_list: name = clone_yaml["metadata"]["name"] size = clone_yaml["spec"]["resources"]["requests"]["storage"] log.info(f"Clone {name} of size {size} created") for pvc_obj in all_pvc_objs: if pvc_obj.name == name: clone_objs.append(pvc_obj) assert len(clone_bound_list) == len( clone_objs ), "Not all clones reached BOUND state, cannot measure time" start_time = helpers.get_provision_time(self.interface, clone_objs, status="start") end_time = helpers.get_provision_time(self.interface, clone_objs, status="end") total_time = (end_time - start_time).total_seconds() speed = round(total_files_size / total_time, 2) csi_creation_time = performance_lib.csi_bulk_pvc_time_measure( self.interface, clone_objs, "create", csi_bulk_start_time) log.info( f"Total creation time = {total_time} secs, csi creation time = {csi_creation_time}," f" data size = {total_files_size} MB, speed = {speed} MB/sec " f"for {self.interface} clone in bulk of {pvc_count} clones.") # Produce ES report # Collecting environment information self.get_env_info() # Initialize the results doc file. full_results = self.init_full_results( ResultsAnalyse( self.uuid, self.crd_data, self.full_log_path, "bulk_clone_perf_fullres", )) full_results.add_key("interface", self.interface) full_results.add_key("bulk_size", pvc_count) full_results.add_key("clone_size", vol_size) full_results.add_key("bulk_creation_time", total_time) full_results.add_key("bulk_csi_creation_time", csi_creation_time) full_results.add_key("data_size(MB)", total_files_size) full_results.add_key("speed", speed) full_results.add_key("es_results_link", full_results.results_link()) # Write the test results into the ES server full_results.es_write() self.results_path = get_full_test_logs_path(cname=self) res_link = full_results.results_link() # write the ES link to the test results in the test log. log.info(f"The result can be found at : {res_link}") # Create text file with results of all subtest (3 - according to the parameters) self.write_result_to_file(res_link) # Finally is used to clean-up the resources created # Irrespective of try block pass/fail finally will be executed. finally: # Cleanup activities log.info( "Cleanup of all the resources created during test execution") if job_pod_file: job_pod_file.delete(namespace=self.namespace) job_pod_file.wait_for_delete(resource_name=job_pod_file.name, namespace=self.namespace) if job_clone_file: job_clone_file.delete(namespace=self.namespace) job_clone_file.wait_for_delete( resource_name=job_clone_file.name, namespace=self.namespace) if job_pvc_file: job_pvc_file.delete(namespace=self.namespace) job_pvc_file.wait_for_delete(resource_name=job_pvc_file.name, namespace=self.namespace) # Check ceph health status utils.ceph_health_check(tries=20)
def test_all_4_type_pvc_creation_deletion_scale(self, namespace, tmp_path): """ Measuring PVC creation time while scaling PVC of all 4 types, A total of 500 times the number of worker nodes will be created, i.e. 375 each pvc type Measure PVC deletion time in scale env """ scale_pvc_count = scale_lib.get_max_pvc_count() log.info(f"Start creating {scale_pvc_count} PVC of all 4 types") cephfs_sc_obj = constants.DEFAULT_STORAGECLASS_CEPHFS rbd_sc_obj = constants.DEFAULT_STORAGECLASS_RBD # Get pvc_dict_list, append all the pvc.yaml dict to pvc_dict_list rbd_pvc_dict_list, cephfs_pvc_dict_list = ([] for i in range(2)) for mode in [constants.ACCESS_MODE_RWO, constants.ACCESS_MODE_RWX]: rbd_pvc_dict_list.extend( scale_lib.construct_pvc_creation_yaml_bulk_for_kube_job( no_of_pvc=int(scale_pvc_count / 4), access_mode=mode, sc_name=rbd_sc_obj, )) cephfs_pvc_dict_list.extend( scale_lib.construct_pvc_creation_yaml_bulk_for_kube_job( no_of_pvc=int(scale_pvc_count / 4), access_mode=mode, sc_name=cephfs_sc_obj, )) # There is 2 kube_job for cephfs and rbd PVCs job_file_rbd = ObjectConfFile( name="rbd_pvc_job", obj_dict_list=rbd_pvc_dict_list, project=self.namespace, tmp_path=tmp_path, ) job_file_cephfs = ObjectConfFile( name="cephfs_pvc_job", obj_dict_list=cephfs_pvc_dict_list, project=self.namespace, tmp_path=tmp_path, ) # Create kube_job job_file_rbd.create(namespace=self.namespace) job_file_cephfs.create(namespace=self.namespace) # Check all the PVC reached Bound state rbd_pvc_name = scale_lib.check_all_pvc_reached_bound_state_in_kube_job( kube_job_obj=job_file_rbd, namespace=self.namespace, no_of_pvc=int(scale_pvc_count / 2), ) fs_pvc_name = scale_lib.check_all_pvc_reached_bound_state_in_kube_job( kube_job_obj=job_file_cephfs, namespace=self.namespace, no_of_pvc=int(scale_pvc_count / 2), ) # Get pvc objs from namespace, which is used to identify backend pv rbd_pvc_obj, cephfs_pvc_obj = ([] for i in range(2)) pvc_objs = pvc.get_all_pvc_objs(namespace=self.namespace) for pvc_obj in pvc_objs: if pvc_obj.backed_sc == constants.DEFAULT_STORAGECLASS_RBD: rbd_pvc_obj.append(pvc_obj) elif pvc_obj.backed_sc == constants.DEFAULT_STORAGECLASS_CEPHFS: cephfs_pvc_obj.append(pvc_obj) # Get PVC creation time fs_pvc_create_time = helpers.measure_pvc_creation_time_bulk( interface=constants.CEPHFS_INTERFACE, pvc_name_list=fs_pvc_name) rbd_pvc_create_time = helpers.measure_pvc_creation_time_bulk( interface=constants.CEPHBLOCKPOOL, pvc_name_list=rbd_pvc_name) fs_pvc_create_time.update(rbd_pvc_create_time) # TODO: Update below code with google API, to record value in spreadsheet # TODO: For now observing Google API limit to write more than 100 writes log_path = f"{ocsci_log_path()}/All-type-PVC" with open(f"{log_path}-creation-time.csv", "w") as fd: csv_obj = csv.writer(fd) for k, v in fs_pvc_create_time.items(): csv_obj.writerow([k, v]) log.info(f"Create data present in {log_path}-creation-time.csv file") # Get pv_name, require pv_name to fetch deletion time data from log rbd_pv_list, fs_pv_list = ([] for i in range(2)) get_rbd_kube_job = job_file_rbd.get(namespace=self.namespace) for i in range(int(scale_pvc_count / 2)): rbd_pv_list.append( get_rbd_kube_job["items"][i]["spec"]["volumeName"]) get_fs_kube_job = job_file_cephfs.get(namespace=self.namespace) for i in range(int(scale_pvc_count / 2)): fs_pv_list.append( get_fs_kube_job["items"][i]["spec"]["volumeName"]) # Delete kube_job job_file_rbd.delete(namespace=self.namespace) job_file_cephfs.delete(namespace=self.namespace) # Adding 1min wait time for PVC deletion logs to be updated # Observed failure when we immediately check the logs for pvc delete time # https://github.com/red-hat-storage/ocs-ci/issues/3371 time.sleep(60) # Get PV deletion time fs_pvc_deletion_time = helpers.measure_pv_deletion_time_bulk( interface=constants.CEPHFS_INTERFACE, pv_name_list=fs_pv_list) rbd_pvc_deletion_time = helpers.measure_pv_deletion_time_bulk( interface=constants.CEPHBLOCKPOOL, pv_name_list=rbd_pv_list) fs_pvc_deletion_time.update(rbd_pvc_deletion_time) # TODO: Update below code with google API, to record value in spreadsheet # TODO: For now observing Google API limit to write more than 100 writes with open(f"{log_path}-deletion-time.csv", "w") as fd: csv_obj = csv.writer(fd) for k, v in fs_pvc_deletion_time.items(): csv_obj.writerow([k, v]) log.info(f"Delete data present in {log_path}-deletion-time.csv file") end_time = default_timer() log.info(f"Elapsed time -- {end_time - self.start_time} seconds")
def test_multiple_pvc_creation_deletion_scale(self, namespace, tmp_path, access_mode, interface): """ Measuring PVC creation time while scaling PVC Measure PVC deletion time after creation test """ scale_pvc_count = scale_lib.get_max_pvc_count() log.info( f"Start creating {access_mode}-{interface} {scale_pvc_count} PVC") if interface == constants.CEPHBLOCKPOOL: sc_name = constants.DEFAULT_STORAGECLASS_RBD elif interface == constants.CEPHFS_INTERFACE: sc_name = constants.DEFAULT_STORAGECLASS_CEPHFS # Get pvc_dict_list, append all the pvc.yaml dict to pvc_dict_list pvc_dict_list1 = scale_lib.construct_pvc_creation_yaml_bulk_for_kube_job( no_of_pvc=int(scale_pvc_count / 2), access_mode=access_mode, sc_name=sc_name) pvc_dict_list2 = scale_lib.construct_pvc_creation_yaml_bulk_for_kube_job( no_of_pvc=int(scale_pvc_count / 2), access_mode=access_mode, sc_name=sc_name) # There is 2 kube_job to reduce the load, observed time_out problems # during delete process of single kube_job and heavy load. job_file1 = ObjectConfFile( name="job_profile_1", obj_dict_list=pvc_dict_list1, project=self.namespace, tmp_path=tmp_path, ) job_file2 = ObjectConfFile( name="job_profile_2", obj_dict_list=pvc_dict_list2, project=self.namespace, tmp_path=tmp_path, ) # Create kube_job job_file1.create(namespace=self.namespace) job_file2.create(namespace=self.namespace) # Check all the PVC reached Bound state pvc_bound_list = scale_lib.check_all_pvc_reached_bound_state_in_kube_job( kube_job_obj=job_file1, namespace=self.namespace, no_of_pvc=int(scale_pvc_count / 2), ) pvc_bound_list.extend( scale_lib.check_all_pvc_reached_bound_state_in_kube_job( kube_job_obj=job_file2, namespace=self.namespace, no_of_pvc=int(scale_pvc_count / 2), )) log.info(f"Number of PVCs in Bound state {len(pvc_bound_list)}") # Get PVC creation time pvc_create_time = helpers.measure_pvc_creation_time_bulk( interface=interface, pvc_name_list=pvc_bound_list, wait_time=300, ) # TODO: Update below code with google API, to record value in spreadsheet # TODO: For now observing Google API limit to write more than 100 writes log_path = f"{ocsci_log_path()}/{interface}-{access_mode}" with open(f"{log_path}-creation-time.csv", "w") as fd: csv_obj = csv.writer(fd) for k, v in pvc_create_time.items(): csv_obj.writerow([k, v]) log.info(f"Create data present in {log_path}-creation-time.csv file") # Get pv_name, require pv_name to fetch deletion time data from log pv_name_list = list() get_kube_job_1 = job_file1.get(namespace=self.namespace) for i in range(int(scale_pvc_count / 2)): pv_name_list.append( get_kube_job_1["items"][i]["spec"]["volumeName"]) get_kube_job_2 = job_file2.get(namespace=self.namespace) for i in range(int(scale_pvc_count / 2)): pv_name_list.append( get_kube_job_2["items"][i]["spec"]["volumeName"]) # Delete kube_job job_file1.delete(namespace=self.namespace) job_file2.delete(namespace=self.namespace) # Adding 1min wait time for PVC deletion logs to be updated # Observed failure when we immediately check the logs for pvc delete time # https://github.com/red-hat-storage/ocs-ci/issues/3371 time.sleep(60) # Get PVC deletion time pvc_deletion_time = helpers.measure_pv_deletion_time_bulk( interface=interface, pv_name_list=pv_name_list) # Update result to csv file. # TODO: Update below code with google API, to record value in spreadsheet # TODO: For now observing Google API limit to write more than 100 writes with open(f"{log_path}-deletion-time.csv", "w") as fd: csv_obj = csv.writer(fd) for k, v in pvc_deletion_time.items(): csv_obj.writerow([k, v]) log.info(f"Delete data present in {log_path}-deletion-time.csv file") end_time = default_timer() log.info(f"Elapsed time -- {end_time - self.start_time} seconds")
def workload_fio_storageutilization( fixture_name, target_percentage, project, fio_pvc_dict, fio_job_dict, fio_configmap_dict, measurement_dir, tmp_path, ): """ This function implements core functionality of fio storage utilization workload fixture. This is necessary because we can't parametrize single general fixture over multiple parameters (it would mess with test case id and polarion test case tracking). """ if fixture_name.endswith("rbd"): storage_class_name = "ocs-storagecluster-ceph-rbd" ceph_pool_name = "ocs-storagecluster-cephblockpool" elif fixture_name.endswith("cephfs"): storage_class_name = "ocs-storagecluster-cephfs" ceph_pool_name = "ocs-storagecluster-cephfilesystem-data0" else: raise UnexpectedVolumeType( "unexpected volume type, ocs-ci code is wrong") # make sure we communicate what is going to happen logger.info((f"starting {fixture_name} fixture, " f"using {storage_class_name} storage class " f"backed by {ceph_pool_name} ceph pool")) pvc_size = get_storageutilization_size(target_percentage, ceph_pool_name) # For cephfs we can't use fill_fs because of BZ 1763808 (the process # will get *Disk quota exceeded* error instead of *No space left on # device* error). # On the other hand, we can't use size={pvc_size} for rbd, as we can't # write pvc_size bytes to a filesystem on a block device of {pvc_size} # size (obviously, some space is used by filesystem metadata). if fixture_name.endswith("rbd"): fio_conf = textwrap.dedent(""" [simple-write] readwrite=write buffered=1 blocksize=4k ioengine=libaio directory=/mnt/target fill_fs=1 """) else: fio_conf = textwrap.dedent(f""" [simple-write] readwrite=write buffered=1 blocksize=4k ioengine=libaio directory=/mnt/target size={pvc_size}G """) # put the dicts together into yaml file of the Job fio_configmap_dict["data"]["workload.fio"] = fio_conf fio_pvc_dict["spec"]["storageClassName"] = storage_class_name fio_pvc_dict["spec"]["resources"]["requests"]["storage"] = f"{pvc_size}Gi" fio_objs = [fio_pvc_dict, fio_configmap_dict, fio_job_dict] fio_job_file = ObjectConfFile(fixture_name, fio_objs, project, tmp_path) # how long do we let the job running while writing data to the volume # TODO: increase this value or make it configurable write_timeout = pvc_size * 30 # seconds logger.info((f"fixture will wait {write_timeout} seconds for the Job " f"to write {pvc_size} Gi data on OCS backed volume")) def write_data(): """ Write data via fio Job (specified in ``tf`` tmp file) to reach desired utilization level, and keep this level for ``minimal_time`` seconds. """ # deploy the fio Job to the cluster fio_job_file.create() # This is a WORKAROUND of particular ocsci design choices: I just wait # for one pod in the namespace, and then ask for the pod again to get # it's name (but it would be much better to just wait for the job to # finish instead, then ask for a name of the successful pod and use it # to get logs ...) ocp_pod = ocp.OCP(kind="Pod", namespace=project.namespace) ocp_pod.wait_for_resource(resource_count=1, condition=constants.STATUS_COMPLETED, timeout=write_timeout, sleep=30) pod_data = ocp_pod.get() # explicit list of assumptions, if these assumptions are not met, the # code won't work and it either means that something went terrible # wrong or that the code needs to be changed assert pod_data['kind'] == "List" pod_dict = pod_data['items'][0] assert pod_dict['kind'] == "Pod" pod_name = pod_dict['metadata']['name'] logger.info(f"Identified pod name of the finished fio Job: {pod_name}") fio_output = ocp_pod.exec_oc_cmd(f"logs {pod_name}", out_yaml_format=False) # parse fio output fio_report = fio_to_dict(fio_output) logger.info(fio_report) # data which will be available to the test via: # fixture_name['result'] result = { 'fio': fio_report, 'pvc_size': pvc_size, 'target_p': target_percentage, 'namespace': project.namespace } return result test_file = os.path.join(measurement_dir, f"{fixture_name}.json") measured_op = measure_operation(write_data, test_file, measure_after=True, minimal_time=480) # we don't need to delete anything if this fixture has been already # executed if measured_op['first_run']: # make sure we communicate what is going to happen logger.info(f"going to delete {fixture_name} Job") fio_job_file.delete() return measured_op