def test_fio_with_block_storage(self): name = 'test_workload' spec = self.pod_obj.data.get('spec') path = ( spec.get('containers')[0].get('volumeMounts')[0].get('mountPath')) work_load = 'fio' storage_type = 'fs' # few io parameters for Fio runtime = 10 size = '200M' wl = workload.WorkLoad(name, path, work_load, storage_type, self.pod_obj) assert wl.setup() io_params = templating.load_yaml(constants.FIO_IO_PARAMS_YAML) io_params['runtime'] = runtime io_params['size'] = size future_result = wl.run(**io_params) timeout = 1200 sample = TimeoutSampler(timeout=timeout, sleep=3, func=future_result.done) assert sample.wait_for_func_status(result=True) try: logger.info(future_result.result()) except exceptions.CommandFailed: logger.exception(f"FIO failed") raise except Exception: logger.exception(f"Found Exception") raise
def test_fio_with_block_storage(self): name = "test_workload" spec = self.pod_obj.data.get("spec") path = spec.get("containers")[0].get("volumeMounts")[0].get( "mountPath") work_load = "fio" storage_type = "fs" # few io parameters for Fio runtime = 10 size = "200M" wl = workload.WorkLoad(name, path, work_load, storage_type, self.pod_obj) assert wl.setup() io_params = templating.load_yaml(constants.FIO_IO_PARAMS_YAML) io_params["runtime"] = runtime io_params["size"] = size future_result = wl.run(**io_params) timeout = 1200 sample = TimeoutSampler(timeout=timeout, sleep=3, func=future_result.done) assert sample.wait_for_func_status(result=True) try: logger.info(future_result.result()) except exceptions.CommandFailed: logger.exception("FIO failed") raise except Exception: logger.exception("Found Exception") raise
def run_git_clone(self): """ Execute git clone on a pod to simulate a Jenkins user """ name = 'test_workload' work_load = 'jenkins' wl = workload.WorkLoad(name=name, work_load=work_load, pod=self) assert wl.setup(), "Setup up for git failed" wl.run()
def run_io(self, storage_type, size, io_direction='rw', rw_ratio=75, jobs=1, runtime=60, depth=4, fio_filename=None): """ Execute FIO on a pod This operation will run in background and will store the results in 'self.thread.result()'. In order to wait for the output and not continue with the test until FIO is done, call self.thread.result() right after calling run_io. See tests/manage/test_pvc_deletion_during_io.py::test_run_io for usage of FIO Args: storage_type (str): 'fs' or 'block' size (str): Size in MB, e.g. '200M' io_direction (str): Determines the operation: 'ro', 'wo', 'rw' (default: 'rw') rw_ratio (int): Determines the reads and writes using a <rw_ratio>%/100-<rw_ratio>% (e.g. the default is 75 which means it is 75%/25% which equivalent to 3 reads are performed for every 1 write) jobs (int): Number of jobs to execute FIO runtime (int): Number of seconds IO should run for depth (int): IO depth fio_filename(str): Name of fio file created on app pod's mount point """ name = 'test_workload' spec = self.pod_data.get('spec') path = ( spec.get('containers')[0].get('volumeMounts')[0].get('mountPath')) work_load = 'fio' # few io parameters for Fio wl = workload.WorkLoad(name, path, work_load, storage_type, self, jobs) assert wl.setup(), "Setup up for FIO failed" if io_direction == 'rw': io_params = templating.load_yaml_to_dict( constants.FIO_IO_RW_PARAMS_YAML) io_params['rwmixread'] = rw_ratio else: io_params = templating.load_yaml_to_dict( constants.FIO_IO_PARAMS_YAML) io_params['runtime'] = runtime io_params['size'] = size if fio_filename: io_params['filename'] = fio_filename io_params['iodepth'] = depth self.fio_thread = wl.run(**io_params)
def workload_setup(self, storage_type, jobs=1): """ Do setup on pod for running FIO Args: storage_type (str): 'fs' or 'block' jobs (int): Number of jobs to execute FIO """ work_load = 'fio' name = f'test_workload_{work_load}' path = self.get_storage_path(storage_type) # few io parameters for Fio self.wl_obj = workload.WorkLoad(name, path, work_load, storage_type, self, jobs) assert self.wl_obj.setup(), f"Setup for FIO failed on pod {self.name}" self.wl_setup_done = True
def factory( num_of_pvcs=100, pvc_size=2, bulk=False, project=None, measure=True, delete=True, file_name=None, fio_percentage=25, verify_fio=False, expand=False, ): """ Args: num_of_pvcs (int) : Number of PVCs / PODs we want to create. pvc_size (int) : Size of each PVC in GB. bulk (bool) : True for bulk operations, False otherwise. project (obj) : Project obj inside which the PODs/PVCs are created. measure (bool) : True if we want to measure the PVC creation/deletion time and POD to PVC attach time, False otherwise. delete (bool) : True if we want to delete PVCs and PODs, False otherwise file_name (str) : Name of the file on which FIO is performed. fio_percentage (float) : Percentage of PVC space we want to be utilized for FIO. verify_fio (bool) : True if we want to verify FIO, False otherwise. expand (bool) : True if we want to verify_fio for expansion of PVCs operation, False otherwise. """ if not project: project = project_factory("longevity") pvc_objs = list() executor = ThreadPoolExecutor(max_workers=1) start_time = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") for interface in (constants.CEPHFILESYSTEM, constants.CEPHBLOCKPOOL): if interface == constants.CEPHFILESYSTEM: access_modes = [ constants.ACCESS_MODE_RWO, constants.ACCESS_MODE_RWX ] num_of_pvc = num_of_pvcs // 2 else: access_modes = [ constants.ACCESS_MODE_RWO, constants.ACCESS_MODE_RWO + "-" + constants.VOLUME_MODE_BLOCK, constants.ACCESS_MODE_RWX + "-" + constants.VOLUME_MODE_BLOCK, ] num_of_pvc = num_of_pvcs - num_of_pvcs // 2 # Create PVCs if num_of_pvc > 0: pvc_objs_tmp = multi_pvc_factory( interface=interface, size=pvc_size, project=project, access_modes=access_modes, status=constants.STATUS_BOUND, num_of_pvc=num_of_pvc, wait_each=not bulk, ) log.info("PVC creation was successful.") pvc_objs.extend(pvc_objs_tmp) if measure: # Measure PVC Creation Time measure_pvc_creation_time(interface, pvc_objs_tmp, start_time) else: log.info( f"Num of PVCs of interface - {interface} = {num_of_pvc}. So no PVCs created." ) # PVC and PV Teardown for pvc_obj in pvc_objs: teardown_factory(pvc_obj) teardown_factory(pvc_obj.backed_pv_obj) # Create PODs pod_objs = list() for pvc_obj in pvc_objs: if pvc_obj.get_pvc_vol_mode == constants.VOLUME_MODE_BLOCK: if not bulk: pod_objs.append( pod_factory( pvc=pvc_obj, raw_block_pv=True, status=constants.STATUS_RUNNING, pod_dict_path=constants.PERF_BLOCK_POD_YAML, )) else: pod_objs.append( pod_factory( pvc=pvc_obj, raw_block_pv=True, pod_dict_path=constants.PERF_BLOCK_POD_YAML, )) else: if not bulk: pod_objs.append( pod_factory( pvc=pvc_obj, status=constants.STATUS_RUNNING, pod_dict_path=constants.PERF_POD_YAML, )) else: pod_objs.append( pod_factory(pvc=pvc_obj, pod_dict_path=constants.PERF_POD_YAML)) log.info(f"POD {pod_objs[-1].name} creation was successful.") log.info("All PODs are created.") if bulk: for pod_obj in pod_objs: executor.submit( helpers.wait_for_resource_state, pod_obj, constants.STATUS_RUNNING, timeout=300, ) log.info(f"POD {pod_obj.name} reached Running State.") log.info("All PODs reached Running State.") if measure: # Measure POD to PVC attach time measure_pod_to_pvc_attach_time(pod_objs) # POD Teardown for pod_obj in pod_objs: teardown_factory(pod_obj) # Run FIO on PODs fio_size = int((fio_percentage / 100) * pvc_size * 1000) for pod_obj in pod_objs: storage_type = ("block" if pod_obj.pvc.get_pvc_vol_mode == constants.VOLUME_MODE_BLOCK else "fs") pod_obj.wl_setup_done = True pod_obj.wl_obj = workload.WorkLoad( "test_workload_fio", pod_obj.get_storage_path(storage_type), "fio", storage_type, pod_obj, 1, ) if not file_name: pod_obj.run_io(storage_type, f"{fio_size}M") else: pod_obj.run_io( storage_type=storage_type, size=f"{fio_size}M", runtime=20, fio_filename=file_name, end_fsync=1, ) if verify_fio: log.info( "Waiting for IO to complete on all pods to utilise 25% of PVC used space" ) for pod_obj in pod_objs: # Wait for IO to finish pod_obj.get_fio_results(3600) log.info(f"IO finished on pod {pod_obj.name}") is_block = (True if pod_obj.pvc.get_pvc_vol_mode == constants.VOLUME_MODE_BLOCK else False) file_name_pod = (file_name if not is_block else pod_obj.get_storage_path( storage_type="block")) # Verify presence of the file file_path = (file_name_pod if is_block else pod.get_file_path( pod_obj, file_name_pod)) log.info(f"Actual file path on the pod {file_path}") assert pod.check_file_existence( pod_obj, file_path), f"File {file_name_pod} does not exist" log.info(f"File {file_name_pod} exists in {pod_obj.name}") if expand and is_block: # Read IO from block PVCs using dd and calculate md5sum. # This dd command reads the data from the device, writes it to # stdout, and reads md5sum from stdin. pod_obj.pvc.md5sum = pod_obj.exec_sh_cmd_on_pod( command=(f"dd iflag=direct if={file_path} bs=10M " f"count={fio_size // 10} | md5sum")) log.info( f"md5sum of {file_name_pod}: {pod_obj.pvc.md5sum}") else: # Calculate md5sum of the file pod_obj.pvc.md5sum = pod.cal_md5sum(pod_obj, file_name_pod) log.info("POD FIO was successful.") if delete: # Delete PODs pod_delete = executor.submit(delete_pods, pod_objs, wait=not bulk) pod_delete.result() log.info("Verified: Pods are deleted.") # Delete PVCs pvc_delete = executor.submit(delete_pvcs, pvc_objs, concurrent=bulk) res = pvc_delete.result() if not res: raise ex.UnexpectedBehaviour("Deletion of PVCs failed") log.info("PVC deletion was successful.") # Validate PV Deletion for pvc_obj in pvc_objs: helpers.validate_pv_delete(pvc_obj.backed_pv) log.info("PV deletion was successful.") if measure: # Measure PVC Deletion Time for interface in (constants.CEPHFILESYSTEM, constants.CEPHBLOCKPOOL): if interface == constants.CEPHFILESYSTEM: measure_pvc_deletion_time( interface, pvc_objs[:num_of_pvcs // 2], ) else: measure_pvc_deletion_time( interface, pvc_objs[num_of_pvcs // 2:], ) log.info(f"Successfully deleted {num_of_pvcs} PVCs") else: return pvc_objs, pod_objs
def expand_verify_pvcs(pvc_objs, pod_objs, pvc_size_new, file_name, fio_size): """ Expands size of each PVC in the provided list of PVCs, Verifies data integrity by checking the existence and md5sum of file in the expanded PVC and Runs FIO on expanded PVCs and verifies results. Args: pvc_objs (list) : List of PVC objects which are to be expanded. pod_objs (list) : List of POD objects attached to the PVCs. pvc_size_new (int) : Size of the expanded PVC in GB. file_name (str) : Name of the file on which FIO is performed. fio_size (int) : Size in MB of FIO. """ # Expand original PVCs log.info("Started expansion of the PVCs.") for pvc_obj in pvc_objs: log.info(f"Expanding size of PVC {pvc_obj.name} to {pvc_size_new}G") pvc_obj.resize_pvc(pvc_size_new, True) log.info("Successfully expanded the PVCs.") # Verify that the fio exists and md5sum matches for pod_no in range(len(pod_objs)): pod_obj = pod_objs[pod_no] if pod_obj.pvc.get_pvc_vol_mode == constants.VOLUME_MODE_BLOCK: pod.verify_data_integrity_after_expansion_for_block_pvc( pod_obj, pvc_objs[pod_no], fio_size) else: pod.verify_data_integrity(pod_obj, file_name, pvc_objs[pod_no].md5sum) # Run IO to utilize 50% of volume log.info( "Run IO on all pods to utilise 50% of the expanded PVC used space") expanded_file_name = "fio_50" for pod_obj in pod_objs: log.info(f"Running IO on pod {pod_obj.name}") log.info(f"File created during IO {expanded_file_name}") fio_size = int(0.50 * pvc_size_new * 1000) storage_type = ("block" if pod_obj.pvc.get_pvc_vol_mode == constants.VOLUME_MODE_BLOCK else "fs") pod_obj.wl_setup_done = True pod_obj.wl_obj = workload.WorkLoad( "test_workload_fio", pod_obj.get_storage_path(storage_type), "fio", storage_type, pod_obj, 1, ) pod_obj.run_io( storage_type=storage_type, size=f"{fio_size}M", runtime=20, fio_filename=expanded_file_name, end_fsync=1, ) log.info("Started IO on all pods to utilise 50% of PVCs") for pod_obj in pod_objs: # Wait for IO to finish pod_obj.get_fio_results(3600) log.info(f"IO finished on pod {pod_obj.name}") is_block = (True if pod_obj.pvc.get_pvc_vol_mode == constants.VOLUME_MODE_BLOCK else False) expanded_file_name_pod = (expanded_file_name if not is_block else pod_obj.get_storage_path( storage_type="block")) # Verify presence of the file expanded_file_path = (expanded_file_name_pod if is_block else pod.get_file_path( pod_obj, expanded_file_name_pod)) log.info(f"Actual file path on the pod {expanded_file_path}") assert pod.check_file_existence( pod_obj, expanded_file_path ), f"File {expanded_file_name_pod} does not exist" log.info(f"File {expanded_file_name_pod} exists in {pod_obj.name}")