def run_fio_on_pvcs(self, pvc_dict_list, pod_factory): total_files_size = 0 searched_pvc_objs = pvc.get_all_pvc_objs(namespace=self.namespace) logging.info(f"Found {len(searched_pvc_objs)} PVCs") for pvc_yaml in pvc_dict_list: pvc_name = pvc_yaml["metadata"]["name"] pvc_size = pvc_yaml["spec"]["resources"]["requests"]["storage"] logging.info(f"Size of pvc {pvc_name} is {pvc_size}") pvc_size_int = int(pvc_size[:-2]) # without "Gi" file_size_mb = int(pvc_size_int * 0.6) * constants.GB2MB total_files_size += file_size_mb file_size_mb_str = str(file_size_mb) + "M" logging.info(f"Writing file of size {file_size_mb_str}") # now find pvc_obj by name and create pod_obj to write to pvc_obj = None for obj in searched_pvc_objs: if obj.name == pvc_name: pvc_obj = obj searched_pvc_objs.remove(obj) break assert pvc_obj is not None, f"Cannot find PVC with name {pvc_name}" logging.info(f"PVC with name {pvc_name} found") pod_obj = pod_factory(interface=self.interface, pvc=pvc_obj, status=constants.STATUS_RUNNING) performance_lib.write_fio_on_pod(pod_obj, file_size_mb_str) return total_files_size
def run_fio_on_pvcs(self, pvc_size): searched_pvc_objs = pvc.get_all_pvc_objs(namespace=self.namespace) pod_objs = pod.get_all_pods(namespace=self.namespace) log.info(f"Found {len(searched_pvc_objs)} PVCs") pvc_size_int = int(pvc_size[:-2]) # without "Gi" file_size_mb = int(pvc_size_int * 0.6) * constants.GB2MB total_files_size = file_size_mb * len(searched_pvc_objs) file_size_mb_str = str(file_size_mb) + "M" log.info(f"Writing file of size {file_size_mb_str} in each PVC") for objs in pod_objs: performance_lib.write_fio_on_pod(objs, file_size_mb_str) return total_files_size
def test_pvc_multiple_clone_performance( self, interface_iterate, teardown_factory, storageclass_factory, pvc_factory, pod_factory, ): """ 1. Creating PVC PVC size is calculated in the test and depends on the storage capacity, but not less then 1 GiB it will use ~75% capacity of the Storage, Min storage capacity 1 TiB 2. Fill the PVC with 70% of data 3. Take a clone of the PVC and measure time and speed of creation by reading start creation and end creation times from relevant logs 4. Repeat the previous step number of times (maximal num_of_clones is 512) 5. Print all measured statistics for all the clones. Raises: StorageNotSufficientException: in case of not enough capacity on the cluster """ num_of_clones = 512 # Getting the total Storage capacity ceph_cluster = CephCluster() ceph_capacity = int(ceph_cluster.get_ceph_capacity()) # Use 70% of the storage capacity in the test capacity_to_use = int(ceph_capacity * 0.7) # since we do not want to use more then 65%, we add 35% to the needed # capacity, and minimum PVC size is 1 GiB need_capacity = int((num_of_clones + 2) * 1.35) # Test will run only on system with enough capacity if capacity_to_use < need_capacity: err_msg = (f"The system have only {ceph_capacity} GiB, " f"we want to use only {capacity_to_use} GiB, " f"and we need {need_capacity} GiB to run the test") log.error(err_msg) raise exceptions.StorageNotSufficientException(err_msg) # Calculating the PVC size in GiB pvc_size = int(capacity_to_use / (num_of_clones + 2)) self.interface = interface_iterate self.sc_obj = storageclass_factory(self.interface) self.pvc_obj = pvc_factory(interface=self.interface, size=pvc_size, status=constants.STATUS_BOUND) self.pod_obj = pod_factory(interface=self.interface, pvc=self.pvc_obj, status=constants.STATUS_RUNNING) # Calculating the file size as 70% of the PVC size filesize = self.pvc_obj.size * 0.70 # Change the file size to MB for the FIO function file_size = f"{int(filesize * constants.GB2MB)}M" file_name = self.pod_obj.name log.info(f"Total capacity size is : {ceph_capacity} GiB, " f"Going to use {need_capacity} GiB, " f"With {num_of_clones} clones to {pvc_size} GiB PVC. " f"File size to be written is : {file_size} " f"with the name of {file_name}") self.params = {} self.params["clonenum"] = f"{num_of_clones}" self.params["filesize"] = file_size self.params["ERRMSG"] = "Error in command" clone_yaml = self.build_params() performance_lib.write_fio_on_pod(self.pod_obj, file_size) # Running the test results = [] for test_num in range(1, int(self.params["clonenum"]) + 1): log.info(f"Starting test number {test_num}") ct = self.create_clone(test_num, clone_yaml) speed = self.params["datasize"] / ct results.append({"Clone Num": test_num, "time": ct, "speed": speed}) log.info( f"Results for clone number {test_num} are : " f"Creation time is {ct} secs, Creation speed {speed} MB/sec") for r in results: log.info( f"Clone number {r['Clone Num']} creation time is {r['time']} secs." ) log.info( f"Clone number {r['Clone Num']} creation speed is {r['speed']} MB/sec." )
def test_pvc_multiple_clone_performance( self, interface_iterate, teardown_factory, storageclass_factory, pvc_factory, pod_factory, ): """ 1. Creating PVC PVC size is calculated in the test and depends on the storage capacity, but not less then 1 GiB it will use ~75% capacity of the Storage, Min storage capacity 1 TiB 2. Fill the PVC with 70% of data 3. Take a clone of the PVC and measure time and speed of creation by reading start creation and end creation times from relevant logs 4. Repeat the previous step number of times (maximal num_of_clones is 512) 5. Print all measured statistics for all the clones. Raises: StorageNotSufficientException: in case of not enough capacity on the cluster """ num_of_clones = 512 # Getting the total Storage capacity ceph_cluster = CephCluster() ceph_capacity = int(ceph_cluster.get_ceph_capacity()) # Use 70% of the storage capacity in the test capacity_to_use = int(ceph_capacity * 0.7) # since we do not want to use more then 65%, we add 35% to the needed # capacity, and minimum PVC size is 1 GiB need_capacity = int((num_of_clones + 2) * 1.35) # Test will run only on system with enough capacity if capacity_to_use < need_capacity: err_msg = (f"The system have only {ceph_capacity} GiB, " f"we want to use only {capacity_to_use} GiB, " f"and we need {need_capacity} GiB to run the test") log.error(err_msg) raise exceptions.StorageNotSufficientException(err_msg) # Calculating the PVC size in GiB pvc_size = int(capacity_to_use / (num_of_clones + 2)) self.interface = interface_iterate self.sc_obj = storageclass_factory(self.interface) if self.interface == constants.CEPHFILESYSTEM: sc = "CephFS" if self.interface == constants.CEPHBLOCKPOOL: sc = "RBD" self.full_log_path = get_full_test_logs_path(cname=self) self.full_log_path += f"-{sc}" self.pvc_obj = pvc_factory(interface=self.interface, size=pvc_size, status=constants.STATUS_BOUND) self.pod_obj = pod_factory(interface=self.interface, pvc=self.pvc_obj, status=constants.STATUS_RUNNING) # Calculating the file size as 70% of the PVC size filesize = self.pvc_obj.size * 0.70 # Change the file size to MB for the FIO function file_size = f"{int(filesize * constants.GB2MB)}M" file_name = self.pod_obj.name log.info(f"Total capacity size is : {ceph_capacity} GiB, " f"Going to use {need_capacity} GiB, " f"With {num_of_clones} clones to {pvc_size} GiB PVC. " f"File size to be written is : {file_size} " f"with the name of {file_name}") self.params = {} self.params["clonenum"] = f"{num_of_clones}" self.params["filesize"] = file_size self.params["ERRMSG"] = "Error in command" clone_yaml = self.build_params() performance_lib.write_fio_on_pod(self.pod_obj, file_size) # Running the test results = [] for test_num in range(1, int(self.params["clonenum"]) + 1): log.info(f"Starting test number {test_num}") ct = self.create_clone(test_num, clone_yaml) speed = self.params["datasize"] / ct results.append({"Clone Num": test_num, "time": ct, "speed": speed}) log.info( f"Results for clone number {test_num} are : " f"Creation time is {ct} secs, Creation speed {speed} MB/sec") for r in results: log.info( f"Clone number {r['Clone Num']} creation time is {r['time']} secs." ) log.info( f"Clone number {r['Clone Num']} creation speed is {r['speed']} MB/sec." ) creation_time_list = [r["time"] for r in results] average_creation_time = statistics.mean(creation_time_list) log.info(f"Average creation time is {average_creation_time} secs.") creation_speed_list = [r["speed"] for r in results] average_creation_speed = statistics.mean(creation_speed_list) log.info(f"Average creation speed is {average_creation_time} MB/sec.") self.results_path = get_full_test_logs_path(cname=self) # Produce ES report # Collecting environment information self.get_env_info() # Initialize the results doc file. full_results = self.init_full_results( ResultsAnalyse( self.uuid, self.crd_data, self.full_log_path, "pvc_multiple_clone_measurement", )) full_results.add_key("interface", self.interface) full_results.add_key("clones_num", num_of_clones) full_results.add_key("clone_size", pvc_size) full_results.add_key("multi_clone_creation_time", creation_time_list) full_results.add_key("multi_clone_creation_time_average", average_creation_time) full_results.add_key("multi_clone_creation_speed", creation_speed_list) full_results.add_key("multi_clone_creation_speed_average", average_creation_speed) # Write the test results into the ES server if full_results.es_write(): res_link = full_results.results_link() log.info(f"The Result can be found at : {res_link}") # Create text file with results of all subtest (4 - according to the parameters) self.write_result_to_file(res_link)
def test_clone_create_delete_performance(self, interface_type, pvc_size, file_size, teardown_factory): """ Write data (60% of PVC capacity) to the PVC created in setup Create single clone for an existing pvc, Measure clone creation time and speed Delete the created clone Measure clone deletion time and speed Note: by increasing max_num_of_clones value you increase number of the clones to be created/deleted """ file_size_for_io = file_size[:-1] performance_lib.write_fio_on_pod(self.pod_obj, file_size_for_io) max_num_of_clones = 1 clone_creation_measures = [] clones_list = [] timeout = 18000 sc_name = self.pvc_obj.backed_sc parent_pvc = self.pvc_obj.name clone_yaml = constants.CSI_RBD_PVC_CLONE_YAML namespace = self.pvc_obj.namespace if interface_type == constants.CEPHFILESYSTEM: clone_yaml = constants.CSI_CEPHFS_PVC_CLONE_YAML file_size_mb = convert_device_size(file_size, "MB") # creating single clone ( or many one by one if max_mum_of_clones > 1) logger.info( f"Start creating {max_num_of_clones} clones on {interface_type} PVC of size {pvc_size} GB." ) for i in range(max_num_of_clones): logger.info(f"Start creation of clone number {i + 1}.") cloned_pvc_obj = pvc.create_pvc_clone(sc_name, parent_pvc, clone_yaml, namespace, storage_size=pvc_size + "Gi") teardown_factory(cloned_pvc_obj) helpers.wait_for_resource_state(cloned_pvc_obj, constants.STATUS_BOUND, timeout) cloned_pvc_obj.reload() logger.info( f"Clone with name {cloned_pvc_obj.name} for {pvc_size} pvc {parent_pvc} was created." ) clones_list.append(cloned_pvc_obj) create_time = helpers.measure_pvc_creation_time( interface_type, cloned_pvc_obj.name) creation_speed = int(file_size_mb / create_time) logger.info( f"Clone number {i+1} creation time is {create_time} secs for {pvc_size} GB pvc." ) logger.info( f"Clone number {i+1} creation speed is {creation_speed} MB/sec for {pvc_size} GB pvc." ) creation_measures = { "clone_num": i + 1, "time": create_time, "speed": creation_speed, } clone_creation_measures.append(creation_measures) # deleting one by one and measuring deletion times and speed for each one of the clones create above # in case of single clone will run one time clone_deletion_measures = [] logger.info( f"Start deleting {max_num_of_clones} clones on {interface_type} PVC of size {pvc_size} GB." ) for i in range(max_num_of_clones): cloned_pvc_obj = clones_list[i] pvc_reclaim_policy = cloned_pvc_obj.reclaim_policy cloned_pvc_obj.delete() logger.info( f"Deletion of clone number {i + 1} , the clone name is {cloned_pvc_obj.name}." ) cloned_pvc_obj.ocp.wait_for_delete(cloned_pvc_obj.name, timeout) if pvc_reclaim_policy == constants.RECLAIM_POLICY_DELETE: helpers.validate_pv_delete(cloned_pvc_obj.backed_pv) delete_time = helpers.measure_pvc_deletion_time( interface_type, cloned_pvc_obj.backed_pv) logger.info( f"Clone number {i + 1} deletion time is {delete_time} secs for {pvc_size} GB pvc." ) deletion_speed = int(file_size_mb / delete_time) logger.info( f"Clone number {i+1} deletion speed is {deletion_speed} MB/sec for {pvc_size} GB pvc." ) deletion_measures = { "clone_num": i + 1, "time": delete_time, "speed": deletion_speed, } clone_deletion_measures.append(deletion_measures) logger.info( f"Printing clone creation time and speed for {max_num_of_clones} clones " f"on {interface_type} PVC of size {pvc_size} GB:") for c in clone_creation_measures: logger.info( f"Clone number {c['clone_num']} creation time is {c['time']} secs for {pvc_size} GB pvc ." ) logger.info( f"Clone number {c['clone_num']} creation speed is {c['speed']} MB/sec for {pvc_size} GB pvc." ) logger.info( f"Clone deletion time and speed for {interface_type} PVC of size {pvc_size} GB are:" ) for d in clone_deletion_measures: logger.info( f"Clone number {d['clone_num']} deletion time is {d['time']} secs for {pvc_size} GB pvc." ) logger.info( f"Clone number {d['clone_num']} deletion speed is {d['speed']} MB/sec for {pvc_size} GB pvc." ) logger.info("test_clones_creation_performance finished successfully.")
def test_clone_create_delete_performance(self, interface_type, pvc_size, file_size, teardown_factory): """ Write data (60% of PVC capacity) to the PVC created in setup Create clones for an existing pvc, Measure clones average creation time and speed Delete the created clone Measure clone average deletion time and speed Note: by increasing max_num_of_clones value you increase number of the clones to be created/deleted """ file_size_for_io = file_size[:-1] performance_lib.write_fio_on_pod(self.pod_object, file_size_for_io) max_num_of_clones = 10 clone_creation_measures = [] csi_clone_creation_measures = [] clones_list = [] timeout = 18000 sc_name = self.pvc_obj.backed_sc parent_pvc = self.pvc_obj.name clone_yaml = constants.CSI_RBD_PVC_CLONE_YAML namespace = self.pvc_obj.namespace if interface_type == constants.CEPHFILESYSTEM: clone_yaml = constants.CSI_CEPHFS_PVC_CLONE_YAML file_size_mb = convert_device_size(file_size, "MB") logger.info( f"Start creating {max_num_of_clones} clones on {interface_type} PVC of size {pvc_size} GB." ) # taking the time, so parsing the provision log will be faster. start_time = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") for i in range(max_num_of_clones): logger.info(f"Start creation of clone number {i + 1}.") cloned_pvc_obj = pvc.create_pvc_clone(sc_name, parent_pvc, clone_yaml, namespace, storage_size=pvc_size + "Gi") teardown_factory(cloned_pvc_obj) helpers.wait_for_resource_state(cloned_pvc_obj, constants.STATUS_BOUND, timeout) cloned_pvc_obj.reload() logger.info( f"Clone with name {cloned_pvc_obj.name} for {pvc_size} pvc {parent_pvc} was created." ) clones_list.append(cloned_pvc_obj) create_time = helpers.measure_pvc_creation_time( interface_type, cloned_pvc_obj.name) creation_speed = int(file_size_mb / create_time) logger.info( f"Clone number {i+1} creation time is {create_time} secs for {pvc_size} GB pvc." ) logger.info( f"Clone number {i+1} creation speed is {creation_speed} MB/sec for {pvc_size} GB pvc." ) creation_measures = { "clone_num": i + 1, "time": create_time, "speed": creation_speed, } clone_creation_measures.append(creation_measures) csi_clone_creation_measures.append( performance_lib.csi_pvc_time_measure(self.interface, cloned_pvc_obj, "create", start_time)) # deleting one by one and measuring deletion times and speed for each one of the clones create above # in case of single clone will run one time clone_deletion_measures = [] csi_clone_deletion_measures = [] logger.info( f"Start deleting {max_num_of_clones} clones on {interface_type} PVC of size {pvc_size} GB." ) for i in range(max_num_of_clones): cloned_pvc_obj = clones_list[i] pvc_reclaim_policy = cloned_pvc_obj.reclaim_policy cloned_pvc_obj.delete() logger.info( f"Deletion of clone number {i + 1} , the clone name is {cloned_pvc_obj.name}." ) cloned_pvc_obj.ocp.wait_for_delete(cloned_pvc_obj.name, timeout) if pvc_reclaim_policy == constants.RECLAIM_POLICY_DELETE: helpers.validate_pv_delete(cloned_pvc_obj.backed_pv) delete_time = helpers.measure_pvc_deletion_time( interface_type, cloned_pvc_obj.backed_pv) logger.info( f"Clone number {i + 1} deletion time is {delete_time} secs for {pvc_size} GB pvc." ) deletion_speed = int(file_size_mb / delete_time) logger.info( f"Clone number {i+1} deletion speed is {deletion_speed} MB/sec for {pvc_size} GB pvc." ) deletion_measures = { "clone_num": i + 1, "time": delete_time, "speed": deletion_speed, } clone_deletion_measures.append(deletion_measures) csi_clone_deletion_measures.append( performance_lib.csi_pvc_time_measure(self.interface, cloned_pvc_obj, "delete", start_time)) logger.info( f"Printing clone creation time and speed for {max_num_of_clones} clones " f"on {interface_type} PVC of size {pvc_size} GB:") for c in clone_creation_measures: logger.info( f"Clone number {c['clone_num']} creation time is {c['time']} secs for {pvc_size} GB pvc ." ) logger.info( f"Clone number {c['clone_num']} creation speed is {c['speed']} MB/sec for {pvc_size} GB pvc." ) logger.info( f"Clone deletion time and speed for {interface_type} PVC of size {pvc_size} GB are:" ) creation_time_list = [r["time"] for r in clone_creation_measures] creation_speed_list = [r["speed"] for r in clone_creation_measures] average_creation_time = statistics.mean(creation_time_list) average_csi_creation_time = statistics.mean( csi_clone_creation_measures) average_creation_speed = statistics.mean(creation_speed_list) logger.info(f"Average creation time is {average_creation_time} secs.") logger.info( f"Average creation speed is {average_creation_speed} Mb/sec.") for d in clone_deletion_measures: logger.info( f"Clone number {d['clone_num']} deletion time is {d['time']} secs for {pvc_size} GB pvc." ) logger.info( f"Clone number {d['clone_num']} deletion speed is {d['speed']} MB/sec for {pvc_size} GB pvc." ) deletion_time_list = [r["time"] for r in clone_deletion_measures] deletion_speed_list = [r["speed"] for r in clone_deletion_measures] average_deletion_time = statistics.mean(deletion_time_list) average_csi_deletion_time = statistics.mean( csi_clone_deletion_measures) average_deletion_speed = statistics.mean(deletion_speed_list) logger.info(f"Average deletion time is {average_deletion_time} secs.") logger.info( f"Average deletion speed is {average_deletion_speed} Mb/sec.") logger.info("test_clones_creation_performance finished successfully.") self.results_path = get_full_test_logs_path(cname=self) # Produce ES report # Collecting environment information self.get_env_info() self.full_log_path = get_full_test_logs_path(cname=self) self.results_path = get_full_test_logs_path(cname=self) self.full_log_path += f"-{self.interface}-{pvc_size}-{file_size}" logger.info(f"Logs file path name is : {self.full_log_path}") # Initialize the results doc file. full_results = self.init_full_results( ResultsAnalyse( self.uuid, self.crd_data, self.full_log_path, "pvc_clone_performance", )) full_results.add_key("interface", self.interface) full_results.add_key("total_clone_number", max_num_of_clones) full_results.add_key("pvc_size", self.pvc_size) full_results.add_key("average_clone_creation_time", average_creation_time) full_results.add_key("average_csi_clone_creation_time", average_csi_creation_time) full_results.add_key("average_clone_deletion_time", average_deletion_time) full_results.add_key("average_csi_clone_deletion_time", average_csi_deletion_time) full_results.add_key("average_clone_creation_speed", average_creation_speed) full_results.add_key("average_clone_deletion_speed", average_deletion_speed) full_results.all_results = { "clone_creation_time": creation_time_list, "csi_clone_creation_time": csi_clone_creation_measures, "clone_deletion_time": deletion_time_list, "csi_clone_deletion_time": csi_clone_deletion_measures, "clone_creation_speed": creation_speed_list, "clone_deletion_speed": deletion_speed_list, } # Write the test results into the ES server if full_results.es_write(): res_link = full_results.results_link() logger.info(f"The Result can be found at : {res_link}") # Create text file with results of all subtest (8 - according to the parameters) self.write_result_to_file(res_link)