def check_nodes_specs(min_memory, min_cpu): """ Check that the cluster worker nodes meet the required minimum CPU and memory Args: min_memory (int): The required minimum memory in bytes min_cpu (int): The required minimum number of vCPUs Returns: bool: True if all nodes meet the required minimum specs, False otherwise """ nodes = get_nodes() log.info(f"Checking following nodes with worker selector (assuming that " f"this is ran in CI and there are no worker nodes without OCS):\n" f"{[node.get().get('metadata').get('name') for node in nodes]}") for node in nodes: real_cpu = int(node.get()["status"]["capacity"]["cpu"]) real_memory = convert_device_size( node.get()["status"]["capacity"]["memory"], "B") if real_cpu < min_cpu or real_memory < min_memory: log.warning( f"Node {node.get().get('metadata').get('name')} specs don't meet " f" the minimum required specs.\n The requirements are: " f"{min_cpu} CPUs and {min_memory} Memory\nThe node has: {real_cpu} " f"CPUs and {real_memory} Memory") return False log.info(f"Cluster worker nodes meet the minimum requirements of " f"{min_cpu} CPUs and {min_memory} Memory") return True
def size(self): """ Returns the PVC size pvc_name in namespace Returns: int: PVC size """ unformatted_size = self.data.get('spec').get('resources').get( 'requests').get('storage') return convert_device_size(unformatted_size, 'GB')
def size(self): """ Returns the PVC size pvc_name in namespace Returns: int: PVC size """ unformatted_size = (self.data.get("spec").get("resources").get( "requests").get("storage")) return convert_device_size(unformatted_size, "GB")
def get_pv_size(pv_obj): """ Get the size of a pv object Args: pv_obj (dict): A dictionary that represent the pv object Returns: int: The size of the pv object """ storage_size = pv_obj.get("spec").get("capacity").get("storage") return convert_device_size(storage_size, "GB")
def get_osd_pods_memory_sum(): """ Get the sum of memory of all OSD pods. This is used to determine the size needed for a PVC so when IO will be running over it the OSDs cache will be filled Returns: int: The sum of the OSD pods memory in GB """ osd_pods = pod.get_osd_pods() num_of_osd_pods = len(osd_pods) osd_pod_mem_size_str = osd_pods[0].get_memory().get('osd') osd_pod_mem_size = convert_device_size( unformatted_size=osd_pod_mem_size_str, units_to_covert_to='GB') return num_of_osd_pods * osd_pod_mem_size
def get_mon_db_size_in_kb(self): """ Get mon db size and returns the size in KB The output of 'du -sh' command contains the size of the directory and its path as string e.g. "67M\t/var/lib/ceph/mon/ceph-c/store.db" The size is extracted by splitting the string with '\t'. The size format for example: 1K, 234M, 2G For uniformity, this test uses KB """ size = self.selected_mon_pod_obj.exec_cmd_on_pod( f"du -sh /var/lib/ceph/mon/ceph-{self.selected_mon_pod}/store.db", out_yaml_format=False, ) size = re.split("\t+", size) assert len(size) > 0, f"Failed to get mon-{self.selected_mon_pod} db size" size = size[0] return convert_device_size(size + "i", "KB")
def test_clone_create_delete_performance(self, secret_factory, interface_type, pvc_size, file_size): """ Write data (60% of PVC capacity) to the PVC created in setup Create clones for an existing pvc, Measure clones average creation time and speed Delete the created clone Measure clone average deletion time and speed Note: by increasing max_num_of_clones value you increase number of the clones to be created/deleted """ # Initialize some variabels self.interface = interface_type self.timeout = 18000 self.pvc_size = pvc_size self.results_path = get_full_test_logs_path(cname=self) file_size_mb = convert_device_size(file_size, "MB") # Initialize the results doc file. full_results = self.init_full_results( ClonesResultsAnalyse( self.uuid, self.crd_data, self.full_log_path, "pvc_clone_performance", )) test_start_time = self.get_time() # Create new pool and sc only for RBD, for CepgFS use thr default if self.interface == constants.CEPHBLOCKPOOL: # Creating new pool to run the test on it self.create_new_pool_and_sc(secret_factory) else: self.sc_obj = ocs.OCS( kind="StorageCluster", metadata={ "namespace": self.namespace, "name": Interfaces_info[self.interface]["sc"], }, ) self.pool_name = "ocs-storagecluster-cephfilesystem" # Create a PVC self.create_pvc_and_wait_for_bound() # Create a POD self.create_pod_and_wait_for_completion(filesize=f"{file_size_mb}M") # taking the time, so parsing the provision log will be faster. start_time = self.get_time("csi") self.clones_list = self.create_and_delete_clones() # Mesure Creation / Deletion time of all clones results_times = performance_lib.get_pvc_provision_times( interface=self.interface, pvc_name=self.clones_list, start_time=start_time, ) test_end_time = self.get_time() logger.info( f"Printing clone creation time and speed for {self.number_of_clones} clones " f"on {self.interface} PVC of size {self.pvc_size} GB:") # Produce ES report speed = True if self.interface == constants.CEPHFILESYSTEM else False full_results.analyse_results(results_times, total_data=file_size_mb, speed=speed) # Add the test time to the ES report full_results.add_key("test_time", { "start": test_start_time, "end": test_end_time }) full_results.add_key("total_clone_number", self.number_of_clones) # Write the test results into the ES server if full_results.es_write(): res_link = full_results.results_link() logger.info(f"The Result can be found at : {res_link}") # Create text file with results of all subtest (8 - according to the parameters) self.write_result_to_file(res_link)
def test_clone_create_delete_performance(self, interface_type, pvc_size, file_size, teardown_factory): """ Write data (60% of PVC capacity) to the PVC created in setup Create single clone for an existing pvc, Measure clone creation time and speed Delete the created clone Measure clone deletion time and speed Note: by increasing max_num_of_clones value you increase number of the clones to be created/deleted """ file_size_for_io = file_size[:-1] performance_lib.write_fio_on_pod(self.pod_obj, file_size_for_io) max_num_of_clones = 1 clone_creation_measures = [] clones_list = [] timeout = 18000 sc_name = self.pvc_obj.backed_sc parent_pvc = self.pvc_obj.name clone_yaml = constants.CSI_RBD_PVC_CLONE_YAML namespace = self.pvc_obj.namespace if interface_type == constants.CEPHFILESYSTEM: clone_yaml = constants.CSI_CEPHFS_PVC_CLONE_YAML file_size_mb = convert_device_size(file_size, "MB") # creating single clone ( or many one by one if max_mum_of_clones > 1) logger.info( f"Start creating {max_num_of_clones} clones on {interface_type} PVC of size {pvc_size} GB." ) for i in range(max_num_of_clones): logger.info(f"Start creation of clone number {i + 1}.") cloned_pvc_obj = pvc.create_pvc_clone(sc_name, parent_pvc, clone_yaml, namespace, storage_size=pvc_size + "Gi") teardown_factory(cloned_pvc_obj) helpers.wait_for_resource_state(cloned_pvc_obj, constants.STATUS_BOUND, timeout) cloned_pvc_obj.reload() logger.info( f"Clone with name {cloned_pvc_obj.name} for {pvc_size} pvc {parent_pvc} was created." ) clones_list.append(cloned_pvc_obj) create_time = helpers.measure_pvc_creation_time( interface_type, cloned_pvc_obj.name) creation_speed = int(file_size_mb / create_time) logger.info( f"Clone number {i+1} creation time is {create_time} secs for {pvc_size} GB pvc." ) logger.info( f"Clone number {i+1} creation speed is {creation_speed} MB/sec for {pvc_size} GB pvc." ) creation_measures = { "clone_num": i + 1, "time": create_time, "speed": creation_speed, } clone_creation_measures.append(creation_measures) # deleting one by one and measuring deletion times and speed for each one of the clones create above # in case of single clone will run one time clone_deletion_measures = [] logger.info( f"Start deleting {max_num_of_clones} clones on {interface_type} PVC of size {pvc_size} GB." ) for i in range(max_num_of_clones): cloned_pvc_obj = clones_list[i] pvc_reclaim_policy = cloned_pvc_obj.reclaim_policy cloned_pvc_obj.delete() logger.info( f"Deletion of clone number {i + 1} , the clone name is {cloned_pvc_obj.name}." ) cloned_pvc_obj.ocp.wait_for_delete(cloned_pvc_obj.name, timeout) if pvc_reclaim_policy == constants.RECLAIM_POLICY_DELETE: helpers.validate_pv_delete(cloned_pvc_obj.backed_pv) delete_time = helpers.measure_pvc_deletion_time( interface_type, cloned_pvc_obj.backed_pv) logger.info( f"Clone number {i + 1} deletion time is {delete_time} secs for {pvc_size} GB pvc." ) deletion_speed = int(file_size_mb / delete_time) logger.info( f"Clone number {i+1} deletion speed is {deletion_speed} MB/sec for {pvc_size} GB pvc." ) deletion_measures = { "clone_num": i + 1, "time": delete_time, "speed": deletion_speed, } clone_deletion_measures.append(deletion_measures) logger.info( f"Printing clone creation time and speed for {max_num_of_clones} clones " f"on {interface_type} PVC of size {pvc_size} GB:") for c in clone_creation_measures: logger.info( f"Clone number {c['clone_num']} creation time is {c['time']} secs for {pvc_size} GB pvc ." ) logger.info( f"Clone number {c['clone_num']} creation speed is {c['speed']} MB/sec for {pvc_size} GB pvc." ) logger.info( f"Clone deletion time and speed for {interface_type} PVC of size {pvc_size} GB are:" ) for d in clone_deletion_measures: logger.info( f"Clone number {d['clone_num']} deletion time is {d['time']} secs for {pvc_size} GB pvc." ) logger.info( f"Clone number {d['clone_num']} deletion speed is {d['speed']} MB/sec for {pvc_size} GB pvc." ) logger.info("test_clones_creation_performance finished successfully.")
def raw_block_pv(self): """ Testing basic creation of app pod with RBD RWX raw block pv support """ worker_nodes = node.get_worker_nodes() pvcs = list() size_mb = "500Mi" size_gb = "10Gi" if config.ENV_DATA["platform"].lower( ) in constants.MANAGED_SERVICE_PLATFORMS: size_tb = str(convert_device_size("50Gi", "TB")) + "Ti" else: size_tb = "1Ti" for size in [size_mb, size_gb, size_tb]: pvcs.append( helpers.create_pvc( sc_name=self.sc_obj.name, size=size, access_mode=constants.ACCESS_MODE_RWX, namespace=self.namespace, volume_mode="Block", )) pvc_mb, pvc_gb, pvc_tb = pvcs[0], pvcs[1], pvcs[2] for pvc in pvcs: helpers.wait_for_resource_state(resource=pvc, state=constants.STATUS_BOUND, timeout=120) pvs = [pvc.backed_pv_obj for pvc in pvcs] pods = list() pod_dict = constants.CSI_RBD_RAW_BLOCK_POD_YAML for pvc in pvc_mb, pvc_gb, pvc_tb: for _ in range(3): pods.append( helpers.create_pod( interface_type=constants.CEPHBLOCKPOOL, pvc_name=pvc.name, namespace=self.namespace, raw_block_pv=True, pod_dict_path=pod_dict, node_name=random.choice(worker_nodes), )) pvc_mb_pods, pvc_gb_pods, pvc_tb_pods = pods[0:3], pods[3:6], pods[6:9] for pod in pods: helpers.wait_for_resource_state(resource=pod, state=constants.STATUS_RUNNING, timeout=120) storage_type = "block" with ThreadPoolExecutor() as p: for pod in pvc_mb_pods: log.info(f"running io on pod {pod.name}") p.submit( pod.run_io, storage_type=storage_type, size=f"{random.randint(10,200)}M", invalidate=0, ) for pod in pvc_gb_pods: log.info(f"running io on pod {pod.name}") p.submit( pod.run_io, storage_type=storage_type, size=f"{random.randint(1,5)}G", invalidate=0, ) for pod in pvc_tb_pods: log.info(f"running io on pod {pod.name}") p.submit( pod.run_io, storage_type=storage_type, size=f"{random.randint(10,15)}G", invalidate=0, ) for pod in pods: get_fio_rw_iops(pod) return pods, pvcs, pvs
def test_clone_create_delete_performance(self, interface_type, pvc_size, file_size, teardown_factory): """ Write data (60% of PVC capacity) to the PVC created in setup Create clones for an existing pvc, Measure clones average creation time and speed Delete the created clone Measure clone average deletion time and speed Note: by increasing max_num_of_clones value you increase number of the clones to be created/deleted """ file_size_for_io = file_size[:-1] performance_lib.write_fio_on_pod(self.pod_object, file_size_for_io) max_num_of_clones = 10 clone_creation_measures = [] csi_clone_creation_measures = [] clones_list = [] timeout = 18000 sc_name = self.pvc_obj.backed_sc parent_pvc = self.pvc_obj.name clone_yaml = constants.CSI_RBD_PVC_CLONE_YAML namespace = self.pvc_obj.namespace if interface_type == constants.CEPHFILESYSTEM: clone_yaml = constants.CSI_CEPHFS_PVC_CLONE_YAML file_size_mb = convert_device_size(file_size, "MB") logger.info( f"Start creating {max_num_of_clones} clones on {interface_type} PVC of size {pvc_size} GB." ) # taking the time, so parsing the provision log will be faster. start_time = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") for i in range(max_num_of_clones): logger.info(f"Start creation of clone number {i + 1}.") cloned_pvc_obj = pvc.create_pvc_clone(sc_name, parent_pvc, clone_yaml, namespace, storage_size=pvc_size + "Gi") teardown_factory(cloned_pvc_obj) helpers.wait_for_resource_state(cloned_pvc_obj, constants.STATUS_BOUND, timeout) cloned_pvc_obj.reload() logger.info( f"Clone with name {cloned_pvc_obj.name} for {pvc_size} pvc {parent_pvc} was created." ) clones_list.append(cloned_pvc_obj) create_time = helpers.measure_pvc_creation_time( interface_type, cloned_pvc_obj.name) creation_speed = int(file_size_mb / create_time) logger.info( f"Clone number {i+1} creation time is {create_time} secs for {pvc_size} GB pvc." ) logger.info( f"Clone number {i+1} creation speed is {creation_speed} MB/sec for {pvc_size} GB pvc." ) creation_measures = { "clone_num": i + 1, "time": create_time, "speed": creation_speed, } clone_creation_measures.append(creation_measures) csi_clone_creation_measures.append( performance_lib.csi_pvc_time_measure(self.interface, cloned_pvc_obj, "create", start_time)) # deleting one by one and measuring deletion times and speed for each one of the clones create above # in case of single clone will run one time clone_deletion_measures = [] csi_clone_deletion_measures = [] logger.info( f"Start deleting {max_num_of_clones} clones on {interface_type} PVC of size {pvc_size} GB." ) for i in range(max_num_of_clones): cloned_pvc_obj = clones_list[i] pvc_reclaim_policy = cloned_pvc_obj.reclaim_policy cloned_pvc_obj.delete() logger.info( f"Deletion of clone number {i + 1} , the clone name is {cloned_pvc_obj.name}." ) cloned_pvc_obj.ocp.wait_for_delete(cloned_pvc_obj.name, timeout) if pvc_reclaim_policy == constants.RECLAIM_POLICY_DELETE: helpers.validate_pv_delete(cloned_pvc_obj.backed_pv) delete_time = helpers.measure_pvc_deletion_time( interface_type, cloned_pvc_obj.backed_pv) logger.info( f"Clone number {i + 1} deletion time is {delete_time} secs for {pvc_size} GB pvc." ) deletion_speed = int(file_size_mb / delete_time) logger.info( f"Clone number {i+1} deletion speed is {deletion_speed} MB/sec for {pvc_size} GB pvc." ) deletion_measures = { "clone_num": i + 1, "time": delete_time, "speed": deletion_speed, } clone_deletion_measures.append(deletion_measures) csi_clone_deletion_measures.append( performance_lib.csi_pvc_time_measure(self.interface, cloned_pvc_obj, "delete", start_time)) logger.info( f"Printing clone creation time and speed for {max_num_of_clones} clones " f"on {interface_type} PVC of size {pvc_size} GB:") for c in clone_creation_measures: logger.info( f"Clone number {c['clone_num']} creation time is {c['time']} secs for {pvc_size} GB pvc ." ) logger.info( f"Clone number {c['clone_num']} creation speed is {c['speed']} MB/sec for {pvc_size} GB pvc." ) logger.info( f"Clone deletion time and speed for {interface_type} PVC of size {pvc_size} GB are:" ) creation_time_list = [r["time"] for r in clone_creation_measures] creation_speed_list = [r["speed"] for r in clone_creation_measures] average_creation_time = statistics.mean(creation_time_list) average_csi_creation_time = statistics.mean( csi_clone_creation_measures) average_creation_speed = statistics.mean(creation_speed_list) logger.info(f"Average creation time is {average_creation_time} secs.") logger.info( f"Average creation speed is {average_creation_speed} Mb/sec.") for d in clone_deletion_measures: logger.info( f"Clone number {d['clone_num']} deletion time is {d['time']} secs for {pvc_size} GB pvc." ) logger.info( f"Clone number {d['clone_num']} deletion speed is {d['speed']} MB/sec for {pvc_size} GB pvc." ) deletion_time_list = [r["time"] for r in clone_deletion_measures] deletion_speed_list = [r["speed"] for r in clone_deletion_measures] average_deletion_time = statistics.mean(deletion_time_list) average_csi_deletion_time = statistics.mean( csi_clone_deletion_measures) average_deletion_speed = statistics.mean(deletion_speed_list) logger.info(f"Average deletion time is {average_deletion_time} secs.") logger.info( f"Average deletion speed is {average_deletion_speed} Mb/sec.") logger.info("test_clones_creation_performance finished successfully.") self.results_path = get_full_test_logs_path(cname=self) # Produce ES report # Collecting environment information self.get_env_info() self.full_log_path = get_full_test_logs_path(cname=self) self.results_path = get_full_test_logs_path(cname=self) self.full_log_path += f"-{self.interface}-{pvc_size}-{file_size}" logger.info(f"Logs file path name is : {self.full_log_path}") # Initialize the results doc file. full_results = self.init_full_results( ResultsAnalyse( self.uuid, self.crd_data, self.full_log_path, "pvc_clone_performance", )) full_results.add_key("interface", self.interface) full_results.add_key("total_clone_number", max_num_of_clones) full_results.add_key("pvc_size", self.pvc_size) full_results.add_key("average_clone_creation_time", average_creation_time) full_results.add_key("average_csi_clone_creation_time", average_csi_creation_time) full_results.add_key("average_clone_deletion_time", average_deletion_time) full_results.add_key("average_csi_clone_deletion_time", average_csi_deletion_time) full_results.add_key("average_clone_creation_speed", average_creation_speed) full_results.add_key("average_clone_deletion_speed", average_deletion_speed) full_results.all_results = { "clone_creation_time": creation_time_list, "csi_clone_creation_time": csi_clone_creation_measures, "clone_deletion_time": deletion_time_list, "csi_clone_deletion_time": csi_clone_deletion_measures, "clone_creation_speed": creation_speed_list, "clone_deletion_speed": deletion_speed_list, } # Write the test results into the ES server if full_results.es_write(): res_link = full_results.results_link() logger.info(f"The Result can be found at : {res_link}") # Create text file with results of all subtest (8 - according to the parameters) self.write_result_to_file(res_link)