def test_create_delete_pvc( self, setup_ui, sc_type, pvc_name, access_mode, pvc_size, vol_mode ): """ Test create and delete pvc via UI """ pvc_ui_obj = PvcUI(setup_ui) pvc_ui_obj.create_pvc_ui(sc_type, pvc_name, access_mode, pvc_size, vol_mode) time.sleep(2) pvc_objs = get_all_pvc_objs(namespace="openshift-storage") pvc = [pvc_obj for pvc_obj in pvc_objs if pvc_obj.name == pvc_name] assert pvc[0].size == int(pvc_size), ( f"size error| expected size:{pvc_size} \n " f"actual size:{str(pvc[0].size)}" ) assert pvc[0].get_pvc_access_mode == access_mode, ( f"access mode error| expected access mode:{access_mode} " f"\n actual access mode:{pvc[0].get_pvc_access_mode}" ) assert pvc[0].backed_sc == sc_type, ( f"storage class error| expected storage class:{sc_type} " f"\n actual storage class:{pvc[0].backed_sc}" ) assert pvc[0].get_pvc_vol_mode == vol_mode, ( f"volume mode error| expected volume mode:{vol_mode} " f"\n actual volume mode:{pvc[0].get_pvc_vol_mode}" ) logger.info("Verifying PVC Details via UI") pvc_ui_obj.verify_pvc_ui( pvc_size=pvc_size, access_mode=access_mode, vol_mode=vol_mode, sc_type=sc_type, ) logger.info("PVC Details Verified via UI..!!") logger.info(f"Delete {pvc_name} pvc") pvc_ui_obj.delete_pvc_ui(pvc_name) time.sleep(5) pvc_objs = get_all_pvc_objs(namespace="openshift-storage") pvcs = [pvc_obj for pvc_obj in pvc_objs if pvc_obj.name == pvc_name] if len(pvcs) > 0: assert f"PVC {pvcs[0].name} does not deleted"
def wait_for_all_resources_deletion( namespace, check_replication_resources_state=True, timeout=900 ): """ Wait for workload and replication resources to be deleted Args: namespace (str): the namespace of the workload check_replication_resources_state (bool): True for checking replication resources state, False otherwise timeout (int): time in seconds to wait for resource deletion """ logger.info("Waiting for all pods to be deleted") all_pods = get_all_pods(namespace=namespace) for pod_obj in all_pods: pod_obj.ocp.wait_for_delete( resource_name=pod_obj.name, timeout=timeout, sleep=5 ) wait_for_replication_resources_deletion( namespace, timeout, check_replication_resources_state ) logger.info("Waiting for all PVCs to be deleted") all_pvcs = get_all_pvc_objs(namespace=namespace) for pvc_obj in all_pvcs: pvc_obj.ocp.wait_for_delete( resource_name=pvc_obj.name, timeout=timeout, sleep=5 )
def test_monitoring_delete_pvc(self): """ Test case to validate whether delete pvcs+configmap and recovery of a node where monitoring pods running has no functional impact """ # Get 'cluster-monitoring-config' configmap ocp_configmap = ocp.OCP( namespace=constants.MONITORING_NAMESPACE, kind="configmap" ) configmap_dict = ocp_configmap.get(resource_name="cluster-monitoring-config") dir_configmap = tempfile.mkdtemp(prefix="configmap_") yaml_file = f"{dir_configmap}/configmap.yaml" templating.dump_data_to_temp_yaml(configmap_dict, yaml_file) # Get prometheus and alertmanager pods prometheus_alertmanager_pods = pod.get_all_pods( namespace=defaults.OCS_MONITORING_NAMESPACE, selector=["prometheus", "alertmanager"], ) # Get all pvc on monitoring namespace pvc_objs_list = pvc.get_all_pvc_objs(namespace=constants.MONITORING_NAMESPACE) # Delete configmap ocp_configmap.delete(resource_name="cluster-monitoring-config") # Delete all pvcs on monitoring namespace pvc.delete_pvcs(pvc_objs=pvc_objs_list) # Check all the prometheus and alertmanager pods are up for pod_obj in prometheus_alertmanager_pods: wait_for_resource_state( resource=pod_obj, state=constants.STATUS_RUNNING, timeout=180 ) # Create configmap ocp_configmap.create(yaml_file=dir_configmap) # Check all the PVCs are up for pvc_obj in pvc_objs_list: wait_for_resource_state( resource=pvc_obj, state=constants.STATUS_BOUND, timeout=180 ) # Check all the prometheus and alertmanager pods are up # and pvc are mounted on monitoring pods for pod_obj in prometheus_alertmanager_pods: wait_for_resource_state( resource=pod_obj, state=constants.STATUS_RUNNING, timeout=180 ) mount_point = pod_obj.exec_cmd_on_pod( command="df -kh", out_yaml_format=False, ) assert "/dev/rbd" in mount_point, f"pvc is not mounted on pod {pod.name}" log.info("Verified all pvc are mounted on monitoring pods") # Validate the prometheus health is ok assert prometheus_health_check(), "Prometheus cluster health is not OK"
def run_fio_on_pvcs(self, pvc_dict_list, pod_factory): total_files_size = 0 searched_pvc_objs = pvc.get_all_pvc_objs(namespace=self.namespace) logging.info(f"Found {len(searched_pvc_objs)} PVCs") for pvc_yaml in pvc_dict_list: pvc_name = pvc_yaml["metadata"]["name"] pvc_size = pvc_yaml["spec"]["resources"]["requests"]["storage"] logging.info(f"Size of pvc {pvc_name} is {pvc_size}") pvc_size_int = int(pvc_size[:-2]) # without "Gi" file_size_mb = int(pvc_size_int * 0.6) * constants.GB2MB total_files_size += file_size_mb file_size_mb_str = str(file_size_mb) + "M" logging.info(f"Writing file of size {file_size_mb_str}") # now find pvc_obj by name and create pod_obj to write to pvc_obj = None for obj in searched_pvc_objs: if obj.name == pvc_name: pvc_obj = obj searched_pvc_objs.remove(obj) break assert pvc_obj is not None, f"Cannot find PVC with name {pvc_name}" logging.info(f"PVC with name {pvc_name} found") pod_obj = pod_factory(interface=self.interface, pvc=pvc_obj, status=constants.STATUS_RUNNING) performance_lib.write_fio_on_pod(pod_obj, file_size_mb_str) return total_files_size
def cleanup(self): """ Function to tear down """ # Delete all pods, pvcs and namespaces for namespace in self.namespace_list: delete_objs_parallel( obj_list=pod.get_all_pods(namespace=namespace.namespace), namespace=namespace.namespace, kind=self.kind, ) delete_objs_parallel( obj_list=pvc.get_all_pvc_objs(namespace=namespace.namespace), namespace=namespace.namespace, kind=constants.PVC, ) ocp = OCP(kind=constants.NAMESPACE) ocp.delete(resource_name=namespace.namespace) # Remove scale label from worker nodes in cleanup scale_workers = machine.get_labeled_nodes(constants.SCALE_LABEL) helpers.remove_label_from_worker_node(node_list=scale_workers, label_key="scale-label") # Delete machineset which will delete respective nodes too for aws-ipi platform if self.ms_name: for name in self.ms_name: machine.delete_custom_machineset(name)
def get_postgres_pvc(self): """ Get all postgres pvc Returns: List: postgres pvc objects list """ return get_all_pvc_objs(namespace=RIPSAW_NAMESPACE)
def cleanup( self, kafka_namespace=constants.AMQ_NAMESPACE, tiller_namespace=AMQ_BENCHMARK_NAMESPACE, ): """ Clean up function, will start to delete from amq cluster operator then amq-connector, persistent, bridge, at the end it will delete the created namespace Args: kafka_namespace (str): Created namespace for amq tiller_namespace (str): Created namespace for benchmark """ if self.amq_is_setup: if self.messaging: self.consumer_pod.delete() self.producer_pod.delete() self.kafka_user.delete() self.kafka_topic.delete() if self.benchmark: # Delete the helm app try: purge_cmd = f"linux-amd64/helm delete benchmark --purge --tiller-namespace {tiller_namespace}" run(purge_cmd, shell=True, cwd=self.dir, check=True) except (CommandFailed, CalledProcessError) as cf: log.error("Failed to delete help app") raise cf # Delete the pods and namespace created self.sa_tiller.delete() self.crb_tiller.delete() run_cmd(f"oc delete project {tiller_namespace}") self.ns_obj.wait_for_delete(resource_name=tiller_namespace) self.kafka_persistent.delete() self.kafka_connect.delete() self.kafka_bridge.delete() run_cmd(f"oc delete -f {self.amq_dir}", shell=True, check=True, cwd=self.dir) ocs_pvc_obj = get_all_pvc_objs(namespace=kafka_namespace) run_cmd(f"oc delete project {kafka_namespace}") self.ns_obj.wait_for_delete(resource_name=kafka_namespace, timeout=90) for pvc in ocs_pvc_obj: logging.info(pvc.name) validate_pv_delete(pvc.backed_pv) # Reset namespace to default switch_to_default_rook_cluster_project()
def validate_pvc(self): """ Check whether all PVCs are in bound state """ ocs_pvc_obj = get_all_pvc_objs(namespace=self.namespace) for pvc_obj in ocs_pvc_obj: assert pvc_obj.status == constants.STATUS_BOUND, { f"PVC {pvc_obj.name} is not Bound" } logger.info(f"PVC {pvc_obj.name} is in Bound state")
def run_fio_on_pvcs(self, pvc_size): searched_pvc_objs = pvc.get_all_pvc_objs(namespace=self.namespace) pod_objs = pod.get_all_pods(namespace=self.namespace) log.info(f"Found {len(searched_pvc_objs)} PVCs") pvc_size_int = int(pvc_size[:-2]) # without "Gi" file_size_mb = int(pvc_size_int * 0.6) * constants.GB2MB total_files_size = file_size_mb * len(searched_pvc_objs) file_size_mb_str = str(file_size_mb) + "M" log.info(f"Writing file of size {file_size_mb_str} in each PVC") for objs in pod_objs: performance_lib.write_fio_on_pod(objs, file_size_mb_str) return total_files_size
def validate_cluster_on_pvc(): """ Validate creation of PVCs for MON and OSD pods. Also validate that those PVCs are attached to the OCS pods Raises: AssertionError: If PVC is not mounted on one or more OCS pods """ # Get the PVCs for selected label (MON/OSD) ns = config.ENV_DATA['cluster_namespace'] ocs_pvc_obj = get_all_pvc_objs(namespace=ns) # Check all pvc's are in bound state pvc_names = [] for pvc_obj in ocs_pvc_obj: if (pvc_obj.name.startswith(constants.DEFAULT_DEVICESET_PVC_NAME) or pvc_obj.name.startswith(constants.DEFAULT_MON_PVC_NAME)): assert pvc_obj.status == constants.STATUS_BOUND, ( f"PVC {pvc_obj.name} is not Bound" ) logger.info(f"PVC {pvc_obj.name} is in Bound state") pvc_names.append(pvc_obj.name) mon_pods = get_pod_name_by_pattern('rook-ceph-mon', ns) if not config.DEPLOYMENT.get('local_storage'): logger.info("Validating all mon pods have PVC") validate_ocs_pods_on_pvc(mon_pods, pvc_names) else: logger.debug( "Skipping validation if all mon pods have PVC because in LSO " "deployment we don't have mon pods backed by PVC" ) logger.info("Validating all osd pods have PVC") osd_deviceset_pods = get_pod_name_by_pattern( 'rook-ceph-osd-prepare-ocs-deviceset', ns ) validate_ocs_pods_on_pvc(osd_deviceset_pods, pvc_names) osd_pods = get_pod_name_by_pattern('rook-ceph-osd', ns, filter='prepare') for ceph_pod in mon_pods + osd_pods: out = run_cmd(f'oc -n {ns} get pods {ceph_pod} -o yaml') out_yaml = yaml.safe_load(out) for vol in out_yaml['spec']['volumes']: if vol.get('persistentVolumeClaim'): claimName = vol.get('persistentVolumeClaim').get('claimName') logger.info(f"{ceph_pod} backed by pvc {claimName}") assert claimName in pvc_names, ( "Ceph Internal Volume not backed by PVC" )
def wait_for_workload_resource_deletion(namespace, timeout=120): """ Wait for workload resources to be deleted Args: namespace (str): the namespace of the workload timeout (int): time in seconds to wait for resource deletion """ logger.info("Waiting for all pods to be deleted") all_pods = get_all_pods(namespace=namespace) for pod_obj in all_pods: pod_obj.ocp.wait_for_delete(resource_name=pod_obj.name, timeout=timeout) logger.info("Waiting for all PVCs to be deleted") all_pvcs = get_all_pvc_objs(namespace=namespace) for pvc_obj in all_pvcs: pvc_obj.ocp.wait_for_delete(resource_name=pvc_obj.name, timeout=timeout)
def validate_cluster_on_pvc(): """ Validate creation of PVCs for MON and OSD pods. Also validate that those PVCs are attached to the OCS pods Raises: AssertionError: If PVC is not mounted on one or more OCS pods """ # Get the PVCs for selected label (MON/OSD) ns = config.ENV_DATA['cluster_namespace'] ocs_pvc_obj = get_all_pvc_objs(namespace=ns) # Check all pvc's are in bound state pvc_names = [] for pvc_obj in ocs_pvc_obj: if (pvc_obj.name.startswith(constants.DEFAULT_DEVICESET_PVC_NAME) or pvc_obj.name.startswith(constants.DEFAULT_MON_PVC_NAME)): assert pvc_obj.status == constants.STATUS_BOUND, ( f"PVC {pvc_obj.name} is not Bound" ) logger.info(f"PVC {pvc_obj.name} is in Bound state") pvc_names.append(pvc_obj.name) mon_pods = get_pod_name_by_pattern('rook-ceph-mon', ns) osd_pods = get_pod_name_by_pattern('rook-ceph-osd', ns, filter='prepare') assert len(mon_pods) + len(osd_pods) == len(pvc_names), ( "Not enough PVC's available for all Ceph Pods" ) for ceph_pod in mon_pods + osd_pods: out = run_cmd(f'oc -n {ns} get pods {ceph_pod} -o yaml') out_yaml = yaml.safe_load(out) for vol in out_yaml['spec']['volumes']: if vol.get('persistentVolumeClaim'): claimName = vol.get('persistentVolumeClaim').get('claimName') logger.info(f"{ceph_pod} backed by pvc {claimName}") assert claimName in pvc_names, ( "Ceph Internal Volume not backed by PVC" )
def get_osd_size(): """ Get osd size from Storage cluster Returns: int: osd size """ # In the case of UI deployment of LSO cluster, the value in StorageCluster CR # is set to 1, so we can not take OSD size from there. For LSO we will return # the size from PVC. if config.DEPLOYMENT.get("local_storage"): ocs_pvc_objects = get_all_pvc_objs( namespace=config.ENV_DATA["cluster_namespace"]) for pvc_obj in ocs_pvc_objects: if pvc_obj.name.startswith(constants.DEFAULT_DEVICESET_PVC_NAME): return int(pvc_obj.data["status"]["capacity"]["storage"][:-2]) sc = get_storage_cluster() return int(sc.get().get("items")[0].get("spec").get("storageDeviceSets") [0].get("dataPVCTemplate").get("spec").get("resources").get( "requests").get("storage")[:-2])
def validate_cluster_on_pvc(label): """ Validate creation of PVCs for MON and OSD pods. Also validate that those PVCs are attached to the OCS pods Args: label(string): Label for MON or OSD PVCs Raises: AssertionError: If PVC is not mounted on one or more OCS pods """ # Get the PVCs for selected label (MON/OSD) ocs_pvc_obj = get_all_pvc_objs(namespace=defaults.ROOK_CLUSTER_NAMESPACE, selector=label) # Check all pvc's are in bound state for pvc_obj in ocs_pvc_obj: assert pvc_obj.status == constants.STATUS_BOUND, ( f"PVC {pvc_obj.name} is not Bound") logger.info(f"PVC {pvc_obj.name} is in Bound state") # Get OCS pod names based on selected label if label == constants.MON_APP_LABEL: ocs_pod_obj = pod.get_mon_pods( mon_label=label, namespace=defaults.ROOK_CLUSTER_NAMESPACE) if label == constants.DEFAULT_DEVICESET_LABEL: ocs_pod_obj = pod.get_osd_pods( osd_label=constants.OSD_APP_LABEL, namespace=defaults.ROOK_CLUSTER_NAMESPACE) # Create a pvc list for requested label pvc_list = [] for pvc_obj in ocs_pvc_obj: pvc_list.append(pvc_obj.name) # Check if PVC is mounted on designated OCS pod # loop over all PVCs in the pvc_list # Each mon and osd pod is expected to have only one Claim attached claim_found = True all_backed_by_pvc = True for pod_obj in ocs_pod_obj: pod_volumes = pod_obj.get().get('spec').get('volumes') claim_spec_exists = False for volumes in pod_volumes: pvc = volumes.get('persistentVolumeClaim') if pvc: claim_name = pvc.get('claimName') claim_spec_exists = True backed_by_pvc = False if claim_name in pvc_list: logger.info( f"OCS pod {pod_obj.name} is backed by PVC {claim_name}" ) # If backed by PVC, set backed_by_pvc = True backed_by_pvc = True # Check if Mon PVC is mounted as /var/lib/ceph/mon/ceph-x if label == constants.MON_APP_LABEL: mount_point = pod_obj.exec_cmd_on_pod(command="df -kh") assert "/var/lib/ceph/mon/ceph" in mount_point, ( f"pvc is not mounted on pod {pod_obj.name}") logger.info(f"PVC {claim_name} is mounted" f" on pod {pod_obj.name}") if not backed_by_pvc: logger.error(f"{pod_obj.name} is not backed by designated PVC ") all_backed_by_pvc = False # If no PVC is mounted, print error and continue checking other pods if not claim_spec_exists: logger.error(f"No PersistentVolumeClaim spec found in " f"OCS pod {pod_obj.name}") claim_found = False # Even if one OCS POD is not backed by a PVC, fail the deployment assert all_backed_by_pvc, ("One or more pods are not backed by a PVC " "please check deployment logs") # Even if one OCS POD as PVC spec, fail the deployment assert claim_found, ( "Claim name doesn't exists in spec of one or more pods " "please check deployment logs")
def test_bulk_clone_performance(self, tmp_path, interface_iterate): """ Creates number of PVCs in a bulk using kube job Write 60% of PVC capacity to each one of the created PVCs Creates 1 clone per each PVC altogether in a bulk Measuring total and csi creation times for bulk of clones """ self.interface = interface_iterate job_pod_file, job_pvc_file, job_clone_file = [None, None, None] log.info(f"Start creating {self.interface} {self.pvc_count} PVC") try: pvc_dict_list = scale_lib.construct_pvc_creation_yaml_bulk_for_kube_job( no_of_pvc=self.pvc_count, access_mode=Interfaces_info[self.interface]["accessmode"], sc_name=Interfaces_info[self.interface]["sc_name"], pvc_size=self.vol_size, ) job_pvc_file = ObjectConfFile( name="job_profile_pvc", obj_dict_list=pvc_dict_list, project=self.namespace, tmp_path=tmp_path, ) # Create kube_job job_pvc_file.create(namespace=self.namespace) # Check all the PVC reached Bound state performance_lib.wait_for_resource_bulk_status( resource="pvc", resource_count=self.pvc_count, namespace=self.namespace, status=constants.STATUS_BOUND, timeout=120, sleep_time=5, ) log.info( f"All the PVCs ({self.pvc_count}) was created and are in Bound state" ) # Getting the list of the PVC names pvc_bound_list = [ p.name for p in pvc.get_all_pvc_objs(namespace=self.namespace) ] # Kube_job to Create pod log.info( "Attaching PODs to the PVCs and filling them with data (60%)") pod_dict_list = self.attach_pvcs_to_pod_dict(pvc_bound_list) job_pod_file = ObjectConfFile( name="job_profile_pod", obj_dict_list=pod_dict_list, project=self.namespace, tmp_path=tmp_path, ) job_pod_file.create(namespace=self.namespace) # Check all PODs are in Completed state performance_lib.wait_for_resource_bulk_status( resource="pod", resource_count=self.pvc_count, namespace=self.namespace, status=constants.STATUS_COMPLETED, timeout=1200, sleep_time=30, ) log.info("All the PODs completed writing data to the PVC's") clone_dict_list = scale_lib.construct_pvc_clone_yaml_bulk_for_kube_job( pvc_dict_list, Interfaces_info[self.interface]["clone_yaml"], Interfaces_info[self.interface]["sc_name"], ) log.info("Created clone dict list") csi_bulk_start_time = self.get_time(time_format="csi") job_clone_file = ObjectConfFile( name="job_profile_clone", obj_dict_list=clone_dict_list, project=self.namespace, tmp_path=tmp_path, ) # Create kube_job that creates clones job_clone_file.create(namespace=self.namespace) log.info("Going to check bound status for clones") # Check all the clones reached Bound state try: performance_lib.wait_for_resource_bulk_status( resource="pvc", resource_count=self.pvc_count * 2, namespace=self.namespace, status=constants.STATUS_BOUND, timeout=1200, sleep_time=30, ) except Exception as ex: log.error("Failed to cvreate clones for PVCs") raise ex log.info( f"All the Clones ({self.pvc_count}) was created and are in Bound state" ) all_pvc_objs = pvc.get_all_pvc_objs(namespace=self.namespace) clone_objs = [ cl for cl in all_pvc_objs if re.match("clone", cl.name) ] for clone_yaml in clone_dict_list: name = clone_yaml["metadata"]["name"] size = clone_yaml["spec"]["resources"]["requests"]["storage"] log.info(f"Clone {name} of size {size} created") start_time = get_provision_time(self.interface, clone_objs, status="start") end_time = get_provision_time(self.interface, clone_objs, status="end") total_time = (end_time - start_time).total_seconds() speed = round(self.total_files_size / total_time, 2) csi_creation_time = performance_lib.csi_bulk_pvc_time_measure( self.interface, clone_objs, "create", csi_bulk_start_time) log.info( f"Total creation time = {total_time} secs, csi creation time = {csi_creation_time}," f" data size = {self.total_files_size} MB, speed = {speed} MB/sec " f"for {self.interface} clone in bulk of {self.pvc_count} clones." ) # Produce ES report # Collecting environment information self.get_env_info() # Initialize the results' doc file. full_results = self.init_full_results( ResultsAnalyse( self.uuid, self.crd_data, self.full_log_path, "bulk_clone_perf_fullres", )) full_results.add_key("interface", self.interface) full_results.add_key("bulk_size", self.pvc_count) full_results.add_key("clone_size", self.vol_size) full_results.add_key("bulk_creation_time", total_time) full_results.add_key("bulk_csi_creation_time", csi_creation_time) full_results.add_key("data_size(MB)", self.total_files_size) full_results.add_key("speed", speed) full_results.add_key("es_results_link", full_results.results_link()) # Write the test results into the ES server full_results.es_write() self.results_path = get_full_test_logs_path(cname=self) res_link = full_results.results_link() # write the ES link to the test results in the test log. log.info(f"The result can be found at : {res_link}") # Create text file with results of all subtest (3 - according to the parameters) self.write_result_to_file(res_link) # Finally, is used to clean up the resources created # Irrespective of try block pass/fail finally will be executed. finally: # Cleanup activities log.info( "Cleanup of all the resources created during test execution") for object_file in [job_pod_file, job_clone_file, job_pvc_file]: if object_file: object_file.delete(namespace=self.namespace) try: object_file.wait_for_delete( resource_name=object_file.name, namespace=self.namespace) except Exception: log.error(f"{object_file['name']} didnt deleted !") # Check ceph health status utils.ceph_health_check(tries=20)
def uninstall_cluster_logging(): """ Function to uninstall cluster-logging from the cluster Deletes the project "openshift-logging" and "openshift-operators-redhat" """ # Validating the pods before deleting the instance pod_list = get_all_pods(namespace=constants.OPENSHIFT_LOGGING_NAMESPACE) for pod in pod_list: logger.info( f"Pods running in the openshift-logging namespace {pod.name}") # Excluding cluster-logging-operator from pod_list and getting pod names pod_names_list = [ pod.name for pod in pod_list if not pod.name.startswith("cluster-logging-operator") ] pvc_objs = get_all_pvc_objs( namespace=constants.OPENSHIFT_LOGGING_NAMESPACE) # Fetch image uuid associated with PVCs to be deleted pvc_uuid_map = {} for pvc_obj in pvc_objs: pvc_uuid_map[pvc_obj.name] = pvc_obj.image_uuid # Checking for used space cbp_name = default_ceph_block_pool() used_space_before_deletion = fetch_used_size(cbp_name) logger.info( f"Used space before deletion of cluster logging {used_space_before_deletion}" ) # Deleting the clusterlogging instance clusterlogging_obj = ocp.OCP( kind=constants.CLUSTER_LOGGING, namespace=constants.OPENSHIFT_LOGGING_NAMESPACE) assert clusterlogging_obj.delete(resource_name="instance") check_pod_vanished(pod_names_list) for pvc_obj in pvc_objs: pv_obj = pvc_obj.backed_pv_obj assert delete_pvcs(pvc_objs=pvc_objs), "PVCs deletion failed" for pvc_obj in pvc_objs: pvc_obj.ocp.wait_for_delete(resource_name=pvc_obj.name, timeout=300) pv_obj.ocp.wait_for_delete(resource_name=pv_obj.name, timeout=300) logger.info("Verified: PVCs are deleted.") logger.info("Verified: PV are deleted") for pvc_name, uuid in pvc_uuid_map.items(): rbd = verify_volume_deleted_in_backend( interface=constants.CEPHBLOCKPOOL, image_uuid=uuid, pool_name=cbp_name) assert rbd, f"Volume associated with PVC {pvc_name} still exists " f"in backend" # Checking for used space after PVC deletion used_space_after_deletion = fetch_used_size(cbp_name, exp_val=30) logger.info( f"Used space after deletion of cluster logging {used_space_after_deletion}" ) if used_space_after_deletion < used_space_before_deletion: logger.info("Expected !!! Space has reclaimed") else: logger.warning( "Unexpected !! No space reclaimed after deletion of PVC") # Deleting the RBAC permission set rbac_role = ocp.OCP( kind=constants.ROLE, namespace=constants.OPENSHIFT_OPERATORS_REDHAT_NAMESPACE) rbac_role.delete(yaml_file=constants.EO_RBAC_YAML) openshift_logging_namespace = ocp.OCP( kind=constants.NAMESPACES, namespace=constants.OPENSHIFT_LOGGING_NAMESPACE) openshift_operators_redhat_namespace = ocp.OCP( kind=constants.NAMESPACES, namespace=constants.OPENSHIFT_OPERATORS_REDHAT_NAMESPACE, ) if openshift_operators_redhat_namespace.get(): assert openshift_operators_redhat_namespace.delete( resource_name=constants.OPENSHIFT_OPERATORS_REDHAT_NAMESPACE) logger.info( "The project openshift-opertors-redhat got deleted successfully") if openshift_logging_namespace.get(): assert openshift_logging_namespace.delete( resource_name=constants.OPENSHIFT_LOGGING_NAMESPACE) logger.info("The namespace openshift-logging got deleted successfully")
def test_create_resize_delete_pvc( self, project_factory, teardown_factory, setup_ui, sc_name, access_mode, pvc_size, vol_mode, ): """ Test create, resize and delete pvc via UI """ # Creating a test project via CLI pro_obj = project_factory() project_name = pro_obj.namespace pvc_ui_obj = PvcUI(setup_ui) # Creating PVC via UI pvc_name = create_unique_resource_name("test", "pvc") pvc_ui_obj.create_pvc_ui( project_name, sc_name, pvc_name, access_mode, pvc_size, vol_mode ) pvc_objs = get_all_pvc_objs(namespace=project_name) pvc = [pvc_obj for pvc_obj in pvc_objs if pvc_obj.name == pvc_name] assert pvc[0].size == int(pvc_size), ( f"size error| expected size:{pvc_size} \n " f"actual size:{str(pvc[0].size)}" ) assert pvc[0].get_pvc_access_mode == access_mode, ( f"access mode error| expected access mode:{access_mode} " f"\n actual access mode:{pvc[0].get_pvc_access_mode}" ) assert pvc[0].backed_sc == sc_name, ( f"storage class error| expected storage class:{sc_name} " f"\n actual storage class:{pvc[0].backed_sc}" ) assert pvc[0].get_pvc_vol_mode == vol_mode, ( f"volume mode error| expected volume mode:{vol_mode} " f"\n actual volume mode:{pvc[0].get_pvc_vol_mode}" ) # Verifying PVC via UI logger.info("Verifying PVC Details via UI") pvc_ui_obj.verify_pvc_ui( pvc_size=pvc_size, access_mode=access_mode, vol_mode=vol_mode, sc_name=sc_name, pvc_name=pvc_name, project_name=project_name, ) logger.info("PVC Details Verified via UI..!!") # Creating Pod via CLI logger.info("Creating Pod") if sc_name in (constants.DEFAULT_STORAGECLASS_RBD,): interface_type = constants.CEPHBLOCKPOOL else: interface_type = constants.CEPHFILESYSTEM new_pod = helpers.create_pod( interface_type=interface_type, pvc_name=pvc_name, namespace=project_name, raw_block_pv=vol_mode == constants.VOLUME_MODE_BLOCK, ) logger.info(f"Waiting for Pod: state= {constants.STATUS_RUNNING}") wait_for_resource_state(resource=new_pod, state=constants.STATUS_RUNNING) # Calling the Teardown Factory Method to make sure Pod is deleted teardown_factory(new_pod) # Expanding the PVC logger.info("Pvc Resizing") new_size = int(pvc_size) + 3 pvc_ui_obj.pvc_resize_ui( pvc_name=pvc_name, new_size=new_size, project_name=project_name ) assert new_size > int( pvc_size ), f"New size of the PVC cannot be less than existing size: new size is {new_size})" ocp_version = get_ocp_version() self.pvc_loc = locators[ocp_version]["pvc"] # Verifying PVC expansion logger.info("Verifying PVC resize") expected_capacity = f"{new_size} GiB" pvc_resize = pvc_ui_obj.verify_pvc_resize_ui( project_name=project_name, pvc_name=pvc_name, expected_capacity=expected_capacity, ) assert pvc_resize, "PVC resize failed" logger.info( "Pvc resize verified..!!" f"New Capacity after PVC resize is {expected_capacity}" ) # Running FIO logger.info("Execute FIO on a Pod") if vol_mode == constants.VOLUME_MODE_BLOCK: storage_type = constants.WORKLOAD_STORAGE_TYPE_BLOCK else: storage_type = constants.WORKLOAD_STORAGE_TYPE_FS new_pod.run_io(storage_type, size=(new_size - 1), invalidate=0, rate="1000m") get_fio_rw_iops(new_pod) logger.info("FIO execution on Pod successfully completed..!!") # Checking if the Pod is deleted or not new_pod.delete(wait=True) new_pod.ocp.wait_for_delete(resource_name=new_pod.name) # Deleting the PVC via UI logger.info(f"Delete {pvc_name} pvc") pvc_ui_obj.delete_pvc_ui(pvc_name, project_name) pvc[0].ocp.wait_for_delete(pvc_name, timeout=120) pvc_objs = get_all_pvc_objs(namespace=project_name) pvcs = [pvc_obj for pvc_obj in pvc_objs if pvc_obj.name == pvc_name] if len(pvcs) > 0: assert f"PVC {pvcs[0].name} does not deleted"
def test_bulk_clone_performance(self, namespace, tmp_path, pod_factory): """ Creates number of PVCs in a bulk using kube job Write 60% of PVC capacity to each one of the created PVCs Creates 1 clone per each PVC altogether in a bulk Measuring time for bulk of clones creation """ pvc_count = 50 log.info(f"Start creating {self.interface} {pvc_count} PVC") if self.interface == constants.CEPHBLOCKPOOL: sc_name = constants.DEFAULT_STORAGECLASS_RBD clone_yaml = constants.CSI_RBD_PVC_CLONE_YAML elif self.interface == constants.CEPHFILESYSTEM: sc_name = constants.DEFAULT_STORAGECLASS_CEPHFS clone_yaml = constants.CSI_CEPHFS_PVC_CLONE_YAML pvc_dict_list = scale_lib.construct_pvc_creation_yaml_bulk_for_kube_job( no_of_pvc=pvc_count, access_mode=constants.ACCESS_MODE_RWO, sc_name=sc_name, pvc_size="5Gi", ) job_pvc_file = ObjectConfFile( name="job_profile_pvc", obj_dict_list=pvc_dict_list, project=self.namespace, tmp_path=tmp_path, ) # Create kube_job job_pvc_file.create(namespace=self.namespace) # Check all the PVC reached Bound state pvc_bound_list = scale_lib.check_all_pvc_reached_bound_state_in_kube_job( kube_job_obj=job_pvc_file, namespace=self.namespace, no_of_pvc=pvc_count, ) logging.info(f"Number of PVCs in Bound state {len(pvc_bound_list)}") total_files_size = self.run_fio_on_pvcs(pvc_dict_list, pod_factory) clone_dict_list = scale_lib.construct_pvc_clone_yaml_bulk_for_kube_job( pvc_dict_list, clone_yaml, sc_name) logging.info("Created clone dict list") job_clone_file = ObjectConfFile( name="job_profile_clone", obj_dict_list=clone_dict_list, project=self.namespace, tmp_path=tmp_path, ) # Create kube_job that creates clones job_clone_file.create(namespace=self.namespace) logging.info("Going to check bound status for clones") # Check all the clones reached Bound state clone_bound_list = scale_lib.check_all_pvc_reached_bound_state_in_kube_job( kube_job_obj=job_clone_file, namespace=self.namespace, no_of_pvc=pvc_count, timeout=200, ) logging.info( f"Number of clones in Bound state {len(clone_bound_list)}") clone_objs = [] all_pvc_objs = pvc.get_all_pvc_objs(namespace=self.namespace) for clone_yaml in clone_dict_list: name = clone_yaml["metadata"]["name"] size = clone_yaml["spec"]["resources"]["requests"]["storage"] logging.info(f"Clone {name} of size {size} created") for pvc_obj in all_pvc_objs: if pvc_obj.name == name: clone_objs.append(pvc_obj) assert len(clone_bound_list) == len( clone_objs ), "Not all clones reached BOUND state, cannot measure time" start_time = helpers.get_provision_time(self.interface, clone_objs, status="start") end_time = helpers.get_provision_time(self.interface, clone_objs, status="end") total_time = (end_time - start_time).total_seconds() speed = round(total_files_size / total_time, 2) logging.info( f"Total creation time = {total_time} secs, data size = {total_files_size} MB, speed = {speed} MB/sec " f"for {self.interface} clone in bulk of {pvc_count} clones.")
def test_bulk_clone_performance(self, namespace, tmp_path): """ Creates number of PVCs in a bulk using kube job Write 60% of PVC capacity to each one of the created PVCs Creates 1 clone per each PVC altogether in a bulk Measuring total and csi creation times for bulk of clones """ pvc_count = 50 vol_size = "5Gi" job_pod_file, job_pvc_file, job_clone_file = [None, None, None] log.info(f"Start creating {self.interface} {pvc_count} PVC") if self.interface == constants.CEPHBLOCKPOOL: sc_name = constants.DEFAULT_STORAGECLASS_RBD clone_yaml = constants.CSI_RBD_PVC_CLONE_YAML elif self.interface == constants.CEPHFILESYSTEM: sc_name = constants.DEFAULT_STORAGECLASS_CEPHFS clone_yaml = constants.CSI_CEPHFS_PVC_CLONE_YAML try: pvc_dict_list = scale_lib.construct_pvc_creation_yaml_bulk_for_kube_job( no_of_pvc=pvc_count, access_mode=constants.ACCESS_MODE_RWO, sc_name=sc_name, pvc_size=vol_size, ) job_pvc_file = ObjectConfFile( name="job_profile_pvc", obj_dict_list=pvc_dict_list, project=self.namespace, tmp_path=tmp_path, ) # Create kube_job job_pvc_file.create(namespace=self.namespace) # Check all the PVC reached Bound state pvc_bound_list = scale_lib.check_all_pvc_reached_bound_state_in_kube_job( kube_job_obj=job_pvc_file, namespace=self.namespace, no_of_pvc=pvc_count, ) log.info(f"Number of PVCs in Bound state {len(pvc_bound_list)}") # Kube_job to Create pod pod_dict_list = scale_lib.attach_multiple_pvc_to_pod_dict( pvc_list=pvc_bound_list, namespace=self.namespace, pvcs_per_pod=1, start_io=False, pod_yaml=constants.NGINX_POD_YAML, ) job_pod_file = ObjectConfFile( name="job_profile_pod", obj_dict_list=pod_dict_list, project=self.namespace, tmp_path=tmp_path, ) job_pod_file.create(namespace=self.namespace) # Check all PODs in Running state scale_lib.check_all_pod_reached_running_state_in_kube_job( kube_job_obj=job_pod_file, namespace=self.namespace, no_of_pod=len(pod_dict_list), timeout=90, ) log.info(f"Number of PODs in Running state {len(pod_dict_list)}") total_files_size = self.run_fio_on_pvcs(vol_size) clone_dict_list = scale_lib.construct_pvc_clone_yaml_bulk_for_kube_job( pvc_dict_list, clone_yaml, sc_name) log.info("Created clone dict list") csi_bulk_start_time = self.get_time(time_format="csi") job_clone_file = ObjectConfFile( name="job_profile_clone", obj_dict_list=clone_dict_list, project=self.namespace, tmp_path=tmp_path, ) # Create kube_job that creates clones job_clone_file.create(namespace=self.namespace) log.info("Going to check bound status for clones") # Check all the clones reached Bound state clone_bound_list = scale_lib.check_all_pvc_reached_bound_state_in_kube_job( kube_job_obj=job_clone_file, namespace=self.namespace, no_of_pvc=pvc_count, timeout=180, ) log.info( f"Number of clones in Bound state {len(clone_bound_list)}") clone_objs = [] all_pvc_objs = pvc.get_all_pvc_objs(namespace=self.namespace) for clone_yaml in clone_dict_list: name = clone_yaml["metadata"]["name"] size = clone_yaml["spec"]["resources"]["requests"]["storage"] log.info(f"Clone {name} of size {size} created") for pvc_obj in all_pvc_objs: if pvc_obj.name == name: clone_objs.append(pvc_obj) assert len(clone_bound_list) == len( clone_objs ), "Not all clones reached BOUND state, cannot measure time" start_time = helpers.get_provision_time(self.interface, clone_objs, status="start") end_time = helpers.get_provision_time(self.interface, clone_objs, status="end") total_time = (end_time - start_time).total_seconds() speed = round(total_files_size / total_time, 2) csi_creation_time = performance_lib.csi_bulk_pvc_time_measure( self.interface, clone_objs, "create", csi_bulk_start_time) log.info( f"Total creation time = {total_time} secs, csi creation time = {csi_creation_time}," f" data size = {total_files_size} MB, speed = {speed} MB/sec " f"for {self.interface} clone in bulk of {pvc_count} clones.") # Produce ES report # Collecting environment information self.get_env_info() # Initialize the results doc file. full_results = self.init_full_results( ResultsAnalyse( self.uuid, self.crd_data, self.full_log_path, "bulk_clone_perf_fullres", )) full_results.add_key("interface", self.interface) full_results.add_key("bulk_size", pvc_count) full_results.add_key("clone_size", vol_size) full_results.add_key("bulk_creation_time", total_time) full_results.add_key("bulk_csi_creation_time", csi_creation_time) full_results.add_key("data_size(MB)", total_files_size) full_results.add_key("speed", speed) full_results.add_key("es_results_link", full_results.results_link()) # Write the test results into the ES server full_results.es_write() self.results_path = get_full_test_logs_path(cname=self) res_link = full_results.results_link() # write the ES link to the test results in the test log. log.info(f"The result can be found at : {res_link}") # Create text file with results of all subtest (3 - according to the parameters) self.write_result_to_file(res_link) # Finally is used to clean-up the resources created # Irrespective of try block pass/fail finally will be executed. finally: # Cleanup activities log.info( "Cleanup of all the resources created during test execution") if job_pod_file: job_pod_file.delete(namespace=self.namespace) job_pod_file.wait_for_delete(resource_name=job_pod_file.name, namespace=self.namespace) if job_clone_file: job_clone_file.delete(namespace=self.namespace) job_clone_file.wait_for_delete( resource_name=job_clone_file.name, namespace=self.namespace) if job_pvc_file: job_pvc_file.delete(namespace=self.namespace) job_pvc_file.wait_for_delete(resource_name=job_pvc_file.name, namespace=self.namespace) # Check ceph health status utils.ceph_health_check(tries=20)
def test_all_4_type_pvc_creation_deletion_scale(self, namespace, tmp_path): """ Measuring PVC creation time while scaling PVC of all 4 types, A total of 500 times the number of worker nodes will be created, i.e. 375 each pvc type Measure PVC deletion time in scale env """ scale_pvc_count = scale_lib.get_max_pvc_count() log.info(f"Start creating {scale_pvc_count} PVC of all 4 types") cephfs_sc_obj = constants.DEFAULT_STORAGECLASS_CEPHFS rbd_sc_obj = constants.DEFAULT_STORAGECLASS_RBD # Get pvc_dict_list, append all the pvc.yaml dict to pvc_dict_list rbd_pvc_dict_list, cephfs_pvc_dict_list = ([] for i in range(2)) for mode in [constants.ACCESS_MODE_RWO, constants.ACCESS_MODE_RWX]: rbd_pvc_dict_list.extend( scale_lib.construct_pvc_creation_yaml_bulk_for_kube_job( no_of_pvc=int(scale_pvc_count / 4), access_mode=mode, sc_name=rbd_sc_obj, )) cephfs_pvc_dict_list.extend( scale_lib.construct_pvc_creation_yaml_bulk_for_kube_job( no_of_pvc=int(scale_pvc_count / 4), access_mode=mode, sc_name=cephfs_sc_obj, )) # There is 2 kube_job for cephfs and rbd PVCs job_file_rbd = ObjectConfFile( name="rbd_pvc_job", obj_dict_list=rbd_pvc_dict_list, project=self.namespace, tmp_path=tmp_path, ) job_file_cephfs = ObjectConfFile( name="cephfs_pvc_job", obj_dict_list=cephfs_pvc_dict_list, project=self.namespace, tmp_path=tmp_path, ) # Create kube_job job_file_rbd.create(namespace=self.namespace) job_file_cephfs.create(namespace=self.namespace) # Check all the PVC reached Bound state rbd_pvc_name = scale_lib.check_all_pvc_reached_bound_state_in_kube_job( kube_job_obj=job_file_rbd, namespace=self.namespace, no_of_pvc=int(scale_pvc_count / 2), ) fs_pvc_name = scale_lib.check_all_pvc_reached_bound_state_in_kube_job( kube_job_obj=job_file_cephfs, namespace=self.namespace, no_of_pvc=int(scale_pvc_count / 2), ) # Get pvc objs from namespace, which is used to identify backend pv rbd_pvc_obj, cephfs_pvc_obj = ([] for i in range(2)) pvc_objs = pvc.get_all_pvc_objs(namespace=self.namespace) for pvc_obj in pvc_objs: if pvc_obj.backed_sc == constants.DEFAULT_STORAGECLASS_RBD: rbd_pvc_obj.append(pvc_obj) elif pvc_obj.backed_sc == constants.DEFAULT_STORAGECLASS_CEPHFS: cephfs_pvc_obj.append(pvc_obj) # Get PVC creation time fs_pvc_create_time = helpers.measure_pvc_creation_time_bulk( interface=constants.CEPHFS_INTERFACE, pvc_name_list=fs_pvc_name) rbd_pvc_create_time = helpers.measure_pvc_creation_time_bulk( interface=constants.CEPHBLOCKPOOL, pvc_name_list=rbd_pvc_name) fs_pvc_create_time.update(rbd_pvc_create_time) # TODO: Update below code with google API, to record value in spreadsheet # TODO: For now observing Google API limit to write more than 100 writes log_path = f"{ocsci_log_path()}/All-type-PVC" with open(f"{log_path}-creation-time.csv", "w") as fd: csv_obj = csv.writer(fd) for k, v in fs_pvc_create_time.items(): csv_obj.writerow([k, v]) log.info(f"Create data present in {log_path}-creation-time.csv file") # Get pv_name, require pv_name to fetch deletion time data from log rbd_pv_list, fs_pv_list = ([] for i in range(2)) get_rbd_kube_job = job_file_rbd.get(namespace=self.namespace) for i in range(int(scale_pvc_count / 2)): rbd_pv_list.append( get_rbd_kube_job["items"][i]["spec"]["volumeName"]) get_fs_kube_job = job_file_cephfs.get(namespace=self.namespace) for i in range(int(scale_pvc_count / 2)): fs_pv_list.append( get_fs_kube_job["items"][i]["spec"]["volumeName"]) # Delete kube_job job_file_rbd.delete(namespace=self.namespace) job_file_cephfs.delete(namespace=self.namespace) # Adding 1min wait time for PVC deletion logs to be updated # Observed failure when we immediately check the logs for pvc delete time # https://github.com/red-hat-storage/ocs-ci/issues/3371 time.sleep(60) # Get PV deletion time fs_pvc_deletion_time = helpers.measure_pv_deletion_time_bulk( interface=constants.CEPHFS_INTERFACE, pv_name_list=fs_pv_list) rbd_pvc_deletion_time = helpers.measure_pv_deletion_time_bulk( interface=constants.CEPHBLOCKPOOL, pv_name_list=rbd_pv_list) fs_pvc_deletion_time.update(rbd_pvc_deletion_time) # TODO: Update below code with google API, to record value in spreadsheet # TODO: For now observing Google API limit to write more than 100 writes with open(f"{log_path}-deletion-time.csv", "w") as fd: csv_obj = csv.writer(fd) for k, v in fs_pvc_deletion_time.items(): csv_obj.writerow([k, v]) log.info(f"Delete data present in {log_path}-deletion-time.csv file") end_time = default_timer() log.info(f"Elapsed time -- {end_time - self.start_time} seconds")
def teardown(self): pvc_objs = get_all_pvc_objs(namespace="openshift-storage") pvcs = [pvc_obj for pvc_obj in pvc_objs if "test-pvc" in pvc_obj.name] delete_pvcs(pvc_objs=pvcs)