def destroy_ocs(self): """ Uninstall ODF Managed Service addon via rosa cli. """ cluster_namespace = config.ENV_DATA["cluster_namespace"] # Deleting PVCs rbd_pvcs = [ p for p in pvc.get_all_pvcs_in_storageclass( constants.CEPHBLOCKPOOL_SC) if not (p.data["metadata"]["namespace"] == cluster_namespace and p.data["metadata"]["labels"]["app"] == "noobaa") ] pvc.delete_pvcs(rbd_pvcs) cephfs_pvcs = pvc.get_all_pvcs_in_storageclass( constants.CEPHFILESYSTEM_SC) pvc.delete_pvcs(cephfs_pvcs) rosa.delete_odf_addon(self.cluster_name)
def wait_for_mirroring_status_ok(replaying_images=None, timeout=300): """ Wait for mirroring status to reach health OK and expected number of replaying images for each of the ODF cluster Args: replaying_images (int): Expected number of images in replaying state timeout (int): time in seconds to wait for mirroring status reach OK Returns: bool: True if status contains expected health and states values Raises: TimeoutExpiredError: In case of unexpected mirroring status """ restore_index = config.cur_index if not replaying_images: replaying_images = 0 for cluster in get_non_acm_cluster_config(): config.switch_ctx(cluster.MULTICLUSTER["multicluster_index"]) replaying_images += len( get_all_pvcs_in_storageclass(constants.CEPHBLOCKPOOL_SC) ) replaying_images -= 2 # Ignore db-noobaa-db-pg-0 PVCs for cluster in get_non_acm_cluster_config(): config.switch_ctx(cluster.MULTICLUSTER["multicluster_index"]) logger.info( f"Validating mirroring status on cluster {cluster.ENV_DATA['cluster_name']}" ) sample = TimeoutSampler( timeout=timeout, sleep=5, func=check_mirroring_status_ok, replaying_images=replaying_images, ) if not sample.wait_for_func_status(result=True): error_msg = ( "The mirroring status does not have expected values within the time" f" limit on cluster {cluster.ENV_DATA['cluster_name']}" ) logger.error(error_msg) raise TimeoutExpiredError(error_msg) config.switch_ctx(restore_index) return True
def uninstall_ocs(): """ The function uninstalls the OCS operator from a openshift cluster and removes all its settings and dependencies """ ocp_obj = ocp.OCP() log.info("deleting volume snapshots") vs_ocp_obj = ocp.OCP(kind=constants.VOLUMESNAPSHOT) vs_list = vs_ocp_obj.get(all_namespaces=True)["items"] for vs in vs_list: vs_obj = ocp.OCP(kind=constants.VOLUMESNAPSHOT, namespace=vs.get("metadata").get("namespace")) vs_obj.delete(resource_name=vs.get("metadata").get("name")) log.info("queering for OCS PVCs") provisioners = constants.OCS_PROVISIONERS sc_list = [ sc for sc in get_all_storageclass() if sc.get("provisioner") in provisioners ] pvc_to_delete = [] for sc in sc_list: pvc_to_delete.extend(pvc for pvc in get_all_pvcs_in_storageclass( sc.get("metadata").get("name")) if "noobaa" not in pvc.name) if config.ENV_DATA["platform"].lower() == constants.ROSA_PLATFORM: log.info("Deleting OCS PVCs") for pvc in pvc_to_delete: log.info(f"Deleting PVC: {pvc.name}") pvc.delete() rosa.delete_odf_addon(config.ENV_DATA["cluster_name"]) return None log.info("Removing monitoring stack from OpenShift Container Storage") remove_monitoring_stack_from_ocs() log.info( "Removing OpenShift Container Platform registry from OpenShift Container Storage" ) remove_ocp_registry_from_ocs(config.ENV_DATA["platform"]) log.info( "Removing the cluster logging operator from OpenShift Container Storage" ) try: remove_cluster_logging_operator_from_ocs() except CommandFailed: log.info("No cluster logging found") log.info("Deleting OCS PVCs") for pvc in pvc_to_delete: log.info(f"Deleting PVC: {pvc.name}") pvc.delete() storage_cluster = ocp.OCP( kind=constants.STORAGECLUSTER, resource_name=constants.DEFAULT_CLUSTERNAME, namespace="openshift-storage", ) log.info("Checking for local storage") lso_sc = None if check_local_volume_local_volume_set(): "Local volume was found. Will be removed later" lso_sc = (storage_cluster.get().get("spec").get("storageDeviceSets")[0] .get("dataPVCTemplate").get("spec").get("storageClassName")) cleanup_policy = (storage_cluster.get().get("metadata").get( "annotations").get("uninstall.ocs.openshift.io/cleanup-policy")) log.info("Deleting storageCluster object") storage_cluster.delete(resource_name=constants.DEFAULT_CLUSTERNAME) if cleanup_policy == "delete": log.info("Cleanup policy set to delete. checking cleanup pods") cleanup_pods = [ pod for pod in get_all_pods() if "cluster-cleanup-job" in pod.name ] for pod in cleanup_pods: while pod.get().get("status").get("phase") != "Succeeded": log.info(f"waiting for cleanup pod {pod.name} to complete") TimeoutSampler(timeout=10, sleep=30) log.info(f"Cleanup pod {pod.name} completed successfully ") # no need to confirm var/vib/rook was deleted from nodes if all cleanup pods are completed. else: log.info("Cleanup policy set to retain. skipping nodes cleanup") log.info("Deleting openshift-storage namespace") ocp_obj.delete_project(constants.OPENSHIFT_STORAGE_NAMESPACE) ocp_obj.wait_for_delete(constants.OPENSHIFT_STORAGE_NAMESPACE) switch_to_project(constants.DEFAULT_NAMESPACE) # step 10: TODO remove crypto from nodes. """for node in storage_node_list: log.info(f"removing encryption from {node}") ocp_obj.exec_oc_debug_cmd(node=node, cmd_list=[])""" if lso_sc is not None: log.info("Removing LSO") try: uninstall_lso(lso_sc) except Exception as e: log.info(f"LSO removal failed.{e}") log.info("deleting noobaa storage class") noobaa_sc = ocp.OCP(kind=constants.STORAGECLASS) noobaa_sc.delete(resource_name=constants.NOOBAA_SC) nodes = get_all_nodes() node_objs = get_node_objs(nodes) log.info("Unlabeling storage nodes") label_nodes(nodes=node_objs, label=constants.OPERATOR_NODE_LABEL[:-3] + "-") label_nodes(nodes=node_objs, label=constants.TOPOLOGY_ROOK_LABEL + "-") log.info("Removing taints from storage nodes") taint_nodes(nodes=nodes, taint_label=constants.OPERATOR_NODE_TAINT + "-") log.info("Deleting remaining OCS PVs (if there are any)") try: rbd_pv = ocp.OCP(kind=constants.PV, resource_name="ocs-storagecluster-ceph-rbd") fs_pv = ocp.OCP(kind=constants.PV, resource_name="ocs-storagecluster-cephfs") rbd_pv.delete() fs_pv.delete() log.info("OCS PVs deleted") except Exception as e: log.info(f"OCS PV(s) not found. {e}") log.info("Removing CRDs") crd_list = [ "backingstores.noobaa.io", "bucketclasses.noobaa.io", "cephblockpools.ceph.rook.io", "cephclusters.ceph.rook.io", "cephfilesystems.ceph.rook.io", "cephnfses.ceph.rook.io", "cephobjectstores.ceph.rook.io", "cephobjectstoreusers.ceph.rook.io", "noobaas.noobaa.io", "ocsinitializations.ocs.openshift.io", "storageclusters.ocs.openshift.io", "cephclients.ceph.rook.io", "cephobjectrealms.ceph.rook.io", "cephobjectzonegroups.ceph.rook.io", "cephobjectzones.ceph.rook.io", "cephrbdmirrors.ceph.rook.io", ] for crd in crd_list: try: ocp_obj.exec_oc_cmd(f"delete crd {crd} --timeout=300m") except Exception: log.info(f"crd {crd} was not found")
def uninstall_ocs(): """ The function uninstalls the OCS operator from a openshift cluster and removes all its settings and dependencies """ ocp_obj = ocp.OCP() provisioners = constants.OCS_PROVISIONERS # List the storage classes sc_list = [ sc for sc in get_all_storageclass() if sc.get('provisioner') in provisioners ] # Query for PVCs and OBCs that are using the storage class provisioners listed in the previous step. pvc_to_delete = [] for sc in sc_list: pvc_to_delete.extend(pvc for pvc in get_all_pvcs_in_storageclass( sc.get('metadata').get('name')) if 'noobaa' not in pvc.name) log.info("Removing monitoring stack from OpenShift Container Storage") remove_monitoring_stack_from_ocs() log.info( "Removing OpenShift Container Platform registry from OpenShift Container Storage" ) remove_ocp_registry_from_ocs(config.ENV_DATA['platform']) log.info( "Removing the cluster logging operator from OpenShift Container Storage" ) try: remove_cluster_logging_operator_from_ocs() except CommandFailed: log.info("No cluster logging found") log.info("Deleting pvcs") for pvc in pvc_to_delete: log.info(f"Deleting pvc: {pvc.name}") pvc.delete() storage_cluster = ocp.OCP(kind=constants.STORAGECLUSTER, resource_name=constants.DEFAULT_CLUSTERNAME, namespace='openshift-storage') log.info("Checking for local storage") lso_sc = None if check_local_volume(): "Local volume was found. Will be removed later" lso_sc = storage_cluster.get().get('spec').get('storageDeviceSets')[ 0].get('dataPVCTemplate').get('spec').get('storageClassName') log.info("Deleting storageCluster object") storage_cluster.delete(resource_name=constants.DEFAULT_CLUSTERNAME) log.info("Removing CRDs") crd_list = [ 'backingstores.noobaa.io', 'bucketclasses.noobaa.io', 'cephblockpools.ceph.rook.io', 'cephfilesystems.ceph.rook.io', 'cephnfses.ceph.rook.io', 'cephobjectstores.ceph.rook.io', 'cephobjectstoreusers.ceph.rook.io', 'noobaas.noobaa.io', 'ocsinitializations.ocs.openshift.io', 'storageclusterinitializations.ocs.openshift.io', 'storageclusters.ocs.openshift.io', 'cephclusters.ceph.rook.io' ] for crd in crd_list: ocp_obj.exec_oc_cmd(f"delete crd {crd} --timeout=300m") log.info("Deleting openshift-storage namespace") ocp_obj.delete_project('openshift-storage') ocp_obj.wait_for_delete('openshift-storage') switch_to_project("default") log.info("Removing rook directory from nodes") nodes_list = get_labeled_nodes(constants.OPERATOR_NODE_LABEL) for node in nodes_list: log.info(f"Removing rook from {node}") ocp_obj.exec_oc_debug_cmd(node=node, cmd_list=["rm -rf /var/lib/rook"]) log.info("Removing LSO ") if lso_sc is not None: uninstall_lso(lso_sc) log.info( "Delete the storage classes with an openshift-storage provisioner list" ) for storage_class in sc_list: log.info( f"Deleting storage class {storage_class.get('metadata').get('name')}" ) sc_obj = ocp.OCP(kind=constants.STORAGECLASS) sc_obj.delete(resource_name=storage_class.get('metadata').get('name')) log.info("Unlabeling storage nodes") nodes_list = get_all_nodes() for node in nodes_list: node_obj = ocp.OCP(kind=constants.NODE, resource_name=node) node_obj.add_label(resource_name=node, label=constants.OPERATOR_NODE_LABEL[:-3] + '-') node_obj.add_label(resource_name=node, label=constants.TOPOLOGY_ROOK_LABEL + '-') log.info("OCS was removed successfully from cluster ")
def test_multiple_sc_comp_rep_data_deletion(self, storageclass_factory, pvc_factory, pod_factory): """ This test function does below, *. Creates 2 Storage Class with creating new rbd pool *. Creates PVCs using new Storage Class *. Mount PVC to an app pod *. Run IO on an app pod *. Delete the pods and pvc *. Verify that the data is deleted """ log.info("Creating storageclasses with compression and replica3") interface_type = constants.CEPHBLOCKPOOL sc_obj1 = storageclass_factory( interface=interface_type, new_rbd_pool=True, replica=3, compression="aggressive", ) log.info("Creating storageclasses with compression and replica2") sc_obj2 = storageclass_factory( interface=interface_type, new_rbd_pool=True, replica=2, compression="aggressive", ) sc_obj_list = [sc_obj1, sc_obj2] pod_obj_list = [] pvc_obj_list = [] log.info("Creating PVCs and PODs") for sc_obj in sc_obj_list: pvc_obj = pvc_factory(interface=interface_type, storageclass=sc_obj) pvc_obj_list.append(pvc_obj) pod_obj_list.append( pod_factory(interface=interface_type, pvc=pvc_obj)) log.info("Running IO on pods") for pod_obj in pod_obj_list: pod_obj.run_io("fs", size="1G") for pod_obj in pod_obj_list: get_fio_rw_iops(pod_obj) log.info("deleting PODs and PVCs") delete_pods(pod_obj_list, wait=True) delete_pvcs(pvc_obj_list, concurrent=True) log.info("Wait for 15 seconds for all data to delete") sleep(15) log.info("Checking stats after deleting PODs and PVCs") for sc_obj in sc_obj_list: pvc_list = get_all_pvcs_in_storageclass(sc_obj.name) if len(pvc_list) == 0: cbp_name = sc_obj.get()["parameters"]["pool"] ceph_pool_byte_used = get_byte_used_by_pool(cbp_name) log.info( f"pool {cbp_name} has {ceph_pool_byte_used} bytes used") if ceph_pool_byte_used > MAX_BYTES_IN_POOL_AFTER_DATA_DELETE: raise PoolDataNotErased( f"Pool {cbp_name} has {ceph_pool_byte_used} bytes which were not deleted" ) else: raise PvcNotDeleted(f"PVC {pvc_list} were not deleted")
def destroy_ocs(self): """ Handle OCS destruction. Remove storage classes, PVCs, Storage Cluster, Openshift-storage namespace, LocalVolume, unlabel worker-storage nodes, delete ocs CRDs, etc. """ cluster_namespace = config.ENV_DATA["cluster_namespace"] # https://access.redhat.com/documentation/en-us/red_hat_openshift_container_storage/4.5/html/deploying_openshift_container_storage_using_bare_metal_infrastructure/assembly_uninstalling-openshift-container-storage_rhocs # Section 3.1 Step 1 # Deleting PVCs rbd_pvcs = [ p for p in pvc.get_all_pvcs_in_storageclass( constants.CEPHBLOCKPOOL_SC) if not (p.data["metadata"]["namespace"] == cluster_namespace and p.data["metadata"]["labels"]["app"] == "noobaa") ] pvc.delete_pvcs(rbd_pvcs) cephfs_pvcs = pvc.get_all_pvcs_in_storageclass( constants.CEPHFILESYSTEM_SC) # Section 3.1 Step 2 # Section 3.3 Step 1 # Removing OpenShift Container Platform registry from OpenShift Container Storage registry_conf_name = "configs.imageregistry.operator.openshift.io" registry_conf = ocp.OCP().exec_oc_cmd( f"get {registry_conf_name} -o yaml") if registry_conf["items"][0]["spec"].get("storage", dict()).get("pvc"): patch = dict(spec=dict(storage=dict(emptyDir=dict(), pvc=None))) ocp.OCP().exec_oc_cmd( f"patch {registry_conf_name} cluster --type merge " f"-p '{json.dumps(patch)}'") # Section 3.3 Step 2 pvc.delete_pvcs(cephfs_pvcs) # Section 3.1 Step 3 try: ocp.OCP().exec_oc_cmd( f"delete -n {cluster_namespace} storagecluster --all --wait=true" ) except (CommandFailed, subprocess.TimeoutExpired): pass # Section 3.1 Step 4 ocp.OCP().exec_oc_cmd("project default") ocp.OCP().exec_oc_cmd( f"delete project {cluster_namespace} --wait=true --timeout=5m") tried = 0 leftovers = True while tried < 5: # We need to loop here until the project can't be found try: ocp.OCP().exec_oc_cmd( f"get project {cluster_namespace}", out_yaml_format=False, ) except CommandFailed: leftovers = False break time.sleep(60) tried += 1 if leftovers: # https://access.redhat.com/documentation/en-us/red_hat_openshift_container_storage/4.5/html/troubleshooting_openshift_container_storage/troubleshooting-and-deleting-remaining-resources-during-uninstall_rhocs leftover_types = [ "cephfilesystem.ceph.rook.io", "cephobjectstore.ceph.rook.io", "cephobjectstoreuser.ceph.rook.io", "storagecluster.ocs.openshift.io", ] patch = dict(metadata=dict(finalizers=None)) for obj_type in leftover_types: try: objs = ocp.OCP(kind=obj_type).get() except CommandFailed: continue for obj in objs["items"]: name = obj["metadata"]["name"] ocp.OCP().exec_oc_cmd( f"oc patch -n {cluster_namespace} {obj_type} {name} --type=merge -p '{json.dumps(patch)}'" ) # Section 3.1 Step 5 nodes = ocp.OCP().exec_oc_cmd( "get node -l cluster.ocs.openshift.io/openshift-storage= -o yaml") for node in nodes["items"]: node_name = node["metadata"]["name"] ocp.OCP().exec_oc_cmd( f"debug node/{node_name} -- chroot /host rm -rfv /var/lib/rook" ) # Section 3.1 Step 6 ocp.OCP().exec_oc_cmd( "delete storageclass openshift-storage.noobaa.io --wait=true --timeout=5m" ) # Section 3.1 Step 7 ocp.OCP().exec_oc_cmd( "label nodes --all cluster.ocs.openshift.io/openshift-storage-") ocp.OCP().exec_oc_cmd("label nodes --all topology.rook.io/rack-") # Section 3.1 Step 8 pvs = ocp.OCP(kind="PersistentVolume").get() for pv in pvs["items"]: pv_name = pv["metadata"]["name"] if pv_name.startswith("ocs-storagecluster-ceph"): ocp.OCP().exec_oc_cmd(f"oc delete pv {pv_name}") # Section 3.1 Step 9 # Note that the below process differs from the documentation slightly. # Instead of deleting all CRDs at once and calling the job done, we # iterate over a list of them, noting which ones don't delete fully and # applying the standard workaround of removing the finalizers from any # CRs and also the CRD. Finally, the documentation leaves out a few # CRDs that we've seen in deployed clusters. crd_types = [ "backingstores.noobaa.io", "bucketclasses.noobaa.io", "cephblockpools.ceph.rook.io", "cephclients.ceph.rook.io", "cephclusters.ceph.rook.io", "cephfilesystems.ceph.rook.io", "cephnfses.ceph.rook.io", "cephobjectrealms.ceph.rook.io", # not in doc "cephobjectstores.ceph.rook.io", "cephobjectstoreusers.ceph.rook.io", "cephobjectzonegroups.ceph.rook.io", # not in doc "cephobjectzones.ceph.rook.io", # not in doc "cephrbdmirrors.ceph.rook.io", # not in doc "noobaas.noobaa.io", "ocsinitializations.ocs.openshift.io", "storageclusterinitializations.ocs.openshift.io", "storageclusters.ocs.openshift.io", ] cr_patch = json.dumps(dict(finalizers=None)) crd_patch = json.dumps(dict(metadata=dict(finalizers=None))) for crd_type in crd_types: try: ocp.OCP().exec_oc_cmd( f"delete crd {crd_type} --wait=true --timeout=30s", out_yaml_format=False, ) except CommandFailed: pass crs = [] try: crs = ocp.OCP(kind=crd_type).get(all_namespaces=True)["items"] except CommandFailed: continue for cr in crs: cr_md = cr["metadata"] ocp.OCP().exec_oc_cmd( f"patch -n {cr_md['namespace']} {crd_type} {cr_md['name']} --type=merge -p '{cr_patch}'" ) try: crs = ocp.OCP(kind=crd_type).get(all_namespaces=True)["items"] except CommandFailed: continue ocp.OCP().exec_oc_cmd( f"patch crd {crd_type} --type=merge -p '{crd_patch}'") # End sections from above documentation ocp.OCP().exec_oc_cmd( f"delete catalogsource {constants.OPERATOR_CATALOG_SOURCE_NAME} " f"-n {constants.MARKETPLACE_NAMESPACE}") storageclasses = ocp.OCP(kind="StorageClass").get(all_namespaces=True) for sc in storageclasses["items"]: if sc["provisioner"].startswith("openshift-storage."): sc_name = sc["metadata"]["name"] ocp.OCP().exec_oc_cmd(f"delete storageclass {sc_name}") volumesnapclasses = ocp.OCP(kind="VolumeSnapshotClass").get( all_namespaces=True) for vsc in volumesnapclasses["items"]: if vsc["driver"].startswith("openshift-storage."): vsc_name = vsc["metadata"]["name"] ocp.OCP().exec_oc_cmd(f"delete volumesnapshotclass {vsc_name}") self.destroy_lso()
def uninstall_ocs(): """ The function uninstalls the OCS operator from a openshift cluster and removes all its settings and dependencies """ ocp_obj = ocp.OCP() provisioners = constants.OCS_PROVISIONERS # List the storage classes sc_list = get_all_storageclass() sc_name_list = [] for storage_class in sc_list: if storage_class.get('provisioner') not in provisioners: sc_list.remove(storage_class) else: sc_name_list.append(storage_class.get('metadata').get('name')) # Query for PVCs and OBCs that are using the storage class provisioners listed in the previous step. pvc_to_delete = [] pvc_name_list = [] for sc in sc_name_list: pvc_to_delete.extend(get_all_pvcs_in_storageclass(sc)) # ignoring all noobaa pvcs & make name list for pvc in pvc_to_delete: if "noobaa" in pvc.name: pvc_to_delete.remove(pvc) else: pvc_name_list.append(pvc.name) pods_to_delete = [] all_pods = get_all_pods() # default openshift-storage namespace all_pods.extend(get_all_pods(namespace=constants.OPENSHIFT_IMAGE_REGISTRY_NAMESPACE)) all_pods.extend(get_all_pods(namespace=constants.OPENSHIFT_MONITORING_NAMESPACE)) for pod_obj in all_pods: try: pvc_name = get_pvc_name(pod_obj) except UnavailableResourceException: continue if pvc_name in pvc_name_list: pods_to_delete.append(pod_obj) log.info("Removing monitoring stack from OpenShift Container Storage") remove_monitoring_stack_from_ocs() log.info("Removing OpenShift Container Platform registry from OpenShift Container Storage") remove_ocp_registry_from_ocs(config.ENV_DATA['platform']) log.info("Removing the cluster logging operator from OpenShift Container Storage") csv = ocp.OCP( kind=constants.CLUSTER_SERVICE_VERSION, namespace=constants.OPENSHIFT_LOGGING_NAMESPACE ) logging_csv = csv.get().get('items') if logging_csv: clusterlogging_obj = ocp.OCP( kind=constants.CLUSTER_LOGGING, namespace=constants.OPENSHIFT_LOGGING_NAMESPACE ) clusterlogging_obj.delete(resource_name='instance') log.info("deleting pvcs") for pvc in pvc_to_delete: log.info(f"deleting pvc: {pvc.name}") pvc.delete() log.info("deleting pods") for pod in pods_to_delete: log.info(f"deleting pod {pod.name}") pod.delete() log.info("removing rook directory from nodes") nodes_list = get_labeled_nodes(constants.OPERATOR_NODE_LABEL) for node in nodes_list: log.info(f"removing rook from {node}") ocp_obj.exec_oc_debug_cmd(node=node, cmd_list=["rm -rf /var/lib/rook"]) log.info("Delete the storage classes with an openshift-storage provisioner list") for storage_class in sc_list: log.info(f"deleting storage class {storage_class.get('metadata').get('name')}") sc_obj = ocp.OCP(kind=constants.STORAGECLASS) sc_obj.delete(resource_name=storage_class.get('metadata').get('name')) log.info("unlabaling storage nodes") nodes_list = get_all_nodes() for node in nodes_list: node_obj = ocp.OCP(kind=constants.NODE, resource_name=node) node_obj.add_label(resource_name=node, label=constants.OPERATOR_NODE_LABEL[:-3] + '-') node_obj.add_label(resource_name=node, label=constants.TOPOLOGY_ROOK_LABEL + '-') log.info("deleting storageCluster object") storage_cluster = ocp.OCP(kind=constants.STORAGECLUSTER, resource_name=constants.DEFAULT_CLUSTERNAME) storage_cluster.delete(resource_name=constants.DEFAULT_CLUSTERNAME) log.info("removing CRDs") crd_list = ['backingstores.noobaa.io', 'bucketclasses.noobaa.io', 'cephblockpools.ceph.rook.io', 'cephfilesystems.ceph.rook.io', 'cephnfses.ceph.rook.io', 'cephobjectstores.ceph.rook.io', 'cephobjectstoreusers.ceph.rook.io', 'noobaas.noobaa.io', 'ocsinitializations.ocs.openshift.io', 'storageclusterinitializations.ocs.openshift.io', 'storageclusters.ocs.openshift.io', 'cephclusters.ceph.rook.io'] for crd in crd_list: ocp_obj.exec_oc_cmd(f"delete crd {crd} --timeout=300m") log.info("deleting openshift-storage namespace") ocp_obj.delete_project('openshift-storage') ocp_obj.wait_for_delete('openshift-storage')