def wait_for_all_nodes_csr(timeout=900, sleep=10): """ Wait for CSR to generate for nodes Args: timeout (int): Time in seconds to wait sleep (int): Sampling time in seconds Returns: bool: True if all nodes are generated CSR Raises: TimeoutExpiredError: in case CSR not found """ pending = False all_nodes = get_all_nodes() for csr_nodes in TimeoutSampler( timeout=timeout, sleep=sleep, func=get_nodes_csr ): pending_nodes = [] logger.debug(f"CSR data: {csr_nodes}") for node in all_nodes: if node not in csr_nodes.keys(): logger.info(f"{node} CSR is not generated") pending = True pending_nodes.append(node) if not pending: logger.info("CSR generated for all nodes in cluster") return logger.warning( f"Nodes {pending_nodes} are not generated CSR. retrying again" )
def create_instance_in_clusterlogging(): """ Creation of instance for clusterlogging that creates PVCs, ElasticSearch, curator fluentd and kibana pods and checks for all the pods and PVCs Args: sc_name (str): Storage class name to create PVCs Returns: dict: Contains all detailed information of the instance such as pods that got created, its resources and limits values, storage class and size details etc. """ nodes_in_cluster = len(get_all_nodes()) inst_data = templating.load_yaml(constants.CL_INSTANCE_YAML) es_node_count = inst_data["spec"]["logStore"]["elasticsearch"]["nodeCount"] if helpers.storagecluster_independent_check(): inst_data["spec"]["logStore"]["elasticsearch"]["storage"][ "storageClassName" ] = constants.DEFAULT_EXTERNAL_MODE_STORAGECLASS_RBD helpers.create_resource(wait=False, **inst_data) oc = ocp.OCP("v1", "ClusterLogging", "openshift-logging") logging_instance = oc.get(resource_name="instance", out_yaml_format="True") if logging_instance: logger.info("Successfully created instance for cluster-logging") logger.debug(logging_instance) else: logger.error("Instance for clusterlogging is not created properly") pod_obj = ocp.OCP(kind=constants.POD, namespace="openshift-logging") pod_status = pod_obj.wait_for_resource( condition=constants.STATUS_RUNNING, resource_count=2 + es_node_count + nodes_in_cluster, timeout=500, sleep=2, ) assert pod_status, "Pods are not in Running state." logger.info("All pods are in Running state") pvc_obj = ocp.OCP(kind=constants.PVC, namespace="openshift-logging") pvc_status = pvc_obj.wait_for_resource( condition=constants.STATUS_BOUND, resource_count=es_node_count, timeout=150, sleep=5, ) assert pvc_status, "PVCs are not in bound state." logger.info("PVCs are Bound") return logging_instance
def create_instance_in_clusterlogging(): """ Creation of instance for clusterlogging that creates PVCs, ElasticSearch, curator fluentd and kibana pods and checks for all the pods and PVCs Args: sc_name (str): Storage class name to create PVCs Returns: dict: Contains all detailed information of the instance such as pods that got created, its resources and limits values, storage class and size details etc. """ nodes_in_cluster = len(get_all_nodes()) inst_data = templating.load_yaml(constants.CL_INSTANCE_YAML) es_node_count = inst_data['spec']['logStore']['elasticsearch']['nodeCount'] helpers.create_resource(wait=False, **inst_data) oc = ocp.OCP('v1', 'ClusterLogging', 'openshift-logging') logging_instance = oc.get(resource_name='instance', out_yaml_format='True') if logging_instance: logger.info("Successfully created instance for cluster-logging") logger.debug(logging_instance) else: logger.error("Instance for clusterlogging is not created properly") pod_obj = ocp.OCP(kind=constants.POD, namespace='openshift-logging') pod_status = pod_obj.wait_for_resource(condition=constants.STATUS_RUNNING, resource_count=2 + es_node_count + nodes_in_cluster, timeout=500, sleep=2) assert pod_status, "Pods are not in Running state." logger.info("All pods are in Running state") pvc_obj = ocp.OCP(kind=constants.PVC, namespace='openshift-logging') pvc_status = pvc_obj.wait_for_resource(condition=constants.STATUS_BOUND, resource_count=es_node_count, timeout=150, sleep=5) assert pvc_status, "PVCs are not in bound state." logger.info("PVCs are Bound") return logging_instance
def uninstall_ocs(): """ The function uninstalls the OCS operator from a openshift cluster and removes all its settings and dependencies """ ocp_obj = ocp.OCP() log.info("deleting volume snapshots") vs_ocp_obj = ocp.OCP(kind=constants.VOLUMESNAPSHOT) vs_list = vs_ocp_obj.get(all_namespaces=True)["items"] for vs in vs_list: vs_obj = ocp.OCP(kind=constants.VOLUMESNAPSHOT, namespace=vs.get("metadata").get("namespace")) vs_obj.delete(resource_name=vs.get("metadata").get("name")) log.info("queering for OCS PVCs") provisioners = constants.OCS_PROVISIONERS sc_list = [ sc for sc in get_all_storageclass() if sc.get("provisioner") in provisioners ] pvc_to_delete = [] for sc in sc_list: pvc_to_delete.extend(pvc for pvc in get_all_pvcs_in_storageclass( sc.get("metadata").get("name")) if "noobaa" not in pvc.name) if config.ENV_DATA["platform"].lower() == constants.ROSA_PLATFORM: log.info("Deleting OCS PVCs") for pvc in pvc_to_delete: log.info(f"Deleting PVC: {pvc.name}") pvc.delete() rosa.delete_odf_addon(config.ENV_DATA["cluster_name"]) return None log.info("Removing monitoring stack from OpenShift Container Storage") remove_monitoring_stack_from_ocs() log.info( "Removing OpenShift Container Platform registry from OpenShift Container Storage" ) remove_ocp_registry_from_ocs(config.ENV_DATA["platform"]) log.info( "Removing the cluster logging operator from OpenShift Container Storage" ) try: remove_cluster_logging_operator_from_ocs() except CommandFailed: log.info("No cluster logging found") log.info("Deleting OCS PVCs") for pvc in pvc_to_delete: log.info(f"Deleting PVC: {pvc.name}") pvc.delete() storage_cluster = ocp.OCP( kind=constants.STORAGECLUSTER, resource_name=constants.DEFAULT_CLUSTERNAME, namespace="openshift-storage", ) log.info("Checking for local storage") lso_sc = None if check_local_volume_local_volume_set(): "Local volume was found. Will be removed later" lso_sc = (storage_cluster.get().get("spec").get("storageDeviceSets")[0] .get("dataPVCTemplate").get("spec").get("storageClassName")) cleanup_policy = (storage_cluster.get().get("metadata").get( "annotations").get("uninstall.ocs.openshift.io/cleanup-policy")) log.info("Deleting storageCluster object") storage_cluster.delete(resource_name=constants.DEFAULT_CLUSTERNAME) if cleanup_policy == "delete": log.info("Cleanup policy set to delete. checking cleanup pods") cleanup_pods = [ pod for pod in get_all_pods() if "cluster-cleanup-job" in pod.name ] for pod in cleanup_pods: while pod.get().get("status").get("phase") != "Succeeded": log.info(f"waiting for cleanup pod {pod.name} to complete") TimeoutSampler(timeout=10, sleep=30) log.info(f"Cleanup pod {pod.name} completed successfully ") # no need to confirm var/vib/rook was deleted from nodes if all cleanup pods are completed. else: log.info("Cleanup policy set to retain. skipping nodes cleanup") log.info("Deleting openshift-storage namespace") ocp_obj.delete_project(constants.OPENSHIFT_STORAGE_NAMESPACE) ocp_obj.wait_for_delete(constants.OPENSHIFT_STORAGE_NAMESPACE) switch_to_project(constants.DEFAULT_NAMESPACE) # step 10: TODO remove crypto from nodes. """for node in storage_node_list: log.info(f"removing encryption from {node}") ocp_obj.exec_oc_debug_cmd(node=node, cmd_list=[])""" if lso_sc is not None: log.info("Removing LSO") try: uninstall_lso(lso_sc) except Exception as e: log.info(f"LSO removal failed.{e}") log.info("deleting noobaa storage class") noobaa_sc = ocp.OCP(kind=constants.STORAGECLASS) noobaa_sc.delete(resource_name=constants.NOOBAA_SC) nodes = get_all_nodes() node_objs = get_node_objs(nodes) log.info("Unlabeling storage nodes") label_nodes(nodes=node_objs, label=constants.OPERATOR_NODE_LABEL[:-3] + "-") label_nodes(nodes=node_objs, label=constants.TOPOLOGY_ROOK_LABEL + "-") log.info("Removing taints from storage nodes") taint_nodes(nodes=nodes, taint_label=constants.OPERATOR_NODE_TAINT + "-") log.info("Deleting remaining OCS PVs (if there are any)") try: rbd_pv = ocp.OCP(kind=constants.PV, resource_name="ocs-storagecluster-ceph-rbd") fs_pv = ocp.OCP(kind=constants.PV, resource_name="ocs-storagecluster-cephfs") rbd_pv.delete() fs_pv.delete() log.info("OCS PVs deleted") except Exception as e: log.info(f"OCS PV(s) not found. {e}") log.info("Removing CRDs") crd_list = [ "backingstores.noobaa.io", "bucketclasses.noobaa.io", "cephblockpools.ceph.rook.io", "cephclusters.ceph.rook.io", "cephfilesystems.ceph.rook.io", "cephnfses.ceph.rook.io", "cephobjectstores.ceph.rook.io", "cephobjectstoreusers.ceph.rook.io", "noobaas.noobaa.io", "ocsinitializations.ocs.openshift.io", "storageclusters.ocs.openshift.io", "cephclients.ceph.rook.io", "cephobjectrealms.ceph.rook.io", "cephobjectzonegroups.ceph.rook.io", "cephobjectzones.ceph.rook.io", "cephrbdmirrors.ceph.rook.io", ] for crd in crd_list: try: ocp_obj.exec_oc_cmd(f"delete crd {crd} --timeout=300m") except Exception: log.info(f"crd {crd} was not found")
def uninstall_ocs(): """ The function uninstalls the OCS operator from a openshift cluster and removes all its settings and dependencies """ ocp_obj = ocp.OCP() provisioners = constants.OCS_PROVISIONERS # List the storage classes sc_list = [ sc for sc in get_all_storageclass() if sc.get('provisioner') in provisioners ] # Query for PVCs and OBCs that are using the storage class provisioners listed in the previous step. pvc_to_delete = [] for sc in sc_list: pvc_to_delete.extend(pvc for pvc in get_all_pvcs_in_storageclass( sc.get('metadata').get('name')) if 'noobaa' not in pvc.name) log.info("Removing monitoring stack from OpenShift Container Storage") remove_monitoring_stack_from_ocs() log.info( "Removing OpenShift Container Platform registry from OpenShift Container Storage" ) remove_ocp_registry_from_ocs(config.ENV_DATA['platform']) log.info( "Removing the cluster logging operator from OpenShift Container Storage" ) try: remove_cluster_logging_operator_from_ocs() except CommandFailed: log.info("No cluster logging found") log.info("Deleting pvcs") for pvc in pvc_to_delete: log.info(f"Deleting pvc: {pvc.name}") pvc.delete() storage_cluster = ocp.OCP(kind=constants.STORAGECLUSTER, resource_name=constants.DEFAULT_CLUSTERNAME, namespace='openshift-storage') log.info("Checking for local storage") lso_sc = None if check_local_volume(): "Local volume was found. Will be removed later" lso_sc = storage_cluster.get().get('spec').get('storageDeviceSets')[ 0].get('dataPVCTemplate').get('spec').get('storageClassName') log.info("Deleting storageCluster object") storage_cluster.delete(resource_name=constants.DEFAULT_CLUSTERNAME) log.info("Removing CRDs") crd_list = [ 'backingstores.noobaa.io', 'bucketclasses.noobaa.io', 'cephblockpools.ceph.rook.io', 'cephfilesystems.ceph.rook.io', 'cephnfses.ceph.rook.io', 'cephobjectstores.ceph.rook.io', 'cephobjectstoreusers.ceph.rook.io', 'noobaas.noobaa.io', 'ocsinitializations.ocs.openshift.io', 'storageclusterinitializations.ocs.openshift.io', 'storageclusters.ocs.openshift.io', 'cephclusters.ceph.rook.io' ] for crd in crd_list: ocp_obj.exec_oc_cmd(f"delete crd {crd} --timeout=300m") log.info("Deleting openshift-storage namespace") ocp_obj.delete_project('openshift-storage') ocp_obj.wait_for_delete('openshift-storage') switch_to_project("default") log.info("Removing rook directory from nodes") nodes_list = get_labeled_nodes(constants.OPERATOR_NODE_LABEL) for node in nodes_list: log.info(f"Removing rook from {node}") ocp_obj.exec_oc_debug_cmd(node=node, cmd_list=["rm -rf /var/lib/rook"]) log.info("Removing LSO ") if lso_sc is not None: uninstall_lso(lso_sc) log.info( "Delete the storage classes with an openshift-storage provisioner list" ) for storage_class in sc_list: log.info( f"Deleting storage class {storage_class.get('metadata').get('name')}" ) sc_obj = ocp.OCP(kind=constants.STORAGECLASS) sc_obj.delete(resource_name=storage_class.get('metadata').get('name')) log.info("Unlabeling storage nodes") nodes_list = get_all_nodes() for node in nodes_list: node_obj = ocp.OCP(kind=constants.NODE, resource_name=node) node_obj.add_label(resource_name=node, label=constants.OPERATOR_NODE_LABEL[:-3] + '-') node_obj.add_label(resource_name=node, label=constants.TOPOLOGY_ROOK_LABEL + '-') log.info("OCS was removed successfully from cluster ")
def uninstall_ocs(): """ The function uninstalls the OCS operator from a openshift cluster and removes all its settings and dependencies """ ocp_obj = ocp.OCP() provisioners = constants.OCS_PROVISIONERS # List the storage classes sc_list = get_all_storageclass() sc_name_list = [] for storage_class in sc_list: if storage_class.get('provisioner') not in provisioners: sc_list.remove(storage_class) else: sc_name_list.append(storage_class.get('metadata').get('name')) # Query for PVCs and OBCs that are using the storage class provisioners listed in the previous step. pvc_to_delete = [] pvc_name_list = [] for sc in sc_name_list: pvc_to_delete.extend(get_all_pvcs_in_storageclass(sc)) # ignoring all noobaa pvcs & make name list for pvc in pvc_to_delete: if "noobaa" in pvc.name: pvc_to_delete.remove(pvc) else: pvc_name_list.append(pvc.name) pods_to_delete = [] all_pods = get_all_pods() # default openshift-storage namespace all_pods.extend(get_all_pods(namespace=constants.OPENSHIFT_IMAGE_REGISTRY_NAMESPACE)) all_pods.extend(get_all_pods(namespace=constants.OPENSHIFT_MONITORING_NAMESPACE)) for pod_obj in all_pods: try: pvc_name = get_pvc_name(pod_obj) except UnavailableResourceException: continue if pvc_name in pvc_name_list: pods_to_delete.append(pod_obj) log.info("Removing monitoring stack from OpenShift Container Storage") remove_monitoring_stack_from_ocs() log.info("Removing OpenShift Container Platform registry from OpenShift Container Storage") remove_ocp_registry_from_ocs(config.ENV_DATA['platform']) log.info("Removing the cluster logging operator from OpenShift Container Storage") csv = ocp.OCP( kind=constants.CLUSTER_SERVICE_VERSION, namespace=constants.OPENSHIFT_LOGGING_NAMESPACE ) logging_csv = csv.get().get('items') if logging_csv: clusterlogging_obj = ocp.OCP( kind=constants.CLUSTER_LOGGING, namespace=constants.OPENSHIFT_LOGGING_NAMESPACE ) clusterlogging_obj.delete(resource_name='instance') log.info("deleting pvcs") for pvc in pvc_to_delete: log.info(f"deleting pvc: {pvc.name}") pvc.delete() log.info("deleting pods") for pod in pods_to_delete: log.info(f"deleting pod {pod.name}") pod.delete() log.info("removing rook directory from nodes") nodes_list = get_labeled_nodes(constants.OPERATOR_NODE_LABEL) for node in nodes_list: log.info(f"removing rook from {node}") ocp_obj.exec_oc_debug_cmd(node=node, cmd_list=["rm -rf /var/lib/rook"]) log.info("Delete the storage classes with an openshift-storage provisioner list") for storage_class in sc_list: log.info(f"deleting storage class {storage_class.get('metadata').get('name')}") sc_obj = ocp.OCP(kind=constants.STORAGECLASS) sc_obj.delete(resource_name=storage_class.get('metadata').get('name')) log.info("unlabaling storage nodes") nodes_list = get_all_nodes() for node in nodes_list: node_obj = ocp.OCP(kind=constants.NODE, resource_name=node) node_obj.add_label(resource_name=node, label=constants.OPERATOR_NODE_LABEL[:-3] + '-') node_obj.add_label(resource_name=node, label=constants.TOPOLOGY_ROOK_LABEL + '-') log.info("deleting storageCluster object") storage_cluster = ocp.OCP(kind=constants.STORAGECLUSTER, resource_name=constants.DEFAULT_CLUSTERNAME) storage_cluster.delete(resource_name=constants.DEFAULT_CLUSTERNAME) log.info("removing CRDs") crd_list = ['backingstores.noobaa.io', 'bucketclasses.noobaa.io', 'cephblockpools.ceph.rook.io', 'cephfilesystems.ceph.rook.io', 'cephnfses.ceph.rook.io', 'cephobjectstores.ceph.rook.io', 'cephobjectstoreusers.ceph.rook.io', 'noobaas.noobaa.io', 'ocsinitializations.ocs.openshift.io', 'storageclusterinitializations.ocs.openshift.io', 'storageclusters.ocs.openshift.io', 'cephclusters.ceph.rook.io'] for crd in crd_list: ocp_obj.exec_oc_cmd(f"delete crd {crd} --timeout=300m") log.info("deleting openshift-storage namespace") ocp_obj.delete_project('openshift-storage') ocp_obj.wait_for_delete('openshift-storage')