def wait_for_all_nodes_csr(timeout=900, sleep=10):
"""
Wait for CSR to generate for nodes
Args:
timeout (int): Time in seconds to wait
sleep (int): Sampling time in seconds
Returns:
bool: True if all nodes are generated CSR
Raises:
TimeoutExpiredError: in case CSR not found
"""
pending = False
all_nodes = get_all_nodes()
for csr_nodes in TimeoutSampler(
timeout=timeout, sleep=sleep, func=get_nodes_csr
):
pending_nodes = []
logger.debug(f"CSR data: {csr_nodes}")
for node in all_nodes:
if node not in csr_nodes.keys():
logger.info(f"{node} CSR is not generated")
pending = True
pending_nodes.append(node)
if not pending:
logger.info("CSR generated for all nodes in cluster")
return
logger.warning(
f"Nodes {pending_nodes} are not generated CSR. retrying again"
)
def create_instance_in_clusterlogging():
"""
Creation of instance for clusterlogging that creates PVCs,
ElasticSearch, curator fluentd and kibana pods and checks for all
the pods and PVCs
Args:
sc_name (str): Storage class name to create PVCs
Returns:
dict: Contains all detailed information of the
instance such as pods that got created, its resources and limits
values, storage class and size details etc.
"""
nodes_in_cluster = len(get_all_nodes())
inst_data = templating.load_yaml(constants.CL_INSTANCE_YAML)
es_node_count = inst_data["spec"]["logStore"]["elasticsearch"]["nodeCount"]
if helpers.storagecluster_independent_check():
inst_data["spec"]["logStore"]["elasticsearch"]["storage"][
"storageClassName"
] = constants.DEFAULT_EXTERNAL_MODE_STORAGECLASS_RBD
helpers.create_resource(wait=False, **inst_data)
oc = ocp.OCP("v1", "ClusterLogging", "openshift-logging")
logging_instance = oc.get(resource_name="instance", out_yaml_format="True")
if logging_instance:
logger.info("Successfully created instance for cluster-logging")
logger.debug(logging_instance)
else:
logger.error("Instance for clusterlogging is not created properly")
pod_obj = ocp.OCP(kind=constants.POD, namespace="openshift-logging")
pod_status = pod_obj.wait_for_resource(
condition=constants.STATUS_RUNNING,
resource_count=2 + es_node_count + nodes_in_cluster,
timeout=500,
sleep=2,
)
assert pod_status, "Pods are not in Running state."
logger.info("All pods are in Running state")
pvc_obj = ocp.OCP(kind=constants.PVC, namespace="openshift-logging")
pvc_status = pvc_obj.wait_for_resource(
condition=constants.STATUS_BOUND,
resource_count=es_node_count,
timeout=150,
sleep=5,
)
assert pvc_status, "PVCs are not in bound state."
logger.info("PVCs are Bound")
return logging_instance
def create_instance_in_clusterlogging():
"""
Creation of instance for clusterlogging that creates PVCs,
ElasticSearch, curator fluentd and kibana pods and checks for all
the pods and PVCs
Args:
sc_name (str): Storage class name to create PVCs
Returns:
dict: Contains all detailed information of the
instance such as pods that got created, its resources and limits
values, storage class and size details etc.
"""
nodes_in_cluster = len(get_all_nodes())
inst_data = templating.load_yaml(constants.CL_INSTANCE_YAML)
es_node_count = inst_data['spec']['logStore']['elasticsearch']['nodeCount']
helpers.create_resource(wait=False, **inst_data)
oc = ocp.OCP('v1', 'ClusterLogging', 'openshift-logging')
logging_instance = oc.get(resource_name='instance', out_yaml_format='True')
if logging_instance:
logger.info("Successfully created instance for cluster-logging")
logger.debug(logging_instance)
else:
logger.error("Instance for clusterlogging is not created properly")
pod_obj = ocp.OCP(kind=constants.POD, namespace='openshift-logging')
pod_status = pod_obj.wait_for_resource(condition=constants.STATUS_RUNNING,
resource_count=2 + es_node_count +
nodes_in_cluster,
timeout=500,
sleep=2)
assert pod_status, "Pods are not in Running state."
logger.info("All pods are in Running state")
pvc_obj = ocp.OCP(kind=constants.PVC, namespace='openshift-logging')
pvc_status = pvc_obj.wait_for_resource(condition=constants.STATUS_BOUND,
resource_count=es_node_count,
timeout=150,
sleep=5)
assert pvc_status, "PVCs are not in bound state."
logger.info("PVCs are Bound")
return logging_instance
def uninstall_ocs():
"""
The function uninstalls the OCS operator from a openshift
cluster and removes all its settings and dependencies
"""
ocp_obj = ocp.OCP()
log.info("deleting volume snapshots")
vs_ocp_obj = ocp.OCP(kind=constants.VOLUMESNAPSHOT)
vs_list = vs_ocp_obj.get(all_namespaces=True)["items"]
for vs in vs_list:
vs_obj = ocp.OCP(kind=constants.VOLUMESNAPSHOT,
namespace=vs.get("metadata").get("namespace"))
vs_obj.delete(resource_name=vs.get("metadata").get("name"))
log.info("queering for OCS PVCs")
provisioners = constants.OCS_PROVISIONERS
sc_list = [
sc for sc in get_all_storageclass()
if sc.get("provisioner") in provisioners
]
pvc_to_delete = []
for sc in sc_list:
pvc_to_delete.extend(pvc for pvc in get_all_pvcs_in_storageclass(
sc.get("metadata").get("name")) if "noobaa" not in pvc.name)
if config.ENV_DATA["platform"].lower() == constants.ROSA_PLATFORM:
log.info("Deleting OCS PVCs")
for pvc in pvc_to_delete:
log.info(f"Deleting PVC: {pvc.name}")
pvc.delete()
rosa.delete_odf_addon(config.ENV_DATA["cluster_name"])
return None
log.info("Removing monitoring stack from OpenShift Container Storage")
remove_monitoring_stack_from_ocs()
log.info(
"Removing OpenShift Container Platform registry from OpenShift Container Storage"
)
remove_ocp_registry_from_ocs(config.ENV_DATA["platform"])
log.info(
"Removing the cluster logging operator from OpenShift Container Storage"
)
try:
remove_cluster_logging_operator_from_ocs()
except CommandFailed:
log.info("No cluster logging found")
log.info("Deleting OCS PVCs")
for pvc in pvc_to_delete:
log.info(f"Deleting PVC: {pvc.name}")
pvc.delete()
storage_cluster = ocp.OCP(
kind=constants.STORAGECLUSTER,
resource_name=constants.DEFAULT_CLUSTERNAME,
namespace="openshift-storage",
)
log.info("Checking for local storage")
lso_sc = None
if check_local_volume_local_volume_set():
"Local volume was found. Will be removed later"
lso_sc = (storage_cluster.get().get("spec").get("storageDeviceSets")[0]
.get("dataPVCTemplate").get("spec").get("storageClassName"))
cleanup_policy = (storage_cluster.get().get("metadata").get(
"annotations").get("uninstall.ocs.openshift.io/cleanup-policy"))
log.info("Deleting storageCluster object")
storage_cluster.delete(resource_name=constants.DEFAULT_CLUSTERNAME)
if cleanup_policy == "delete":
log.info("Cleanup policy set to delete. checking cleanup pods")
cleanup_pods = [
pod for pod in get_all_pods() if "cluster-cleanup-job" in pod.name
]
for pod in cleanup_pods:
while pod.get().get("status").get("phase") != "Succeeded":
log.info(f"waiting for cleanup pod {pod.name} to complete")
TimeoutSampler(timeout=10, sleep=30)
log.info(f"Cleanup pod {pod.name} completed successfully ")
# no need to confirm var/vib/rook was deleted from nodes if all cleanup pods are completed.
else:
log.info("Cleanup policy set to retain. skipping nodes cleanup")
log.info("Deleting openshift-storage namespace")
ocp_obj.delete_project(constants.OPENSHIFT_STORAGE_NAMESPACE)
ocp_obj.wait_for_delete(constants.OPENSHIFT_STORAGE_NAMESPACE)
switch_to_project(constants.DEFAULT_NAMESPACE)
# step 10: TODO remove crypto from nodes.
"""for node in storage_node_list:
log.info(f"removing encryption from {node}")
ocp_obj.exec_oc_debug_cmd(node=node, cmd_list=[])"""
if lso_sc is not None:
log.info("Removing LSO")
try:
uninstall_lso(lso_sc)
except Exception as e:
log.info(f"LSO removal failed.{e}")
log.info("deleting noobaa storage class")
noobaa_sc = ocp.OCP(kind=constants.STORAGECLASS)
noobaa_sc.delete(resource_name=constants.NOOBAA_SC)
nodes = get_all_nodes()
node_objs = get_node_objs(nodes)
log.info("Unlabeling storage nodes")
label_nodes(nodes=node_objs,
label=constants.OPERATOR_NODE_LABEL[:-3] + "-")
label_nodes(nodes=node_objs, label=constants.TOPOLOGY_ROOK_LABEL + "-")
log.info("Removing taints from storage nodes")
taint_nodes(nodes=nodes, taint_label=constants.OPERATOR_NODE_TAINT + "-")
log.info("Deleting remaining OCS PVs (if there are any)")
try:
rbd_pv = ocp.OCP(kind=constants.PV,
resource_name="ocs-storagecluster-ceph-rbd")
fs_pv = ocp.OCP(kind=constants.PV,
resource_name="ocs-storagecluster-cephfs")
rbd_pv.delete()
fs_pv.delete()
log.info("OCS PVs deleted")
except Exception as e:
log.info(f"OCS PV(s) not found. {e}")
log.info("Removing CRDs")
crd_list = [
"backingstores.noobaa.io",
"bucketclasses.noobaa.io",
"cephblockpools.ceph.rook.io",
"cephclusters.ceph.rook.io",
"cephfilesystems.ceph.rook.io",
"cephnfses.ceph.rook.io",
"cephobjectstores.ceph.rook.io",
"cephobjectstoreusers.ceph.rook.io",
"noobaas.noobaa.io",
"ocsinitializations.ocs.openshift.io",
"storageclusters.ocs.openshift.io",
"cephclients.ceph.rook.io",
"cephobjectrealms.ceph.rook.io",
"cephobjectzonegroups.ceph.rook.io",
"cephobjectzones.ceph.rook.io",
"cephrbdmirrors.ceph.rook.io",
]
for crd in crd_list:
try:
ocp_obj.exec_oc_cmd(f"delete crd {crd} --timeout=300m")
except Exception:
log.info(f"crd {crd} was not found")
def uninstall_ocs():
"""
The function uninstalls the OCS operator from a openshift
cluster and removes all its settings and dependencies
"""
ocp_obj = ocp.OCP()
provisioners = constants.OCS_PROVISIONERS
# List the storage classes
sc_list = [
sc for sc in get_all_storageclass()
if sc.get('provisioner') in provisioners
]
# Query for PVCs and OBCs that are using the storage class provisioners listed in the previous step.
pvc_to_delete = []
for sc in sc_list:
pvc_to_delete.extend(pvc for pvc in get_all_pvcs_in_storageclass(
sc.get('metadata').get('name')) if 'noobaa' not in pvc.name)
log.info("Removing monitoring stack from OpenShift Container Storage")
remove_monitoring_stack_from_ocs()
log.info(
"Removing OpenShift Container Platform registry from OpenShift Container Storage"
)
remove_ocp_registry_from_ocs(config.ENV_DATA['platform'])
log.info(
"Removing the cluster logging operator from OpenShift Container Storage"
)
try:
remove_cluster_logging_operator_from_ocs()
except CommandFailed:
log.info("No cluster logging found")
log.info("Deleting pvcs")
for pvc in pvc_to_delete:
log.info(f"Deleting pvc: {pvc.name}")
pvc.delete()
storage_cluster = ocp.OCP(kind=constants.STORAGECLUSTER,
resource_name=constants.DEFAULT_CLUSTERNAME,
namespace='openshift-storage')
log.info("Checking for local storage")
lso_sc = None
if check_local_volume():
"Local volume was found. Will be removed later"
lso_sc = storage_cluster.get().get('spec').get('storageDeviceSets')[
0].get('dataPVCTemplate').get('spec').get('storageClassName')
log.info("Deleting storageCluster object")
storage_cluster.delete(resource_name=constants.DEFAULT_CLUSTERNAME)
log.info("Removing CRDs")
crd_list = [
'backingstores.noobaa.io', 'bucketclasses.noobaa.io',
'cephblockpools.ceph.rook.io', 'cephfilesystems.ceph.rook.io',
'cephnfses.ceph.rook.io', 'cephobjectstores.ceph.rook.io',
'cephobjectstoreusers.ceph.rook.io', 'noobaas.noobaa.io',
'ocsinitializations.ocs.openshift.io',
'storageclusterinitializations.ocs.openshift.io',
'storageclusters.ocs.openshift.io', 'cephclusters.ceph.rook.io'
]
for crd in crd_list:
ocp_obj.exec_oc_cmd(f"delete crd {crd} --timeout=300m")
log.info("Deleting openshift-storage namespace")
ocp_obj.delete_project('openshift-storage')
ocp_obj.wait_for_delete('openshift-storage')
switch_to_project("default")
log.info("Removing rook directory from nodes")
nodes_list = get_labeled_nodes(constants.OPERATOR_NODE_LABEL)
for node in nodes_list:
log.info(f"Removing rook from {node}")
ocp_obj.exec_oc_debug_cmd(node=node, cmd_list=["rm -rf /var/lib/rook"])
log.info("Removing LSO ")
if lso_sc is not None:
uninstall_lso(lso_sc)
log.info(
"Delete the storage classes with an openshift-storage provisioner list"
)
for storage_class in sc_list:
log.info(
f"Deleting storage class {storage_class.get('metadata').get('name')}"
)
sc_obj = ocp.OCP(kind=constants.STORAGECLASS)
sc_obj.delete(resource_name=storage_class.get('metadata').get('name'))
log.info("Unlabeling storage nodes")
nodes_list = get_all_nodes()
for node in nodes_list:
node_obj = ocp.OCP(kind=constants.NODE, resource_name=node)
node_obj.add_label(resource_name=node,
label=constants.OPERATOR_NODE_LABEL[:-3] + '-')
node_obj.add_label(resource_name=node,
label=constants.TOPOLOGY_ROOK_LABEL + '-')
log.info("OCS was removed successfully from cluster ")
def uninstall_ocs():
"""
The function uninstalls the OCS operator from a openshift
cluster and removes all its settings and dependencies
"""
ocp_obj = ocp.OCP()
provisioners = constants.OCS_PROVISIONERS
# List the storage classes
sc_list = get_all_storageclass()
sc_name_list = []
for storage_class in sc_list:
if storage_class.get('provisioner') not in provisioners:
sc_list.remove(storage_class)
else:
sc_name_list.append(storage_class.get('metadata').get('name'))
# Query for PVCs and OBCs that are using the storage class provisioners listed in the previous step.
pvc_to_delete = []
pvc_name_list = []
for sc in sc_name_list:
pvc_to_delete.extend(get_all_pvcs_in_storageclass(sc))
# ignoring all noobaa pvcs & make name list
for pvc in pvc_to_delete:
if "noobaa" in pvc.name:
pvc_to_delete.remove(pvc)
else:
pvc_name_list.append(pvc.name)
pods_to_delete = []
all_pods = get_all_pods() # default openshift-storage namespace
all_pods.extend(get_all_pods(namespace=constants.OPENSHIFT_IMAGE_REGISTRY_NAMESPACE))
all_pods.extend(get_all_pods(namespace=constants.OPENSHIFT_MONITORING_NAMESPACE))
for pod_obj in all_pods:
try:
pvc_name = get_pvc_name(pod_obj)
except UnavailableResourceException:
continue
if pvc_name in pvc_name_list:
pods_to_delete.append(pod_obj)
log.info("Removing monitoring stack from OpenShift Container Storage")
remove_monitoring_stack_from_ocs()
log.info("Removing OpenShift Container Platform registry from OpenShift Container Storage")
remove_ocp_registry_from_ocs(config.ENV_DATA['platform'])
log.info("Removing the cluster logging operator from OpenShift Container Storage")
csv = ocp.OCP(
kind=constants.CLUSTER_SERVICE_VERSION,
namespace=constants.OPENSHIFT_LOGGING_NAMESPACE
)
logging_csv = csv.get().get('items')
if logging_csv:
clusterlogging_obj = ocp.OCP(
kind=constants.CLUSTER_LOGGING, namespace=constants.OPENSHIFT_LOGGING_NAMESPACE
)
clusterlogging_obj.delete(resource_name='instance')
log.info("deleting pvcs")
for pvc in pvc_to_delete:
log.info(f"deleting pvc: {pvc.name}")
pvc.delete()
log.info("deleting pods")
for pod in pods_to_delete:
log.info(f"deleting pod {pod.name}")
pod.delete()
log.info("removing rook directory from nodes")
nodes_list = get_labeled_nodes(constants.OPERATOR_NODE_LABEL)
for node in nodes_list:
log.info(f"removing rook from {node}")
ocp_obj.exec_oc_debug_cmd(node=node, cmd_list=["rm -rf /var/lib/rook"])
log.info("Delete the storage classes with an openshift-storage provisioner list")
for storage_class in sc_list:
log.info(f"deleting storage class {storage_class.get('metadata').get('name')}")
sc_obj = ocp.OCP(kind=constants.STORAGECLASS)
sc_obj.delete(resource_name=storage_class.get('metadata').get('name'))
log.info("unlabaling storage nodes")
nodes_list = get_all_nodes()
for node in nodes_list:
node_obj = ocp.OCP(kind=constants.NODE, resource_name=node)
node_obj.add_label(resource_name=node, label=constants.OPERATOR_NODE_LABEL[:-3] + '-')
node_obj.add_label(resource_name=node, label=constants.TOPOLOGY_ROOK_LABEL + '-')
log.info("deleting storageCluster object")
storage_cluster = ocp.OCP(kind=constants.STORAGECLUSTER, resource_name=constants.DEFAULT_CLUSTERNAME)
storage_cluster.delete(resource_name=constants.DEFAULT_CLUSTERNAME)
log.info("removing CRDs")
crd_list = ['backingstores.noobaa.io', 'bucketclasses.noobaa.io', 'cephblockpools.ceph.rook.io',
'cephfilesystems.ceph.rook.io', 'cephnfses.ceph.rook.io',
'cephobjectstores.ceph.rook.io', 'cephobjectstoreusers.ceph.rook.io', 'noobaas.noobaa.io',
'ocsinitializations.ocs.openshift.io', 'storageclusterinitializations.ocs.openshift.io',
'storageclusters.ocs.openshift.io', 'cephclusters.ceph.rook.io']
for crd in crd_list:
ocp_obj.exec_oc_cmd(f"delete crd {crd} --timeout=300m")
log.info("deleting openshift-storage namespace")
ocp_obj.delete_project('openshift-storage')
ocp_obj.wait_for_delete('openshift-storage')
