def add_capacity(osd_size_capacity_requested): """ Add storage capacity to the cluster Args: osd_size_capacity_requested(int): Requested osd size capacity Returns: new storage device set count (int) : Returns True if all OSDs are in Running state Note: "StoragedeviceSets->count" represents the set of 3 OSDs. That is, if there are 3 OSDs in the system then count will be 1. If there are 6 OSDs then count is 2 and so on. By changing this value,we can add extra devices to the cluster. For example, if we want to expand the cluster by 3 more osds in a cluster that already has 3 osds, we can set count as 2. So, with each increase of count by 1, we get 3 OSDs extra added to the cluster. This is how we are going to 'add capacity' via automation. As we know that OCS has 3 way replica. That is, same data is placed in 3 OSDs. Because of this, the total usable capacity for apps from 3 OSDs will be the size of one OSD (all osds are of same size). If we want to add more capacity to the cluster then we need to add 3 OSDs of same size as that of the original OSD. add_capacity needs to accept the 'capacity_to_add' as an argument. From this we need to arrive at storagedeviceSets -> count and then "Patch" this count to get the required capacity to add. To do so, we use following formula: storageDeviceSets->count = (capacity reqested / osd capacity ) + existing count storageDeviceSets """ osd_size_existing = get_osd_size() device_sets_required = int(osd_size_capacity_requested / osd_size_existing) old_storage_devices_sets_count = get_deviceset_count() new_storage_devices_sets_count = int(device_sets_required + old_storage_devices_sets_count) lvpresent = localstorage.check_local_volume() if lvpresent: final_device_list = localstorage.get_new_device_paths(device_sets_required, osd_size_capacity_requested) param = f"""[{{ "op": "replace", "path": "/spec/storageClassDevices/0/devicePaths", "value": {final_device_list}}}]""" log.info(f"Final device list : {final_device_list}") lvcr = localstorage.get_local_volume_cr() log.info("Patching Local Volume CR...") lvcr.patch( resource_name=lvcr.get()['items'][0]['metadata']['name'], params=param.strip('\n'), format_type='json' ) localstorage.check_pvs_created(int(len(final_device_list) / new_storage_devices_sets_count)) sc = get_storage_cluster() # adding the storage capacity to the cluster params = f"""[{{ "op": "replace", "path": "/spec/storageDeviceSets/0/count", "value": {new_storage_devices_sets_count}}}]""" sc.patch( resource_name=sc.get()['items'][0]['metadata']['name'], params=params.strip('\n'), format_type='json' ) return new_storage_devices_sets_count
def uninstall_ocs(): """ The function uninstalls the OCS operator from a openshift cluster and removes all its settings and dependencies """ ocp_obj = ocp.OCP() log.info("deleting volume snapshots") vs_ocp_obj = ocp.OCP(kind=constants.VOLUMESNAPSHOT) vs_list = vs_ocp_obj.get(all_namespaces=True)["items"] for vs in vs_list: vs_obj = ocp.OCP( kind=constants.VOLUMESNAPSHOT, namespace=vs.get("metadata").get("namespace") ) vs_obj.delete(resource_name=vs.get("metadata").get("name")) log.info("queering for OCS PVCs") provisioners = constants.OCS_PROVISIONERS sc_list = [ sc for sc in get_all_storageclass() if sc.get("provisioner") in provisioners ] pvc_to_delete = [] for sc in sc_list: pvc_to_delete.extend( pvc for pvc in get_all_pvcs_in_storageclass(sc.get("metadata").get("name")) if "noobaa" not in pvc.name ) log.info("Removing monitoring stack from OpenShift Container Storage") remove_monitoring_stack_from_ocs() log.info( "Removing OpenShift Container Platform registry from OpenShift Container Storage" ) remove_ocp_registry_from_ocs(config.ENV_DATA["platform"]) log.info("Removing the cluster logging operator from OpenShift Container Storage") try: remove_cluster_logging_operator_from_ocs() except CommandFailed: log.info("No cluster logging found") log.info("Deleting OCS PVCs") for pvc in pvc_to_delete: log.info(f"Deleting PVC: {pvc.name}") pvc.delete() storage_cluster = ocp.OCP( kind=constants.STORAGECLUSTER, resource_name=constants.DEFAULT_CLUSTERNAME, namespace="openshift-storage", ) log.info("Checking for local storage") lso_sc = None if check_local_volume(): "Local volume was found. Will be removed later" lso_sc = ( storage_cluster.get() .get("spec") .get("storageDeviceSets")[0] .get("dataPVCTemplate") .get("spec") .get("storageClassName") ) cleanup_policy = ( storage_cluster.get() .get("metadata") .get("annotations") .get("uninstall.ocs.openshift.io/cleanup-policy") ) log.info("Deleting storageCluster object") storage_cluster.delete(resource_name=constants.DEFAULT_CLUSTERNAME) if cleanup_policy == "delete": log.info("Cleanup policy set to delete. checking cleanup pods") cleanup_pods = [ pod for pod in get_all_pods() if "cluster-cleanup-job" in pod.name ] for pod in cleanup_pods: while pod.get().get("status").get("phase") != "Succeeded": log.info(f"waiting for cleanup pod {pod.name} to complete") TimeoutSampler(timeout=10, sleep=30) log.info(f"Cleanup pod {pod.name} completed successfully ") # no need to confirm var/vib/rook was deleted from nodes if all cleanup pods are completed. else: log.info("Cleanup policy set to retain. skipping nodes cleanup") log.info("Deleting openshift-storage namespace") ocp_obj.delete_project(constants.OPENSHIFT_STORAGE_NAMESPACE) ocp_obj.wait_for_delete(constants.OPENSHIFT_STORAGE_NAMESPACE) switch_to_project(constants.DEFAULT_NAMESPACE) # step 10: TODO remove crypto from nodes. """for node in storage_node_list: log.info(f"removing encryption from {node}") ocp_obj.exec_oc_debug_cmd(node=node, cmd_list=[])""" if lso_sc is not None: log.info("Removing LSO") try: uninstall_lso(lso_sc) except Exception as e: log.info(f"LSO removal failed.{e}") log.info("deleting noobaa storage class") noobaa_sc = ocp.OCP(kind=constants.STORAGECLASS) noobaa_sc.delete(resource_name=constants.NOOBAA_SC) nodes = get_all_nodes() node_objs = get_node_objs(nodes) log.info("Unlabeling storage nodes") label_nodes(nodes=node_objs, label=constants.OPERATOR_NODE_LABEL[:-3] + "-") label_nodes(nodes=node_objs, label=constants.TOPOLOGY_ROOK_LABEL + "-") log.info("Removing taints from storage nodes") taint_nodes(nodes=nodes, taint_label=constants.OCS_TAINT + "-") log.info("Deleting remaining OCS PVs (if there are any)") try: rbd_pv = ocp.OCP(kind=constants.PV, resource_name="ocs-storagecluster-ceph-rbd") fs_pv = ocp.OCP(kind=constants.PV, resource_name="ocs-storagecluster-cephfs") rbd_pv.delete() fs_pv.delete() log.info("OCS PVs deleted") except Exception as e: log.info(f"OCS PV(s) not found. {e}") log.info("Removing CRDs") crd_list = [ "backingstores.noobaa.io", "bucketclasses.noobaa.io", "cephblockpools.ceph.rook.io", "cephclusters.ceph.rook.io", "cephfilesystems.ceph.rook.io", "cephnfses.ceph.rook.io", "cephobjectstores.ceph.rook.io", "cephobjectstoreusers.ceph.rook.io", "noobaas.noobaa.io", "ocsinitializations.ocs.openshift.io", "storageclusters.ocs.openshift.io", "cephclients.ceph.rook.io", "cephobjectrealms.ceph.rook.io", "cephobjectzonegroups.ceph.rook.io", "cephobjectzones.ceph.rook.io", "cephrbdmirrors.ceph.rook.io", ] for crd in crd_list: try: ocp_obj.exec_oc_cmd(f"delete crd {crd} --timeout=300m") except Exception: log.info(f"crd {crd} was not found")
def uninstall_ocs(): """ The function uninstalls the OCS operator from a openshift cluster and removes all its settings and dependencies """ ocp_obj = ocp.OCP() provisioners = constants.OCS_PROVISIONERS # List the storage classes sc_list = [ sc for sc in get_all_storageclass() if sc.get('provisioner') in provisioners ] # Query for PVCs and OBCs that are using the storage class provisioners listed in the previous step. pvc_to_delete = [] for sc in sc_list: pvc_to_delete.extend(pvc for pvc in get_all_pvcs_in_storageclass( sc.get('metadata').get('name')) if 'noobaa' not in pvc.name) log.info("Removing monitoring stack from OpenShift Container Storage") remove_monitoring_stack_from_ocs() log.info( "Removing OpenShift Container Platform registry from OpenShift Container Storage" ) remove_ocp_registry_from_ocs(config.ENV_DATA['platform']) log.info( "Removing the cluster logging operator from OpenShift Container Storage" ) try: remove_cluster_logging_operator_from_ocs() except CommandFailed: log.info("No cluster logging found") log.info("Deleting pvcs") for pvc in pvc_to_delete: log.info(f"Deleting pvc: {pvc.name}") pvc.delete() storage_cluster = ocp.OCP(kind=constants.STORAGECLUSTER, resource_name=constants.DEFAULT_CLUSTERNAME, namespace='openshift-storage') log.info("Checking for local storage") lso_sc = None if check_local_volume(): "Local volume was found. Will be removed later" lso_sc = storage_cluster.get().get('spec').get('storageDeviceSets')[ 0].get('dataPVCTemplate').get('spec').get('storageClassName') log.info("Deleting storageCluster object") storage_cluster.delete(resource_name=constants.DEFAULT_CLUSTERNAME) log.info("Removing CRDs") crd_list = [ 'backingstores.noobaa.io', 'bucketclasses.noobaa.io', 'cephblockpools.ceph.rook.io', 'cephfilesystems.ceph.rook.io', 'cephnfses.ceph.rook.io', 'cephobjectstores.ceph.rook.io', 'cephobjectstoreusers.ceph.rook.io', 'noobaas.noobaa.io', 'ocsinitializations.ocs.openshift.io', 'storageclusterinitializations.ocs.openshift.io', 'storageclusters.ocs.openshift.io', 'cephclusters.ceph.rook.io' ] for crd in crd_list: ocp_obj.exec_oc_cmd(f"delete crd {crd} --timeout=300m") log.info("Deleting openshift-storage namespace") ocp_obj.delete_project('openshift-storage') ocp_obj.wait_for_delete('openshift-storage') switch_to_project("default") log.info("Removing rook directory from nodes") nodes_list = get_labeled_nodes(constants.OPERATOR_NODE_LABEL) for node in nodes_list: log.info(f"Removing rook from {node}") ocp_obj.exec_oc_debug_cmd(node=node, cmd_list=["rm -rf /var/lib/rook"]) log.info("Removing LSO ") if lso_sc is not None: uninstall_lso(lso_sc) log.info( "Delete the storage classes with an openshift-storage provisioner list" ) for storage_class in sc_list: log.info( f"Deleting storage class {storage_class.get('metadata').get('name')}" ) sc_obj = ocp.OCP(kind=constants.STORAGECLASS) sc_obj.delete(resource_name=storage_class.get('metadata').get('name')) log.info("Unlabeling storage nodes") nodes_list = get_all_nodes() for node in nodes_list: node_obj = ocp.OCP(kind=constants.NODE, resource_name=node) node_obj.add_label(resource_name=node, label=constants.OPERATOR_NODE_LABEL[:-3] + '-') node_obj.add_label(resource_name=node, label=constants.TOPOLOGY_ROOK_LABEL + '-') log.info("OCS was removed successfully from cluster ")
def add_capacity(osd_size_capacity_requested): """ Add storage capacity to the cluster Args: osd_size_capacity_requested(int): Requested osd size capacity Returns: new storage device set count (int) : Returns True if all OSDs are in Running state Note: "StoragedeviceSets->count" represents the set of 3 OSDs. That is, if there are 3 OSDs in the system then count will be 1. If there are 6 OSDs then count is 2 and so on. By changing this value,we can add extra devices to the cluster. For example, if we want to expand the cluster by 3 more osds in a cluster that already has 3 osds, we can set count as 2. So, with each increase of count by 1, we get 3 OSDs extra added to the cluster. This is how we are going to 'add capacity' via automation. As we know that OCS has 3 way replica. That is, same data is placed in 3 OSDs. Because of this, the total usable capacity for apps from 3 OSDs will be the size of one OSD (all osds are of same size). If we want to add more capacity to the cluster then we need to add 3 OSDs of same size as that of the original OSD. add_capacity needs to accept the 'capacity_to_add' as an argument. From this we need to arrive at storagedeviceSets -> count and then "Patch" this count to get the required capacity to add. To do so, we use following formula: storageDeviceSets->count = (capacity reqested / osd capacity ) + existing count storageDeviceSets """ osd_size_existing = get_osd_size() device_sets_required = int(osd_size_capacity_requested / osd_size_existing) old_storage_devices_sets_count = get_deviceset_count() new_storage_devices_sets_count = int(device_sets_required + old_storage_devices_sets_count) lvpresent = localstorage.check_local_volume() ocp_version = get_ocp_version() platform = config.ENV_DATA.get("platform", "").lower() is_lso = config.DEPLOYMENT.get("local_storage") if (ocp_version == "4.7" and (platform == constants.AWS_PLATFORM or platform == constants.VSPHERE_PLATFORM) and (not is_lso)): logging.info("Add capacity via UI") setup_ui = login_ui() add_ui_obj = AddReplaceDeviceUI(setup_ui) add_ui_obj.add_capacity_ui() close_browser(setup_ui) else: if lvpresent: ocp_obj = OCP(kind="localvolume", namespace=config.ENV_DATA["local_storage_namespace"]) localvolume_data = ocp_obj.get(resource_name="local-block") device_list = localvolume_data["spec"]["storageClassDevices"][0][ "devicePaths"] final_device_list = localstorage.get_new_device_paths( device_sets_required, osd_size_capacity_requested) device_list.sort() final_device_list.sort() if device_list == final_device_list: raise ResourceNotFoundError("No Extra device found") param = f"""[{{ "op": "replace", "path": "/spec/storageClassDevices/0/devicePaths", "value": {final_device_list}}}]""" log.info(f"Final device list : {final_device_list}") lvcr = localstorage.get_local_volume_cr() log.info("Patching Local Volume CR...") lvcr.patch( resource_name=lvcr.get()["items"][0]["metadata"]["name"], params=param.strip("\n"), format_type="json", ) localstorage.check_pvs_created( int(len(final_device_list) / new_storage_devices_sets_count)) sc = get_storage_cluster() # adding the storage capacity to the cluster params = f"""[{{ "op": "replace", "path": "/spec/storageDeviceSets/0/count", "value": {new_storage_devices_sets_count}}}]""" sc.patch( resource_name=sc.get()["items"][0]["metadata"]["name"], params=params.strip("\n"), format_type="json", ) return new_storage_devices_sets_count