def add_capacity(osd_size_capacity_requested):
"""
Add storage capacity to the cluster
Args:
osd_size_capacity_requested(int): Requested osd size capacity
Returns:
new storage device set count (int) : Returns True if all OSDs are in Running state
Note:
"StoragedeviceSets->count" represents the set of 3 OSDs.
That is, if there are 3 OSDs in the system then count will be 1.
If there are 6 OSDs then count is 2 and so on.
By changing this value,we can add extra devices to the cluster.
For example, if we want to expand the cluster by 3 more osds in a cluster that already has 3 osds,
we can set count as 2. So, with each increase of count by 1,
we get 3 OSDs extra added to the cluster.
This is how we are going to 'add capacity' via automation.
As we know that OCS has 3 way replica. That is, same data is placed in 3 OSDs.
Because of this, the total usable capacity for apps from 3 OSDs
will be the size of one OSD (all osds are of same size).
If we want to add more capacity to the cluster then we need to add 3 OSDs of same size
as that of the original OSD. add_capacity needs to accept the 'capacity_to_add' as an argument.
From this we need to arrive at storagedeviceSets -> count and then
"Patch" this count to get the required capacity to add.
To do so, we use following formula:
storageDeviceSets->count = (capacity reqested / osd capacity ) + existing count storageDeviceSets
"""
osd_size_existing = get_osd_size()
device_sets_required = int(osd_size_capacity_requested / osd_size_existing)
old_storage_devices_sets_count = get_deviceset_count()
new_storage_devices_sets_count = int(device_sets_required + old_storage_devices_sets_count)
lvpresent = localstorage.check_local_volume()
if lvpresent:
final_device_list = localstorage.get_new_device_paths(device_sets_required, osd_size_capacity_requested)
param = f"""[{{ "op": "replace", "path": "/spec/storageClassDevices/0/devicePaths",
"value": {final_device_list}}}]"""
log.info(f"Final device list : {final_device_list}")
lvcr = localstorage.get_local_volume_cr()
log.info("Patching Local Volume CR...")
lvcr.patch(
resource_name=lvcr.get()['items'][0]['metadata']['name'],
params=param.strip('\n'),
format_type='json'
)
localstorage.check_pvs_created(int(len(final_device_list) / new_storage_devices_sets_count))
sc = get_storage_cluster()
# adding the storage capacity to the cluster
params = f"""[{{ "op": "replace", "path": "/spec/storageDeviceSets/0/count",
"value": {new_storage_devices_sets_count}}}]"""
sc.patch(
resource_name=sc.get()['items'][0]['metadata']['name'],
params=params.strip('\n'),
format_type='json'
)
return new_storage_devices_sets_count
def uninstall_ocs():
"""
The function uninstalls the OCS operator from a openshift
cluster and removes all its settings and dependencies
"""
ocp_obj = ocp.OCP()
log.info("deleting volume snapshots")
vs_ocp_obj = ocp.OCP(kind=constants.VOLUMESNAPSHOT)
vs_list = vs_ocp_obj.get(all_namespaces=True)["items"]
for vs in vs_list:
vs_obj = ocp.OCP(
kind=constants.VOLUMESNAPSHOT, namespace=vs.get("metadata").get("namespace")
)
vs_obj.delete(resource_name=vs.get("metadata").get("name"))
log.info("queering for OCS PVCs")
provisioners = constants.OCS_PROVISIONERS
sc_list = [
sc for sc in get_all_storageclass() if sc.get("provisioner") in provisioners
]
pvc_to_delete = []
for sc in sc_list:
pvc_to_delete.extend(
pvc
for pvc in get_all_pvcs_in_storageclass(sc.get("metadata").get("name"))
if "noobaa" not in pvc.name
)
log.info("Removing monitoring stack from OpenShift Container Storage")
remove_monitoring_stack_from_ocs()
log.info(
"Removing OpenShift Container Platform registry from OpenShift Container Storage"
)
remove_ocp_registry_from_ocs(config.ENV_DATA["platform"])
log.info("Removing the cluster logging operator from OpenShift Container Storage")
try:
remove_cluster_logging_operator_from_ocs()
except CommandFailed:
log.info("No cluster logging found")
log.info("Deleting OCS PVCs")
for pvc in pvc_to_delete:
log.info(f"Deleting PVC: {pvc.name}")
pvc.delete()
storage_cluster = ocp.OCP(
kind=constants.STORAGECLUSTER,
resource_name=constants.DEFAULT_CLUSTERNAME,
namespace="openshift-storage",
)
log.info("Checking for local storage")
lso_sc = None
if check_local_volume():
"Local volume was found. Will be removed later"
lso_sc = (
storage_cluster.get()
.get("spec")
.get("storageDeviceSets")[0]
.get("dataPVCTemplate")
.get("spec")
.get("storageClassName")
)
cleanup_policy = (
storage_cluster.get()
.get("metadata")
.get("annotations")
.get("uninstall.ocs.openshift.io/cleanup-policy")
)
log.info("Deleting storageCluster object")
storage_cluster.delete(resource_name=constants.DEFAULT_CLUSTERNAME)
if cleanup_policy == "delete":
log.info("Cleanup policy set to delete. checking cleanup pods")
cleanup_pods = [
pod for pod in get_all_pods() if "cluster-cleanup-job" in pod.name
]
for pod in cleanup_pods:
while pod.get().get("status").get("phase") != "Succeeded":
log.info(f"waiting for cleanup pod {pod.name} to complete")
TimeoutSampler(timeout=10, sleep=30)
log.info(f"Cleanup pod {pod.name} completed successfully ")
# no need to confirm var/vib/rook was deleted from nodes if all cleanup pods are completed.
else:
log.info("Cleanup policy set to retain. skipping nodes cleanup")
log.info("Deleting openshift-storage namespace")
ocp_obj.delete_project(constants.OPENSHIFT_STORAGE_NAMESPACE)
ocp_obj.wait_for_delete(constants.OPENSHIFT_STORAGE_NAMESPACE)
switch_to_project(constants.DEFAULT_NAMESPACE)
# step 10: TODO remove crypto from nodes.
"""for node in storage_node_list:
log.info(f"removing encryption from {node}")
ocp_obj.exec_oc_debug_cmd(node=node, cmd_list=[])"""
if lso_sc is not None:
log.info("Removing LSO")
try:
uninstall_lso(lso_sc)
except Exception as e:
log.info(f"LSO removal failed.{e}")
log.info("deleting noobaa storage class")
noobaa_sc = ocp.OCP(kind=constants.STORAGECLASS)
noobaa_sc.delete(resource_name=constants.NOOBAA_SC)
nodes = get_all_nodes()
node_objs = get_node_objs(nodes)
log.info("Unlabeling storage nodes")
label_nodes(nodes=node_objs, label=constants.OPERATOR_NODE_LABEL[:-3] + "-")
label_nodes(nodes=node_objs, label=constants.TOPOLOGY_ROOK_LABEL + "-")
log.info("Removing taints from storage nodes")
taint_nodes(nodes=nodes, taint_label=constants.OCS_TAINT + "-")
log.info("Deleting remaining OCS PVs (if there are any)")
try:
rbd_pv = ocp.OCP(kind=constants.PV, resource_name="ocs-storagecluster-ceph-rbd")
fs_pv = ocp.OCP(kind=constants.PV, resource_name="ocs-storagecluster-cephfs")
rbd_pv.delete()
fs_pv.delete()
log.info("OCS PVs deleted")
except Exception as e:
log.info(f"OCS PV(s) not found. {e}")
log.info("Removing CRDs")
crd_list = [
"backingstores.noobaa.io",
"bucketclasses.noobaa.io",
"cephblockpools.ceph.rook.io",
"cephclusters.ceph.rook.io",
"cephfilesystems.ceph.rook.io",
"cephnfses.ceph.rook.io",
"cephobjectstores.ceph.rook.io",
"cephobjectstoreusers.ceph.rook.io",
"noobaas.noobaa.io",
"ocsinitializations.ocs.openshift.io",
"storageclusters.ocs.openshift.io",
"cephclients.ceph.rook.io",
"cephobjectrealms.ceph.rook.io",
"cephobjectzonegroups.ceph.rook.io",
"cephobjectzones.ceph.rook.io",
"cephrbdmirrors.ceph.rook.io",
]
for crd in crd_list:
try:
ocp_obj.exec_oc_cmd(f"delete crd {crd} --timeout=300m")
except Exception:
log.info(f"crd {crd} was not found")
def uninstall_ocs():
"""
The function uninstalls the OCS operator from a openshift
cluster and removes all its settings and dependencies
"""
ocp_obj = ocp.OCP()
provisioners = constants.OCS_PROVISIONERS
# List the storage classes
sc_list = [
sc for sc in get_all_storageclass()
if sc.get('provisioner') in provisioners
]
# Query for PVCs and OBCs that are using the storage class provisioners listed in the previous step.
pvc_to_delete = []
for sc in sc_list:
pvc_to_delete.extend(pvc for pvc in get_all_pvcs_in_storageclass(
sc.get('metadata').get('name')) if 'noobaa' not in pvc.name)
log.info("Removing monitoring stack from OpenShift Container Storage")
remove_monitoring_stack_from_ocs()
log.info(
"Removing OpenShift Container Platform registry from OpenShift Container Storage"
)
remove_ocp_registry_from_ocs(config.ENV_DATA['platform'])
log.info(
"Removing the cluster logging operator from OpenShift Container Storage"
)
try:
remove_cluster_logging_operator_from_ocs()
except CommandFailed:
log.info("No cluster logging found")
log.info("Deleting pvcs")
for pvc in pvc_to_delete:
log.info(f"Deleting pvc: {pvc.name}")
pvc.delete()
storage_cluster = ocp.OCP(kind=constants.STORAGECLUSTER,
resource_name=constants.DEFAULT_CLUSTERNAME,
namespace='openshift-storage')
log.info("Checking for local storage")
lso_sc = None
if check_local_volume():
"Local volume was found. Will be removed later"
lso_sc = storage_cluster.get().get('spec').get('storageDeviceSets')[
0].get('dataPVCTemplate').get('spec').get('storageClassName')
log.info("Deleting storageCluster object")
storage_cluster.delete(resource_name=constants.DEFAULT_CLUSTERNAME)
log.info("Removing CRDs")
crd_list = [
'backingstores.noobaa.io', 'bucketclasses.noobaa.io',
'cephblockpools.ceph.rook.io', 'cephfilesystems.ceph.rook.io',
'cephnfses.ceph.rook.io', 'cephobjectstores.ceph.rook.io',
'cephobjectstoreusers.ceph.rook.io', 'noobaas.noobaa.io',
'ocsinitializations.ocs.openshift.io',
'storageclusterinitializations.ocs.openshift.io',
'storageclusters.ocs.openshift.io', 'cephclusters.ceph.rook.io'
]
for crd in crd_list:
ocp_obj.exec_oc_cmd(f"delete crd {crd} --timeout=300m")
log.info("Deleting openshift-storage namespace")
ocp_obj.delete_project('openshift-storage')
ocp_obj.wait_for_delete('openshift-storage')
switch_to_project("default")
log.info("Removing rook directory from nodes")
nodes_list = get_labeled_nodes(constants.OPERATOR_NODE_LABEL)
for node in nodes_list:
log.info(f"Removing rook from {node}")
ocp_obj.exec_oc_debug_cmd(node=node, cmd_list=["rm -rf /var/lib/rook"])
log.info("Removing LSO ")
if lso_sc is not None:
uninstall_lso(lso_sc)
log.info(
"Delete the storage classes with an openshift-storage provisioner list"
)
for storage_class in sc_list:
log.info(
f"Deleting storage class {storage_class.get('metadata').get('name')}"
)
sc_obj = ocp.OCP(kind=constants.STORAGECLASS)
sc_obj.delete(resource_name=storage_class.get('metadata').get('name'))
log.info("Unlabeling storage nodes")
nodes_list = get_all_nodes()
for node in nodes_list:
node_obj = ocp.OCP(kind=constants.NODE, resource_name=node)
node_obj.add_label(resource_name=node,
label=constants.OPERATOR_NODE_LABEL[:-3] + '-')
node_obj.add_label(resource_name=node,
label=constants.TOPOLOGY_ROOK_LABEL + '-')
log.info("OCS was removed successfully from cluster ")
def add_capacity(osd_size_capacity_requested):
"""
Add storage capacity to the cluster
Args:
osd_size_capacity_requested(int): Requested osd size capacity
Returns:
new storage device set count (int) : Returns True if all OSDs are in Running state
Note:
"StoragedeviceSets->count" represents the set of 3 OSDs.
That is, if there are 3 OSDs in the system then count will be 1.
If there are 6 OSDs then count is 2 and so on.
By changing this value,we can add extra devices to the cluster.
For example, if we want to expand the cluster by 3 more osds in a cluster that already has 3 osds,
we can set count as 2. So, with each increase of count by 1,
we get 3 OSDs extra added to the cluster.
This is how we are going to 'add capacity' via automation.
As we know that OCS has 3 way replica. That is, same data is placed in 3 OSDs.
Because of this, the total usable capacity for apps from 3 OSDs
will be the size of one OSD (all osds are of same size).
If we want to add more capacity to the cluster then we need to add 3 OSDs of same size
as that of the original OSD. add_capacity needs to accept the 'capacity_to_add' as an argument.
From this we need to arrive at storagedeviceSets -> count and then
"Patch" this count to get the required capacity to add.
To do so, we use following formula:
storageDeviceSets->count = (capacity reqested / osd capacity ) + existing count storageDeviceSets
"""
osd_size_existing = get_osd_size()
device_sets_required = int(osd_size_capacity_requested / osd_size_existing)
old_storage_devices_sets_count = get_deviceset_count()
new_storage_devices_sets_count = int(device_sets_required +
old_storage_devices_sets_count)
lvpresent = localstorage.check_local_volume()
ocp_version = get_ocp_version()
platform = config.ENV_DATA.get("platform", "").lower()
is_lso = config.DEPLOYMENT.get("local_storage")
if (ocp_version == "4.7" and (platform == constants.AWS_PLATFORM
or platform == constants.VSPHERE_PLATFORM)
and (not is_lso)):
logging.info("Add capacity via UI")
setup_ui = login_ui()
add_ui_obj = AddReplaceDeviceUI(setup_ui)
add_ui_obj.add_capacity_ui()
close_browser(setup_ui)
else:
if lvpresent:
ocp_obj = OCP(kind="localvolume",
namespace=config.ENV_DATA["local_storage_namespace"])
localvolume_data = ocp_obj.get(resource_name="local-block")
device_list = localvolume_data["spec"]["storageClassDevices"][0][
"devicePaths"]
final_device_list = localstorage.get_new_device_paths(
device_sets_required, osd_size_capacity_requested)
device_list.sort()
final_device_list.sort()
if device_list == final_device_list:
raise ResourceNotFoundError("No Extra device found")
param = f"""[{{ "op": "replace", "path": "/spec/storageClassDevices/0/devicePaths",
"value": {final_device_list}}}]"""
log.info(f"Final device list : {final_device_list}")
lvcr = localstorage.get_local_volume_cr()
log.info("Patching Local Volume CR...")
lvcr.patch(
resource_name=lvcr.get()["items"][0]["metadata"]["name"],
params=param.strip("\n"),
format_type="json",
)
localstorage.check_pvs_created(
int(len(final_device_list) / new_storage_devices_sets_count))
sc = get_storage_cluster()
# adding the storage capacity to the cluster
params = f"""[{{ "op": "replace", "path": "/spec/storageDeviceSets/0/count",
"value": {new_storage_devices_sets_count}}}]"""
sc.patch(
resource_name=sc.get()["items"][0]["metadata"]["name"],
params=params.strip("\n"),
format_type="json",
)
return new_storage_devices_sets_count