def test_noobaa_rebuild(self, bucket_factory):
"""
Test case to verify noobaa rebuild. Verifies KCS: https://access.redhat.com/solutions/5948631
1. Stop the noobaa-operator by setting the replicas of noobaa-operator deployment to 0.
2. Delete the noobaa deployments/statefulsets.
3. Delete the PVC db-noobaa-db-0.
4. Patch existing backingstores and bucketclasses to remove finalizer
5. Delete the backingstores/bucketclass.
6. Delete the noobaa secrets.
7. Restart noobaa-operator by setting the replicas back to 1.
8. Monitor the pods in openshift-storage for noobaa pods to be Running.
"""
dep_ocp = OCP(kind=constants.DEPLOYMENT,
namespace=constants.OPENSHIFT_STORAGE_NAMESPACE)
state_ocp = OCP(kind=constants.STATEFULSET,
namespace=constants.OPENSHIFT_STORAGE_NAMESPACE)
noobaa_pvc_obj = get_pvc_objs(pvc_names=["db-noobaa-db-pg-0"])
# Scale down noobaa operator
logger.info(
f"Scaling down {constants.NOOBAA_OPERATOR_DEPLOYMENT} deployment to replica: 0"
)
dep_ocp.exec_oc_cmd(
f"scale deployment {constants.NOOBAA_OPERATOR_DEPLOYMENT} --replicas=0"
)
# Delete noobaa deployments and statefulsets
logger.info("Deleting noobaa deployments and statefulsets")
dep_ocp.delete(resource_name=constants.NOOBAA_ENDPOINT_DEPLOYMENT)
state_ocp.delete(resource_name=constants.NOOBAA_DB_STATEFULSET)
state_ocp.delete(resource_name=constants.NOOBAA_CORE_STATEFULSET)
# Delete noobaa-db pvc
pvc_obj = OCP(kind=constants.PVC,
namespace=constants.OPENSHIFT_STORAGE_NAMESPACE)
logger.info("Deleting noobaa-db pvc")
pvc_obj.delete(resource_name=noobaa_pvc_obj[0].name, wait=True)
pvc_obj.wait_for_delete(resource_name=noobaa_pvc_obj[0].name,
timeout=300)
# Patch and delete existing backingstores
params = '{"metadata": {"finalizers":null}}'
bs_obj = OCP(kind=constants.BACKINGSTORE,
namespace=constants.OPENSHIFT_STORAGE_NAMESPACE)
for bs in bs_obj.get()["items"]:
assert bs_obj.patch(
resource_name=bs["metadata"]["name"],
params=params,
format_type="merge",
), "Failed to change the parameter in backingstore"
logger.info(f"Deleting backingstore: {bs['metadata']['name']}")
bs_obj.delete(resource_name=bs["metadata"]["name"])
# Patch and delete existing bucketclass
bc_obj = OCP(kind=constants.BUCKETCLASS,
namespace=constants.OPENSHIFT_STORAGE_NAMESPACE)
for bc in bc_obj.get()["items"]:
assert bc_obj.patch(
resource_name=bc["metadata"]["name"],
params=params,
format_type="merge",
), "Failed to change the parameter in bucketclass"
logger.info(f"Deleting bucketclass: {bc['metadata']['name']}")
bc_obj.delete(resource_name=bc["metadata"]["name"])
# Delete noobaa secrets
logger.info("Deleting noobaa related secrets")
dep_ocp.exec_oc_cmd(
"delete secrets noobaa-admin noobaa-endpoints noobaa-operator noobaa-server"
)
# Scale back noobaa-operator deployment
logger.info(
f"Scaling back {constants.NOOBAA_OPERATOR_DEPLOYMENT} deployment to replica: 1"
)
dep_ocp.exec_oc_cmd(
f"scale deployment {constants.NOOBAA_OPERATOR_DEPLOYMENT} --replicas=1"
)
# Wait and validate noobaa PVC is in bound state
pvc_obj.wait_for_resource(
condition=constants.STATUS_BOUND,
resource_name=noobaa_pvc_obj[0].name,
timeout=600,
sleep=120,
)
# Validate noobaa pods are up and running
pod_obj = OCP(kind=constants.POD,
namespace=defaults.ROOK_CLUSTER_NAMESPACE)
noobaa_pods = get_noobaa_pods()
pod_obj.wait_for_resource(
condition=constants.STATUS_RUNNING,
resource_count=len(noobaa_pods),
selector=constants.NOOBAA_APP_LABEL,
timeout=900,
)
# Verify everything running fine
logger.info(
"Verifying all resources are Running and matches expected result")
self.sanity_helpers.health_check(tries=120)
# Verify default backingstore/bucketclass
default_bs = OCP(kind=constants.BACKINGSTORE,
namespace=constants.OPENSHIFT_STORAGE_NAMESPACE).get(
resource_name=DEFAULT_NOOBAA_BACKINGSTORE)
default_bc = OCP(kind=constants.BUCKETCLASS,
namespace=constants.OPENSHIFT_STORAGE_NAMESPACE).get(
resource_name=DEFAULT_NOOBAA_BUCKETCLASS)
assert (default_bs["status"]["phase"] == default_bc["status"]["phase"]
== constants.STATUS_READY
), "Failed: Default bs/bc are not in ready state"
# Create OBCs
logger.info("Creating OBCs after noobaa rebuild")
bucket_factory(amount=3, interface="OC", verify_health=True)
def test_noobaa_db_backup_and_recovery(
self,
pvc_factory,
pod_factory,
snapshot_factory,
bucket_factory,
rgw_bucket_factory,
):
"""
Test case to verify noobaa backup and recovery
1. Take snapshot db-noobaa-db-0 PVC and retore it to PVC
2. Scale down the statefulset noobaa-db
3. Get the yaml of the current PVC, db-noobaa-db-0 and
change the parameter persistentVolumeReclaimPolicy to Retain for restored PVC
4. Delete both PVCs, the PV for the original claim db-noobaa-db-0 will be removed.
The PV for claim db-noobaa-db-0-snapshot-restore will move to ‘Released’
5. Edit again restore PV and remove the claimRef section.
The volume will transition to Available.
6. Edit the yaml db-noobaa-db-0.yaml and change the setting volumeName to restored PVC.
7. Scale up the stateful set again and the pod should be running
"""
# Initialise variable
self.noobaa_db_sst_name = "noobaa-db-pg"
# Get noobaa pods before execution
noobaa_pods = get_noobaa_pods()
# Get noobaa PVC before execution
noobaa_pvc_obj = get_pvc_objs(pvc_names=["db-noobaa-db-pg-0"])
noobaa_pv_name = noobaa_pvc_obj[0].get("spec").get("spec").get(
"volumeName")
# Take snapshot db-noobaa-db-0 PVC
log.info(f"Creating snapshot of the {noobaa_pvc_obj[0].name} PVC")
snap_obj = snapshot_factory(
pvc_obj=noobaa_pvc_obj[0],
wait=True,
snapshot_name=f"{noobaa_pvc_obj[0].name}-snapshot",
)
log.info(
f"Successfully created snapshot {snap_obj.name} and in Ready state"
)
# Restore it to PVC
log.info(f"Restoring snapshot {snap_obj.name} to create new PVC")
sc_name = noobaa_pvc_obj[0].get().get("spec").get("storageClassName")
pvc_size = (noobaa_pvc_obj[0].get().get("spec").get("resources").get(
"requests").get("storage"))
self.restore_pvc_obj = create_restore_pvc(
sc_name=sc_name,
snap_name=snap_obj.name,
namespace=snap_obj.namespace,
size=pvc_size,
pvc_name=f"{snap_obj.name}-restore",
volume_mode=snap_obj.parent_volume_mode,
access_mode=snap_obj.parent_access_mode,
)
wait_for_resource_state(self.restore_pvc_obj, constants.STATUS_BOUND)
self.restore_pvc_obj.reload()
log.info(f"Succeesfuly created PVC {self.restore_pvc_obj.name} "
f"from snapshot {snap_obj.name}")
# Scale down the statefulset noobaa-db
modify_statefulset_replica_count(
statefulset_name=self.noobaa_db_sst_name, replica_count=0
), f"Failed to scale down the statefulset {self.noobaa_db_sst_name}"
# Get the noobaa-db PVC
pvc_obj = OCP(kind=constants.PVC,
namespace=constants.OPENSHIFT_STORAGE_NAMESPACE)
noobaa_pvc_yaml = pvc_obj.get(resource_name=noobaa_pvc_obj[0].name)
# Get the restored noobaa PVC and
# change the parameter persistentVolumeReclaimPolicy to Retain
restored_noobaa_pvc_obj = get_pvc_objs(
pvc_names=[f"{snap_obj.name}-restore"])
restored_noobaa_pv_name = (restored_noobaa_pvc_obj[0].get("spec").get(
"spec").get("volumeName"))
pv_obj = OCP(kind=constants.PV,
namespace=constants.OPENSHIFT_STORAGE_NAMESPACE)
params = '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}'
assert pv_obj.patch(
resource_name=restored_noobaa_pv_name, params=params), (
"Failed to change the parameter persistentVolumeReclaimPolicy"
f" to Retain {restored_noobaa_pv_name}")
# Delete both PVCs
delete_pvcs(pvc_objs=[noobaa_pvc_obj[0], restored_noobaa_pvc_obj[0]])
# Validate original claim db-noobaa-db-0 removed
assert validate_pv_delete(
pv_name=noobaa_pv_name
), f"PV not deleted, still exist {noobaa_pv_name}"
# Validate PV for claim db-noobaa-db-0-snapshot-restore is in Released state
pv_obj.wait_for_resource(condition=constants.STATUS_RELEASED,
resource_name=restored_noobaa_pv_name)
# Edit again restore PV and remove the claimRef section
log.info(
f"Remove the claimRef section from PVC {restored_noobaa_pv_name}")
params = '[{"op": "remove", "path": "/spec/claimRef"}]'
pv_obj.patch(resource_name=restored_noobaa_pv_name,
params=params,
format_type="json")
log.info(
f"Successfully removed claimRef section from PVC {restored_noobaa_pv_name}"
)
# Validate PV is in Available state
pv_obj.wait_for_resource(condition=constants.STATUS_AVAILABLE,
resource_name=restored_noobaa_pv_name)
# Edit the yaml db-noobaa-db-0.yaml and change the
# setting volumeName to restored PVC
noobaa_pvc_yaml["spec"]["volumeName"] = restored_noobaa_pv_name
noobaa_pvc_yaml = OCS(**noobaa_pvc_yaml)
noobaa_pvc_yaml.create()
# Validate noobaa PVC is in bound state
pvc_obj.wait_for_resource(
condition=constants.STATUS_BOUND,
resource_name=noobaa_pvc_obj[0].name,
timeout=120,
)
# Scale up the statefulset again
assert modify_statefulset_replica_count(
statefulset_name=self.noobaa_db_sst_name, replica_count=1
), f"Failed to scale up the statefulset {self.noobaa_db_sst_name}"
# Validate noobaa pod is up and running
pod_obj = OCP(kind=constants.POD,
namespace=defaults.ROOK_CLUSTER_NAMESPACE)
pod_obj.wait_for_resource(
condition=constants.STATUS_RUNNING,
resource_count=len(noobaa_pods),
selector=constants.NOOBAA_APP_LABEL,
)
# Change the parameter persistentVolumeReclaimPolicy to Delete again
params = '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}'
assert pv_obj.patch(
resource_name=restored_noobaa_pv_name, params=params), (
"Failed to change the parameter persistentVolumeReclaimPolicy"
f" to Delete {restored_noobaa_pv_name}")
log.info(
"Changed the parameter persistentVolumeReclaimPolicy to Delete again"
)
# Verify all storage pods are running
wait_for_storage_pods()
# Creating Resources
log.info("Creating Resources using sanity helpers")
self.sanity_helpers.create_resources(pvc_factory, pod_factory,
bucket_factory,
rgw_bucket_factory)
# Deleting Resources
self.sanity_helpers.delete_resources()
# Verify everything running fine
log.info(
"Verifying All resources are Running and matches expected result")
self.sanity_helpers.health_check(tries=120)
def factory(snapshot_factory=snapshot_factory):
# Get noobaa pods before execution
noobaa_pods = pod.get_noobaa_pods()
# Get noobaa PVC before execution
noobaa_pvc_obj = pvc.get_pvc_objs(pvc_names=["db-noobaa-db-pg-0"])
noobaa_pv_name = noobaa_pvc_obj[0].get("spec").get("spec").get(
"volumeName")
# Take snapshot db-noobaa-db-0 PVC
logger.info(f"Creating snapshot of the {noobaa_pvc_obj[0].name} PVC")
snap_obj = snapshot_factory(
pvc_obj=noobaa_pvc_obj[0],
wait=True,
snapshot_name=f"{noobaa_pvc_obj[0].name}-snapshot",
)
logger.info(
f"Successfully created snapshot {snap_obj.name} and in Ready state"
)
# Restore it to PVC
logger.info(f"Restoring snapshot {snap_obj.name} to create new PVC")
sc_name = noobaa_pvc_obj[0].get().get("spec").get("storageClassName")
pvc_size = (noobaa_pvc_obj[0].get().get("spec").get("resources").get(
"requests").get("storage"))
restore_pvc_obj = pvc.create_restore_pvc(
sc_name=sc_name,
snap_name=snap_obj.name,
namespace=snap_obj.namespace,
size=pvc_size,
pvc_name=f"{snap_obj.name}-restore",
volume_mode=snap_obj.parent_volume_mode,
access_mode=snap_obj.parent_access_mode,
)
restore_pvc_objs.append(restore_pvc_obj)
wait_for_resource_state(restore_pvc_obj, constants.STATUS_BOUND)
restore_pvc_obj.reload()
logger.info(f"Succeesfuly created PVC {restore_pvc_obj.name} "
f"from snapshot {snap_obj.name}")
# Scale down the statefulset noobaa-db
modify_statefulset_replica_count(
statefulset_name=constants.NOOBAA_DB_STATEFULSET, replica_count=0
), f"Failed to scale down the statefulset {constants.NOOBAA_DB_STATEFULSET}"
# Get the noobaa-db PVC
pvc_obj = OCP(kind=constants.PVC,
namespace=constants.OPENSHIFT_STORAGE_NAMESPACE)
noobaa_pvc_yaml = pvc_obj.get(resource_name=noobaa_pvc_obj[0].name)
# Get the restored noobaa PVC and
# change the parameter persistentVolumeReclaimPolicy to Retain
restored_noobaa_pvc_obj = pvc.get_pvc_objs(
pvc_names=[f"{snap_obj.name}-restore"])
restored_noobaa_pv_name = (restored_noobaa_pvc_obj[0].get("spec").get(
"spec").get("volumeName"))
pv_obj = OCP(kind=constants.PV,
namespace=constants.OPENSHIFT_STORAGE_NAMESPACE)
params = '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}'
assert pv_obj.patch(
resource_name=restored_noobaa_pv_name, params=params), (
"Failed to change the parameter persistentVolumeReclaimPolicy"
f" to Retain {restored_noobaa_pv_name}")
# Delete both PVCs
pvc.delete_pvcs(
pvc_objs=[noobaa_pvc_obj[0], restored_noobaa_pvc_obj[0]])
# Validate original claim db-noobaa-db-0 removed
assert validate_pv_delete(
pv_name=noobaa_pv_name
), f"PV not deleted, still exist {noobaa_pv_name}"
# Validate PV for claim db-noobaa-db-0-snapshot-restore is in Released state
pv_obj.wait_for_resource(condition=constants.STATUS_RELEASED,
resource_name=restored_noobaa_pv_name)
# Edit again restore PV and remove the claimRef section
logger.info(
f"Remove the claimRef section from PVC {restored_noobaa_pv_name}")
params = '[{"op": "remove", "path": "/spec/claimRef"}]'
pv_obj.patch(resource_name=restored_noobaa_pv_name,
params=params,
format_type="json")
logger.info(
f"Successfully removed claimRef section from PVC {restored_noobaa_pv_name}"
)
# Validate PV is in Available state
pv_obj.wait_for_resource(condition=constants.STATUS_AVAILABLE,
resource_name=restored_noobaa_pv_name)
# Edit the yaml db-noobaa-db-0.yaml and change the
# setting volumeName to restored PVC
noobaa_pvc_yaml["spec"]["volumeName"] = restored_noobaa_pv_name
noobaa_pvc_yaml = OCS(**noobaa_pvc_yaml)
noobaa_pvc_yaml.create()
# Validate noobaa PVC is in bound state
pvc_obj.wait_for_resource(
condition=constants.STATUS_BOUND,
resource_name=noobaa_pvc_obj[0].name,
timeout=120,
)
# Scale up the statefulset again
assert modify_statefulset_replica_count(
statefulset_name=constants.NOOBAA_DB_STATEFULSET, replica_count=1
), f"Failed to scale up the statefulset {constants.NOOBAA_DB_STATEFULSET}"
# Validate noobaa pod is up and running
pod_obj = OCP(kind=constants.POD,
namespace=defaults.ROOK_CLUSTER_NAMESPACE)
pod_obj.wait_for_resource(
condition=constants.STATUS_RUNNING,
resource_count=len(noobaa_pods),
selector=constants.NOOBAA_APP_LABEL,
)
# Change the parameter persistentVolumeReclaimPolicy to Delete again
params = '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}'
assert pv_obj.patch(
resource_name=restored_noobaa_pv_name, params=params), (
"Failed to change the parameter persistentVolumeReclaimPolicy"
f" to Delete {restored_noobaa_pv_name}")
logger.info(
"Changed the parameter persistentVolumeReclaimPolicy to Delete again"
)
def test_clone_pvc(
self,
project_factory,
teardown_factory,
setup_ui_class,
sc_name,
access_mode,
clone_access_mode,
):
"""
Test to verify PVC clone from UI
"""
pvc_size = "1"
vol_mode = constants.VOLUME_MODE_FILESYSTEM
# Creating a project from CLI
pro_obj = project_factory()
project_name = pro_obj.namespace
pvc_ui_obj = PvcUI(setup_ui_class)
if config.DEPLOYMENT["external_mode"]:
if sc_name == constants.CEPHFILESYSTEM_SC:
sc_name = constants.DEFAULT_EXTERNAL_MODE_STORAGECLASS_CEPHFS
elif sc_name == constants.CEPHBLOCKPOOL_SC:
sc_name = constants.DEFAULT_EXTERNAL_MODE_STORAGECLASS_RBD
# Creating PVC from UI
pvc_name = create_unique_resource_name("test", "pvc")
pvc_ui_obj.create_pvc_ui(
project_name, sc_name, pvc_name, access_mode, pvc_size, vol_mode
)
teardown_factory(get_pvc_objs(pvc_names=[pvc_name], namespace=project_name)[0])
# Verifying PVC details in UI
logger.info("Verifying PVC details in UI")
pvc_ui_obj.verify_pvc_ui(
pvc_size=pvc_size,
access_mode=access_mode,
vol_mode=vol_mode,
sc_name=sc_name,
pvc_name=pvc_name,
project_name=project_name,
)
logger.info("Verified PVC details in UI")
# Clone PVC from UI
clone_pvc_name = f"{pvc_name}-clone"
pvc_ui_obj.pvc_clone_ui(
project_name=project_name,
pvc_name=pvc_name,
cloned_pvc_access_mode=clone_access_mode,
cloned_pvc_name=clone_pvc_name,
)
teardown_factory(
get_pvc_objs(pvc_names=[clone_pvc_name], namespace=project_name)[0]
)
# Verifying cloned PVC details in UI
logger.info("Verifying cloned PVC details in UI")
pvc_ui_obj.verify_pvc_ui(
pvc_size=pvc_size,
access_mode=clone_access_mode,
vol_mode=vol_mode,
sc_name=sc_name,
pvc_name=clone_pvc_name,
project_name=project_name,
)
logger.info("Verified cloned PVC details in UI")
def test_clone_pvc(
self,
project_factory,
teardown_factory,
setup_ui,
sc_name,
access_mode,
clone_access_mode,
):
"""
Test to verify PVC clone from UI
"""
pvc_size = "1"
vol_mode = constants.VOLUME_MODE_FILESYSTEM
# Creating a project from CLI
pro_obj = project_factory()
project_name = pro_obj.namespace
pvc_ui_obj = PvcUI(setup_ui)
# Creating PVC from UI
pvc_name = create_unique_resource_name("test", "pvc")
pvc_ui_obj.create_pvc_ui(project_name, sc_name, pvc_name, access_mode,
pvc_size, vol_mode)
teardown_factory(
get_pvc_objs(pvc_names=[pvc_name], namespace=project_name)[0])
# Verifying PVC details in UI
logger.info("Verifying PVC details in UI")
pvc_ui_obj.verify_pvc_ui(
pvc_size=pvc_size,
access_mode=access_mode,
vol_mode=vol_mode,
sc_name=sc_name,
pvc_name=pvc_name,
project_name=project_name,
)
logger.info("Verified PVC details in UI")
# Clone PVC from UI
clone_pvc_name = f"{pvc_name}-clone"
pvc_ui_obj.pvc_clone_ui(
project_name=project_name,
pvc_name=pvc_name,
cloned_pvc_access_mode=clone_access_mode,
cloned_pvc_name=clone_pvc_name,
)
teardown_factory(
get_pvc_objs(pvc_names=[clone_pvc_name],
namespace=project_name)[0])
# Verifying cloned PVC details in UI
logger.info("Verifying cloned PVC details in UI")
pvc_ui_obj.verify_pvc_ui(
pvc_size=pvc_size,
access_mode=clone_access_mode,
vol_mode=vol_mode,
sc_name=sc_name,
pvc_name=clone_pvc_name,
project_name=project_name,
)
logger.info("Verified cloned PVC details in UI")