def test_noobaa_rebuild(self, bucket_factory): """ Test case to verify noobaa rebuild. Verifies KCS: https://access.redhat.com/solutions/5948631 1. Stop the noobaa-operator by setting the replicas of noobaa-operator deployment to 0. 2. Delete the noobaa deployments/statefulsets. 3. Delete the PVC db-noobaa-db-0. 4. Patch existing backingstores and bucketclasses to remove finalizer 5. Delete the backingstores/bucketclass. 6. Delete the noobaa secrets. 7. Restart noobaa-operator by setting the replicas back to 1. 8. Monitor the pods in openshift-storage for noobaa pods to be Running. """ dep_ocp = OCP(kind=constants.DEPLOYMENT, namespace=constants.OPENSHIFT_STORAGE_NAMESPACE) state_ocp = OCP(kind=constants.STATEFULSET, namespace=constants.OPENSHIFT_STORAGE_NAMESPACE) noobaa_pvc_obj = get_pvc_objs(pvc_names=["db-noobaa-db-pg-0"]) # Scale down noobaa operator logger.info( f"Scaling down {constants.NOOBAA_OPERATOR_DEPLOYMENT} deployment to replica: 0" ) dep_ocp.exec_oc_cmd( f"scale deployment {constants.NOOBAA_OPERATOR_DEPLOYMENT} --replicas=0" ) # Delete noobaa deployments and statefulsets logger.info("Deleting noobaa deployments and statefulsets") dep_ocp.delete(resource_name=constants.NOOBAA_ENDPOINT_DEPLOYMENT) state_ocp.delete(resource_name=constants.NOOBAA_DB_STATEFULSET) state_ocp.delete(resource_name=constants.NOOBAA_CORE_STATEFULSET) # Delete noobaa-db pvc pvc_obj = OCP(kind=constants.PVC, namespace=constants.OPENSHIFT_STORAGE_NAMESPACE) logger.info("Deleting noobaa-db pvc") pvc_obj.delete(resource_name=noobaa_pvc_obj[0].name, wait=True) pvc_obj.wait_for_delete(resource_name=noobaa_pvc_obj[0].name, timeout=300) # Patch and delete existing backingstores params = '{"metadata": {"finalizers":null}}' bs_obj = OCP(kind=constants.BACKINGSTORE, namespace=constants.OPENSHIFT_STORAGE_NAMESPACE) for bs in bs_obj.get()["items"]: assert bs_obj.patch( resource_name=bs["metadata"]["name"], params=params, format_type="merge", ), "Failed to change the parameter in backingstore" logger.info(f"Deleting backingstore: {bs['metadata']['name']}") bs_obj.delete(resource_name=bs["metadata"]["name"]) # Patch and delete existing bucketclass bc_obj = OCP(kind=constants.BUCKETCLASS, namespace=constants.OPENSHIFT_STORAGE_NAMESPACE) for bc in bc_obj.get()["items"]: assert bc_obj.patch( resource_name=bc["metadata"]["name"], params=params, format_type="merge", ), "Failed to change the parameter in bucketclass" logger.info(f"Deleting bucketclass: {bc['metadata']['name']}") bc_obj.delete(resource_name=bc["metadata"]["name"]) # Delete noobaa secrets logger.info("Deleting noobaa related secrets") dep_ocp.exec_oc_cmd( "delete secrets noobaa-admin noobaa-endpoints noobaa-operator noobaa-server" ) # Scale back noobaa-operator deployment logger.info( f"Scaling back {constants.NOOBAA_OPERATOR_DEPLOYMENT} deployment to replica: 1" ) dep_ocp.exec_oc_cmd( f"scale deployment {constants.NOOBAA_OPERATOR_DEPLOYMENT} --replicas=1" ) # Wait and validate noobaa PVC is in bound state pvc_obj.wait_for_resource( condition=constants.STATUS_BOUND, resource_name=noobaa_pvc_obj[0].name, timeout=600, sleep=120, ) # Validate noobaa pods are up and running pod_obj = OCP(kind=constants.POD, namespace=defaults.ROOK_CLUSTER_NAMESPACE) noobaa_pods = get_noobaa_pods() pod_obj.wait_for_resource( condition=constants.STATUS_RUNNING, resource_count=len(noobaa_pods), selector=constants.NOOBAA_APP_LABEL, timeout=900, ) # Verify everything running fine logger.info( "Verifying all resources are Running and matches expected result") self.sanity_helpers.health_check(tries=120) # Verify default backingstore/bucketclass default_bs = OCP(kind=constants.BACKINGSTORE, namespace=constants.OPENSHIFT_STORAGE_NAMESPACE).get( resource_name=DEFAULT_NOOBAA_BACKINGSTORE) default_bc = OCP(kind=constants.BUCKETCLASS, namespace=constants.OPENSHIFT_STORAGE_NAMESPACE).get( resource_name=DEFAULT_NOOBAA_BUCKETCLASS) assert (default_bs["status"]["phase"] == default_bc["status"]["phase"] == constants.STATUS_READY ), "Failed: Default bs/bc are not in ready state" # Create OBCs logger.info("Creating OBCs after noobaa rebuild") bucket_factory(amount=3, interface="OC", verify_health=True)
def test_noobaa_db_backup_and_recovery( self, pvc_factory, pod_factory, snapshot_factory, bucket_factory, rgw_bucket_factory, ): """ Test case to verify noobaa backup and recovery 1. Take snapshot db-noobaa-db-0 PVC and retore it to PVC 2. Scale down the statefulset noobaa-db 3. Get the yaml of the current PVC, db-noobaa-db-0 and change the parameter persistentVolumeReclaimPolicy to Retain for restored PVC 4. Delete both PVCs, the PV for the original claim db-noobaa-db-0 will be removed. The PV for claim db-noobaa-db-0-snapshot-restore will move to ‘Released’ 5. Edit again restore PV and remove the claimRef section. The volume will transition to Available. 6. Edit the yaml db-noobaa-db-0.yaml and change the setting volumeName to restored PVC. 7. Scale up the stateful set again and the pod should be running """ # Initialise variable self.noobaa_db_sst_name = "noobaa-db-pg" # Get noobaa pods before execution noobaa_pods = get_noobaa_pods() # Get noobaa PVC before execution noobaa_pvc_obj = get_pvc_objs(pvc_names=["db-noobaa-db-pg-0"]) noobaa_pv_name = noobaa_pvc_obj[0].get("spec").get("spec").get( "volumeName") # Take snapshot db-noobaa-db-0 PVC log.info(f"Creating snapshot of the {noobaa_pvc_obj[0].name} PVC") snap_obj = snapshot_factory( pvc_obj=noobaa_pvc_obj[0], wait=True, snapshot_name=f"{noobaa_pvc_obj[0].name}-snapshot", ) log.info( f"Successfully created snapshot {snap_obj.name} and in Ready state" ) # Restore it to PVC log.info(f"Restoring snapshot {snap_obj.name} to create new PVC") sc_name = noobaa_pvc_obj[0].get().get("spec").get("storageClassName") pvc_size = (noobaa_pvc_obj[0].get().get("spec").get("resources").get( "requests").get("storage")) self.restore_pvc_obj = create_restore_pvc( sc_name=sc_name, snap_name=snap_obj.name, namespace=snap_obj.namespace, size=pvc_size, pvc_name=f"{snap_obj.name}-restore", volume_mode=snap_obj.parent_volume_mode, access_mode=snap_obj.parent_access_mode, ) wait_for_resource_state(self.restore_pvc_obj, constants.STATUS_BOUND) self.restore_pvc_obj.reload() log.info(f"Succeesfuly created PVC {self.restore_pvc_obj.name} " f"from snapshot {snap_obj.name}") # Scale down the statefulset noobaa-db modify_statefulset_replica_count( statefulset_name=self.noobaa_db_sst_name, replica_count=0 ), f"Failed to scale down the statefulset {self.noobaa_db_sst_name}" # Get the noobaa-db PVC pvc_obj = OCP(kind=constants.PVC, namespace=constants.OPENSHIFT_STORAGE_NAMESPACE) noobaa_pvc_yaml = pvc_obj.get(resource_name=noobaa_pvc_obj[0].name) # Get the restored noobaa PVC and # change the parameter persistentVolumeReclaimPolicy to Retain restored_noobaa_pvc_obj = get_pvc_objs( pvc_names=[f"{snap_obj.name}-restore"]) restored_noobaa_pv_name = (restored_noobaa_pvc_obj[0].get("spec").get( "spec").get("volumeName")) pv_obj = OCP(kind=constants.PV, namespace=constants.OPENSHIFT_STORAGE_NAMESPACE) params = '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}' assert pv_obj.patch( resource_name=restored_noobaa_pv_name, params=params), ( "Failed to change the parameter persistentVolumeReclaimPolicy" f" to Retain {restored_noobaa_pv_name}") # Delete both PVCs delete_pvcs(pvc_objs=[noobaa_pvc_obj[0], restored_noobaa_pvc_obj[0]]) # Validate original claim db-noobaa-db-0 removed assert validate_pv_delete( pv_name=noobaa_pv_name ), f"PV not deleted, still exist {noobaa_pv_name}" # Validate PV for claim db-noobaa-db-0-snapshot-restore is in Released state pv_obj.wait_for_resource(condition=constants.STATUS_RELEASED, resource_name=restored_noobaa_pv_name) # Edit again restore PV and remove the claimRef section log.info( f"Remove the claimRef section from PVC {restored_noobaa_pv_name}") params = '[{"op": "remove", "path": "/spec/claimRef"}]' pv_obj.patch(resource_name=restored_noobaa_pv_name, params=params, format_type="json") log.info( f"Successfully removed claimRef section from PVC {restored_noobaa_pv_name}" ) # Validate PV is in Available state pv_obj.wait_for_resource(condition=constants.STATUS_AVAILABLE, resource_name=restored_noobaa_pv_name) # Edit the yaml db-noobaa-db-0.yaml and change the # setting volumeName to restored PVC noobaa_pvc_yaml["spec"]["volumeName"] = restored_noobaa_pv_name noobaa_pvc_yaml = OCS(**noobaa_pvc_yaml) noobaa_pvc_yaml.create() # Validate noobaa PVC is in bound state pvc_obj.wait_for_resource( condition=constants.STATUS_BOUND, resource_name=noobaa_pvc_obj[0].name, timeout=120, ) # Scale up the statefulset again assert modify_statefulset_replica_count( statefulset_name=self.noobaa_db_sst_name, replica_count=1 ), f"Failed to scale up the statefulset {self.noobaa_db_sst_name}" # Validate noobaa pod is up and running pod_obj = OCP(kind=constants.POD, namespace=defaults.ROOK_CLUSTER_NAMESPACE) pod_obj.wait_for_resource( condition=constants.STATUS_RUNNING, resource_count=len(noobaa_pods), selector=constants.NOOBAA_APP_LABEL, ) # Change the parameter persistentVolumeReclaimPolicy to Delete again params = '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' assert pv_obj.patch( resource_name=restored_noobaa_pv_name, params=params), ( "Failed to change the parameter persistentVolumeReclaimPolicy" f" to Delete {restored_noobaa_pv_name}") log.info( "Changed the parameter persistentVolumeReclaimPolicy to Delete again" ) # Verify all storage pods are running wait_for_storage_pods() # Creating Resources log.info("Creating Resources using sanity helpers") self.sanity_helpers.create_resources(pvc_factory, pod_factory, bucket_factory, rgw_bucket_factory) # Deleting Resources self.sanity_helpers.delete_resources() # Verify everything running fine log.info( "Verifying All resources are Running and matches expected result") self.sanity_helpers.health_check(tries=120)
def factory(snapshot_factory=snapshot_factory): # Get noobaa pods before execution noobaa_pods = pod.get_noobaa_pods() # Get noobaa PVC before execution noobaa_pvc_obj = pvc.get_pvc_objs(pvc_names=["db-noobaa-db-pg-0"]) noobaa_pv_name = noobaa_pvc_obj[0].get("spec").get("spec").get( "volumeName") # Take snapshot db-noobaa-db-0 PVC logger.info(f"Creating snapshot of the {noobaa_pvc_obj[0].name} PVC") snap_obj = snapshot_factory( pvc_obj=noobaa_pvc_obj[0], wait=True, snapshot_name=f"{noobaa_pvc_obj[0].name}-snapshot", ) logger.info( f"Successfully created snapshot {snap_obj.name} and in Ready state" ) # Restore it to PVC logger.info(f"Restoring snapshot {snap_obj.name} to create new PVC") sc_name = noobaa_pvc_obj[0].get().get("spec").get("storageClassName") pvc_size = (noobaa_pvc_obj[0].get().get("spec").get("resources").get( "requests").get("storage")) restore_pvc_obj = pvc.create_restore_pvc( sc_name=sc_name, snap_name=snap_obj.name, namespace=snap_obj.namespace, size=pvc_size, pvc_name=f"{snap_obj.name}-restore", volume_mode=snap_obj.parent_volume_mode, access_mode=snap_obj.parent_access_mode, ) restore_pvc_objs.append(restore_pvc_obj) wait_for_resource_state(restore_pvc_obj, constants.STATUS_BOUND) restore_pvc_obj.reload() logger.info(f"Succeesfuly created PVC {restore_pvc_obj.name} " f"from snapshot {snap_obj.name}") # Scale down the statefulset noobaa-db modify_statefulset_replica_count( statefulset_name=constants.NOOBAA_DB_STATEFULSET, replica_count=0 ), f"Failed to scale down the statefulset {constants.NOOBAA_DB_STATEFULSET}" # Get the noobaa-db PVC pvc_obj = OCP(kind=constants.PVC, namespace=constants.OPENSHIFT_STORAGE_NAMESPACE) noobaa_pvc_yaml = pvc_obj.get(resource_name=noobaa_pvc_obj[0].name) # Get the restored noobaa PVC and # change the parameter persistentVolumeReclaimPolicy to Retain restored_noobaa_pvc_obj = pvc.get_pvc_objs( pvc_names=[f"{snap_obj.name}-restore"]) restored_noobaa_pv_name = (restored_noobaa_pvc_obj[0].get("spec").get( "spec").get("volumeName")) pv_obj = OCP(kind=constants.PV, namespace=constants.OPENSHIFT_STORAGE_NAMESPACE) params = '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}' assert pv_obj.patch( resource_name=restored_noobaa_pv_name, params=params), ( "Failed to change the parameter persistentVolumeReclaimPolicy" f" to Retain {restored_noobaa_pv_name}") # Delete both PVCs pvc.delete_pvcs( pvc_objs=[noobaa_pvc_obj[0], restored_noobaa_pvc_obj[0]]) # Validate original claim db-noobaa-db-0 removed assert validate_pv_delete( pv_name=noobaa_pv_name ), f"PV not deleted, still exist {noobaa_pv_name}" # Validate PV for claim db-noobaa-db-0-snapshot-restore is in Released state pv_obj.wait_for_resource(condition=constants.STATUS_RELEASED, resource_name=restored_noobaa_pv_name) # Edit again restore PV and remove the claimRef section logger.info( f"Remove the claimRef section from PVC {restored_noobaa_pv_name}") params = '[{"op": "remove", "path": "/spec/claimRef"}]' pv_obj.patch(resource_name=restored_noobaa_pv_name, params=params, format_type="json") logger.info( f"Successfully removed claimRef section from PVC {restored_noobaa_pv_name}" ) # Validate PV is in Available state pv_obj.wait_for_resource(condition=constants.STATUS_AVAILABLE, resource_name=restored_noobaa_pv_name) # Edit the yaml db-noobaa-db-0.yaml and change the # setting volumeName to restored PVC noobaa_pvc_yaml["spec"]["volumeName"] = restored_noobaa_pv_name noobaa_pvc_yaml = OCS(**noobaa_pvc_yaml) noobaa_pvc_yaml.create() # Validate noobaa PVC is in bound state pvc_obj.wait_for_resource( condition=constants.STATUS_BOUND, resource_name=noobaa_pvc_obj[0].name, timeout=120, ) # Scale up the statefulset again assert modify_statefulset_replica_count( statefulset_name=constants.NOOBAA_DB_STATEFULSET, replica_count=1 ), f"Failed to scale up the statefulset {constants.NOOBAA_DB_STATEFULSET}" # Validate noobaa pod is up and running pod_obj = OCP(kind=constants.POD, namespace=defaults.ROOK_CLUSTER_NAMESPACE) pod_obj.wait_for_resource( condition=constants.STATUS_RUNNING, resource_count=len(noobaa_pods), selector=constants.NOOBAA_APP_LABEL, ) # Change the parameter persistentVolumeReclaimPolicy to Delete again params = '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' assert pv_obj.patch( resource_name=restored_noobaa_pv_name, params=params), ( "Failed to change the parameter persistentVolumeReclaimPolicy" f" to Delete {restored_noobaa_pv_name}") logger.info( "Changed the parameter persistentVolumeReclaimPolicy to Delete again" )
def test_clone_pvc( self, project_factory, teardown_factory, setup_ui_class, sc_name, access_mode, clone_access_mode, ): """ Test to verify PVC clone from UI """ pvc_size = "1" vol_mode = constants.VOLUME_MODE_FILESYSTEM # Creating a project from CLI pro_obj = project_factory() project_name = pro_obj.namespace pvc_ui_obj = PvcUI(setup_ui_class) if config.DEPLOYMENT["external_mode"]: if sc_name == constants.CEPHFILESYSTEM_SC: sc_name = constants.DEFAULT_EXTERNAL_MODE_STORAGECLASS_CEPHFS elif sc_name == constants.CEPHBLOCKPOOL_SC: sc_name = constants.DEFAULT_EXTERNAL_MODE_STORAGECLASS_RBD # Creating PVC from UI pvc_name = create_unique_resource_name("test", "pvc") pvc_ui_obj.create_pvc_ui( project_name, sc_name, pvc_name, access_mode, pvc_size, vol_mode ) teardown_factory(get_pvc_objs(pvc_names=[pvc_name], namespace=project_name)[0]) # Verifying PVC details in UI logger.info("Verifying PVC details in UI") pvc_ui_obj.verify_pvc_ui( pvc_size=pvc_size, access_mode=access_mode, vol_mode=vol_mode, sc_name=sc_name, pvc_name=pvc_name, project_name=project_name, ) logger.info("Verified PVC details in UI") # Clone PVC from UI clone_pvc_name = f"{pvc_name}-clone" pvc_ui_obj.pvc_clone_ui( project_name=project_name, pvc_name=pvc_name, cloned_pvc_access_mode=clone_access_mode, cloned_pvc_name=clone_pvc_name, ) teardown_factory( get_pvc_objs(pvc_names=[clone_pvc_name], namespace=project_name)[0] ) # Verifying cloned PVC details in UI logger.info("Verifying cloned PVC details in UI") pvc_ui_obj.verify_pvc_ui( pvc_size=pvc_size, access_mode=clone_access_mode, vol_mode=vol_mode, sc_name=sc_name, pvc_name=clone_pvc_name, project_name=project_name, ) logger.info("Verified cloned PVC details in UI")
def test_clone_pvc( self, project_factory, teardown_factory, setup_ui, sc_name, access_mode, clone_access_mode, ): """ Test to verify PVC clone from UI """ pvc_size = "1" vol_mode = constants.VOLUME_MODE_FILESYSTEM # Creating a project from CLI pro_obj = project_factory() project_name = pro_obj.namespace pvc_ui_obj = PvcUI(setup_ui) # Creating PVC from UI pvc_name = create_unique_resource_name("test", "pvc") pvc_ui_obj.create_pvc_ui(project_name, sc_name, pvc_name, access_mode, pvc_size, vol_mode) teardown_factory( get_pvc_objs(pvc_names=[pvc_name], namespace=project_name)[0]) # Verifying PVC details in UI logger.info("Verifying PVC details in UI") pvc_ui_obj.verify_pvc_ui( pvc_size=pvc_size, access_mode=access_mode, vol_mode=vol_mode, sc_name=sc_name, pvc_name=pvc_name, project_name=project_name, ) logger.info("Verified PVC details in UI") # Clone PVC from UI clone_pvc_name = f"{pvc_name}-clone" pvc_ui_obj.pvc_clone_ui( project_name=project_name, pvc_name=pvc_name, cloned_pvc_access_mode=clone_access_mode, cloned_pvc_name=clone_pvc_name, ) teardown_factory( get_pvc_objs(pvc_names=[clone_pvc_name], namespace=project_name)[0]) # Verifying cloned PVC details in UI logger.info("Verifying cloned PVC details in UI") pvc_ui_obj.verify_pvc_ui( pvc_size=pvc_size, access_mode=clone_access_mode, vol_mode=vol_mode, sc_name=sc_name, pvc_name=clone_pvc_name, project_name=project_name, ) logger.info("Verified cloned PVC details in UI")