def test_monitoring_delete_pvc(self): """ Test case to validate whether delete pvcs+configmap and recovery of a node where monitoring pods running has no functional impact """ # Get 'cluster-monitoring-config' configmap ocp_configmap = ocp.OCP( namespace=constants.MONITORING_NAMESPACE, kind="configmap" ) configmap_dict = ocp_configmap.get(resource_name="cluster-monitoring-config") dir_configmap = tempfile.mkdtemp(prefix="configmap_") yaml_file = f"{dir_configmap}/configmap.yaml" templating.dump_data_to_temp_yaml(configmap_dict, yaml_file) # Get prometheus and alertmanager pods prometheus_alertmanager_pods = pod.get_all_pods( namespace=defaults.OCS_MONITORING_NAMESPACE, selector=["prometheus", "alertmanager"], ) # Get all pvc on monitoring namespace pvc_objs_list = pvc.get_all_pvc_objs(namespace=constants.MONITORING_NAMESPACE) # Delete configmap ocp_configmap.delete(resource_name="cluster-monitoring-config") # Delete all pvcs on monitoring namespace pvc.delete_pvcs(pvc_objs=pvc_objs_list) # Check all the prometheus and alertmanager pods are up for pod_obj in prometheus_alertmanager_pods: wait_for_resource_state( resource=pod_obj, state=constants.STATUS_RUNNING, timeout=180 ) # Create configmap ocp_configmap.create(yaml_file=dir_configmap) # Check all the PVCs are up for pvc_obj in pvc_objs_list: wait_for_resource_state( resource=pvc_obj, state=constants.STATUS_BOUND, timeout=180 ) # Check all the prometheus and alertmanager pods are up # and pvc are mounted on monitoring pods for pod_obj in prometheus_alertmanager_pods: wait_for_resource_state( resource=pod_obj, state=constants.STATUS_RUNNING, timeout=180 ) mount_point = pod_obj.exec_cmd_on_pod( command="df -kh", out_yaml_format=False, ) assert "/dev/rbd" in mount_point, f"pvc is not mounted on pod {pod.name}" log.info("Verified all pvc are mounted on monitoring pods") # Validate the prometheus health is ok assert prometheus_health_check(), "Prometheus cluster health is not OK"
def finalizer(): # Delete pods for pod_obj in cls_ref.pod_objs: pod_obj.delete() # Delete newly created PVCs delete_pvcs(cls_ref.pvc_objs_new)
def create_pvc_delete(self, multi_pvc_factory, project=None): """ Creates and deletes all types of PVCs """ # Create rbd pvcs pvc_objs_rbd = create_pvcs(multi_pvc_factory=multi_pvc_factory, interface='CephBlockPool', project=project, status="", storageclass=None) # Create cephfs pvcs pvc_objs_cephfs = create_pvcs(multi_pvc_factory=multi_pvc_factory, interface='CephFileSystem', project=project, status="", storageclass=None) all_pvc_to_delete = pvc_objs_rbd + pvc_objs_cephfs # Check pvc status for pvc_obj in all_pvc_to_delete: helpers.wait_for_resource_state(resource=pvc_obj, state=constants.STATUS_BOUND, timeout=300) # Start deleting PVC delete_pvcs(all_pvc_to_delete) # Check PVCs are deleted for pvc_obj in all_pvc_to_delete: pvc_obj.ocp.wait_for_delete(resource_name=pvc_obj.name) logger.info("All PVCs are deleted as expected")
def cleanup( self, kafka_namespace=constants.AMQ_NAMESPACE, tiller_namespace=AMQ_BENCHMARK_NAMESPACE, ): """ Clean up function, will start to delete from amq cluster operator then amq-connector, persistent, bridge, at the end it will delete the created namespace Args: kafka_namespace (str): Created namespace for amq tiller_namespace (str): Created namespace for benchmark """ ocs_pvc_obj = get_all_pvc_objs(namespace=kafka_namespace) if self.amq_is_setup: if self.messaging: self.consumer_pod.delete() self.producer_pod.delete() self.kafka_user.delete() self.kafka_topic.delete() if self.benchmark: # Delete the helm app try: purge_cmd = f"linux-amd64/helm delete benchmark --purge --tiller-namespace {tiller_namespace}" run(purge_cmd, shell=True, cwd=self.dir, check=True) except (CommandFailed, CalledProcessError) as cf: log.error("Failed to delete help app") raise cf # Delete the pods and namespace created self.sa_tiller.delete() self.crb_tiller.delete() run_cmd(f"oc delete project {tiller_namespace}") self.ns_obj.wait_for_delete(resource_name=tiller_namespace) self.kafka_connect.delete() self.kafka_bridge.delete() self.kafka_persistent.delete() delete_pvcs(pvc_objs=ocs_pvc_obj) cmd = f"oc delete -f {self.strimzi_kafka_operator}" log.info(f"Executing cmd: {cmd}") run(cmd, shell=True, check=True, cwd=self.dir) for pvc in ocs_pvc_obj: logging.info(pvc.name) validate_pv_delete(pvc.backed_pv) run_cmd(f"oc delete project {kafka_namespace}") self.ns_obj.wait_for_delete(resource_name=kafka_namespace, timeout=90) # Reset namespace to default switch_to_default_rook_cluster_project()
def test_bulk_pvc_creation_after_deletion_performance( self, teardown_factory): """ Measuring PVC creation time of bulk of 75% of initial PVC bulk (120) in the same rate after deleting ( serial deletion) 75% of the initial PVCs. Args: teardown_factory: A fixture used when we want a new resource that was created during the tests to be removed in the teardown phase. Returns: """ initial_number_of_pvcs = 120 number_of_pvcs = math.ceil(initial_number_of_pvcs * 0.75) log.info(f"Start creating new {initial_number_of_pvcs} PVCs in a bulk") pvc_objs, _ = helpers.create_multiple_pvcs( sc_name=self.sc_obj.name, namespace=defaults.ROOK_CLUSTER_NAMESPACE, number_of_pvc=initial_number_of_pvcs, size=self.pvc_size, burst=True, ) for pvc_obj in pvc_objs: teardown_factory(pvc_obj) with ThreadPoolExecutor() as executor: for pvc_obj in pvc_objs: executor.submit(helpers.wait_for_resource_state, pvc_obj, constants.STATUS_BOUND) executor.submit(pvc_obj.reload) log.info("Deleting 75% of the PVCs - 90 PVCs") assert pvc.delete_pvcs(pvc_objs[:number_of_pvcs], True), "Deletion of 75% of PVCs failed" log.info("Re-creating the 90 PVCs") pvc_objs, _ = helpers.create_multiple_pvcs( sc_name=self.sc_obj.name, namespace=defaults.ROOK_CLUSTER_NAMESPACE, number_of_pvc=number_of_pvcs, size=self.pvc_size, burst=True, ) start_time = helpers.get_provision_time(self.interface, pvc_objs, status="start") end_time = helpers.get_provision_time(self.interface, pvc_objs, status="end") total = end_time - start_time total_time = total.total_seconds() logging.info( f"Deletion time of {number_of_pvcs} is {total_time} seconds.") if total_time > 50: raise ex.PerformanceException( f"{number_of_pvcs} PVCs creation (after initial deletion of " f"75% of PVCs) time is {total_time} and greater than 50 seconds." ) logging.info( f"{number_of_pvcs} PVCs creation time took less than a 50 seconds")
def teardown(self): """ Delete pvc and config map created """ assert ocp.delete(resource_name='cluster-monitoring-config') pvc_obj_list = get_list_pvc_objs_created_on_monitoring_pods() assert delete_pvcs(pvc_obj_list)
def destroy_ocs(self): """ Uninstall ODF Managed Service addon via rosa cli. """ cluster_namespace = config.ENV_DATA["cluster_namespace"] # Deleting PVCs rbd_pvcs = [ p for p in pvc.get_all_pvcs_in_storageclass( constants.CEPHBLOCKPOOL_SC) if not (p.data["metadata"]["namespace"] == cluster_namespace and p.data["metadata"]["labels"]["app"] == "noobaa") ] pvc.delete_pvcs(rbd_pvcs) cephfs_pvcs = pvc.get_all_pvcs_in_storageclass( constants.CEPHFILESYSTEM_SC) pvc.delete_pvcs(cephfs_pvcs) rosa.delete_odf_addon(self.cluster_name)
def test_multiple_pvc_creation_after_deletion_performance( self, teardown_factory ): """ Measuring PVC creation time of 75% of initial PVCs (120) in the same rate after deleting 75% of the initial PVCs """ initial_number_of_pvcs = 120 number_of_pvcs = math.ceil(initial_number_of_pvcs * 0.75) log.info('Start creating new 120 PVCs') pvc_objs = helpers.create_multiple_pvcs( sc_name=self.sc_obj.name, namespace=defaults.ROOK_CLUSTER_NAMESPACE, number_of_pvc=initial_number_of_pvcs, size=self.pvc_size, ) for pvc_obj in pvc_objs: teardown_factory(pvc_obj) with ThreadPoolExecutor() as executor: for pvc_obj in pvc_objs: executor.submit( helpers.wait_for_resource_state, pvc_obj, constants.STATUS_BOUND ) executor.submit(pvc_obj.reload) log.info('Deleting 75% of the PVCs - 90 PVCs') assert pvc.delete_pvcs(pvc_objs[:number_of_pvcs], True), ( "Deletion of 75% of PVCs failed" ) log.info('Re-creating the 90 PVCs') pvc_objs = helpers.create_multiple_pvcs( sc_name=self.sc_obj.name, namespace=defaults.ROOK_CLUSTER_NAMESPACE, number_of_pvc=number_of_pvcs, size=self.pvc_size, ) start_time = helpers.get_start_creation_time( self.interface, pvc_objs[0].name ) end_time = helpers.get_end_creation_time( self.interface, pvc_objs[number_of_pvcs - 1].name, ) total = end_time - start_time total_time = total.total_seconds() if total_time > 45: raise ex.PerformanceException( f"{number_of_pvcs} PVCs creation (after initial deletion of " f"75%) time is {total_time} and greater than 45 seconds" ) logging.info( f"{number_of_pvcs} PVCs creation time took less than a 45 seconds" )
def teardown(): # Delete created app pods and pvcs assert pod.delete_pods(pod_objs) assert pvc.delete_pvcs(pvc_objs) # Switch to default project ret = ocp.switch_to_default_rook_cluster_project() assert ret, 'Failed to switch to default rook cluster project' # Delete created projects for prj in namespace_list: prj.delete(resource_name=prj.namespace)
def teardown(self): """ Delete PVCs Delete project """ # Delete newly created PVCs assert delete_pvcs(self.pvc_objs_new), 'Failed to delete PVCs' log.info(f'Newly created {self.number_of_pvc} PVCs are now deleted.') # Switch to default project ret = ocp.switch_to_default_rook_cluster_project() assert ret, 'Failed to switch to default rook cluster project' # Delete project created for the test case self.project_obj.delete(resource_name=self.namespace)
def teardown(): """ Tearing down the environment """ global RBD_SECRET_OBJ, CEPHFS_SECRET_OBJ log.info("Deleting PVC") assert pvc.delete_pvcs(PVC_OBJS) log.info("Deleting CEPH BLOCK POOL") assert helpers.delete_cephblockpools(POOL_OBJS) log.info("Deleting RBD Secret") RBD_SECRET_OBJ.delete() log.info("Deleting CEPHFS Secret") CEPHFS_SECRET_OBJ.delete() log.info("Deleting RBD Storageclass") assert helpers.delete_storageclasses(SC_RBD_OBJS) log.info("Deleting CephFS Storageclass") assert helpers.delete_storageclasses([SC_CEPHFS_OBJ])
def teardown(): """ Tearing down the environment """ global RBD_SECRET_OBJ, CEPHFS_SECRET_OBJ log.info("Deleting PVC") assert pvc.delete_pvcs(PVC_OBJS) log.info("Deleting CEPH BLOCK POOL") assert helpers.delete_cephblockpool() log.info("Deleting RBD Secret") RBD_SECRET_OBJ.delete() log.info("Deleting CEPHFS Secret") CEPHFS_SECRET_OBJ.delete() log.info("Deleting CEPH FILESYSTEM") assert helpers.delete_all_cephfilesystem() log.info("Deleting Storageclass") assert helpers.delete_all_storageclass()
def test_bulk_pvc_creation_after_deletion_performance( self, teardown_factory): """ Measuring PVC creation time of bulk of 75% of initial PVC bulk (120) in the same rate after deleting ( serial deletion) 75% of the initial PVCs and sends results to the Elastic Search DB Args: teardown_factory: A fixture used when we want a new resource that was created during the tests to be removed in the teardown phase. Returns: """ initial_number_of_pvcs = 120 number_of_pvcs = math.ceil(initial_number_of_pvcs * 0.75) log.info(f"Start creating new {initial_number_of_pvcs} PVCs in a bulk") pvc_objs, _ = helpers.create_multiple_pvcs( sc_name=self.sc_obj.name, namespace=self.namespace, number_of_pvc=initial_number_of_pvcs, size=self.pvc_size, burst=True, ) for pvc_obj in pvc_objs: teardown_factory(pvc_obj) with ThreadPoolExecutor() as executor: for pvc_obj in pvc_objs: executor.submit(helpers.wait_for_resource_state, pvc_obj, constants.STATUS_BOUND) executor.submit(pvc_obj.reload) log.info("Deleting 75% of the PVCs - 90 PVCs") assert pvc.delete_pvcs(pvc_objs[:number_of_pvcs], True), "Deletion of 75% of PVCs failed" log.info("Re-creating the 90 PVCs") pvc_objs, _ = helpers.create_multiple_pvcs( sc_name=self.sc_obj.name, namespace=self.namespace, number_of_pvc=number_of_pvcs, size=self.pvc_size, burst=True, ) start_time = helpers.get_provision_time(self.interface, pvc_objs, status="start") end_time = helpers.get_provision_time(self.interface, pvc_objs, status="end") total = end_time - start_time total_time = total.total_seconds() logging.info( f"Creation after deletion time of {number_of_pvcs} is {total_time} seconds." ) for pvc_obj in pvc_objs: teardown_factory(pvc_obj) with ThreadPoolExecutor() as executor: for pvc_obj in pvc_objs: executor.submit(helpers.wait_for_resource_state, pvc_obj, constants.STATUS_BOUND) executor.submit(pvc_obj.reload) if total_time > 50: raise ex.PerformanceException( f"{number_of_pvcs} PVCs creation (after initial deletion of " f"75% of PVCs) time is {total_time} and greater than 50 seconds." ) logging.info( f"{number_of_pvcs} PVCs creation time took less than a 50 seconds") # Produce ES report # Collecting environment information self.get_env_info() # Initialize the results doc file. full_results = self.init_full_results( ResultsAnalyse( self.uuid, self.crd_data, self.full_log_path, "bulk_pvc_creation_after_deletion_measurement", )) full_results.add_key("interface", self.interface) full_results.add_key("number_of_pvcs", number_of_pvcs) full_results.add_key("pvc_size", self.pvc_size) full_results.add_key("creation_after_deletion_time", total_time) # Write the test results into the ES server full_results.es_write()
def teardown(self): pvc_objs = get_all_pvc_objs(namespace="openshift-storage") pvcs = [pvc_obj for pvc_obj in pvc_objs if "test-pvc" in pvc_obj.name] delete_pvcs(pvc_objs=pvcs)
def destroy_ocs(self): """ Handle OCS destruction. Remove storage classes, PVCs, Storage Cluster, Openshift-storage namespace, LocalVolume, unlabel worker-storage nodes, delete ocs CRDs, etc. """ cluster_namespace = config.ENV_DATA["cluster_namespace"] # https://access.redhat.com/documentation/en-us/red_hat_openshift_container_storage/4.5/html/deploying_openshift_container_storage_using_bare_metal_infrastructure/assembly_uninstalling-openshift-container-storage_rhocs # Section 3.1 Step 1 # Deleting PVCs rbd_pvcs = [ p for p in pvc.get_all_pvcs_in_storageclass( constants.CEPHBLOCKPOOL_SC) if not (p.data["metadata"]["namespace"] == cluster_namespace and p.data["metadata"]["labels"]["app"] == "noobaa") ] pvc.delete_pvcs(rbd_pvcs) cephfs_pvcs = pvc.get_all_pvcs_in_storageclass( constants.CEPHFILESYSTEM_SC) # Section 3.1 Step 2 # Section 3.3 Step 1 # Removing OpenShift Container Platform registry from OpenShift Container Storage registry_conf_name = "configs.imageregistry.operator.openshift.io" registry_conf = ocp.OCP().exec_oc_cmd( f"get {registry_conf_name} -o yaml") if registry_conf["items"][0]["spec"].get("storage", dict()).get("pvc"): patch = dict(spec=dict(storage=dict(emptyDir=dict(), pvc=None))) ocp.OCP().exec_oc_cmd( f"patch {registry_conf_name} cluster --type merge " f"-p '{json.dumps(patch)}'") # Section 3.3 Step 2 pvc.delete_pvcs(cephfs_pvcs) # Section 3.1 Step 3 try: ocp.OCP().exec_oc_cmd( f"delete -n {cluster_namespace} storagecluster --all --wait=true" ) except (CommandFailed, subprocess.TimeoutExpired): pass # Section 3.1 Step 4 ocp.OCP().exec_oc_cmd("project default") ocp.OCP().exec_oc_cmd( f"delete project {cluster_namespace} --wait=true --timeout=5m") tried = 0 leftovers = True while tried < 5: # We need to loop here until the project can't be found try: ocp.OCP().exec_oc_cmd( f"get project {cluster_namespace}", out_yaml_format=False, ) except CommandFailed: leftovers = False break time.sleep(60) tried += 1 if leftovers: # https://access.redhat.com/documentation/en-us/red_hat_openshift_container_storage/4.5/html/troubleshooting_openshift_container_storage/troubleshooting-and-deleting-remaining-resources-during-uninstall_rhocs leftover_types = [ "cephfilesystem.ceph.rook.io", "cephobjectstore.ceph.rook.io", "cephobjectstoreuser.ceph.rook.io", "storagecluster.ocs.openshift.io", ] patch = dict(metadata=dict(finalizers=None)) for obj_type in leftover_types: try: objs = ocp.OCP(kind=obj_type).get() except CommandFailed: continue for obj in objs["items"]: name = obj["metadata"]["name"] ocp.OCP().exec_oc_cmd( f"oc patch -n {cluster_namespace} {obj_type} {name} --type=merge -p '{json.dumps(patch)}'" ) # Section 3.1 Step 5 nodes = ocp.OCP().exec_oc_cmd( "get node -l cluster.ocs.openshift.io/openshift-storage= -o yaml") for node in nodes["items"]: node_name = node["metadata"]["name"] ocp.OCP().exec_oc_cmd( f"debug node/{node_name} -- chroot /host rm -rfv /var/lib/rook" ) # Section 3.1 Step 6 ocp.OCP().exec_oc_cmd( "delete storageclass openshift-storage.noobaa.io --wait=true --timeout=5m" ) # Section 3.1 Step 7 ocp.OCP().exec_oc_cmd( "label nodes --all cluster.ocs.openshift.io/openshift-storage-") ocp.OCP().exec_oc_cmd("label nodes --all topology.rook.io/rack-") # Section 3.1 Step 8 pvs = ocp.OCP(kind="PersistentVolume").get() for pv in pvs["items"]: pv_name = pv["metadata"]["name"] if pv_name.startswith("ocs-storagecluster-ceph"): ocp.OCP().exec_oc_cmd(f"oc delete pv {pv_name}") # Section 3.1 Step 9 # Note that the below process differs from the documentation slightly. # Instead of deleting all CRDs at once and calling the job done, we # iterate over a list of them, noting which ones don't delete fully and # applying the standard workaround of removing the finalizers from any # CRs and also the CRD. Finally, the documentation leaves out a few # CRDs that we've seen in deployed clusters. crd_types = [ "backingstores.noobaa.io", "bucketclasses.noobaa.io", "cephblockpools.ceph.rook.io", "cephclients.ceph.rook.io", "cephclusters.ceph.rook.io", "cephfilesystems.ceph.rook.io", "cephnfses.ceph.rook.io", "cephobjectrealms.ceph.rook.io", # not in doc "cephobjectstores.ceph.rook.io", "cephobjectstoreusers.ceph.rook.io", "cephobjectzonegroups.ceph.rook.io", # not in doc "cephobjectzones.ceph.rook.io", # not in doc "cephrbdmirrors.ceph.rook.io", # not in doc "noobaas.noobaa.io", "ocsinitializations.ocs.openshift.io", "storageclusterinitializations.ocs.openshift.io", "storageclusters.ocs.openshift.io", ] cr_patch = json.dumps(dict(finalizers=None)) crd_patch = json.dumps(dict(metadata=dict(finalizers=None))) for crd_type in crd_types: try: ocp.OCP().exec_oc_cmd( f"delete crd {crd_type} --wait=true --timeout=30s", out_yaml_format=False, ) except CommandFailed: pass crs = [] try: crs = ocp.OCP(kind=crd_type).get(all_namespaces=True)["items"] except CommandFailed: continue for cr in crs: cr_md = cr["metadata"] ocp.OCP().exec_oc_cmd( f"patch -n {cr_md['namespace']} {crd_type} {cr_md['name']} --type=merge -p '{cr_patch}'" ) try: crs = ocp.OCP(kind=crd_type).get(all_namespaces=True)["items"] except CommandFailed: continue ocp.OCP().exec_oc_cmd( f"patch crd {crd_type} --type=merge -p '{crd_patch}'") # End sections from above documentation ocp.OCP().exec_oc_cmd( f"delete catalogsource {constants.OPERATOR_CATALOG_SOURCE_NAME} " f"-n {constants.MARKETPLACE_NAMESPACE}") storageclasses = ocp.OCP(kind="StorageClass").get(all_namespaces=True) for sc in storageclasses["items"]: if sc["provisioner"].startswith("openshift-storage."): sc_name = sc["metadata"]["name"] ocp.OCP().exec_oc_cmd(f"delete storageclass {sc_name}") volumesnapclasses = ocp.OCP(kind="VolumeSnapshotClass").get( all_namespaces=True) for vsc in volumesnapclasses["items"]: if vsc["driver"].startswith("openshift-storage."): vsc_name = vsc["metadata"]["name"] ocp.OCP().exec_oc_cmd(f"delete volumesnapshotclass {vsc_name}") self.destroy_lso()
def finalizer(): # Delete newly created PVCs assert delete_pvcs(cls_ref.pvc_objs_new), 'Failed to delete PVCs' log.info(f'Newly created {cls_ref.num_of_pvcs} PVCs are now deleted.')
def uninstall_cluster_logging(): """ Function to uninstall cluster-logging from the cluster Deletes the project "openshift-logging" and "openshift-operators-redhat" """ # Validating the pods before deleting the instance pod_list = get_all_pods(namespace=constants.OPENSHIFT_LOGGING_NAMESPACE) for pod in pod_list: logger.info( f"Pods running in the openshift-logging namespace {pod.name}") # Excluding cluster-logging-operator from pod_list and getting pod names pod_names_list = [ pod.name for pod in pod_list if not pod.name.startswith("cluster-logging-operator") ] pvc_objs = get_all_pvc_objs( namespace=constants.OPENSHIFT_LOGGING_NAMESPACE) # Fetch image uuid associated with PVCs to be deleted pvc_uuid_map = {} for pvc_obj in pvc_objs: pvc_uuid_map[pvc_obj.name] = pvc_obj.image_uuid # Checking for used space cbp_name = default_ceph_block_pool() used_space_before_deletion = fetch_used_size(cbp_name) logger.info( f"Used space before deletion of cluster logging {used_space_before_deletion}" ) # Deleting the clusterlogging instance clusterlogging_obj = ocp.OCP( kind=constants.CLUSTER_LOGGING, namespace=constants.OPENSHIFT_LOGGING_NAMESPACE) assert clusterlogging_obj.delete(resource_name="instance") check_pod_vanished(pod_names_list) for pvc_obj in pvc_objs: pv_obj = pvc_obj.backed_pv_obj assert delete_pvcs(pvc_objs=pvc_objs), "PVCs deletion failed" for pvc_obj in pvc_objs: pvc_obj.ocp.wait_for_delete(resource_name=pvc_obj.name, timeout=300) pv_obj.ocp.wait_for_delete(resource_name=pv_obj.name, timeout=300) logger.info("Verified: PVCs are deleted.") logger.info("Verified: PV are deleted") for pvc_name, uuid in pvc_uuid_map.items(): rbd = verify_volume_deleted_in_backend( interface=constants.CEPHBLOCKPOOL, image_uuid=uuid, pool_name=cbp_name) assert rbd, f"Volume associated with PVC {pvc_name} still exists " f"in backend" # Checking for used space after PVC deletion used_space_after_deletion = fetch_used_size(cbp_name, exp_val=30) logger.info( f"Used space after deletion of cluster logging {used_space_after_deletion}" ) if used_space_after_deletion < used_space_before_deletion: logger.info("Expected !!! Space has reclaimed") else: logger.warning( "Unexpected !! No space reclaimed after deletion of PVC") # Deleting the RBAC permission set rbac_role = ocp.OCP( kind=constants.ROLE, namespace=constants.OPENSHIFT_OPERATORS_REDHAT_NAMESPACE) rbac_role.delete(yaml_file=constants.EO_RBAC_YAML) openshift_logging_namespace = ocp.OCP( kind=constants.NAMESPACES, namespace=constants.OPENSHIFT_LOGGING_NAMESPACE) openshift_operators_redhat_namespace = ocp.OCP( kind=constants.NAMESPACES, namespace=constants.OPENSHIFT_OPERATORS_REDHAT_NAMESPACE, ) if openshift_operators_redhat_namespace.get(): assert openshift_operators_redhat_namespace.delete( resource_name=constants.OPENSHIFT_OPERATORS_REDHAT_NAMESPACE) logger.info( "The project openshift-opertors-redhat got deleted successfully") if openshift_logging_namespace.get(): assert openshift_logging_namespace.delete( resource_name=constants.OPENSHIFT_LOGGING_NAMESPACE) logger.info("The namespace openshift-logging got deleted successfully")
def test_noobaa_db_backup_and_recovery( self, pvc_factory, pod_factory, snapshot_factory, bucket_factory, rgw_bucket_factory, ): """ Test case to verify noobaa backup and recovery 1. Take snapshot db-noobaa-db-0 PVC and retore it to PVC 2. Scale down the statefulset noobaa-db 3. Get the yaml of the current PVC, db-noobaa-db-0 and change the parameter persistentVolumeReclaimPolicy to Retain for restored PVC 4. Delete both PVCs, the PV for the original claim db-noobaa-db-0 will be removed. The PV for claim db-noobaa-db-0-snapshot-restore will move to ‘Released’ 5. Edit again restore PV and remove the claimRef section. The volume will transition to Available. 6. Edit the yaml db-noobaa-db-0.yaml and change the setting volumeName to restored PVC. 7. Scale up the stateful set again and the pod should be running """ # Initialise variable self.noobaa_db_sst_name = "noobaa-db-pg" # Get noobaa pods before execution noobaa_pods = get_noobaa_pods() # Get noobaa PVC before execution noobaa_pvc_obj = get_pvc_objs(pvc_names=["db-noobaa-db-pg-0"]) noobaa_pv_name = noobaa_pvc_obj[0].get("spec").get("spec").get( "volumeName") # Take snapshot db-noobaa-db-0 PVC log.info(f"Creating snapshot of the {noobaa_pvc_obj[0].name} PVC") snap_obj = snapshot_factory( pvc_obj=noobaa_pvc_obj[0], wait=True, snapshot_name=f"{noobaa_pvc_obj[0].name}-snapshot", ) log.info( f"Successfully created snapshot {snap_obj.name} and in Ready state" ) # Restore it to PVC log.info(f"Restoring snapshot {snap_obj.name} to create new PVC") sc_name = noobaa_pvc_obj[0].get().get("spec").get("storageClassName") pvc_size = (noobaa_pvc_obj[0].get().get("spec").get("resources").get( "requests").get("storage")) self.restore_pvc_obj = create_restore_pvc( sc_name=sc_name, snap_name=snap_obj.name, namespace=snap_obj.namespace, size=pvc_size, pvc_name=f"{snap_obj.name}-restore", volume_mode=snap_obj.parent_volume_mode, access_mode=snap_obj.parent_access_mode, ) wait_for_resource_state(self.restore_pvc_obj, constants.STATUS_BOUND) self.restore_pvc_obj.reload() log.info(f"Succeesfuly created PVC {self.restore_pvc_obj.name} " f"from snapshot {snap_obj.name}") # Scale down the statefulset noobaa-db modify_statefulset_replica_count( statefulset_name=self.noobaa_db_sst_name, replica_count=0 ), f"Failed to scale down the statefulset {self.noobaa_db_sst_name}" # Get the noobaa-db PVC pvc_obj = OCP(kind=constants.PVC, namespace=constants.OPENSHIFT_STORAGE_NAMESPACE) noobaa_pvc_yaml = pvc_obj.get(resource_name=noobaa_pvc_obj[0].name) # Get the restored noobaa PVC and # change the parameter persistentVolumeReclaimPolicy to Retain restored_noobaa_pvc_obj = get_pvc_objs( pvc_names=[f"{snap_obj.name}-restore"]) restored_noobaa_pv_name = (restored_noobaa_pvc_obj[0].get("spec").get( "spec").get("volumeName")) pv_obj = OCP(kind=constants.PV, namespace=constants.OPENSHIFT_STORAGE_NAMESPACE) params = '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}' assert pv_obj.patch( resource_name=restored_noobaa_pv_name, params=params), ( "Failed to change the parameter persistentVolumeReclaimPolicy" f" to Retain {restored_noobaa_pv_name}") # Delete both PVCs delete_pvcs(pvc_objs=[noobaa_pvc_obj[0], restored_noobaa_pvc_obj[0]]) # Validate original claim db-noobaa-db-0 removed assert validate_pv_delete( pv_name=noobaa_pv_name ), f"PV not deleted, still exist {noobaa_pv_name}" # Validate PV for claim db-noobaa-db-0-snapshot-restore is in Released state pv_obj.wait_for_resource(condition=constants.STATUS_RELEASED, resource_name=restored_noobaa_pv_name) # Edit again restore PV and remove the claimRef section log.info( f"Remove the claimRef section from PVC {restored_noobaa_pv_name}") params = '[{"op": "remove", "path": "/spec/claimRef"}]' pv_obj.patch(resource_name=restored_noobaa_pv_name, params=params, format_type="json") log.info( f"Successfully removed claimRef section from PVC {restored_noobaa_pv_name}" ) # Validate PV is in Available state pv_obj.wait_for_resource(condition=constants.STATUS_AVAILABLE, resource_name=restored_noobaa_pv_name) # Edit the yaml db-noobaa-db-0.yaml and change the # setting volumeName to restored PVC noobaa_pvc_yaml["spec"]["volumeName"] = restored_noobaa_pv_name noobaa_pvc_yaml = OCS(**noobaa_pvc_yaml) noobaa_pvc_yaml.create() # Validate noobaa PVC is in bound state pvc_obj.wait_for_resource( condition=constants.STATUS_BOUND, resource_name=noobaa_pvc_obj[0].name, timeout=120, ) # Scale up the statefulset again assert modify_statefulset_replica_count( statefulset_name=self.noobaa_db_sst_name, replica_count=1 ), f"Failed to scale up the statefulset {self.noobaa_db_sst_name}" # Validate noobaa pod is up and running pod_obj = OCP(kind=constants.POD, namespace=defaults.ROOK_CLUSTER_NAMESPACE) pod_obj.wait_for_resource( condition=constants.STATUS_RUNNING, resource_count=len(noobaa_pods), selector=constants.NOOBAA_APP_LABEL, ) # Change the parameter persistentVolumeReclaimPolicy to Delete again params = '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' assert pv_obj.patch( resource_name=restored_noobaa_pv_name, params=params), ( "Failed to change the parameter persistentVolumeReclaimPolicy" f" to Delete {restored_noobaa_pv_name}") log.info( "Changed the parameter persistentVolumeReclaimPolicy to Delete again" ) # Verify all storage pods are running wait_for_storage_pods() # Creating Resources log.info("Creating Resources using sanity helpers") self.sanity_helpers.create_resources(pvc_factory, pod_factory, bucket_factory, rgw_bucket_factory) # Deleting Resources self.sanity_helpers.delete_resources() # Verify everything running fine log.info( "Verifying All resources are Running and matches expected result") self.sanity_helpers.health_check(tries=120)
def test_multiple_sc_comp_rep_data_deletion(self, storageclass_factory, pvc_factory, pod_factory): """ This test function does below, *. Creates 2 Storage Class with creating new rbd pool *. Creates PVCs using new Storage Class *. Mount PVC to an app pod *. Run IO on an app pod *. Delete the pods and pvc *. Verify that the data is deleted """ log.info("Creating storageclasses with compression and replica3") interface_type = constants.CEPHBLOCKPOOL sc_obj1 = storageclass_factory( interface=interface_type, new_rbd_pool=True, replica=3, compression="aggressive", ) log.info("Creating storageclasses with compression and replica2") sc_obj2 = storageclass_factory( interface=interface_type, new_rbd_pool=True, replica=2, compression="aggressive", ) sc_obj_list = [sc_obj1, sc_obj2] pod_obj_list = [] pvc_obj_list = [] log.info("Creating PVCs and PODs") for sc_obj in sc_obj_list: pvc_obj = pvc_factory(interface=interface_type, storageclass=sc_obj) pvc_obj_list.append(pvc_obj) pod_obj_list.append( pod_factory(interface=interface_type, pvc=pvc_obj)) log.info("Running IO on pods") for pod_obj in pod_obj_list: pod_obj.run_io("fs", size="1G") for pod_obj in pod_obj_list: get_fio_rw_iops(pod_obj) log.info("deleting PODs and PVCs") delete_pods(pod_obj_list, wait=True) delete_pvcs(pvc_obj_list, concurrent=True) log.info("Wait for 15 seconds for all data to delete") sleep(15) log.info("Checking stats after deleting PODs and PVCs") for sc_obj in sc_obj_list: pvc_list = get_all_pvcs_in_storageclass(sc_obj.name) if len(pvc_list) == 0: cbp_name = sc_obj.get()["parameters"]["pool"] ceph_pool_byte_used = get_byte_used_by_pool(cbp_name) log.info( f"pool {cbp_name} has {ceph_pool_byte_used} bytes used") if ceph_pool_byte_used > MAX_BYTES_IN_POOL_AFTER_DATA_DELETE: raise PoolDataNotErased( f"Pool {cbp_name} has {ceph_pool_byte_used} bytes which were not deleted" ) else: raise PvcNotDeleted(f"PVC {pvc_list} were not deleted")
def factory(snapshot_factory=snapshot_factory): # Get noobaa pods before execution noobaa_pods = pod.get_noobaa_pods() # Get noobaa PVC before execution noobaa_pvc_obj = pvc.get_pvc_objs(pvc_names=["db-noobaa-db-pg-0"]) noobaa_pv_name = noobaa_pvc_obj[0].get("spec").get("spec").get( "volumeName") # Take snapshot db-noobaa-db-0 PVC logger.info(f"Creating snapshot of the {noobaa_pvc_obj[0].name} PVC") snap_obj = snapshot_factory( pvc_obj=noobaa_pvc_obj[0], wait=True, snapshot_name=f"{noobaa_pvc_obj[0].name}-snapshot", ) logger.info( f"Successfully created snapshot {snap_obj.name} and in Ready state" ) # Restore it to PVC logger.info(f"Restoring snapshot {snap_obj.name} to create new PVC") sc_name = noobaa_pvc_obj[0].get().get("spec").get("storageClassName") pvc_size = (noobaa_pvc_obj[0].get().get("spec").get("resources").get( "requests").get("storage")) restore_pvc_obj = pvc.create_restore_pvc( sc_name=sc_name, snap_name=snap_obj.name, namespace=snap_obj.namespace, size=pvc_size, pvc_name=f"{snap_obj.name}-restore", volume_mode=snap_obj.parent_volume_mode, access_mode=snap_obj.parent_access_mode, ) restore_pvc_objs.append(restore_pvc_obj) wait_for_resource_state(restore_pvc_obj, constants.STATUS_BOUND) restore_pvc_obj.reload() logger.info(f"Succeesfuly created PVC {restore_pvc_obj.name} " f"from snapshot {snap_obj.name}") # Scale down the statefulset noobaa-db modify_statefulset_replica_count( statefulset_name=constants.NOOBAA_DB_STATEFULSET, replica_count=0 ), f"Failed to scale down the statefulset {constants.NOOBAA_DB_STATEFULSET}" # Get the noobaa-db PVC pvc_obj = OCP(kind=constants.PVC, namespace=constants.OPENSHIFT_STORAGE_NAMESPACE) noobaa_pvc_yaml = pvc_obj.get(resource_name=noobaa_pvc_obj[0].name) # Get the restored noobaa PVC and # change the parameter persistentVolumeReclaimPolicy to Retain restored_noobaa_pvc_obj = pvc.get_pvc_objs( pvc_names=[f"{snap_obj.name}-restore"]) restored_noobaa_pv_name = (restored_noobaa_pvc_obj[0].get("spec").get( "spec").get("volumeName")) pv_obj = OCP(kind=constants.PV, namespace=constants.OPENSHIFT_STORAGE_NAMESPACE) params = '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}' assert pv_obj.patch( resource_name=restored_noobaa_pv_name, params=params), ( "Failed to change the parameter persistentVolumeReclaimPolicy" f" to Retain {restored_noobaa_pv_name}") # Delete both PVCs pvc.delete_pvcs( pvc_objs=[noobaa_pvc_obj[0], restored_noobaa_pvc_obj[0]]) # Validate original claim db-noobaa-db-0 removed assert validate_pv_delete( pv_name=noobaa_pv_name ), f"PV not deleted, still exist {noobaa_pv_name}" # Validate PV for claim db-noobaa-db-0-snapshot-restore is in Released state pv_obj.wait_for_resource(condition=constants.STATUS_RELEASED, resource_name=restored_noobaa_pv_name) # Edit again restore PV and remove the claimRef section logger.info( f"Remove the claimRef section from PVC {restored_noobaa_pv_name}") params = '[{"op": "remove", "path": "/spec/claimRef"}]' pv_obj.patch(resource_name=restored_noobaa_pv_name, params=params, format_type="json") logger.info( f"Successfully removed claimRef section from PVC {restored_noobaa_pv_name}" ) # Validate PV is in Available state pv_obj.wait_for_resource(condition=constants.STATUS_AVAILABLE, resource_name=restored_noobaa_pv_name) # Edit the yaml db-noobaa-db-0.yaml and change the # setting volumeName to restored PVC noobaa_pvc_yaml["spec"]["volumeName"] = restored_noobaa_pv_name noobaa_pvc_yaml = OCS(**noobaa_pvc_yaml) noobaa_pvc_yaml.create() # Validate noobaa PVC is in bound state pvc_obj.wait_for_resource( condition=constants.STATUS_BOUND, resource_name=noobaa_pvc_obj[0].name, timeout=120, ) # Scale up the statefulset again assert modify_statefulset_replica_count( statefulset_name=constants.NOOBAA_DB_STATEFULSET, replica_count=1 ), f"Failed to scale up the statefulset {constants.NOOBAA_DB_STATEFULSET}" # Validate noobaa pod is up and running pod_obj = OCP(kind=constants.POD, namespace=defaults.ROOK_CLUSTER_NAMESPACE) pod_obj.wait_for_resource( condition=constants.STATUS_RUNNING, resource_count=len(noobaa_pods), selector=constants.NOOBAA_APP_LABEL, ) # Change the parameter persistentVolumeReclaimPolicy to Delete again params = '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' assert pv_obj.patch( resource_name=restored_noobaa_pv_name, params=params), ( "Failed to change the parameter persistentVolumeReclaimPolicy" f" to Delete {restored_noobaa_pv_name}") logger.info( "Changed the parameter persistentVolumeReclaimPolicy to Delete again" )
def test_bulk_pvc_creation_after_deletion_performance( self, interface_iterate, storageclass_factory ): """ Measuring PVC creation time of bulk of 75% of initial PVC bulk (120) in the same rate after deleting ( serial deletion) 75% of the initial PVCs and sends results to the Elastic Search DB """ self.interface = interface_iterate self.sc_obj = storageclass_factory(self.interface) initial_number_of_pvcs = 120 number_of_pvcs = math.ceil(initial_number_of_pvcs * 0.75) # Getting the test start time self.test_start_time = self.get_time() log.info(f"Start creating new {initial_number_of_pvcs} PVCs in a bulk") self.pvc_bulk_create_and_wait_for_bound(initial_number_of_pvcs) log.info(f"Deleting 75% of the PVCs - {number_of_pvcs} PVCs") assert pvc.delete_pvcs( self.pvc_objs[:number_of_pvcs], True ), "Deletion of 75% of PVCs failed" # save the list of pvcs which not deleted, for the teardown phase original_pvcs = self.pvc_objs[number_of_pvcs:] log.info(f"Re-creating the {number_of_pvcs} PVCs") csi_bulk_start_time = self.get_time(time_format="csi") self.pvc_bulk_create_and_wait_for_bound(number_of_pvcs) # Get the bulk recraation time - total time. total_time = self.get_bulk_creation_time() log.info( f"Creation after deletion time of {number_of_pvcs} is {total_time} seconds." ) if total_time > 50: raise ex.PerformanceException( f"{number_of_pvcs} PVCs creation (after initial deletion of " f"75% of PVCs) time is {total_time} and greater than 50 seconds." ) log.info(f"{number_of_pvcs} PVCs creation time took less than a 50 seconds") csi_creation_times = performance_lib.csi_bulk_pvc_time_measure( self.interface, self.pvc_objs, "create", csi_bulk_start_time ) # Getting the end time of the test self.test_end_time = self.get_time() # update the list of pvcs for the teardown process self.pvc_objs += original_pvcs # Produce ES report self.results_path = os.path.join( "/", *self.results_path, "test_bulk_pvc_creation_after_deletion_performance", ) # Collecting environment information self.get_env_info() # Initialize the results doc file. full_results = self.init_full_results( ResultsAnalyse( self.uuid, self.crd_data, self.full_log_path, "bulk_pvc_creation_after_deletion_measurement", ) ) # Add the test time to the ES report full_results.add_key( "test_time", {"start": self.test_start_time, "end": self.test_end_time} ) full_results.add_key("number_of_pvcs", number_of_pvcs) full_results.add_key("creation_after_deletion_time", total_time) full_results.add_key("creation_after_deletion_csi_time", csi_creation_times) # Write the test results into the ES server if full_results.es_write(): res_link = full_results.results_link() log.info(f"The Result can be found at : {res_link}") # Create text file with results of all subtest (2 - according to the parameters) self.write_result_to_file(res_link)