class TestNodeReplacementWithIO(ManageTest): """ Knip-894 Node replacement proactive with IO """ @pytest.fixture(autouse=True) def init_sanity(self): """ Initialize Sanity instance """ self.sanity_helpers = Sanity() def test_nodereplacement_proactive_with_io_running( self, pvc_factory, pod_factory, dc_pod_factory ): """ Knip-894 Node Replacement proactive when IO running in the background """ # Get worker nodes worker_node_list = node.get_worker_nodes() log.info(f"Current available worker nodes are {worker_node_list}") osd_node_name = select_osd_node_name() log.info("Creating dc pod backed with rbd pvc and running io in bg") for worker_node in worker_node_list: if worker_node != osd_node_name: rbd_dc_pod = dc_pod_factory( interface=constants.CEPHBLOCKPOOL, node_name=worker_node, size=20 ) pod.run_io_in_bg(rbd_dc_pod, expect_to_fail=False, fedora_dc=True) log.info("Creating dc pod backed with cephfs pvc and running io in bg") for worker_node in worker_node_list: if worker_node != osd_node_name: cephfs_dc_pod = dc_pod_factory( interface=constants.CEPHFILESYSTEM, node_name=worker_node, size=20 ) pod.run_io_in_bg(cephfs_dc_pod, expect_to_fail=False, fedora_dc=True) delete_and_create_osd_node(osd_node_name) # Creating Resources log.info("Creating Resources using sanity helpers") self.sanity_helpers.create_resources(pvc_factory, pod_factory) # Deleting Resources self.sanity_helpers.delete_resources() # Verify everything running fine log.info("Verifying All resources are Running and matches expected result") self.sanity_helpers.health_check(tries=120)
class TestNoobaaBackupAndRecovery(E2ETest): """ Test to verify noobaa backup and recovery """ @pytest.fixture(autouse=True) def init_sanity(self): """ Initialize Sanity instance """ self.sanity_helpers = Sanity() def test_noobaa_db_backup_and_recovery( self, pvc_factory, pod_factory, snapshot_factory, bucket_factory, rgw_bucket_factory, noobaa_db_backup_and_recovery, ): """ Test case to verify noobaa backup and recovery 1. Take snapshot db-noobaa-db-0 PVC and retore it to PVC 2. Scale down the statefulset noobaa-db 3. Get the yaml of the current PVC, db-noobaa-db-0 and change the parameter persistentVolumeReclaimPolicy to Retain for restored PVC 4. Delete both PVCs, the PV for the original claim db-noobaa-db-0 will be removed. The PV for claim db-noobaa-db-0-snapshot-restore will move to ‘Released’ 5. Edit again restore PV and remove the claimRef section. The volume will transition to Available. 6. Edit the yaml db-noobaa-db-0.yaml and change the setting volumeName to restored PVC. 7. Scale up the stateful set again and the pod should be running """ noobaa_db_backup_and_recovery(snapshot_factory=snapshot_factory) # Verify all storage pods are running wait_for_storage_pods() # Creating Resources log.info("Creating Resources using sanity helpers") self.sanity_helpers.create_resources( pvc_factory, pod_factory, bucket_factory, rgw_bucket_factory ) # Deleting Resources self.sanity_helpers.delete_resources() # Verify everything running fine log.info("Verifying All resources are Running and matches expected result") self.sanity_helpers.health_check(tries=120)
class TestRollingWorkerNodeShutdownAndRecovery(ManageTest): """ Test rolling shutdown and recovery of OCS pods running worker nodes """ @pytest.fixture(autouse=True) def init_sanity(self): """ Initialize Sanity instance """ self.sanity_helpers = Sanity() @pytest.fixture(autouse=True) def teardown(self, request, nodes): """ Make sure all nodes are up again """ def finalizer(): nodes.restart_nodes_by_stop_and_start_teardown() request.addfinalizer(finalizer) def test_rolling_shutdown_and_recovery( self, nodes, pvc_factory, pod_factory, bucket_factory, rgw_bucket_factory ): SECONDS_TO_WAIT = 180 """ Test rolling shutdown and recovery of OCS worker nodes """ # Get OCS worker node objects ocs_node_objs = get_ocs_nodes() # Start rolling shutdown and recovery of OCS worker nodes log.info("ShutDown OCS worker") for node_obj in ocs_node_objs: nodes.stop_nodes(nodes=[node_obj]) log.info(f"Keeping node in stopped state for {SECONDS_TO_WAIT} mins") time.sleep(SECONDS_TO_WAIT) nodes.start_nodes(nodes=[node_obj]) self.sanity_helpers.health_check(cluster_check=False, tries=60) log.info("Checking storage pods status") # Validate storage pods are running wait_for_pods_to_be_running(timeout=600) # Check basic cluster functionality by creating some resources self.sanity_helpers.create_resources( pvc_factory, pod_factory, bucket_factory, rgw_bucket_factory )
class TestDiskFailures(ManageTest): """ Test class for detach and attach worker volume """ def detach_volume_and_wait_for_attach(self, nodes, data_volume, worker_node): """ Detach an EBS volume from an AWS instance and wait for the volume to be re-attached Args: node (OCS): The OCS object representing the node data_volume (Volume): The ec2 volume to delete worker_node (OCS): The OCS object of the EC2 instance """ try: # Detach volume (logging is done inside the function) nodes.detach_volume(data_volume, worker_node) except AWSTimeoutException as e: if "Volume state: in-use" in e: logger.info( f"Volume {data_volume} re-attached successfully to worker" f" node {worker_node}") else: raise else: """ Wait for worker volume to be re-attached automatically to the node """ assert nodes.wait_for_volume_attach(data_volume), ( f"Volume {data_volume} failed to be re-attached to worker " f"node {worker_node}") @pytest.fixture(autouse=True) def teardown(self, request, nodes): """ Restart nodes that are in status NotReady, for situations in which the test failed before restarting the node after detach volume, which leaves nodes in NotReady """ def finalizer(): not_ready_nodes = [ n for n in node.get_node_objs() if n.ocp.get_resource_status( n.name) == constants.NODE_NOT_READY ] logger.warning( f"Nodes in NotReady status found: {[n.name for n in not_ready_nodes]}" ) if not_ready_nodes: nodes.restart_nodes(not_ready_nodes) node.wait_for_nodes_status() # Restart node if the osd stays at CLBO state osd_pods_obj_list = get_osd_pods() for pod in osd_pods_obj_list: if (pod.get().get("status").get("containerStatuses")[0].get( "state") == constants.STATUS_CLBO): node_obj = get_pod_node(pod) nodes.restart_nodes([node_obj]) node.wait_for_nodes_status([node_obj.name]) request.addfinalizer(finalizer) @pytest.fixture(autouse=True) def init_sanity(self): """ Initialize Sanity instance """ self.sanity_helpers = Sanity() @cloud_platform_required @pytest.mark.polarion_id("OCS-1085") @bugzilla("1825675") def test_detach_attach_worker_volume(self, nodes, pvc_factory, pod_factory): """ Detach and attach worker volume - Detach the data volume from one of the worker nodes - Wait for the volumes to be re-attached back to the worker node - Validate cluster functionality, without checking cluster and Ceph health (as one node volume is detached, the cluster will be unhealthy) by creating resources and running IO - Restart the node so the volume will get re-mounted """ # Get a data volume data_volume = nodes.get_data_volumes()[0] # Get the worker node according to the volume attachment worker = nodes.get_node_by_attached_volume(data_volume) # Detach volume and wait for the volume to attach self.detach_volume_and_wait_for_attach(nodes, data_volume, worker) # Validate cluster is still functional # In case the selected node that its volume disk was detached was the one # running the ceph tools pod, we'll need to wait for a new ct pod to start. # For that, a function that connects to the ct pod is being used to check if # it's alive assert (wait_for_ct_pod_recovery() ), "Ceph tools pod failed to come up on another node" self.sanity_helpers.create_resources(pvc_factory, pod_factory) # Restart the instance so the volume will get re-mounted nodes.restart_nodes([worker]) # Cluster health check # W/A: For the investigation of BZ 1825675, timeout is increased to see if cluster # becomes healthy eventually # TODO: Remove 'tries=100' self.sanity_helpers.health_check(tries=100) @cloud_platform_required @pytest.mark.polarion_id("OCS-1086") def test_detach_attach_2_data_volumes(self, nodes, pvc_factory, pod_factory): """ Detach and attach disk from 2 worker nodes - Detach the data 2 of the data volumes from their worker nodes - Wait for the volumes to be re-attached back to the worker nodes - Restart the nodes so the volume will get re-mounted in each node - Check cluster health and functionality to make sure detach, attach and restart did not affect the cluster """ # Get 2 data volumes data_volumes = nodes.get_data_volumes()[:2] workers_and_volumes = [{ "worker": nodes.get_node_by_attached_volume(vol), "volume": vol } for vol in data_volumes] for worker_and_volume in workers_and_volumes: # Detach volume and wait for the volume to attach self.detach_volume_and_wait_for_attach(nodes, worker_and_volume["volume"], worker_and_volume["worker"]) # Restart the instances so the volume will get re-mounted nodes.restart_nodes([ worker_and_volume["worker"] for worker_and_volume in workers_and_volumes ]) # Validate cluster is still functional self.sanity_helpers.health_check() self.sanity_helpers.create_resources(pvc_factory, pod_factory) @bugzilla("1830702") @vsphere_platform_required @pytest.mark.polarion_id("OCS-2172") def test_recovery_from_volume_deletion(self, nodes, pvc_factory, pod_factory): """ Test cluster recovery from disk deletion from the platform side. Based on documented procedure detailed in https://bugzilla.redhat.com/show_bug.cgi?id=1823183 """ logger.info("Picking a PV which to be deleted from the platform side") osd_pvs = get_deviceset_pvs() osd_pv = random.choice(osd_pvs) osd_pv_name = osd_pv.name # get the claim name logger.info(f"Getting the claim name for OSD PV {osd_pv_name}") claim_name = osd_pv.get().get("spec").get("claimRef").get("name") # Get the backing volume name logger.info(f"Getting the backing volume name for PV {osd_pv_name}") backing_volume = nodes.get_data_volumes(pvs=[osd_pv])[0] # Get the corresponding PVC logger.info(f"Getting the corresponding PVC of PV {osd_pv_name}") osd_pvcs = get_deviceset_pvcs() osd_pvcs_count = len(osd_pvcs) osd_pvc = [ ds for ds in osd_pvcs if ds.get().get("metadata").get("name") == claim_name ][0] # Get the corresponding OSD pod and ID logger.info(f"Getting the OSD pod using PVC {osd_pvc.name}") osd_pods = get_osd_pods() osd_pods_count = len(osd_pods) osd_pod = [ osd_pod for osd_pod in osd_pods if osd_pod.get().get("metadata").get("labels").get( constants.CEPH_ROOK_IO_PVC_LABEL) == claim_name ][0] logger.info(f"OSD_POD {osd_pod.name}") osd_id = osd_pod.get().get("metadata").get("labels").get("ceph-osd-id") # Get the node that has the OSD pod running on logger.info( f"Getting the node that has the OSD pod {osd_pod.name} running on") osd_node = get_pod_node(osd_pod) osd_prepare_pods = get_osd_prepare_pods() osd_prepare_pod = [ pod for pod in osd_prepare_pods if pod.get().get("metadata").get( "labels").get(constants.CEPH_ROOK_IO_PVC_LABEL) == claim_name ][0] osd_prepare_job_name = (osd_prepare_pod.get().get("metadata").get( "labels").get("job-name")) osd_prepare_job = get_job_obj(osd_prepare_job_name) # Get the corresponding OSD deployment logger.info(f"Getting the OSD deployment for OSD PVC {claim_name}") osd_deployment = [ osd_pod for osd_pod in get_osd_deployments() if osd_pod.get().get("metadata").get("labels").get( constants.CEPH_ROOK_IO_PVC_LABEL) == claim_name ][0] osd_deployment_name = osd_deployment.name # Delete the volume from the platform side logger.info(f"Deleting {backing_volume} from the platform side") nodes.detach_volume(backing_volume, osd_node) # Scale down OSD deployment logger.info(f"Scaling down OSD deployment {osd_deployment_name} to 0") ocp.OCP().exec_oc_cmd( f"scale --replicas=0 deployment/{osd_deployment_name}") # Force delete OSD pod if necessary osd_pod_name = osd_pod.name logger.info(f"Waiting for OSD pod {osd_pod.name} to get deleted") try: osd_pod.ocp.wait_for_delete(resource_name=osd_pod_name) except TimeoutError: osd_pod.delete(force=True) osd_pod.ocp.wait_for_delete(resource_name=osd_pod_name) # Run ocs-osd-removal job ocp_version = float(get_ocp_version()) if ocp_version >= 4.6: cmd = f"process ocs-osd-removal -p FAILED_OSD_IDS={osd_id} -o yaml" else: cmd = f"process ocs-osd-removal -p FAILED_OSD_ID={osd_id} -o yaml" logger.info(f"Executing OSD removal job on OSD-{osd_id}") ocp_obj = ocp.OCP(namespace=config.ENV_DATA["cluster_namespace"]) osd_removal_job_yaml = ocp_obj.exec_oc_cmd(cmd) osd_removal_job = OCS(**osd_removal_job_yaml) osd_removal_job.create(do_reload=False) # Get ocs-osd-removal pod name logger.info("Getting the ocs-osd-removal pod name") osd_removal_pod_name = get_osd_removal_pod_name(osd_id) osd_removal_pod_obj = get_pod_obj(osd_removal_pod_name, namespace="openshift-storage") osd_removal_pod_obj.ocp.wait_for_resource( condition=constants.STATUS_COMPLETED, resource_name=osd_removal_pod_name) # Verify OSD removal from the ocs-osd-removal pod logs logger.info( f"Verifying removal of OSD from {osd_removal_pod_name} pod logs") logs = get_pod_logs(osd_removal_pod_name) pattern = f"purged osd.{osd_id}" assert re.search(pattern, logs) osd_pvc_name = osd_pvc.name if ocp_version < 4.6: # Delete the OSD prepare job logger.info(f"Deleting OSD prepare job {osd_prepare_job_name}") osd_prepare_job.delete() osd_prepare_job.ocp.wait_for_delete( resource_name=osd_prepare_job_name, timeout=120) # Delete the OSD PVC logger.info(f"Deleting OSD PVC {osd_pvc_name}") osd_pvc.delete() osd_pvc.ocp.wait_for_delete(resource_name=osd_pvc_name) # Delete the OSD deployment logger.info(f"Deleting OSD deployment {osd_deployment_name}") osd_deployment.delete() osd_deployment.ocp.wait_for_delete( resource_name=osd_deployment_name, timeout=120) else: # If ocp version is '4.6' and above the osd removal job should # delete the OSD prepare job, OSD PVC, OSD deployment logger.info( f"Verifying deletion of OSD prepare job {osd_prepare_job_name}" ) osd_prepare_job.ocp.wait_for_delete( resource_name=osd_prepare_job_name, timeout=30) logger.info(f"Verifying deletion of OSD PVC {osd_pvc_name}") osd_pvc.ocp.wait_for_delete(resource_name=osd_pvc_name, timeout=30) logger.info( f"Verifying deletion of OSD deployment {osd_deployment_name}") osd_deployment.ocp.wait_for_delete( resource_name=osd_deployment_name, timeout=30) # Delete PV logger.info(f"Verifying deletion of PV {osd_pv_name}") try: osd_pv.ocp.wait_for_delete(resource_name=osd_pv_name) except TimeoutError: osd_pv.delete() osd_pv.ocp.wait_for_delete(resource_name=osd_pv_name) if ocp_version < 4.6: # Delete the rook ceph operator pod to trigger reconciliation rook_operator_pod = get_operator_pods()[0] logger.info( f"deleting Rook Ceph operator pod {rook_operator_pod.name}") rook_operator_pod.delete() # Delete the OSD removal job logger.info(f"Deleting OSD removal job ocs-osd-removal-{osd_id}") osd_removal_job = get_job_obj(f"ocs-osd-removal-{osd_id}") osd_removal_job.delete() osd_removal_job.ocp.wait_for_delete( resource_name=f"ocs-osd-removal-{osd_id}") timeout = 600 # Wait for OSD PVC to get created and reach Bound state logger.info( "Waiting for a new OSD PVC to get created and reach Bound state") assert osd_pvc.ocp.wait_for_resource( timeout=timeout, condition=constants.STATUS_BOUND, selector=constants.OSD_PVC_GENERIC_LABEL, resource_count=osd_pvcs_count, ), (f"Cluster recovery failed after {timeout} seconds. " f"Expected to have {osd_pvcs_count} OSD PVCs in status Bound. Current OSD PVCs status: " f"{[pvc.ocp.get_resource(pvc.get().get('metadata').get('name'), 'STATUS') for pvc in get_deviceset_pvcs()]}" ) # Wait for OSD pod to get created and reach Running state logger.info( "Waiting for a new OSD pod to get created and reach Running state") assert osd_pod.ocp.wait_for_resource( timeout=timeout, condition=constants.STATUS_RUNNING, selector=constants.OSD_APP_LABEL, resource_count=osd_pods_count, ), (f"Cluster recovery failed after {timeout} seconds. " f"Expected to have {osd_pods_count} OSD pods in status Running. Current OSD pods status: " f"{[osd_pod.ocp.get_resource(pod.get().get('metadata').get('name'), 'STATUS') for pod in get_osd_pods()]}" ) # We need to silence the old osd crash warning due to BZ https://bugzilla.redhat.com/show_bug.cgi?id=1896810 # This is a workaround - issue for tracking: https://github.com/red-hat-storage/ocs-ci/issues/3438 if ocp_version >= 4.6: silence_osd_crash = cluster.wait_for_silence_ceph_osd_crash_warning( osd_pod_name) if not silence_osd_crash: logger.info("Didn't find ceph osd crash warning") # Validate cluster is still functional self.sanity_helpers.health_check(tries=100) self.sanity_helpers.create_resources(pvc_factory, pod_factory)
class TestAutomatedRecoveryFromFailedNodes(ManageTest): """ Knip-678 Automated recovery from failed nodes """ @pytest.fixture(autouse=True) def teardown(self, request): def finalizer(): worker_nodes = get_worker_nodes() # Removing created label on all worker nodes remove_label_from_worker_node(worker_nodes, label_key="dc") request.addfinalizer(finalizer) @pytest.fixture(autouse=True) def init_sanity(self): """ Initialize Sanity instance """ self.sanity_helpers = Sanity() @pytest.mark.parametrize( argnames=["interface"], argvalues=[ pytest.param(*["rbd"], marks=pytest.mark.polarion_id("OCS-2100")), pytest.param(*["cephfs"], marks=pytest.mark.polarion_id("OCS-2101")), ], ) def test_automated_recovery_from_failed_nodes_IPI_proactive( self, interface, pvc_factory, pod_factory, dc_pod_factory ): """ Knip-678 Automated recovery from failed nodes Proactive case - IPI """ # Get OSD running nodes osd_running_nodes = get_osd_running_nodes() log.info(f"OSDs are running on nodes {osd_running_nodes}") # Label osd nodes with fedora app label_worker_node(osd_running_nodes, label_key="dc", label_value="fedora") # Create DC app pods log.info("Creating DC based app pods") interface = ( constants.CEPHBLOCKPOOL if interface == "rbd" else constants.CEPHFILESYSTEM ) dc_pod_obj = [] for i in range(2): dc_pod = dc_pod_factory(interface=interface, node_selector={"dc": "fedora"}) pod.run_io_in_bg(dc_pod, fedora_dc=True) dc_pod_obj.append(dc_pod) # Get app pods running nodes dc_pod_node_name = get_app_pod_running_nodes(dc_pod_obj) log.info(f"DC app pod running nodes are {dc_pod_node_name}") # Get both osd and app pod running node common_nodes = get_both_osd_and_app_pod_running_node( osd_running_nodes, dc_pod_node_name ) msg = "Common OSD and app running node(s) NOT found" assert len(common_nodes) > 0, msg log.info(f"Common OSD and app pod running nodes are {common_nodes}") # Get the machine name using the node name machine_name = machine.get_machine_from_node_name(common_nodes[0]) log.info(f"{common_nodes[0]} associated machine is {machine_name}") # Get the machineset name using machine name machineset_name = machine.get_machineset_from_machine_name(machine_name) log.info(f"{common_nodes[0]} associated machineset is {machineset_name}") # Add a new node and label it add_new_node_and_label_it(machineset_name) # Delete the machine machine.delete_machine(machine_name) log.info(f"Successfully deleted machine {machine_name}") # DC app pods on the failed node will get automatically created on # other running node. Waiting for all dc app pod to reach running # state pod.wait_for_dc_app_pods_to_reach_running_state(dc_pod_obj) log.info("All the dc pods reached running state") pod.wait_for_storage_pods() # Check basic cluster functionality by creating resources # (pools, storageclasses, PVCs, pods - both CephFS and RBD), # run IO and delete the resources self.sanity_helpers.create_resources(pvc_factory, pod_factory) self.sanity_helpers.delete_resources() # Perform cluster and Ceph health checks self.sanity_helpers.health_check()
class TestNodeReplacement(ManageTest): """ Knip-894 Node replacement - AWS-IPI-Reactive """ @pytest.fixture(autouse=True) def teardown(self, request): def finalizer(): worker_nodes = get_worker_nodes() # Removing created label on all worker nodes remove_label_from_worker_node(worker_nodes, label_key="dc") # Verify OSD encrypted if config.ENV_DATA.get("encryption_at_rest"): osd_encryption_verification() request.addfinalizer(finalizer) @pytest.fixture(autouse=True) def init_sanity(self): """ Initialize Sanity instance """ self.sanity_helpers = Sanity() @pytest.mark.parametrize( argnames=["interface", "failure"], argvalues=[ pytest.param(*["rbd", "power off"], marks=pytest.mark.polarion_id("OCS-2118")), pytest.param(*["rbd", "network failure"], marks=pytest.mark.polarion_id("OCS-2120")), pytest.param(*["cephfs", "power off"], marks=pytest.mark.polarion_id("OCS-2119")), pytest.param( *["cephfs", "network failure"], marks=pytest.mark.polarion_id("OCS-2121"), ), ], ) def test_node_replacement_reactive_aws_ipi( self, nodes, pvc_factory, pod_factory, dc_pod_factory, failure, interface, bucket_factory, rgw_bucket_factory, ): """ Knip-894 Node replacement - AWS-IPI-Reactive """ # Get worker nodes initial_nodes = get_worker_nodes() # Get OSD running nodes osd_running_nodes = get_osd_running_nodes() log.info(f"OSDs are running on nodes {osd_running_nodes}") # Label osd nodes with fedora app label_worker_node(osd_running_nodes, label_key="dc", label_value="fedora") # Create DC app pods log.info("Creating DC based app pods") if interface == "rbd": interface = constants.CEPHBLOCKPOOL elif interface == "cephfs": interface = constants.CEPHFILESYSTEM dc_pod_obj = [] for i in range(2): dc_pod = dc_pod_factory(interface=interface, node_selector={"dc": "fedora"}) pod.run_io_in_bg(dc_pod, fedora_dc=True) dc_pod_obj.append(dc_pod) # Get app pods running nodes dc_pod_node_name = get_app_pod_running_nodes(dc_pod_obj) log.info(f"DC app pod running nodes are {dc_pod_node_name}") # Get both osd and app pod running node common_nodes = get_both_osd_and_app_pod_running_node( osd_running_nodes, dc_pod_node_name) log.info(f"Both OSD and app pod is running on nodes {common_nodes}") # Get the machine name using the node name machine_name = machine.get_machine_from_node_name(common_nodes[0]) log.info(f"{common_nodes[0]} associated machine is {machine_name}") # Get the machineset name using machine name machineset_name = machine.get_machineset_from_machine_name( machine_name) log.info( f"{common_nodes[0]} associated machineset is {machineset_name}") # Get the failure node obj failure_node_obj = get_node_objs(node_names=[common_nodes[0]]) # Induce failure on the selected failure node log.info(f"Inducing failure on node {failure_node_obj[0].name}") if failure == "power off": # Power off AWS worker node instance nodes.stop_nodes(failure_node_obj, wait=True) log.info( f"Successfully powered off node: {failure_node_obj[0].name}") elif failure == "network failure": # Induce Network failure node_network_failure([failure_node_obj[0].name]) # Add annotation to the failed node annotation = "machine.openshift.io/exclude-node-draining=''" machine.add_annotation_to_machine(annotation=annotation, machine_name=machine_name) # Delete the machine machine.delete_machine(machine_name) log.info(f"Successfully deleted machine {machine_name}") # Wait for the new machine to spin log.info("Waiting for the new node to be in ready state") machine.wait_for_new_node_to_be_ready(machineset_name) # Get the node name of new spun node nodes_after_new_spun_node = get_worker_nodes() new_spun_node = list( set(nodes_after_new_spun_node) - set(initial_nodes)) log.info(f"New spun node is {new_spun_node}") # Label it node_obj = ocp.OCP(kind="node") node_obj.add_label(resource_name=new_spun_node[0], label=constants.OPERATOR_NODE_LABEL) log.info( f"Successfully labeled {new_spun_node} with OCS storage label") # DC app pods on the failed node will get automatically created on other # running node. Waiting for all dc app pod to reach running state pod.wait_for_dc_app_pods_to_reach_running_state(dc_pod_obj, timeout=1200) log.info("All the dc pods reached running state") pod.wait_for_storage_pods() # Check basic cluster functionality by creating resources # (pools, storageclasses, PVCs, pods - both CephFS and RBD), # run IO and delete the resources self.sanity_helpers.create_resources(pvc_factory, pod_factory, bucket_factory, rgw_bucket_factory) self.sanity_helpers.delete_resources() # Perform cluster and Ceph health checks self.sanity_helpers.health_check()
class TestDiskFailures(ManageTest): """ Test class for detach and attach worker volume """ def detach_volume_and_wait_for_attach(self, nodes, data_volume, worker_node): """ Detach an EBS volume from an AWS instance and wait for the volume to be re-attached Args: node (OCS): The OCS object representing the node data_volume (Volume): The ec2 volume to delete worker_node (OCS): The OCS object of the EC2 instance """ try: # Detach volume (logging is done inside the function) nodes.detach_volume(data_volume, worker_node) except AWSTimeoutException as e: if "Volume state: in-use" in e: logger.info( f"Volume {data_volume} re-attached successfully to worker" f" node {worker_node}") else: raise else: """ Wait for worker volume to be re-attached automatically to the node """ assert nodes.wait_for_volume_attach(data_volume), ( f"Volume {data_volume} failed to be re-attached to worker " f"node {worker_node}") @pytest.fixture(autouse=True) def teardown(self, request, nodes): """ Restart nodes that are in status NotReady, for situations in which the test failed before restarting the node after detach volume, which leaves nodes in NotReady """ def finalizer(): not_ready_nodes = [ n for n in node.get_node_objs() if n.ocp.get_resource_status( n.name) == constants.NODE_NOT_READY ] logger.warning( f"Nodes in NotReady status found: {[n.name for n in not_ready_nodes]}" ) if not_ready_nodes: nodes.restart_nodes(not_ready_nodes) node.wait_for_nodes_status() # Restart node if the osd stays at CLBO state osd_pods_obj_list = get_osd_pods() for pod in osd_pods_obj_list: if (pod.get().get("status").get("containerStatuses")[0].get( "state") == constants.STATUS_CLBO): node_obj = get_pod_node(pod) nodes.restart_nodes([node_obj]) node.wait_for_nodes_status([node_obj.name]) # Verify OSD encrypted if config.ENV_DATA.get("encryption_at_rest"): osd_encryption_verification() logger.info("Clear crash warnings and osd removal leftovers") clear_crash_warning_and_osd_removal_leftovers() request.addfinalizer(finalizer) @pytest.fixture(autouse=True) def init_sanity(self): """ Initialize Sanity instance """ self.sanity_helpers = Sanity() @cloud_platform_required @pytest.mark.polarion_id("OCS-1085") @bugzilla("1825675") def test_detach_attach_worker_volume(self, nodes, pvc_factory, pod_factory, bucket_factory, rgw_bucket_factory): """ Detach and attach worker volume - Detach the data volume from one of the worker nodes - Wait for the volumes to be re-attached back to the worker node - Validate cluster functionality, without checking cluster and Ceph health (as one node volume is detached, the cluster will be unhealthy) by creating resources and running IO - Restart the node so the volume will get re-mounted """ # Get a data volume data_volume = nodes.get_data_volumes()[0] # Get the worker node according to the volume attachment worker = nodes.get_node_by_attached_volume(data_volume) # Detach volume and wait for the volume to attach self.detach_volume_and_wait_for_attach(nodes, data_volume, worker) # Validate cluster is still functional # In case the selected node that its volume disk was detached was the one # running the ceph tools pod, we'll need to wait for a new ct pod to start. # For that, a function that connects to the ct pod is being used to check if # it's alive assert (wait_for_ct_pod_recovery() ), "Ceph tools pod failed to come up on another node" self.sanity_helpers.create_resources(pvc_factory, pod_factory, bucket_factory, rgw_bucket_factory) # Restart the instance so the volume will get re-mounted nodes.restart_nodes([worker]) # Cluster health check # W/A: For the investigation of BZ 1825675, timeout is increased to see if cluster # becomes healthy eventually # TODO: Remove 'tries=100' self.sanity_helpers.health_check(tries=100) @cloud_platform_required @pytest.mark.polarion_id("OCS-1086") @skipif_ibm_cloud def test_detach_attach_2_data_volumes(self, nodes, pvc_factory, pod_factory, bucket_factory, rgw_bucket_factory): """ Detach and attach disk from 2 worker nodes - Detach the data 2 of the data volumes from their worker nodes - Wait for the volumes to be re-attached back to the worker nodes - Restart the nodes so the volume will get re-mounted in each node - Check cluster health and functionality to make sure detach, attach and restart did not affect the cluster """ # Get 2 data volumes data_volumes = nodes.get_data_volumes()[:2] workers_and_volumes = [{ "worker": nodes.get_node_by_attached_volume(vol), "volume": vol } for vol in data_volumes] for worker_and_volume in workers_and_volumes: # Detach volume and wait for the volume to attach self.detach_volume_and_wait_for_attach(nodes, worker_and_volume["volume"], worker_and_volume["worker"]) # Restart the instances so the volume will get re-mounted nodes.restart_nodes([ worker_and_volume["worker"] for worker_and_volume in workers_and_volumes ]) # Validate cluster is still functional self.sanity_helpers.health_check() self.sanity_helpers.create_resources(pvc_factory, pod_factory, bucket_factory, rgw_bucket_factory) @bugzilla("1830702") @vsphere_platform_required @pytest.mark.polarion_id("OCS-2172") def test_recovery_from_volume_deletion(self, nodes, pvc_factory, pod_factory, bucket_factory, rgw_bucket_factory): """ Test cluster recovery from disk deletion from the platform side. Based on documented procedure detailed in https://bugzilla.redhat.com/show_bug.cgi?id=1823183 """ osd_operations.osd_device_replacement(nodes) self.sanity_helpers.create_resources(pvc_factory, pod_factory, bucket_factory, rgw_bucket_factory)
class TestAutomatedRecoveryFromFailedNodes(ManageTest): """ Knip-678 Automated recovery from failed nodes - Reactive """ threads = [] @pytest.fixture(autouse=True) def teardown(self, request): def finalizer(): worker_nodes = get_worker_nodes() # Removing created label on all worker nodes remove_label_from_worker_node(worker_nodes, label_key="dc") for thread in self.threads: thread.join() ceph_health_check() request.addfinalizer(finalizer) @pytest.fixture(autouse=True) def init_sanity(self): """ Initialize Sanity instance """ self.sanity_helpers = Sanity() @pytest.mark.parametrize( argnames=["interface", "failure"], argvalues=[ pytest.param( *["rbd", "shutdown"], marks=[ pytest.mark.polarion_id("OCS-2102"), pytest.mark.bugzilla("1845666"), ], ), pytest.param(*["rbd", "terminate"], marks=pytest.mark.polarion_id("OCS-2103")), pytest.param( *["cephfs", "shutdown"], marks=[ pytest.mark.polarion_id("OCS-2104"), pytest.mark.bugzilla("1845666"), ], ), pytest.param(*["cephfs", "terminate"], marks=pytest.mark.polarion_id("OCS-2105")), ], ) def test_automated_recovery_from_failed_nodes_IPI_reactive( self, nodes, pvc_factory, pod_factory, failure, dc_pod_factory, interface, bucket_factory, rgw_bucket_factory, ): """ Knip-678 Automated recovery from failed nodes Reactive case - IPI """ # Get OSD running nodes osd_running_nodes = get_osd_running_nodes() log.info(f"OSDs are running on nodes {osd_running_nodes}") # Label osd nodes with fedora app label_worker_node(osd_running_nodes, label_key="dc", label_value="fedora") # Create DC app pods log.info("Creating DC based app pods") if interface == "rbd": interface = constants.CEPHBLOCKPOOL elif interface == "cephfs": interface = constants.CEPHFILESYSTEM dc_pod_obj = [] for i in range(2): dc_pod = dc_pod_factory(interface=interface, node_selector={"dc": "fedora"}) self.threads.append(pod.run_io_in_bg(dc_pod, fedora_dc=True)) dc_pod_obj.append(dc_pod) # Get app pods running nodes dc_pod_node_name = get_app_pod_running_nodes(dc_pod_obj) log.info(f"DC app pod running nodes are {dc_pod_node_name}") # Get both osd and app pod running node common_nodes = get_both_osd_and_app_pod_running_node( osd_running_nodes, dc_pod_node_name) log.info(f"Both OSD and app pod is running on nodes {common_nodes}") # Get the machine name using the node name machine_name = machine.get_machine_from_node_name(common_nodes[0]) log.info(f"{common_nodes[0]} associated machine is {machine_name}") # Get the machineset name using machine name machineset_name = machine.get_machineset_from_machine_name( machine_name) log.info( f"{common_nodes[0]} associated machineset is {machineset_name}") # Add a new node and label it add_new_node_and_label_it(machineset_name) # Get the failure node obj failure_node_obj = get_node_objs(node_names=[common_nodes[0]]) # Induce failure on the selected failure node log.info(f"Inducing failure on node {failure_node_obj[0].name}") if failure == "shutdown": nodes.stop_nodes(failure_node_obj, wait=True) log.info(f"Successfully powered off node: " f"{failure_node_obj[0].name}") elif failure == "terminate": nodes.terminate_nodes(failure_node_obj, wait=True) log.info(f"Successfully terminated node : " f"{failure_node_obj[0].name} instance") try: # DC app pods on the failed node will get automatically created on other # running node. Waiting for all dc app pod to reach running state pod.wait_for_dc_app_pods_to_reach_running_state(dc_pod_obj, timeout=720) log.info("All the dc pods reached running state") pod.wait_for_storage_pods() except ResourceWrongStatusException: if failure == "shutdown": nodes.terminate_nodes(failure_node_obj, wait=True) log.info(f"Successfully terminated node : " f"{failure_node_obj[0].name} instance") raise # Check basic cluster functionality by creating resources # (pools, storageclasses, PVCs, pods - both CephFS and RBD), # run IO and delete the resources self.sanity_helpers.create_resources(pvc_factory, pod_factory, bucket_factory, rgw_bucket_factory) self.sanity_helpers.delete_resources() # Perform cluster and Ceph health checks self.sanity_helpers.health_check()
class TestNodesMaintenance(ManageTest): """ Test basic flows of maintenance (unschedule and drain) and activate operations, followed by cluster functionality and health checks """ @pytest.fixture(autouse=True) def init_sanity(self): """ Initialize Sanity instance """ if storagecluster_independent_check(): self.sanity_helpers = SanityExternalCluster() else: self.sanity_helpers = Sanity() @pytest.fixture(autouse=True) def health_checker(self): """ Check Ceph health """ try: status = ceph_health_check_base() if status: log.info("Health check passed") except CephHealthException as e: # skip because ceph is not in good health pytest.skip(str(e)) @tier1 @pytest.mark.parametrize( argnames=["node_type"], argvalues=[ pytest.param(*["worker"], marks=pytest.mark.polarion_id("OCS-1269")), pytest.param(*["master"], marks=pytest.mark.polarion_id("OCS-1272")), ], ) def test_node_maintenance( self, reduce_and_resume_cluster_load, node_type, pvc_factory, pod_factory, bucket_factory, rgw_bucket_factory, ): """ OCS-1269/OCS-1272: - Maintenance (mark as unscheduable and drain) 1 worker/master node - Check cluster functionality by creating resources (pools, storageclasses, PVCs, pods - both CephFS and RBD) - Mark the node as scheduable - Check cluster and Ceph health """ # Get 1 node of the type needed for the test iteration typed_nodes = get_nodes(node_type=node_type, num_of_nodes=1) assert typed_nodes, f"Failed to find a {node_type} node for the test" typed_node_name = typed_nodes[0].name # Maintenance the node (unschedule and drain) drain_nodes([typed_node_name]) # Check basic cluster functionality by creating resources # (pools, storageclasses, PVCs, pods - both CephFS and RBD), # run IO and delete the resources self.sanity_helpers.create_resources(pvc_factory, pod_factory, bucket_factory, rgw_bucket_factory) self.sanity_helpers.delete_resources() # Mark the node back to schedulable schedule_nodes([typed_node_name]) # Perform cluster and Ceph health checks self.sanity_helpers.health_check(tries=90) @tier4 @tier4b @skipif_bm @pytest.mark.parametrize( argnames=["node_type"], argvalues=[ pytest.param(*["worker"], marks=pytest.mark.polarion_id("OCS-1292")), pytest.param(*["master"], marks=pytest.mark.polarion_id("OCS-1293")), ], ) def test_node_maintenance_restart_activate( self, nodes, pvc_factory, pod_factory, node_type, bucket_factory, rgw_bucket_factory, ): """ OCS-1292/OCS-1293: - Maintenance (mark as unscheduable and drain) 1 worker/master node - Restart the node - Mark the node as scheduable - Check cluster and Ceph health - Check cluster functionality by creating and deleting resources (pools, storageclasses, PVCs, pods - both CephFS and RBD) """ # Get 1 node of the type needed for the test iteration typed_nodes = get_nodes(node_type=node_type, num_of_nodes=1) assert typed_nodes, f"Failed to find a {node_type} node for the test" typed_node_name = typed_nodes[0].name reboot_events_cmd = ( f"get events -A --field-selector involvedObject.name=" f"{typed_node_name},reason=Rebooted -o yaml") # Find the number of reboot events in 'typed_node_name' num_events = len( typed_nodes[0].ocp.exec_oc_cmd(reboot_events_cmd)["items"]) # Maintenance the node (unschedule and drain). The function contains logging drain_nodes([typed_node_name]) # Restarting the node nodes.restart_nodes(nodes=typed_nodes, wait=False) try: wait_for_nodes_status( node_names=[typed_node_name], status=constants.NODE_NOT_READY_SCHEDULING_DISABLED, ) except ResourceWrongStatusException: # Sometimes, the node will be back to running state quickly so # that the status change won't be detected. Verify the node was # actually restarted by checking the reboot events count new_num_events = len( typed_nodes[0].ocp.exec_oc_cmd(reboot_events_cmd)["items"]) assert new_num_events > num_events, ( f"Reboot event not found." f"Node {typed_node_name} did not restart.") wait_for_nodes_status( node_names=[typed_node_name], status=constants.NODE_READY_SCHEDULING_DISABLED, ) # Mark the node back to schedulable schedule_nodes([typed_node_name]) # Check cluster and Ceph health and checking basic cluster # functionality by creating resources (pools, storageclasses, # PVCs, pods - both CephFS and RBD), run IO and delete the resources self.sanity_helpers.health_check() self.sanity_helpers.create_resources(pvc_factory, pod_factory, bucket_factory, rgw_bucket_factory) self.sanity_helpers.delete_resources() @tier3 @pytest.mark.parametrize( argnames=["nodes_type"], argvalues=[ pytest.param(*["worker"], marks=pytest.mark.polarion_id("OCS-1273")), pytest.param(*["master"], marks=pytest.mark.polarion_id("OCS-1271")), ], ) def test_2_nodes_maintenance_same_type(self, nodes_type): """ OCS-1273/OCs-1271: - Try draining 2 nodes from the same type - should fail - Check cluster and Ceph health """ # Get 2 nodes typed_nodes = get_nodes(node_type=nodes_type, num_of_nodes=2) assert typed_nodes, f"Failed to find a {nodes_type} node for the test" typed_node_names = [typed_node.name for typed_node in typed_nodes] # Try draining 2 nodes - should fail try: drain_nodes(typed_node_names) except TimeoutExpired: log.info( f"Draining of nodes {typed_node_names} failed as expected") schedule_nodes(typed_node_names) # Perform cluster and Ceph health checks self.sanity_helpers.health_check() @tier2 @pytest.mark.polarion_id("OCS-1274") def test_2_nodes_different_types(self, pvc_factory, pod_factory, bucket_factory, rgw_bucket_factory): """ OCS-1274: - Maintenance (mark as unscheduable and drain) 1 worker node and 1 master node - Check cluster functionality by creating resources (pools, storageclasses, PVCs, pods - both CephFS and RBD) - Mark the nodes as scheduable - Check cluster and Ceph health """ # Get 1 node from each type nodes = [ get_nodes(node_type=node_type, num_of_nodes=1)[0] for node_type in ["worker", "master"] ] assert nodes, "Failed to find a nodes for the test" node_names = [typed_node.name for typed_node in nodes] # Maintenance the nodes (unschedule and drain) drain_nodes(node_names) # Check basic cluster functionality by creating resources # (pools, storageclasses, PVCs, pods - both CephFS and RBD), # run IO and delete the resources self.sanity_helpers.create_resources(pvc_factory, pod_factory, bucket_factory, rgw_bucket_factory) self.sanity_helpers.delete_resources() # Mark the nodes back to schedulable schedule_nodes(node_names) # Perform cluster and Ceph health checks self.sanity_helpers.health_check() @tier4 @tier4b @aws_platform_required @ipi_deployment_required @pytest.mark.parametrize( argnames=["interface"], argvalues=[ pytest.param(*["rbd"], marks=pytest.mark.polarion_id("OCS-2128")), pytest.param(*["cephfs"], marks=pytest.mark.polarion_id("OCS-2129")), ], ) def test_simultaneous_drain_of_two_ocs_nodes( self, pvc_factory, pod_factory, dc_pod_factory, interface, bucket_factory, rgw_bucket_factory, ): """ OCS-2128/OCS-2129: - Create PVCs and start IO on DC based app pods - Add one extra node in two of the AZs and label the nodes with OCS storage label - Maintenance (mark as unscheduable and drain) 2 worker nodes simultaneously - Confirm that OCS and DC pods are in running state - Remove unscheduled nodes - Check cluster functionality by creating resources (pools, storageclasses, PVCs, pods - both CephFS and RBD) - Check cluster and Ceph health """ # Get OSD running nodes osd_running_worker_nodes = get_osd_running_nodes() log.info(f"OSDs are running on nodes {osd_running_worker_nodes}") # Label osd nodes with fedora app label_worker_node(osd_running_worker_nodes, label_key="dc", label_value="fedora") log.info("Successfully labeled worker nodes with {dc:fedora}") # Create DC app pods log.info("Creating DC based app pods and starting IO in background") interface = (constants.CEPHBLOCKPOOL if interface == "rbd" else constants.CEPHFILESYSTEM) dc_pod_obj = [] for i in range(2): dc_pod = dc_pod_factory(interface=interface, node_selector={"dc": "fedora"}) pod.run_io_in_bg(dc_pod, fedora_dc=True) dc_pod_obj.append(dc_pod) # Get the machine name using the node name machine_names = [ machine.get_machine_from_node_name(osd_running_worker_node) for osd_running_worker_node in osd_running_worker_nodes[:2] ] log.info(f"{osd_running_worker_nodes} associated " f"machine are {machine_names}") # Get the machineset name using machine name machineset_names = [ machine.get_machineset_from_machine_name(machine_name) for machine_name in machine_names ] log.info(f"{osd_running_worker_nodes} associated machineset " f"is {machineset_names}") # Add a new node and label it add_new_node_and_label_it(machineset_names[0]) add_new_node_and_label_it(machineset_names[1]) # Drain 2 nodes drain_nodes(osd_running_worker_nodes[:2]) # Check the pods should be in running state all_pod_obj = pod.get_all_pods(wait=True) for pod_obj in all_pod_obj: if ("-1-deploy" or "ocs-deviceset") not in pod_obj.name: try: helpers.wait_for_resource_state( resource=pod_obj, state=constants.STATUS_RUNNING, timeout=200) except ResourceWrongStatusException: # 'rook-ceph-crashcollector' on the failed node stucks at # pending state. BZ 1810014 tracks it. # Ignoring 'rook-ceph-crashcollector' pod health check as # WA and deleting its deployment so that the pod # disappears. Will revert this WA once the BZ is fixed if "rook-ceph-crashcollector" in pod_obj.name: ocp_obj = ocp.OCP( namespace=defaults.ROOK_CLUSTER_NAMESPACE) pod_name = pod_obj.name deployment_name = "-".join(pod_name.split("-")[:-2]) command = f"delete deployment {deployment_name}" ocp_obj.exec_oc_cmd(command=command) log.info(f"Deleted deployment for pod {pod_obj.name}") # DC app pods on the drained node will get automatically created on other # running node in same AZ. Waiting for all dc app pod to reach running state pod.wait_for_dc_app_pods_to_reach_running_state(dc_pod_obj, timeout=1200) log.info("All the dc pods reached running state") # Remove unscheduled nodes # In scenarios where the drain is attempted on >3 worker setup, # post completion of drain we are removing the unscheduled nodes so # that we maintain 3 worker nodes. log.info(f"Removing scheduled nodes {osd_running_worker_nodes[:2]}") remove_node_objs = get_node_objs(osd_running_worker_nodes[:2]) remove_nodes(remove_node_objs) # Check basic cluster functionality by creating resources # (pools, storageclasses, PVCs, pods - both CephFS and RBD), # run IO and delete the resources self.sanity_helpers.create_resources(pvc_factory, pod_factory, bucket_factory, rgw_bucket_factory) self.sanity_helpers.delete_resources() # Perform cluster and Ceph health checks self.sanity_helpers.health_check() @bugzilla("1861104") @pytest.mark.polarion_id("OCS-2524") @tier4b def test_pdb_check_simultaneous_node_drains( self, pvc_factory, pod_factory, bucket_factory, rgw_bucket_factory, node_drain_teardown, ): """ - Check for OSD PDBs before drain - Maintenance (mark as unschedulable and drain) 2 worker node with delay of 30 secs - Drain will be completed on worker node A - Drain will be pending on worker node B due to blocking PDBs - Check the OSD PDBs - Mark the node A as schedulable - Let drain finish on Node B - Mark the node B as schedulable - Check cluster and Ceph health """ # Validate OSD PDBs before drain operation assert (not validate_existence_of_blocking_pdb() ), "Blocking PDBs exist, Can't perform drain" # Get 2 worker nodes to drain typed_nodes = get_nodes(num_of_nodes=2) assert len( typed_nodes) == 2, "Failed to find worker nodes for the test" node_A = typed_nodes[0].name node_B = typed_nodes[1].name # Drain Node A and validate blocking PDBs drain_nodes([node_A]) assert (validate_existence_of_blocking_pdb() ), "Blocking PDBs not created post drain" # Inducing delay between 2 drains # Node-B drain expected to be in pending due to blocking PDBs time.sleep(30) try: drain_nodes([node_B]) except TimeoutExpired: # Mark the node-A back to schedulable and let drain finish in Node-B schedule_nodes([node_A]) time.sleep(40) # Validate OSD PDBs assert (validate_existence_of_blocking_pdb() ), "Blocking PDBs not created post second drain" # Mark the node-B back to schedulable and recover the cluster schedule_nodes([node_B]) sample = TimeoutSampler( timeout=100, sleep=10, func=validate_existence_of_blocking_pdb, ) if not sample.wait_for_func_status(result=False): log.error("Blocking PDBs still exist") # wait for storage pods pod.wait_for_storage_pods() # Perform cluster and Ceph health checks self.sanity_helpers.health_check(tries=50) # Check basic cluster functionality by creating resources # (pools, storageclasses, PVCs, pods - both CephFS and RBD), # run IO and delete the resources self.sanity_helpers.create_resources(pvc_factory, pod_factory, bucket_factory, rgw_bucket_factory) self.sanity_helpers.delete_resources()
class TestNodesRestart(ManageTest): """ Test ungraceful cluster shutdown """ @pytest.fixture(autouse=True) def init_sanity(self): """ Initialize Sanity instance """ self.sanity_helpers = Sanity() @pytest.fixture(autouse=True) def teardown(self, request, nodes): """ Make sure all nodes are up again """ def finalizer(): nodes.restart_nodes_by_stop_and_start_teardown() request.addfinalizer(finalizer) @pytest.mark.parametrize( argnames=["force"], argvalues=[ pytest.param(*[True], marks=pytest.mark.polarion_id("OCS-894")), pytest.param( *[False], marks=[ pytest.mark.polarion_id("OCS-895"), cloud_platform_required ], ), ], ) def test_nodes_restart(self, nodes, pvc_factory, pod_factory, force, bucket_factory, rgw_bucket_factory): """ Test nodes restart (from the platform layer, i.e, EC2 instances, VMWare VMs) """ ocp_nodes = get_node_objs() nodes.restart_nodes_by_stop_and_start(nodes=ocp_nodes, force=force) self.sanity_helpers.health_check() self.sanity_helpers.create_resources(pvc_factory, pod_factory, bucket_factory, rgw_bucket_factory) @bugzilla("1754287") @pytest.mark.polarion_id("OCS-2015") def test_rolling_nodes_restart(self, nodes, pvc_factory, pod_factory, bucket_factory, rgw_bucket_factory): """ Test restart nodes one after the other and check health status in between """ ocp_nodes = get_node_objs() for node in ocp_nodes: nodes.restart_nodes(nodes=[node], wait=False) self.sanity_helpers.health_check(cluster_check=False, tries=60) self.sanity_helpers.create_resources(pvc_factory, pod_factory, bucket_factory, rgw_bucket_factory) @pytest.mark.parametrize( argnames=["interface", "operation"], argvalues=[ pytest.param(*["rbd", "create_resources"], marks=pytest.mark.polarion_id("OCS-1138")), pytest.param(*["rbd", "delete_resources"], marks=pytest.mark.polarion_id("OCS-1241")), pytest.param( *["cephfs", "create_resources"], marks=pytest.mark.polarion_id("OCS-1139"), ), pytest.param( *["cephfs", "delete_resources"], marks=pytest.mark.polarion_id("OCS-1242"), ), ], ) def test_pv_provisioning_under_degraded_state_stop_provisioner_pod_node( self, nodes, pvc_factory, pod_factory, interface, operation, bucket_factory, rgw_bucket_factory, ): """ Test PV provisioning under degraded state - stop the node that has the provisioner pod running on OCS-1138: - Stop 1 worker node that has the RBD provisioner pod running on - Wait for the RBD pod provisioner to come up again to running status - Validate cluster functionality, without checking cluster and Ceph health by creating resources and running IO - Start the worker node - Check cluster and Ceph health OCS-1241: - Stop 1 worker node that has the RBD provisioner pod running on - Wait for the RBD pod provisioner to come up again to running status - Validate cluster functionality, without checking cluster and Ceph health by deleting resources and running IO - Start the worker node - Check cluster and Ceph health OCS-1139: - Stop 1 worker node that has the CephFS provisioner pod running on - Wait for the CephFS pod provisioner to come up again to running status - Validate cluster functionality, without checking cluster and Ceph health by creating resources and running IO - Start the worker node - Check cluster and Ceph health OCS-1242: - Stop 1 worker node that has the CephFS provisioner pod running on - Wait for the CephFS pod provisioner to come up again to running status - Validate cluster functionality, without checking cluster and Ceph health by deleting resources and running IO - Start the worker node - Check cluster and Ceph health """ if operation == "delete_resources": # Create resources that their deletion will be tested later self.sanity_helpers.create_resources(pvc_factory, pod_factory, bucket_factory, rgw_bucket_factory) provisioner_pods = None # Get the provisioner pod according to the interface if interface == "rbd": provisioner_pods = pod.get_rbdfsplugin_provisioner_pods() elif interface == "cephfs": provisioner_pods = pod.get_cephfsplugin_provisioner_pods() provisioner_pod = provisioner_pods[0] # Making sure that the node is not running the rook operator pod: provisioner_node = pod.get_pod_node(provisioner_pod) rook_operator_pod = pod.get_operator_pods()[0] operator_node = pod.get_pod_node(rook_operator_pod) if operator_node.get().get("metadata").get( "name") == provisioner_node.get().get("metadata").get("name"): provisioner_pod = provisioner_pods[1] provisioner_pod_name = provisioner_pod.name logger.info( f"{interface} provisioner pod found: {provisioner_pod_name}") # Get the node name that has the provisioner pod running on provisioner_node = pod.get_pod_node(provisioner_pod) provisioner_node_name = provisioner_node.get().get("metadata").get( "name") logger.info( f"{interface} provisioner pod is running on node {provisioner_node_name}" ) # Stopping the nodes nodes.stop_nodes(nodes=[provisioner_node]) # Wait for the provisioner pod to get to running status selector = (constants.CSI_RBDPLUGIN_PROVISIONER_LABEL if (interface == "rbd") else constants.CSI_CEPHFSPLUGIN_PROVISIONER_LABEL) # Wait for the provisioner pod to reach Terminating status logger.info( f"Waiting for pod {provisioner_pod_name} to reach status Terminating" ) assert provisioner_pod.ocp.wait_for_resource( timeout=600, resource_name=provisioner_pod.name, condition=constants.STATUS_TERMINATING, ), f"{interface} provisioner pod failed to reach status Terminating" logger.info( f"Pod {provisioner_pod_name} has reached status Terminating") # Wait for the provisioner pod to be started and reach running status logger.info( f"Waiting for {interface} provisioner pod to reach status Running") # After this change https://github.com/rook/rook/pull/3642/, there are # 2 provisioners for each interface assert provisioner_pod.ocp.wait_for_resource( timeout=600, condition=constants.STATUS_RUNNING, selector=selector, resource_count=2, ), f"{interface} provisioner pod failed to reach status Running" logger.info(f"{interface} provisioner pod has reached status Running") if operation == "create_resources": # Cluster validation (resources creation and IO running) self.sanity_helpers.create_resources(pvc_factory, pod_factory, bucket_factory, rgw_bucket_factory) elif operation == "delete_resources": # Cluster validation (resources creation and IO running) self.sanity_helpers.delete_resources() # Starting the nodes nodes.start_nodes(nodes=[provisioner_node]) # Checking cluster and Ceph health self.sanity_helpers.health_check() @pytest.mark.parametrize( argnames=["operation"], argvalues=[ pytest.param(*["create_resources"], marks=[pytest.mark.polarion_id("OCS-2016")]), pytest.param(*["delete_resources"], marks=[pytest.mark.polarion_id("OCS-2017")]), ], ) def test_pv_provisioning_under_degraded_state_stop_rook_operator_pod_node( self, nodes, pvc_factory, pod_factory, operation, bucket_factory, rgw_bucket_factory, ): """ Test PV provisioning under degraded state - stop the node that has the rook operator pod running on OCS-2016: - Stop 1 worker node that has the rook ceph operator pod running on - Wait for the rook ceph operator pod to come up again to running status - Validate cluster functionality, without checking cluster and Ceph health by creating resources and running IO - Start the worker node - Check cluster and Ceph health OCS-2017: - Stop 1 worker node that has the rook ceph operator pod running on - Wait for the rook ceph operator pod to come up again to running status - Validate cluster functionality, without checking cluster and Ceph health by deleting resources - Start the worker node - Check cluster and Ceph health """ if operation == "delete_resources": # Create resources that their deletion will be tested later self.sanity_helpers.create_resources(pvc_factory, pod_factory, bucket_factory, rgw_bucket_factory) rook_operator_pods = pod.get_operator_pods() rook_operator_pod = rook_operator_pods[0] rook_operator_pod_name = rook_operator_pod.name logger.info(f"rook operator pod found: {rook_operator_pod_name}") # Get the node name that has the rook operator pod running on operator_node = pod.get_pod_node(rook_operator_pod) operator_node_name = operator_node.get().get("metadata").get("name") logger.info( f"{rook_operator_pod_name} pod is running on node {operator_node_name}" ) # Stopping the node nodes.stop_nodes(nodes=[operator_node]) # Wait for the rook operator pod to get to running status selector = constants.OPERATOR_LABEL # Wait for the rook operator pod to reach Terminating status logger.info( f"Waiting for pod {rook_operator_pod_name} to reach status Terminating" ) assert rook_operator_pod.ocp.wait_for_resource( timeout=600, resource_name=rook_operator_pod_name, condition=constants.STATUS_TERMINATING, ), "rook operator pod failed to reach status Terminating" logger.info( f"Pod {rook_operator_pod_name} has reached status Terminating") # Wait for the rook operator pod to be started and reach running status logger.info( f"Waiting for pod {rook_operator_pod_name} to reach status Running" ) assert rook_operator_pod.ocp.wait_for_resource( timeout=600, condition=constants.STATUS_RUNNING, selector=selector, resource_count=1, ), "rook operator pod failed to reach status Running" logger.info("rook operator pod has reached status Running") assert (wait_for_ct_pod_recovery() ), "Ceph tools pod failed to come up on another node" if operation == "create_resources": # Cluster validation (resources creation and IO running) self.sanity_helpers.create_resources(pvc_factory, pod_factory, bucket_factory, rgw_bucket_factory) elif operation == "delete_resources": # Cluster validation (resources creation and IO running) self.sanity_helpers.delete_resources() # Starting the nodes nodes.start_nodes(nodes=[operator_node]) # Checking cluster and Ceph health self.sanity_helpers.health_check() @skipif_no_lso @bugzilla("1873938") @pytest.mark.polarion_id("OCS-2448") def test_pv_after_reboot_node(self, nodes): """ Verify unexpected PV is not created after node reboot on LSO cluster """ pv_before_reset = get_pv_names() worker_nodes = get_nodes(node_type=constants.WORKER_MACHINE, num_of_nodes=3) ocp_obj = OCP(kind=constants.PV) for worker_node in worker_nodes: # Restart one worker node nodes.restart_nodes(nodes=[worker_node], wait=True) self.sanity_helpers.health_check(cluster_check=False, tries=60) logger.info(f"Verify PV after reboot {worker_node}") pv_after_reset = get_pv_names() pv_diff = set(pv_after_reset) - set(pv_before_reset) pv_new = [] for pv in pv_diff: pv_obj = ocp_obj.get(resource_name=pv) if pv_obj["spec"]["storageClassName"] == "localblock": pv_new.append(pv) assert ( not pv_new ), f"Unexpected PV {pv_new} created after reboot {worker_node}" logger.info("SUCCESS - No new PV was created.")
class TestRestoreCephMonQuorum(E2ETest): """ The objective of this test case is to verify that mons can be brought to quorum successfully by following the steps mentioned in https://access.redhat.com/solutions/5898541, when mon(s) are out of quorum on a cluster """ @pytest.fixture(autouse=True) def init_sanity(self): """ Initialize Sanity instance """ self.sanity_helpers = Sanity() @pytest.fixture(autouse=True) def base_set_up(self): """ Preconditions: a) Have a cluster with FIPS and Hugepages enabled b) Create some resources s3 objects, buckets and write data's to it. c) Create RGW kafka notification & see the objects are notified to kafka d) Perform mcg bucket replication (bidirectional) and see the objects are synced. e) Perform noobaa caching f) there are snapshots and clones for few PVCs g) Background IOs (fio pods) running """ # ToDo: Create objects and write data's to it # ToDo: Create RGW kafka notification # ToDo: Perform mcg bucket replication # Todo: Noobaa caching # ToDo: Create PVC, snapshot and restore into new PVC, clone PVC # ToDo: Run background IOs @pytest.fixture(autouse=True) def rook_operator_teardown(self, request): def finalizer(): op_obj = OCP( kind=constants.DEPLOYMENT, namespace=constants.OPENSHIFT_STORAGE_NAMESPACE, ) pod_obj = OCP( kind=constants.POD, namespace=constants.OPENSHIFT_STORAGE_NAMESPACE ) operator_obj = op_obj.get(resource_name=constants.ROOK_CEPH_OPERATOR) if operator_obj.get("spec").get("replicas") != 1: modify_deployment_replica_count( deployment_name=constants.ROOK_CEPH_OPERATOR, replica_count=1 ), "Failed to scale up rook-ceph-operator to 1" log.info("Validate all mons are up and running") try: pod_obj.wait_for_resource( condition=constants.STATUS_RUNNING, selector=constants.MON_APP_LABEL, resource_count=3, timeout=60, sleep=5, ) except (TimeoutExpiredError, ResourceWrongStatusException) as ex: log.warning(ex) op_obj.delete(resource_name=constants.ROOK_CEPH_OPERATOR) for pod in get_mon_pods(): pod.delete() pod_obj.wait_for_resource( condition=constants.STATUS_RUNNING, selector=constants.MON_APP_LABEL, resource_count=3, timeout=360, sleep=5, ) log.info("All mons are up and running") request.addfinalizer(finalizer) def test_restore_ceph_mon_quorum( self, pvc_factory, pod_factory, bucket_factory, rgw_bucket_factory ): """ Test Procedure: 1) With all the preconditions met, take mons out of quorum 2) Follow the procedure mentioned in https://access.redhat.com/solutions/5898541 to recover the quorum 3) Check for data access and integrity of block, object and file based data - No DU/DL/DC 4) Make sure basic functionality working fine """ # Take mons out of the quorum and confirm it ( self.mon_pod_obj_list, mon_pod_running, ceph_mon_daemon_id, ) = induce_mon_quorum_loss() # Recover mon quorum recover_mon_quorum(self.mon_pod_obj_list, mon_pod_running, ceph_mon_daemon_id) # Validate storage pods are running wait_for_storage_pods() # Remove crash list from ceph health log.info("Silence the ceph warnings by “archiving” the crash") tool_pod = get_ceph_tools_pod() tool_pod.exec_ceph_cmd(ceph_cmd="ceph crash archive-all", format=None) log.info("Removed ceph crash warnings. Check for ceph and cluster health") # Validate cluster health self.sanity_helpers.health_check(tries=40) # ToDo: Common system test case validation: Check for data integrity and corruption after mon recovery # Creating Resources log.info("Creating Resources using sanity helpers") self.sanity_helpers.create_resources( pvc_factory, pod_factory, bucket_factory, rgw_bucket_factory ) # Deleting Resources self.sanity_helpers.delete_resources()
class TestNoobaaBackupAndRecovery(E2ETest): """ Test to verify noobaa backup and recovery """ @pytest.fixture(autouse=True) def init_sanity(self): """ Initialize Sanity instance """ self.sanity_helpers = Sanity() @pytest.fixture(autouse=True) def teardown_fixture(self, request): def finalizer(): # Get the statefulset replica count sst_obj = OCP( kind=constants.STATEFULSET, namespace=constants.OPENSHIFT_STORAGE_NAMESPACE, ) noobaa_db_sst_obj = sst_obj.get( resource_name=self.noobaa_db_sst_name) if noobaa_db_sst_obj["spec"]["replicas"] != 1: modify_statefulset_replica_count( statefulset_name=self.noobaa_db_sst_name, replica_count=1 ), f"Failed to scale up the statefulset {self.noobaa_db_sst_name}" try: self.restore_pvc_obj.delete() except CommandFailed as ex: if f'"{ self.restore_pvc_obj.name}" not found' not in str(ex): raise ex request.addfinalizer(finalizer) def test_noobaa_db_backup_and_recovery( self, pvc_factory, pod_factory, snapshot_factory, bucket_factory, rgw_bucket_factory, ): """ Test case to verify noobaa backup and recovery 1. Take snapshot db-noobaa-db-0 PVC and retore it to PVC 2. Scale down the statefulset noobaa-db 3. Get the yaml of the current PVC, db-noobaa-db-0 and change the parameter persistentVolumeReclaimPolicy to Retain for restored PVC 4. Delete both PVCs, the PV for the original claim db-noobaa-db-0 will be removed. The PV for claim db-noobaa-db-0-snapshot-restore will move to ‘Released’ 5. Edit again restore PV and remove the claimRef section. The volume will transition to Available. 6. Edit the yaml db-noobaa-db-0.yaml and change the setting volumeName to restored PVC. 7. Scale up the stateful set again and the pod should be running """ # Initialise variable self.noobaa_db_sst_name = "noobaa-db-pg" # Get noobaa pods before execution noobaa_pods = get_noobaa_pods() # Get noobaa PVC before execution noobaa_pvc_obj = get_pvc_objs(pvc_names=["db-noobaa-db-pg-0"]) noobaa_pv_name = noobaa_pvc_obj[0].get("spec").get("spec").get( "volumeName") # Take snapshot db-noobaa-db-0 PVC log.info(f"Creating snapshot of the {noobaa_pvc_obj[0].name} PVC") snap_obj = snapshot_factory( pvc_obj=noobaa_pvc_obj[0], wait=True, snapshot_name=f"{noobaa_pvc_obj[0].name}-snapshot", ) log.info( f"Successfully created snapshot {snap_obj.name} and in Ready state" ) # Restore it to PVC log.info(f"Restoring snapshot {snap_obj.name} to create new PVC") sc_name = noobaa_pvc_obj[0].get().get("spec").get("storageClassName") pvc_size = (noobaa_pvc_obj[0].get().get("spec").get("resources").get( "requests").get("storage")) self.restore_pvc_obj = create_restore_pvc( sc_name=sc_name, snap_name=snap_obj.name, namespace=snap_obj.namespace, size=pvc_size, pvc_name=f"{snap_obj.name}-restore", volume_mode=snap_obj.parent_volume_mode, access_mode=snap_obj.parent_access_mode, ) wait_for_resource_state(self.restore_pvc_obj, constants.STATUS_BOUND) self.restore_pvc_obj.reload() log.info(f"Succeesfuly created PVC {self.restore_pvc_obj.name} " f"from snapshot {snap_obj.name}") # Scale down the statefulset noobaa-db modify_statefulset_replica_count( statefulset_name=self.noobaa_db_sst_name, replica_count=0 ), f"Failed to scale down the statefulset {self.noobaa_db_sst_name}" # Get the noobaa-db PVC pvc_obj = OCP(kind=constants.PVC, namespace=constants.OPENSHIFT_STORAGE_NAMESPACE) noobaa_pvc_yaml = pvc_obj.get(resource_name=noobaa_pvc_obj[0].name) # Get the restored noobaa PVC and # change the parameter persistentVolumeReclaimPolicy to Retain restored_noobaa_pvc_obj = get_pvc_objs( pvc_names=[f"{snap_obj.name}-restore"]) restored_noobaa_pv_name = (restored_noobaa_pvc_obj[0].get("spec").get( "spec").get("volumeName")) pv_obj = OCP(kind=constants.PV, namespace=constants.OPENSHIFT_STORAGE_NAMESPACE) params = '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}' assert pv_obj.patch( resource_name=restored_noobaa_pv_name, params=params), ( "Failed to change the parameter persistentVolumeReclaimPolicy" f" to Retain {restored_noobaa_pv_name}") # Delete both PVCs delete_pvcs(pvc_objs=[noobaa_pvc_obj[0], restored_noobaa_pvc_obj[0]]) # Validate original claim db-noobaa-db-0 removed assert validate_pv_delete( pv_name=noobaa_pv_name ), f"PV not deleted, still exist {noobaa_pv_name}" # Validate PV for claim db-noobaa-db-0-snapshot-restore is in Released state pv_obj.wait_for_resource(condition=constants.STATUS_RELEASED, resource_name=restored_noobaa_pv_name) # Edit again restore PV and remove the claimRef section log.info( f"Remove the claimRef section from PVC {restored_noobaa_pv_name}") params = '[{"op": "remove", "path": "/spec/claimRef"}]' pv_obj.patch(resource_name=restored_noobaa_pv_name, params=params, format_type="json") log.info( f"Successfully removed claimRef section from PVC {restored_noobaa_pv_name}" ) # Validate PV is in Available state pv_obj.wait_for_resource(condition=constants.STATUS_AVAILABLE, resource_name=restored_noobaa_pv_name) # Edit the yaml db-noobaa-db-0.yaml and change the # setting volumeName to restored PVC noobaa_pvc_yaml["spec"]["volumeName"] = restored_noobaa_pv_name noobaa_pvc_yaml = OCS(**noobaa_pvc_yaml) noobaa_pvc_yaml.create() # Validate noobaa PVC is in bound state pvc_obj.wait_for_resource( condition=constants.STATUS_BOUND, resource_name=noobaa_pvc_obj[0].name, timeout=120, ) # Scale up the statefulset again assert modify_statefulset_replica_count( statefulset_name=self.noobaa_db_sst_name, replica_count=1 ), f"Failed to scale up the statefulset {self.noobaa_db_sst_name}" # Validate noobaa pod is up and running pod_obj = OCP(kind=constants.POD, namespace=defaults.ROOK_CLUSTER_NAMESPACE) pod_obj.wait_for_resource( condition=constants.STATUS_RUNNING, resource_count=len(noobaa_pods), selector=constants.NOOBAA_APP_LABEL, ) # Change the parameter persistentVolumeReclaimPolicy to Delete again params = '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' assert pv_obj.patch( resource_name=restored_noobaa_pv_name, params=params), ( "Failed to change the parameter persistentVolumeReclaimPolicy" f" to Delete {restored_noobaa_pv_name}") log.info( "Changed the parameter persistentVolumeReclaimPolicy to Delete again" ) # Verify all storage pods are running wait_for_storage_pods() # Creating Resources log.info("Creating Resources using sanity helpers") self.sanity_helpers.create_resources(pvc_factory, pod_factory, bucket_factory, rgw_bucket_factory) # Deleting Resources self.sanity_helpers.delete_resources() # Verify everything running fine log.info( "Verifying All resources are Running and matches expected result") self.sanity_helpers.health_check(tries=120)
class TestRestartMgrWhileTwoMonsDown(ManageTest): """ Restart mgr pod while two mon pods are down """ @pytest.fixture(autouse=True) def init_sanity(self): """ Initialize Sanity instance """ self.sanity_helpers = Sanity() @pytest.fixture(scope="function", autouse=True) def teardown(self, request): """ Verify all pods on openshift-storage project on Running state """ def finalizer(): for mon_scale in self.mons_scale: self.oc.exec_oc_cmd( f"scale --replicas=1 deployment/{mon_scale}") wait_for_pods_to_be_running(timeout=600) request.addfinalizer(finalizer) def test_restart_mgr_while_two_mons_down(self, pvc_factory, pod_factory, bucket_factory, rgw_bucket_factory): """ Test Procedure: 1.Scaling down two mons: oc scale --replicas=0 deploy/rook-ceph-mon-a oc scale --replicas=0 deploy/rook-ceph-mon-b 2.Restarting mgr oc delete pod -l app=rook-ceph-mgr 3.sleep 5 seconds 4.Scaling mons back up oc scale --replicas=1 deploy/rook-ceph-mon-a oc scale --replicas=1 deploy/rook-ceph-mon-b 5.sleep 10 6.Waiting for mgr pod move to running state: oc get pod -l app=rook-ceph-mgr """ self.oc = ocp.OCP(kind=constants.DEPLOYMENT, namespace=config.ENV_DATA["cluster_namespace"]) mons = [ mon["metadata"]["name"] for mon in get_deployments_having_label( constants.MON_APP_LABEL, defaults.ROOK_CLUSTER_NAMESPACE) ] self.mons_scale = mons[0:2] tries = 11 for index in range(1, tries): log.info(f"Scaling down two mons {self.mons_scale}, index={index}") for mon_scale in self.mons_scale: self.oc.exec_oc_cmd( f"scale --replicas=0 deployment/{mon_scale}") log.info(f"Restarting mgr pod, index={index}") mgr_pod = get_mgr_pods() mgr_pod[0].delete(wait=True) time.sleep(5) log.info(f"Scaling up two mons {self.mons_scale}, index={index}") for mon_scale in self.mons_scale: self.oc.exec_oc_cmd( f"scale --replicas=1 deployment/{mon_scale}") time.sleep(10) log.info( f"Waiting for mgr pod move to Running state, index={index}") mgr_pod_obj = ocp.OCP(kind=constants.POD, namespace=defaults.ROOK_CLUSTER_NAMESPACE) assert mgr_pod_obj.wait_for_resource( condition=constants.STATUS_RUNNING, selector=constants.MGR_APP_LABEL, resource_count=1, timeout=100, ), f"Mgr pod did'nt move to Running state after 100 seconds, index={index}" log.info("Creating Resources using sanity helpers") self.sanity_helpers.create_resources(pvc_factory, pod_factory, bucket_factory, rgw_bucket_factory) log.info("Deleting Resources using sanity helpers") self.sanity_helpers.delete_resources()
class TestHugePages(E2ETest): """ Enable huge pages post ODF installation """ @pytest.fixture(scope="function", autouse=True) def huge_pages_setup(self, request): """ Initializes sanity """ self.sanity_helpers = Sanity() def finalizer(): """ Removes huge pages on worker nodes and verifies all pods are up """ disable_huge_pages() wait_for_nodes_status(status=constants.NODE_READY, timeout=600) nodes = get_nodes() for node in nodes: assert (node.get()["status"]["allocatable"]["hugepages-2Mi"] == "0"), f"Huge pages is not applied on {node.name}" log.info("Wait for all pods to be in running state") wait_for_pods_to_be_running(timeout=600) sanity_helpers.ceph_health_check(tries=120) request.addfinalizer(finalizer) def test_hugepages_post_odf_deployment( self, pvc_factory, pod_factory, bucket_factory, rgw_bucket_factory, node_restart_teardown, ): """ Test to verify that after enabling huge pages the nodes come up with higher page size and all odf cluster pods come back up. """ # Applies huge pages on the cluster nodes enable_huge_pages() log.info("Wait for all worker node to be READY state") wait_for_nodes_status(status=constants.NODE_READY, timeout=600) nodes = get_nodes() for node in nodes: assert (node.get()["status"]["allocatable"]["hugepages-2Mi"] == "64Mi"), f"Huge pages is not applied on {node.name}" log.info("Wait for all storage cluster pods to be in running state") wait_for_pods_to_be_running(timeout=600) # Creating Resources log.info("Creating Resources using sanity helpers") self.sanity_helpers.create_resources(pvc_factory, pod_factory, bucket_factory, rgw_bucket_factory, False) # Deleting Resources log.info("Deleting the resources created") self.sanity_helpers.delete_resources() # Verify everything running fine log.info( "Verifying All resources are Running and matches expected result") self.sanity_helpers.health_check(tries=120)