def remove_global_id_reclaim(): """ Removes global id warning by re-spinning client and mon pods """ csi_pods = [] interfaces = [constants.CEPHBLOCKPOOL, constants.CEPHFILESYSTEM] for interface in interfaces: plugin_pods = get_plugin_pods(interface) csi_pods += plugin_pods cephfs_provisioner_pods = get_cephfsplugin_provisioner_pods() rbd_provisioner_pods = get_rbdfsplugin_provisioner_pods() csi_pods += cephfs_provisioner_pods csi_pods += rbd_provisioner_pods for csi_pod in csi_pods: csi_pod.delete() for mds_pod in get_mds_pods(): mds_pod.delete() for mds_pod in get_mds_pods(): wait_for_resource_state(resource=mds_pod, state=constants.STATUS_RUNNING) for mon in get_mon_pods(namespace=constants.OPENSHIFT_STORAGE_NAMESPACE): mon.delete() mon_pods = get_mon_pods(namespace=constants.OPENSHIFT_STORAGE_NAMESPACE) for mon in mon_pods: wait_for_resource_state(resource=mon, state=constants.STATUS_RUNNING)
def verify_multus_network(): """ Verify Multus network(s) created successfully and are present on relevant pods. """ with open(constants.MULTUS_YAML, mode="r") as f: multus_public_data = yaml.load(f) multus_namespace = multus_public_data["metadata"]["namespace"] multus_name = multus_public_data["metadata"]["name"] multus_public_network_name = f"{multus_namespace}/{multus_name}" log.info("Verifying multus NetworkAttachmentDefinitions") ocp.OCP( resource_name=multus_public_network_name, kind="network-attachment-definitions", namespace=constants.OPENSHIFT_STORAGE_NAMESPACE, ) # TODO: also check if private NAD exists log.info("Verifying multus public network exists on ceph pods") osd_pods = get_osd_pods() for _pod in osd_pods: assert (_pod.data["metadata"]["annotations"] ["k8s.v1.cni.cncf.io/networks"] == multus_public_network_name) # TODO: also check private network if it exists on OSD pods mon_pods = get_mon_pods() mds_pods = get_mds_pods() mgr_pods = get_mgr_pods() rgw_pods = get_rgw_pods() ceph_pods = [*mon_pods, *mds_pods, *mgr_pods, *rgw_pods] for _pod in ceph_pods: assert (_pod.data["metadata"]["annotations"] ["k8s.v1.cni.cncf.io/networks"] == multus_public_network_name) log.info("Verifying multus public network exists on CSI pods") csi_pods = [] interfaces = [constants.CEPHBLOCKPOOL, constants.CEPHFILESYSTEM] for interface in interfaces: plugin_pods = get_plugin_pods(interface) csi_pods += plugin_pods cephfs_provisioner_pods = get_cephfsplugin_provisioner_pods() rbd_provisioner_pods = get_rbdfsplugin_provisioner_pods() csi_pods += cephfs_provisioner_pods csi_pods += rbd_provisioner_pods for _pod in csi_pods: assert (_pod.data["metadata"]["annotations"] ["k8s.v1.cni.cncf.io/networks"] == multus_public_network_name) log.info("Verifying StorageCluster multus network data") sc = get_storage_cluster() sc_data = sc.get().get("items")[0] network_data = sc_data["spec"]["network"] assert network_data["provider"] == "multus" selectors = network_data["selectors"] assert selectors[ "public"] == f"{defaults.ROOK_CLUSTER_NAMESPACE}/ocs-public"
def set_resource(self, resource, leader_type="provisioner"): self.resource = resource if (config.ENV_DATA["platform"] in constants.MANAGED_SERVICE_PLATFORMS) and (resource in CEPH_PODS): # If the platform is Managed Services, then the ceph pods will be present in the provider cluster. # Consumer cluster will be the primary cluster context in a multicluster run. Setting 'cluster_kubeconfig' # attribute to use as the value of the parameter '--kubeconfig' in the 'oc' commands to get ceph pods. provider_kubeconfig = os.path.join( config.clusters[ config.get_provider_index()].ENV_DATA["cluster_path"], config.clusters[config.get_provider_index()].RUN.get( "kubeconfig_location"), ) self.cluster_kubeconfig = provider_kubeconfig resource_count = 0 if self.resource == "mgr": self.resource_obj = pod.get_mgr_pods() self.selector = constants.MGR_APP_LABEL if self.resource == "mon": self.resource_obj = pod.get_mon_pods() self.selector = constants.MON_APP_LABEL if self.resource == "osd": self.resource_obj = pod.get_osd_pods() self.selector = constants.OSD_APP_LABEL if self.resource == "mds": self.resource_obj = pod.get_mds_pods() self.selector = constants.MDS_APP_LABEL if self.resource == "cephfsplugin": self.resource_obj = pod.get_plugin_pods( interface=constants.CEPHFILESYSTEM) self.selector = constants.CSI_CEPHFSPLUGIN_LABEL if self.resource == "rbdplugin": self.resource_obj = pod.get_plugin_pods( interface=constants.CEPHBLOCKPOOL) self.selector = constants.CSI_RBDPLUGIN_LABEL if self.resource == "cephfsplugin_provisioner": self.resource_obj = [ pod.get_plugin_provisioner_leader( interface=constants.CEPHFILESYSTEM, leader_type=leader_type) ] self.selector = constants.CSI_CEPHFSPLUGIN_PROVISIONER_LABEL resource_count = len(pod.get_cephfsplugin_provisioner_pods()) if self.resource == "rbdplugin_provisioner": self.resource_obj = [ pod.get_plugin_provisioner_leader( interface=constants.CEPHBLOCKPOOL, leader_type=leader_type) ] self.selector = constants.CSI_RBDPLUGIN_PROVISIONER_LABEL resource_count = len(pod.get_rbdfsplugin_provisioner_pods()) if self.resource == "operator": self.resource_obj = pod.get_operator_pods() self.selector = constants.OPERATOR_LABEL self.resource_count = resource_count or len(self.resource_obj)
def set_resource(self, resource, leader_type="provisioner"): self.resource = resource resource_count = 0 if self.resource == "mgr": self.resource_obj = pod.get_mgr_pods() self.selector = constants.MGR_APP_LABEL if self.resource == "mon": self.resource_obj = pod.get_mon_pods() self.selector = constants.MON_APP_LABEL if self.resource == "osd": self.resource_obj = pod.get_osd_pods() self.selector = constants.OSD_APP_LABEL if self.resource == "mds": self.resource_obj = pod.get_mds_pods() self.selector = constants.MDS_APP_LABEL if self.resource == "cephfsplugin": self.resource_obj = pod.get_plugin_pods( interface=constants.CEPHFILESYSTEM) self.selector = constants.CSI_CEPHFSPLUGIN_LABEL if self.resource == "rbdplugin": self.resource_obj = pod.get_plugin_pods( interface=constants.CEPHBLOCKPOOL) self.selector = constants.CSI_RBDPLUGIN_LABEL if self.resource == "cephfsplugin_provisioner": self.resource_obj = [ pod.get_plugin_provisioner_leader( interface=constants.CEPHFILESYSTEM, leader_type=leader_type) ] self.selector = constants.CSI_CEPHFSPLUGIN_PROVISIONER_LABEL resource_count = len(pod.get_cephfsplugin_provisioner_pods()) if self.resource == "rbdplugin_provisioner": self.resource_obj = [ pod.get_plugin_provisioner_leader( interface=constants.CEPHBLOCKPOOL, leader_type=leader_type) ] self.selector = constants.CSI_RBDPLUGIN_PROVISIONER_LABEL resource_count = len(pod.get_rbdfsplugin_provisioner_pods()) if self.resource == "operator": self.resource_obj = pod.get_operator_pods() self.selector = constants.OPERATOR_LABEL self.resource_count = resource_count or len(self.resource_obj)
def set_resource(self, resource): self.resource = resource resource_count = 0 if self.resource == 'mgr': self.resource_obj = pod.get_mgr_pods() self.selector = constants.MGR_APP_LABEL if self.resource == 'mon': self.resource_obj = pod.get_mon_pods() self.selector = constants.MON_APP_LABEL if self.resource == 'osd': self.resource_obj = pod.get_osd_pods() self.selector = constants.OSD_APP_LABEL if self.resource == 'mds': self.resource_obj = pod.get_mds_pods() self.selector = constants.MDS_APP_LABEL if self.resource == 'cephfsplugin': self.resource_obj = pod.get_plugin_pods( interface=constants.CEPHFILESYSTEM) self.selector = constants.CSI_CEPHFSPLUGIN_LABEL if self.resource == 'rbdplugin': self.resource_obj = pod.get_plugin_pods( interface=constants.CEPHBLOCKPOOL) self.selector = constants.CSI_RBDPLUGIN_LABEL if self.resource == 'cephfsplugin_provisioner': self.resource_obj = [ pod.plugin_provisioner_leader( interface=constants.CEPHFILESYSTEM) ] self.selector = constants.CSI_CEPHFSPLUGIN_PROVISIONER_LABEL resource_count = len(pod.get_cephfsplugin_provisioner_pods()) if self.resource == 'rbdplugin_provisioner': self.resource_obj = [ pod.plugin_provisioner_leader( interface=constants.CEPHBLOCKPOOL) ] self.selector = constants.CSI_RBDPLUGIN_PROVISIONER_LABEL resource_count = len(pod.get_rbdfsplugin_provisioner_pods()) if self.resource == 'operator': self.resource_obj = pod.get_operator_pods() self.selector = constants.OPERATOR_LABEL self.resource_count = resource_count or len(self.resource_obj)
def test_pv_provisioning_under_degraded_state_stop_provisioner_pod_node( self, nodes, pvc_factory, pod_factory, interface, operation, bucket_factory, rgw_bucket_factory, ): """ Test PV provisioning under degraded state - stop the node that has the provisioner pod running on OCS-1138: - Stop 1 worker node that has the RBD provisioner pod running on - Wait for the RBD pod provisioner to come up again to running status - Validate cluster functionality, without checking cluster and Ceph health by creating resources and running IO - Start the worker node - Check cluster and Ceph health OCS-1241: - Stop 1 worker node that has the RBD provisioner pod running on - Wait for the RBD pod provisioner to come up again to running status - Validate cluster functionality, without checking cluster and Ceph health by deleting resources and running IO - Start the worker node - Check cluster and Ceph health OCS-1139: - Stop 1 worker node that has the CephFS provisioner pod running on - Wait for the CephFS pod provisioner to come up again to running status - Validate cluster functionality, without checking cluster and Ceph health by creating resources and running IO - Start the worker node - Check cluster and Ceph health OCS-1242: - Stop 1 worker node that has the CephFS provisioner pod running on - Wait for the CephFS pod provisioner to come up again to running status - Validate cluster functionality, without checking cluster and Ceph health by deleting resources and running IO - Start the worker node - Check cluster and Ceph health """ if operation == "delete_resources": # Create resources that their deletion will be tested later self.sanity_helpers.create_resources(pvc_factory, pod_factory, bucket_factory, rgw_bucket_factory) provisioner_pods = None # Get the provisioner pod according to the interface if interface == "rbd": provisioner_pods = pod.get_rbdfsplugin_provisioner_pods() elif interface == "cephfs": provisioner_pods = pod.get_cephfsplugin_provisioner_pods() provisioner_pod = provisioner_pods[0] # Making sure that the node is not running the rook operator pod: provisioner_node = pod.get_pod_node(provisioner_pod) rook_operator_pod = pod.get_operator_pods()[0] operator_node = pod.get_pod_node(rook_operator_pod) if operator_node.get().get("metadata").get( "name") == provisioner_node.get().get("metadata").get("name"): provisioner_pod = provisioner_pods[1] provisioner_pod_name = provisioner_pod.name logger.info( f"{interface} provisioner pod found: {provisioner_pod_name}") # Get the node name that has the provisioner pod running on provisioner_node = pod.get_pod_node(provisioner_pod) provisioner_node_name = provisioner_node.get().get("metadata").get( "name") logger.info( f"{interface} provisioner pod is running on node {provisioner_node_name}" ) # Stopping the nodes nodes.stop_nodes(nodes=[provisioner_node]) # Wait for the provisioner pod to get to running status selector = (constants.CSI_RBDPLUGIN_PROVISIONER_LABEL if (interface == "rbd") else constants.CSI_CEPHFSPLUGIN_PROVISIONER_LABEL) # Wait for the provisioner pod to reach Terminating status logger.info( f"Waiting for pod {provisioner_pod_name} to reach status Terminating" ) assert provisioner_pod.ocp.wait_for_resource( timeout=600, resource_name=provisioner_pod.name, condition=constants.STATUS_TERMINATING, ), f"{interface} provisioner pod failed to reach status Terminating" logger.info( f"Pod {provisioner_pod_name} has reached status Terminating") # Wait for the provisioner pod to be started and reach running status logger.info( f"Waiting for {interface} provisioner pod to reach status Running") # After this change https://github.com/rook/rook/pull/3642/, there are # 2 provisioners for each interface assert provisioner_pod.ocp.wait_for_resource( timeout=600, condition=constants.STATUS_RUNNING, selector=selector, resource_count=2, ), f"{interface} provisioner pod failed to reach status Running" logger.info(f"{interface} provisioner pod has reached status Running") if operation == "create_resources": # Cluster validation (resources creation and IO running) self.sanity_helpers.create_resources(pvc_factory, pod_factory, bucket_factory, rgw_bucket_factory) elif operation == "delete_resources": # Cluster validation (resources creation and IO running) self.sanity_helpers.delete_resources() # Starting the nodes nodes.start_nodes(nodes=[provisioner_node]) # Checking cluster and Ceph health self.sanity_helpers.health_check()
def test_pv_provisioning_under_degraded_state(self, resources, instances, aws_obj, interface, operation): """ Test PV provisioning under degraded state OCS-1138: - Stop 1 ec2 instance worker node that has the RBD provisioner pod running on - Wait for the RBD pod provisioner to come up again to running status - Validate cluster functionality, without checking cluster and Ceph health by creating resources and running IO - Start the worker node ec2 instance - Check cluster and Ceph health OCS-1241: - Stop 1 ec2 instance worker node that has the RBD provisioner pod running on - Wait for the RBD pod provisioner to come up again to running status - Validate cluster functionality, without checking cluster and Ceph health by deleting resources and running IO - Start the worker node ec2 instance - Check cluster and Ceph health OCS-1139: - Stop 1 ec2 instance worker node that has the CephFS provisioner pod running on - Wait for the CephFS pod provisioner to come up again to running status - Validate cluster functionality, without checking cluster and Ceph health by creating resources and running IO - Start the worker node ec2 instance - Check cluster and Ceph health OCS-1242: - Stop 1 ec2 instance worker node that has the CephFS provisioner pod running on - Wait for the CephFS pod provisioner to come up again to running status - Validate cluster functionality, without checking cluster and Ceph health by deleting resources and running IO - Start the worker node ec2 instance - Check cluster and Ceph health """ if operation == 'delete_resources': # Create resources that their deletion will be tested later sanity_helpers.create_resources(resources=resources) provisioner_pod = None # Get the provisioner pod according to the interface if interface == 'rbd': provisioner_pod = pod.get_rbdfsplugin_provisioner_pods()[0] elif interface == 'cephfs': provisioner_pod = pod.get_cephfsplugin_provisioner_pods()[0] provisioner_pod_name = provisioner_pod.name logger.info( f"{interface} provisioner pod found: {provisioner_pod_name}") # Get the node name that has the provisioner pod running on provisioner_node = pod.get_pod_node(provisioner_pod) provisioner_node_name = provisioner_node.get('metadata').get('name') logger.info( f"{interface} provisioner pod is running on node {provisioner_node_name}" ) # Get the ec2 instance of the node instances = aws.get_instances_ids_and_names([provisioner_node]) # Stopping the nodes aws_obj.stop_ec2_instances(instances=instances, wait=True) # Wait for the provisioner pod to get to running status selector = constants.CSI_RBDPLUGIN_PROVISIONER_LABEL if ( interface == 'rbd') else constants.CSI_CEPHFSPLUGIN_PROVISIONER_LABEL # Wait for the provisioner pod to reach Terminating status logger.info( f"Waiting for pod {provisioner_pod_name} to reach status Terminating" ) assert provisioner_pod.ocp.wait_for_resource( timeout=300, resource_name=provisioner_pod.name, condition=constants.STATUS_TERMINATING ), f"{interface} provisioner pod failed to reach status Terminating" # Wait for the provisioner pod to be started and reach running status logger.info( f"Waiting for pod {provisioner_pod_name} to reach status Running") assert provisioner_pod.ocp.wait_for_resource( timeout=600, condition=constants.STATUS_RUNNING, selector=selector, resource_count=1 ), f"{interface} provisioner pod failed to reach status Running" if operation == 'create_resources': # Cluster validation (resources creation and IO running) sanity_helpers.create_resources(resources=resources) elif operation == 'delete_resources': # Cluster validation (resources creation and IO running) sanity_helpers.delete_resources(resources=resources) # Starting the nodes aws_obj.start_ec2_instances(instances=instances, wait=True) # Checking cluster and Ceph health sanity_helpers.health_check(nodes=[provisioner_node_name])
def set_resource(self, resource, leader_type="provisioner", cluster_index=None): self.resource = resource if (config.ENV_DATA["platform"] in constants.MANAGED_SERVICE_PLATFORMS) and ( resource in CEPH_PODS ): # If the platform is Managed Services, then the ceph pods will be present in the provider cluster. # Consumer cluster will be the primary cluster context in a multicluster run. Setting 'cluster_kubeconfig' # attribute to use as the value of the parameter '--kubeconfig' in the 'oc' commands to get ceph pods. provider_kubeconfig = os.path.join( config.clusters[config.get_provider_index()].ENV_DATA["cluster_path"], config.clusters[config.get_provider_index()].RUN.get( "kubeconfig_location" ), ) self.cluster_kubeconfig = provider_kubeconfig elif config.ENV_DATA["platform"] in constants.MANAGED_SERVICE_PLATFORMS: # cluster_index is used to identify the the cluster in which the pod is residing. If cluster_index is not # passed, assume that the context is already changed to the cluster where the pod is residing. cluster_index = ( cluster_index if cluster_index is not None else config.cur_index ) self.cluster_kubeconfig = os.path.join( config.clusters[cluster_index].ENV_DATA["cluster_path"], config.clusters[cluster_index].RUN.get("kubeconfig_location"), ) resource_count = 0 if self.resource == "mgr": self.resource_obj = pod.get_mgr_pods() self.selector = constants.MGR_APP_LABEL if self.resource == "mon": self.resource_obj = pod.get_mon_pods() self.selector = constants.MON_APP_LABEL if self.resource == "osd": self.resource_obj = pod.get_osd_pods() self.selector = constants.OSD_APP_LABEL if self.resource == "mds": self.resource_obj = pod.get_mds_pods() self.selector = constants.MDS_APP_LABEL if self.resource == "cephfsplugin": self.resource_obj = pod.get_plugin_pods(interface=constants.CEPHFILESYSTEM) self.selector = constants.CSI_CEPHFSPLUGIN_LABEL if self.resource == "rbdplugin": self.resource_obj = pod.get_plugin_pods(interface=constants.CEPHBLOCKPOOL) self.selector = constants.CSI_RBDPLUGIN_LABEL if self.resource == "cephfsplugin_provisioner": self.resource_obj = [ pod.get_plugin_provisioner_leader( interface=constants.CEPHFILESYSTEM, leader_type=leader_type ) ] self.selector = constants.CSI_CEPHFSPLUGIN_PROVISIONER_LABEL resource_count = len(pod.get_cephfsplugin_provisioner_pods()) if self.resource == "rbdplugin_provisioner": self.resource_obj = [ pod.get_plugin_provisioner_leader( interface=constants.CEPHBLOCKPOOL, leader_type=leader_type ) ] self.selector = constants.CSI_RBDPLUGIN_PROVISIONER_LABEL resource_count = len(pod.get_rbdfsplugin_provisioner_pods()) if self.resource == "operator": self.resource_obj = pod.get_operator_pods() self.selector = constants.OPERATOR_LABEL if self.resource == "ocs_operator": self.resource_obj = [pod.get_ocs_operator_pod()] self.selector = constants.OCS_OPERATOR_LABEL if self.resource == "alertmanager_managed_ocs_alertmanager": self.resource_obj = pod.get_alertmanager_managed_ocs_alertmanager_pods() self.selector = constants.MANAGED_ALERTMANAGER_LABEL if self.resource == "ocs_osd_controller_manager": self.resource_obj = [pod.get_ocs_osd_controller_manager_pod()] self.selector = constants.MANAGED_CONTROLLER_LABEL # Setting resource_count because odf-operator-controller-manager pod also have the same label. resource_count = len( pod.get_pods_having_label( constants.MANAGED_CONTROLLER_LABEL, config.ENV_DATA["cluster_namespace"], ) ) if self.resource == "prometheus_managed_ocs_prometheus": self.resource_obj = [pod.get_prometheus_managed_ocs_prometheus_pod()] self.selector = constants.MANAGED_PROMETHEUS_LABEL if self.resource == "prometheus_operator": self.resource_obj = [pod.get_prometheus_operator_pod()] self.selector = constants.PROMETHEUS_OPERATOR_LABEL if self.resource == "ocs_provider_server": self.resource_obj = [pod.get_ocs_provider_server_pod()] self.selector = constants.PROVIDER_SERVER_LABEL self.resource_count = resource_count or len(self.resource_obj)
def test_pv_provisioning_under_degraded_state(self, nodes, pvc_factory, pod_factory, interface, operation): """ Test PV provisioning under degraded state OCS-1138: - Stop 1 worker node that has the RBD provisioner pod running on - Wait for the RBD pod provisioner to come up again to running status - Validate cluster functionality, without checking cluster and Ceph health by creating resources and running IO - Start the worker node - Check cluster and Ceph health OCS-1241: - Stop 1 worker node that has the RBD provisioner pod running on - Wait for the RBD pod provisioner to come up again to running status - Validate cluster functionality, without checking cluster and Ceph health by deleting resources and running IO - Start the worker node - Check cluster and Ceph health OCS-1139: - Stop 1 worker node that has the CephFS provisioner pod running on - Wait for the CephFS pod provisioner to come up again to running status - Validate cluster functionality, without checking cluster and Ceph health by creating resources and running IO - Start the worker node - Check cluster and Ceph health OCS-1242: - Stop 1 worker node that has the CephFS provisioner pod running on - Wait for the CephFS pod provisioner to come up again to running status - Validate cluster functionality, without checking cluster and Ceph health by deleting resources and running IO - Start the worker node - Check cluster and Ceph health """ if operation == 'delete_resources': # Create resources that their deletion will be tested later self.sanity_helpers.create_resources(pvc_factory, pod_factory) provisioner_pods = None # Get the provisioner pod according to the interface if interface == 'rbd': provisioner_pods = pod.get_rbdfsplugin_provisioner_pods() elif interface == 'cephfs': provisioner_pods = pod.get_cephfsplugin_provisioner_pods() provisioner_pod = provisioner_pods[0] # Workaround for BZ 1778488 - https://github.com/red-hat-storage/ocs-ci/issues/1222 provisioner_node = pod.get_pod_node(provisioner_pod) rook_operator_pod = pod.get_operator_pods()[0] operator_node = pod.get_pod_node(rook_operator_pod) if operator_node.get().get('metadata').get( 'name') == provisioner_node.get().get('metadata').get('name'): provisioner_pod = provisioner_pods[1] # End of workaround for BZ 1778488 provisioner_pod_name = provisioner_pod.name logger.info( f"{interface} provisioner pod found: {provisioner_pod_name}") # Get the node name that has the provisioner pod running on provisioner_node = pod.get_pod_node(provisioner_pod) provisioner_node_name = provisioner_node.get().get('metadata').get( 'name') logger.info( f"{interface} provisioner pod is running on node {provisioner_node_name}" ) # Stopping the nodes nodes.stop_nodes(nodes=[provisioner_node]) # Wait for the provisioner pod to get to running status selector = constants.CSI_RBDPLUGIN_PROVISIONER_LABEL if ( interface == 'rbd') else constants.CSI_CEPHFSPLUGIN_PROVISIONER_LABEL # Wait for the provisioner pod to reach Terminating status logger.info( f"Waiting for pod {provisioner_pod_name} to reach status Terminating" ) assert provisioner_pod.ocp.wait_for_resource( timeout=600, resource_name=provisioner_pod.name, condition=constants.STATUS_TERMINATING ), f"{interface} provisioner pod failed to reach status Terminating" logger.info( f"Pod {provisioner_pod_name} has reached status Terminating") # Wait for the provisioner pod to be started and reach running status logger.info( f"Waiting for pod {provisioner_pod_name} to reach status Running") logger.info(f"Pod {provisioner_pod_name} has reached status Running") # After this change https://github.com/rook/rook/pull/3642/, there are # 2 provisioners for each interface assert provisioner_pod.ocp.wait_for_resource( timeout=600, condition=constants.STATUS_RUNNING, selector=selector, resource_count=2 ), f"{interface} provisioner pod failed to reach status Running" assert wait_for_ct_pod_recovery( ), "Ceph tools pod failed to come up on another node" if operation == 'create_resources': # Cluster validation (resources creation and IO running) self.sanity_helpers.create_resources(pvc_factory, pod_factory) elif operation == 'delete_resources': # Cluster validation (resources creation and IO running) self.sanity_helpers.delete_resources() # Starting the nodes nodes.start_nodes(nodes=[provisioner_node]) # Checking cluster and Ceph health self.sanity_helpers.health_check()