def remove_global_id_reclaim():
"""
Removes global id warning by re-spinning client and mon pods
"""
csi_pods = []
interfaces = [constants.CEPHBLOCKPOOL, constants.CEPHFILESYSTEM]
for interface in interfaces:
plugin_pods = get_plugin_pods(interface)
csi_pods += plugin_pods
cephfs_provisioner_pods = get_cephfsplugin_provisioner_pods()
rbd_provisioner_pods = get_rbdfsplugin_provisioner_pods()
csi_pods += cephfs_provisioner_pods
csi_pods += rbd_provisioner_pods
for csi_pod in csi_pods:
csi_pod.delete()
for mds_pod in get_mds_pods():
mds_pod.delete()
for mds_pod in get_mds_pods():
wait_for_resource_state(resource=mds_pod, state=constants.STATUS_RUNNING)
for mon in get_mon_pods(namespace=constants.OPENSHIFT_STORAGE_NAMESPACE):
mon.delete()
mon_pods = get_mon_pods(namespace=constants.OPENSHIFT_STORAGE_NAMESPACE)
for mon in mon_pods:
wait_for_resource_state(resource=mon, state=constants.STATUS_RUNNING)
def verify_multus_network():
"""
Verify Multus network(s) created successfully and are present on relevant pods.
"""
with open(constants.MULTUS_YAML, mode="r") as f:
multus_public_data = yaml.load(f)
multus_namespace = multus_public_data["metadata"]["namespace"]
multus_name = multus_public_data["metadata"]["name"]
multus_public_network_name = f"{multus_namespace}/{multus_name}"
log.info("Verifying multus NetworkAttachmentDefinitions")
ocp.OCP(
resource_name=multus_public_network_name,
kind="network-attachment-definitions",
namespace=constants.OPENSHIFT_STORAGE_NAMESPACE,
)
# TODO: also check if private NAD exists
log.info("Verifying multus public network exists on ceph pods")
osd_pods = get_osd_pods()
for _pod in osd_pods:
assert (_pod.data["metadata"]["annotations"]
["k8s.v1.cni.cncf.io/networks"] == multus_public_network_name)
# TODO: also check private network if it exists on OSD pods
mon_pods = get_mon_pods()
mds_pods = get_mds_pods()
mgr_pods = get_mgr_pods()
rgw_pods = get_rgw_pods()
ceph_pods = [*mon_pods, *mds_pods, *mgr_pods, *rgw_pods]
for _pod in ceph_pods:
assert (_pod.data["metadata"]["annotations"]
["k8s.v1.cni.cncf.io/networks"] == multus_public_network_name)
log.info("Verifying multus public network exists on CSI pods")
csi_pods = []
interfaces = [constants.CEPHBLOCKPOOL, constants.CEPHFILESYSTEM]
for interface in interfaces:
plugin_pods = get_plugin_pods(interface)
csi_pods += plugin_pods
cephfs_provisioner_pods = get_cephfsplugin_provisioner_pods()
rbd_provisioner_pods = get_rbdfsplugin_provisioner_pods()
csi_pods += cephfs_provisioner_pods
csi_pods += rbd_provisioner_pods
for _pod in csi_pods:
assert (_pod.data["metadata"]["annotations"]
["k8s.v1.cni.cncf.io/networks"] == multus_public_network_name)
log.info("Verifying StorageCluster multus network data")
sc = get_storage_cluster()
sc_data = sc.get().get("items")[0]
network_data = sc_data["spec"]["network"]
assert network_data["provider"] == "multus"
selectors = network_data["selectors"]
assert selectors[
"public"] == f"{defaults.ROOK_CLUSTER_NAMESPACE}/ocs-public"
def set_resource(self, resource, leader_type="provisioner"):
self.resource = resource
if (config.ENV_DATA["platform"]
in constants.MANAGED_SERVICE_PLATFORMS) and (resource
in CEPH_PODS):
# If the platform is Managed Services, then the ceph pods will be present in the provider cluster.
# Consumer cluster will be the primary cluster context in a multicluster run. Setting 'cluster_kubeconfig'
# attribute to use as the value of the parameter '--kubeconfig' in the 'oc' commands to get ceph pods.
provider_kubeconfig = os.path.join(
config.clusters[
config.get_provider_index()].ENV_DATA["cluster_path"],
config.clusters[config.get_provider_index()].RUN.get(
"kubeconfig_location"),
)
self.cluster_kubeconfig = provider_kubeconfig
resource_count = 0
if self.resource == "mgr":
self.resource_obj = pod.get_mgr_pods()
self.selector = constants.MGR_APP_LABEL
if self.resource == "mon":
self.resource_obj = pod.get_mon_pods()
self.selector = constants.MON_APP_LABEL
if self.resource == "osd":
self.resource_obj = pod.get_osd_pods()
self.selector = constants.OSD_APP_LABEL
if self.resource == "mds":
self.resource_obj = pod.get_mds_pods()
self.selector = constants.MDS_APP_LABEL
if self.resource == "cephfsplugin":
self.resource_obj = pod.get_plugin_pods(
interface=constants.CEPHFILESYSTEM)
self.selector = constants.CSI_CEPHFSPLUGIN_LABEL
if self.resource == "rbdplugin":
self.resource_obj = pod.get_plugin_pods(
interface=constants.CEPHBLOCKPOOL)
self.selector = constants.CSI_RBDPLUGIN_LABEL
if self.resource == "cephfsplugin_provisioner":
self.resource_obj = [
pod.get_plugin_provisioner_leader(
interface=constants.CEPHFILESYSTEM,
leader_type=leader_type)
]
self.selector = constants.CSI_CEPHFSPLUGIN_PROVISIONER_LABEL
resource_count = len(pod.get_cephfsplugin_provisioner_pods())
if self.resource == "rbdplugin_provisioner":
self.resource_obj = [
pod.get_plugin_provisioner_leader(
interface=constants.CEPHBLOCKPOOL, leader_type=leader_type)
]
self.selector = constants.CSI_RBDPLUGIN_PROVISIONER_LABEL
resource_count = len(pod.get_rbdfsplugin_provisioner_pods())
if self.resource == "operator":
self.resource_obj = pod.get_operator_pods()
self.selector = constants.OPERATOR_LABEL
self.resource_count = resource_count or len(self.resource_obj)
def set_resource(self, resource, leader_type="provisioner"):
self.resource = resource
resource_count = 0
if self.resource == "mgr":
self.resource_obj = pod.get_mgr_pods()
self.selector = constants.MGR_APP_LABEL
if self.resource == "mon":
self.resource_obj = pod.get_mon_pods()
self.selector = constants.MON_APP_LABEL
if self.resource == "osd":
self.resource_obj = pod.get_osd_pods()
self.selector = constants.OSD_APP_LABEL
if self.resource == "mds":
self.resource_obj = pod.get_mds_pods()
self.selector = constants.MDS_APP_LABEL
if self.resource == "cephfsplugin":
self.resource_obj = pod.get_plugin_pods(
interface=constants.CEPHFILESYSTEM)
self.selector = constants.CSI_CEPHFSPLUGIN_LABEL
if self.resource == "rbdplugin":
self.resource_obj = pod.get_plugin_pods(
interface=constants.CEPHBLOCKPOOL)
self.selector = constants.CSI_RBDPLUGIN_LABEL
if self.resource == "cephfsplugin_provisioner":
self.resource_obj = [
pod.get_plugin_provisioner_leader(
interface=constants.CEPHFILESYSTEM,
leader_type=leader_type)
]
self.selector = constants.CSI_CEPHFSPLUGIN_PROVISIONER_LABEL
resource_count = len(pod.get_cephfsplugin_provisioner_pods())
if self.resource == "rbdplugin_provisioner":
self.resource_obj = [
pod.get_plugin_provisioner_leader(
interface=constants.CEPHBLOCKPOOL, leader_type=leader_type)
]
self.selector = constants.CSI_RBDPLUGIN_PROVISIONER_LABEL
resource_count = len(pod.get_rbdfsplugin_provisioner_pods())
if self.resource == "operator":
self.resource_obj = pod.get_operator_pods()
self.selector = constants.OPERATOR_LABEL
self.resource_count = resource_count or len(self.resource_obj)
def set_resource(self, resource):
self.resource = resource
resource_count = 0
if self.resource == 'mgr':
self.resource_obj = pod.get_mgr_pods()
self.selector = constants.MGR_APP_LABEL
if self.resource == 'mon':
self.resource_obj = pod.get_mon_pods()
self.selector = constants.MON_APP_LABEL
if self.resource == 'osd':
self.resource_obj = pod.get_osd_pods()
self.selector = constants.OSD_APP_LABEL
if self.resource == 'mds':
self.resource_obj = pod.get_mds_pods()
self.selector = constants.MDS_APP_LABEL
if self.resource == 'cephfsplugin':
self.resource_obj = pod.get_plugin_pods(
interface=constants.CEPHFILESYSTEM)
self.selector = constants.CSI_CEPHFSPLUGIN_LABEL
if self.resource == 'rbdplugin':
self.resource_obj = pod.get_plugin_pods(
interface=constants.CEPHBLOCKPOOL)
self.selector = constants.CSI_RBDPLUGIN_LABEL
if self.resource == 'cephfsplugin_provisioner':
self.resource_obj = [
pod.plugin_provisioner_leader(
interface=constants.CEPHFILESYSTEM)
]
self.selector = constants.CSI_CEPHFSPLUGIN_PROVISIONER_LABEL
resource_count = len(pod.get_cephfsplugin_provisioner_pods())
if self.resource == 'rbdplugin_provisioner':
self.resource_obj = [
pod.plugin_provisioner_leader(
interface=constants.CEPHBLOCKPOOL)
]
self.selector = constants.CSI_RBDPLUGIN_PROVISIONER_LABEL
resource_count = len(pod.get_rbdfsplugin_provisioner_pods())
if self.resource == 'operator':
self.resource_obj = pod.get_operator_pods()
self.selector = constants.OPERATOR_LABEL
self.resource_count = resource_count or len(self.resource_obj)
def test_pv_provisioning_under_degraded_state_stop_provisioner_pod_node(
self,
nodes,
pvc_factory,
pod_factory,
interface,
operation,
bucket_factory,
rgw_bucket_factory,
):
"""
Test PV provisioning under degraded state -
stop the node that has the provisioner pod running on
OCS-1138:
- Stop 1 worker node that has the RBD provisioner
pod running on
- Wait for the RBD pod provisioner to come up again to running status
- Validate cluster functionality, without checking cluster and Ceph
health by creating resources and running IO
- Start the worker node
- Check cluster and Ceph health
OCS-1241:
- Stop 1 worker node that has the RBD provisioner
pod running on
- Wait for the RBD pod provisioner to come up again to running status
- Validate cluster functionality, without checking cluster and Ceph
health by deleting resources and running IO
- Start the worker node
- Check cluster and Ceph health
OCS-1139:
- Stop 1 worker node that has the CephFS provisioner
pod running on
- Wait for the CephFS pod provisioner to come up again to running status
- Validate cluster functionality, without checking cluster and Ceph
health by creating resources and running IO
- Start the worker node
- Check cluster and Ceph health
OCS-1242:
- Stop 1 worker node that has the CephFS provisioner
pod running on
- Wait for the CephFS pod provisioner to come up again to running status
- Validate cluster functionality, without checking cluster and Ceph
health by deleting resources and running IO
- Start the worker node
- Check cluster and Ceph health
"""
if operation == "delete_resources":
# Create resources that their deletion will be tested later
self.sanity_helpers.create_resources(pvc_factory, pod_factory,
bucket_factory,
rgw_bucket_factory)
provisioner_pods = None
# Get the provisioner pod according to the interface
if interface == "rbd":
provisioner_pods = pod.get_rbdfsplugin_provisioner_pods()
elif interface == "cephfs":
provisioner_pods = pod.get_cephfsplugin_provisioner_pods()
provisioner_pod = provisioner_pods[0]
# Making sure that the node is not running the rook operator pod:
provisioner_node = pod.get_pod_node(provisioner_pod)
rook_operator_pod = pod.get_operator_pods()[0]
operator_node = pod.get_pod_node(rook_operator_pod)
if operator_node.get().get("metadata").get(
"name") == provisioner_node.get().get("metadata").get("name"):
provisioner_pod = provisioner_pods[1]
provisioner_pod_name = provisioner_pod.name
logger.info(
f"{interface} provisioner pod found: {provisioner_pod_name}")
# Get the node name that has the provisioner pod running on
provisioner_node = pod.get_pod_node(provisioner_pod)
provisioner_node_name = provisioner_node.get().get("metadata").get(
"name")
logger.info(
f"{interface} provisioner pod is running on node {provisioner_node_name}"
)
# Stopping the nodes
nodes.stop_nodes(nodes=[provisioner_node])
# Wait for the provisioner pod to get to running status
selector = (constants.CSI_RBDPLUGIN_PROVISIONER_LABEL if
(interface == "rbd") else
constants.CSI_CEPHFSPLUGIN_PROVISIONER_LABEL)
# Wait for the provisioner pod to reach Terminating status
logger.info(
f"Waiting for pod {provisioner_pod_name} to reach status Terminating"
)
assert provisioner_pod.ocp.wait_for_resource(
timeout=600,
resource_name=provisioner_pod.name,
condition=constants.STATUS_TERMINATING,
), f"{interface} provisioner pod failed to reach status Terminating"
logger.info(
f"Pod {provisioner_pod_name} has reached status Terminating")
# Wait for the provisioner pod to be started and reach running status
logger.info(
f"Waiting for {interface} provisioner pod to reach status Running")
# After this change https://github.com/rook/rook/pull/3642/, there are
# 2 provisioners for each interface
assert provisioner_pod.ocp.wait_for_resource(
timeout=600,
condition=constants.STATUS_RUNNING,
selector=selector,
resource_count=2,
), f"{interface} provisioner pod failed to reach status Running"
logger.info(f"{interface} provisioner pod has reached status Running")
if operation == "create_resources":
# Cluster validation (resources creation and IO running)
self.sanity_helpers.create_resources(pvc_factory, pod_factory,
bucket_factory,
rgw_bucket_factory)
elif operation == "delete_resources":
# Cluster validation (resources creation and IO running)
self.sanity_helpers.delete_resources()
# Starting the nodes
nodes.start_nodes(nodes=[provisioner_node])
# Checking cluster and Ceph health
self.sanity_helpers.health_check()
def test_pv_provisioning_under_degraded_state(self, resources, instances,
aws_obj, interface,
operation):
"""
Test PV provisioning under degraded state
OCS-1138:
- Stop 1 ec2 instance worker node that has the RBD provisioner
pod running on
- Wait for the RBD pod provisioner to come up again to running status
- Validate cluster functionality, without checking cluster and Ceph
health by creating resources and running IO
- Start the worker node ec2 instance
- Check cluster and Ceph health
OCS-1241:
- Stop 1 ec2 instance worker node that has the RBD provisioner
pod running on
- Wait for the RBD pod provisioner to come up again to running status
- Validate cluster functionality, without checking cluster and Ceph
health by deleting resources and running IO
- Start the worker node ec2 instance
- Check cluster and Ceph health
OCS-1139:
- Stop 1 ec2 instance worker node that has the CephFS provisioner
pod running on
- Wait for the CephFS pod provisioner to come up again to running status
- Validate cluster functionality, without checking cluster and Ceph
health by creating resources and running IO
- Start the worker node ec2 instance
- Check cluster and Ceph health
OCS-1242:
- Stop 1 ec2 instance worker node that has the CephFS provisioner
pod running on
- Wait for the CephFS pod provisioner to come up again to running status
- Validate cluster functionality, without checking cluster and Ceph
health by deleting resources and running IO
- Start the worker node ec2 instance
- Check cluster and Ceph health
"""
if operation == 'delete_resources':
# Create resources that their deletion will be tested later
sanity_helpers.create_resources(resources=resources)
provisioner_pod = None
# Get the provisioner pod according to the interface
if interface == 'rbd':
provisioner_pod = pod.get_rbdfsplugin_provisioner_pods()[0]
elif interface == 'cephfs':
provisioner_pod = pod.get_cephfsplugin_provisioner_pods()[0]
provisioner_pod_name = provisioner_pod.name
logger.info(
f"{interface} provisioner pod found: {provisioner_pod_name}")
# Get the node name that has the provisioner pod running on
provisioner_node = pod.get_pod_node(provisioner_pod)
provisioner_node_name = provisioner_node.get('metadata').get('name')
logger.info(
f"{interface} provisioner pod is running on node {provisioner_node_name}"
)
# Get the ec2 instance of the node
instances = aws.get_instances_ids_and_names([provisioner_node])
# Stopping the nodes
aws_obj.stop_ec2_instances(instances=instances, wait=True)
# Wait for the provisioner pod to get to running status
selector = constants.CSI_RBDPLUGIN_PROVISIONER_LABEL if (
interface
== 'rbd') else constants.CSI_CEPHFSPLUGIN_PROVISIONER_LABEL
# Wait for the provisioner pod to reach Terminating status
logger.info(
f"Waiting for pod {provisioner_pod_name} to reach status Terminating"
)
assert provisioner_pod.ocp.wait_for_resource(
timeout=300,
resource_name=provisioner_pod.name,
condition=constants.STATUS_TERMINATING
), f"{interface} provisioner pod failed to reach status Terminating"
# Wait for the provisioner pod to be started and reach running status
logger.info(
f"Waiting for pod {provisioner_pod_name} to reach status Running")
assert provisioner_pod.ocp.wait_for_resource(
timeout=600,
condition=constants.STATUS_RUNNING,
selector=selector,
resource_count=1
), f"{interface} provisioner pod failed to reach status Running"
if operation == 'create_resources':
# Cluster validation (resources creation and IO running)
sanity_helpers.create_resources(resources=resources)
elif operation == 'delete_resources':
# Cluster validation (resources creation and IO running)
sanity_helpers.delete_resources(resources=resources)
# Starting the nodes
aws_obj.start_ec2_instances(instances=instances, wait=True)
# Checking cluster and Ceph health
sanity_helpers.health_check(nodes=[provisioner_node_name])
def set_resource(self, resource, leader_type="provisioner", cluster_index=None):
self.resource = resource
if (config.ENV_DATA["platform"] in constants.MANAGED_SERVICE_PLATFORMS) and (
resource in CEPH_PODS
):
# If the platform is Managed Services, then the ceph pods will be present in the provider cluster.
# Consumer cluster will be the primary cluster context in a multicluster run. Setting 'cluster_kubeconfig'
# attribute to use as the value of the parameter '--kubeconfig' in the 'oc' commands to get ceph pods.
provider_kubeconfig = os.path.join(
config.clusters[config.get_provider_index()].ENV_DATA["cluster_path"],
config.clusters[config.get_provider_index()].RUN.get(
"kubeconfig_location"
),
)
self.cluster_kubeconfig = provider_kubeconfig
elif config.ENV_DATA["platform"] in constants.MANAGED_SERVICE_PLATFORMS:
# cluster_index is used to identify the the cluster in which the pod is residing. If cluster_index is not
# passed, assume that the context is already changed to the cluster where the pod is residing.
cluster_index = (
cluster_index if cluster_index is not None else config.cur_index
)
self.cluster_kubeconfig = os.path.join(
config.clusters[cluster_index].ENV_DATA["cluster_path"],
config.clusters[cluster_index].RUN.get("kubeconfig_location"),
)
resource_count = 0
if self.resource == "mgr":
self.resource_obj = pod.get_mgr_pods()
self.selector = constants.MGR_APP_LABEL
if self.resource == "mon":
self.resource_obj = pod.get_mon_pods()
self.selector = constants.MON_APP_LABEL
if self.resource == "osd":
self.resource_obj = pod.get_osd_pods()
self.selector = constants.OSD_APP_LABEL
if self.resource == "mds":
self.resource_obj = pod.get_mds_pods()
self.selector = constants.MDS_APP_LABEL
if self.resource == "cephfsplugin":
self.resource_obj = pod.get_plugin_pods(interface=constants.CEPHFILESYSTEM)
self.selector = constants.CSI_CEPHFSPLUGIN_LABEL
if self.resource == "rbdplugin":
self.resource_obj = pod.get_plugin_pods(interface=constants.CEPHBLOCKPOOL)
self.selector = constants.CSI_RBDPLUGIN_LABEL
if self.resource == "cephfsplugin_provisioner":
self.resource_obj = [
pod.get_plugin_provisioner_leader(
interface=constants.CEPHFILESYSTEM, leader_type=leader_type
)
]
self.selector = constants.CSI_CEPHFSPLUGIN_PROVISIONER_LABEL
resource_count = len(pod.get_cephfsplugin_provisioner_pods())
if self.resource == "rbdplugin_provisioner":
self.resource_obj = [
pod.get_plugin_provisioner_leader(
interface=constants.CEPHBLOCKPOOL, leader_type=leader_type
)
]
self.selector = constants.CSI_RBDPLUGIN_PROVISIONER_LABEL
resource_count = len(pod.get_rbdfsplugin_provisioner_pods())
if self.resource == "operator":
self.resource_obj = pod.get_operator_pods()
self.selector = constants.OPERATOR_LABEL
if self.resource == "ocs_operator":
self.resource_obj = [pod.get_ocs_operator_pod()]
self.selector = constants.OCS_OPERATOR_LABEL
if self.resource == "alertmanager_managed_ocs_alertmanager":
self.resource_obj = pod.get_alertmanager_managed_ocs_alertmanager_pods()
self.selector = constants.MANAGED_ALERTMANAGER_LABEL
if self.resource == "ocs_osd_controller_manager":
self.resource_obj = [pod.get_ocs_osd_controller_manager_pod()]
self.selector = constants.MANAGED_CONTROLLER_LABEL
# Setting resource_count because odf-operator-controller-manager pod also have the same label.
resource_count = len(
pod.get_pods_having_label(
constants.MANAGED_CONTROLLER_LABEL,
config.ENV_DATA["cluster_namespace"],
)
)
if self.resource == "prometheus_managed_ocs_prometheus":
self.resource_obj = [pod.get_prometheus_managed_ocs_prometheus_pod()]
self.selector = constants.MANAGED_PROMETHEUS_LABEL
if self.resource == "prometheus_operator":
self.resource_obj = [pod.get_prometheus_operator_pod()]
self.selector = constants.PROMETHEUS_OPERATOR_LABEL
if self.resource == "ocs_provider_server":
self.resource_obj = [pod.get_ocs_provider_server_pod()]
self.selector = constants.PROVIDER_SERVER_LABEL
self.resource_count = resource_count or len(self.resource_obj)
def test_pv_provisioning_under_degraded_state(self, nodes, pvc_factory,
pod_factory, interface,
operation):
"""
Test PV provisioning under degraded state
OCS-1138:
- Stop 1 worker node that has the RBD provisioner
pod running on
- Wait for the RBD pod provisioner to come up again to running status
- Validate cluster functionality, without checking cluster and Ceph
health by creating resources and running IO
- Start the worker node
- Check cluster and Ceph health
OCS-1241:
- Stop 1 worker node that has the RBD provisioner
pod running on
- Wait for the RBD pod provisioner to come up again to running status
- Validate cluster functionality, without checking cluster and Ceph
health by deleting resources and running IO
- Start the worker node
- Check cluster and Ceph health
OCS-1139:
- Stop 1 worker node that has the CephFS provisioner
pod running on
- Wait for the CephFS pod provisioner to come up again to running status
- Validate cluster functionality, without checking cluster and Ceph
health by creating resources and running IO
- Start the worker node
- Check cluster and Ceph health
OCS-1242:
- Stop 1 worker node that has the CephFS provisioner
pod running on
- Wait for the CephFS pod provisioner to come up again to running status
- Validate cluster functionality, without checking cluster and Ceph
health by deleting resources and running IO
- Start the worker node
- Check cluster and Ceph health
"""
if operation == 'delete_resources':
# Create resources that their deletion will be tested later
self.sanity_helpers.create_resources(pvc_factory, pod_factory)
provisioner_pods = None
# Get the provisioner pod according to the interface
if interface == 'rbd':
provisioner_pods = pod.get_rbdfsplugin_provisioner_pods()
elif interface == 'cephfs':
provisioner_pods = pod.get_cephfsplugin_provisioner_pods()
provisioner_pod = provisioner_pods[0]
# Workaround for BZ 1778488 - https://github.com/red-hat-storage/ocs-ci/issues/1222
provisioner_node = pod.get_pod_node(provisioner_pod)
rook_operator_pod = pod.get_operator_pods()[0]
operator_node = pod.get_pod_node(rook_operator_pod)
if operator_node.get().get('metadata').get(
'name') == provisioner_node.get().get('metadata').get('name'):
provisioner_pod = provisioner_pods[1]
# End of workaround for BZ 1778488
provisioner_pod_name = provisioner_pod.name
logger.info(
f"{interface} provisioner pod found: {provisioner_pod_name}")
# Get the node name that has the provisioner pod running on
provisioner_node = pod.get_pod_node(provisioner_pod)
provisioner_node_name = provisioner_node.get().get('metadata').get(
'name')
logger.info(
f"{interface} provisioner pod is running on node {provisioner_node_name}"
)
# Stopping the nodes
nodes.stop_nodes(nodes=[provisioner_node])
# Wait for the provisioner pod to get to running status
selector = constants.CSI_RBDPLUGIN_PROVISIONER_LABEL if (
interface
== 'rbd') else constants.CSI_CEPHFSPLUGIN_PROVISIONER_LABEL
# Wait for the provisioner pod to reach Terminating status
logger.info(
f"Waiting for pod {provisioner_pod_name} to reach status Terminating"
)
assert provisioner_pod.ocp.wait_for_resource(
timeout=600,
resource_name=provisioner_pod.name,
condition=constants.STATUS_TERMINATING
), f"{interface} provisioner pod failed to reach status Terminating"
logger.info(
f"Pod {provisioner_pod_name} has reached status Terminating")
# Wait for the provisioner pod to be started and reach running status
logger.info(
f"Waiting for pod {provisioner_pod_name} to reach status Running")
logger.info(f"Pod {provisioner_pod_name} has reached status Running")
# After this change https://github.com/rook/rook/pull/3642/, there are
# 2 provisioners for each interface
assert provisioner_pod.ocp.wait_for_resource(
timeout=600,
condition=constants.STATUS_RUNNING,
selector=selector,
resource_count=2
), f"{interface} provisioner pod failed to reach status Running"
assert wait_for_ct_pod_recovery(
), "Ceph tools pod failed to come up on another node"
if operation == 'create_resources':
# Cluster validation (resources creation and IO running)
self.sanity_helpers.create_resources(pvc_factory, pod_factory)
elif operation == 'delete_resources':
# Cluster validation (resources creation and IO running)
self.sanity_helpers.delete_resources()
# Starting the nodes
nodes.start_nodes(nodes=[provisioner_node])
# Checking cluster and Ceph health
self.sanity_helpers.health_check()
