def _get_newly_deployed_gluster_pod(self, g_pod_list_before):
# Fetch pod after delete
g_pod_list_after = [
pod["pod_name"]
for pod in openshift_ops.get_ocp_gluster_pod_details(self._master)]
# Fetch the new gluster pod
g_new_pod = list(set(g_pod_list_after) - set(g_pod_list_before))
self.assertTrue(g_new_pod, "No new gluster pod deployed after delete")
return g_new_pod
def run(target, command, user=None, log_level=None, orig_run=g.run):
"""Function that runs a command on a host or in a pod via a host.
Wraps glusto's run function.
Args:
target (str|Pod): If target is str object and
it equals to 'auto_get_gluster_endpoint', then
Gluster endpoint gets autocalculated to be any of
Gluster PODs or nodes depending on the deployment type of
a Gluster cluster.
If it is str object with other value, then it is considered to be
an endpoint for command.
If 'target' is of the 'Pod' type,
then command will run on the specified POD.
command (str|list): Command to run.
user (str|None): user to be passed on to glusto's run method
log_level (str|None): log level to be passed on to glusto's run method
orig_run (function): The default implementation of the
run method. Will be used when target is not a pod.
Returns:
A tuple of the command's return code, stdout, and stderr.
"""
# NOTE: orig_run captures the glusto run method at function
# definition time in order to capture the method before
# any additional monkeypatching by other code
ocp_client_node = list(g.config['ocp_servers']['client'].keys())[0]
with mock.patch.object(g, 'run', new=orig_run):
gluster_pods = openshift_ops.get_ocp_gluster_pod_details(
ocp_client_node)
if target == 'auto_get_gluster_endpoint':
if gluster_pods:
target = Pod(ocp_client_node, gluster_pods[0]["pod_name"])
else:
target = list(g.config.get("gluster_servers", {}).keys())[0]
elif not isinstance(target, Pod) and gluster_pods:
for g_pod in gluster_pods:
if target in (g_pod['pod_host_ip'], g_pod['pod_hostname']):
target = Pod(ocp_client_node, g_pod['pod_name'])
break
if isinstance(target, Pod):
prefix = ['oc', 'rsh', target.podname]
if isinstance(command, six.string_types):
cmd = ' '.join(prefix + [command])
else:
cmd = prefix + command
# unpack the tuple to make sure our return value exactly matches
# our docstring
return g.run(target.node, cmd, user=user, log_level=log_level)
else:
return orig_run(target, command, user=user, log_level=log_level)
def test_verify_metrics_data_during_gluster_pod_respin(self):
# Add check for CRS version
switch_oc_project(self.master, self.registry_project_name)
if not self.is_containerized_gluster():
self.skipTest("Skipping this test case as CRS version check "
"can not be implemented")
# Verify multipath and iscsi for cassandra pod
switch_oc_project(self.master, self.metrics_project_name)
hawkular_cassandra, pvc_name, iqn, _, node = (
self.verify_cassandra_pod_multipath_and_iscsi())
# Get the ip of active path
device_and_ip = get_iscsi_block_devices_by_path(node, iqn)
mpath = get_mpath_name_from_device_name(node,
list(device_and_ip.keys())[0])
active_passive_dict = get_active_and_enabled_devices_from_mpath(
node, mpath)
node_ip = device_and_ip[active_passive_dict['active'][0]]
# Get the name of gluster pod from the ip
switch_oc_project(self.master, self.registry_project_name)
gluster_pods = get_ocp_gluster_pod_details(self.master)
pod_name = list(
filter(lambda pod: (pod["pod_host_ip"] == node_ip),
gluster_pods))[0]["pod_name"]
err_msg = "Failed to get the gluster pod name {} with active path"
self.assertTrue(pod_name, err_msg.format(pod_name))
# Delete the pod
oc_delete(self.master, 'pod', pod_name)
wait_for_resource_absence(self.master, 'pod', pod_name)
# Wait for new pod to come up
pod_count = len(self.registry_servers_info.keys())
selector = "glusterfs-node=pod"
wait_for_pods_be_ready(self.master, pod_count, selector)
# Validate cassandra pod state, multipath and issci
switch_oc_project(self.master, self.metrics_project_name)
wait_for_pod_be_ready(self.master, hawkular_cassandra, timeout=2)
self.verify_iscsi_sessions_and_multipath(
pvc_name,
self.metrics_rc_hawkular_cassandra,
rtype='rc',
heketi_server_url=self.registry_heketi_server_url,
is_registry_gluster=True)
def test_resping_gluster_pod(self):
"""Validate gluster pod restart with no disruption to elasticsearch pod
"""
restart_custom = ":status.containerStatuses[0].restartCount"
# Fetch pod and validate iscsi and multipath
es_pod, _ = self._get_es_pod_and_verify_iscsi_sessions()
# Fetch the restart count for the es pod
restart_count_before = openshift_ops.oc_get_custom_resource(
self._master, "pod", restart_custom, es_pod)[0]
# Switch to gluster project
openshift_ops.switch_oc_project(self._master,
self._registry_project_name)
# Fetch the gluster pod list before
g_pod_list_before = [
pod["pod_name"]
for pod in openshift_ops.get_ocp_gluster_pod_details(self._master)
]
# Respin a gluster pod
openshift_ops.oc_delete(self._master, "pod", g_pod_list_before[0])
self.addCleanup(self._guster_pod_delete_cleanup, g_pod_list_before)
# Wait for pod to get absent
openshift_ops.wait_for_resource_absence(self._master, "pod",
g_pod_list_before[0])
# Fetch gluster pod after delete
g_new_pod = self._get_newly_deployed_gluster_pod(g_pod_list_before)
openshift_ops.wait_for_pod_be_ready(self._master, g_new_pod[0])
# Switch to logging project
openshift_ops.switch_oc_project(self._master,
self._logging_project_name)
# Fetch the restart count for the es pod
restart_count_after = openshift_ops.oc_get_custom_resource(
self._master, "pod", restart_custom, es_pod)[0]
self.assertEqual(
restart_count_before, restart_count_after,
"Failed disruption to es pod found expecting restart count before"
" {} and after {} for es pod to be equal after gluster pod"
" respin".format(restart_count_before, restart_count_after))
def setUp(self):
super(TestNodeRestart, self).setUp()
self.oc_node = self.ocp_master_node[0]
self.gluster_pod_list = [
pod["pod_name"]
for pod in get_ocp_gluster_pod_details(self.oc_node)]
if not self.gluster_pod_list:
self.skipTest("Standalone Gluster is not supported by this test.")
self.gluster_pod_name = self.gluster_pod_list[0]
if get_openshift_storage_version() < "3.11.1":
self.skipTest("Skipping this test case due to bug BZ-1635736")
self.sc_name = self.create_storage_class()
self.pvc_names = self._create_volumes_with_io(3)
def setUp(self):
super(TestHeketiLvmWrapper, self).setUp()
self.oc_node = self.ocp_master_node[0]
self.pod_name = openshift_ops.get_ocp_gluster_pod_details(self.oc_node)
self.h_pod_name = openshift_ops.get_pod_name_from_dc(
self.oc_node, self.heketi_dc_name)
self.volume_size = 2
ocp_version = openshift_version.get_openshift_version()
if ocp_version < "3.11.170":
self.skipTest("Heketi LVM Wrapper functionality does not "
"support on OCP {}".format(ocp_version.v_str))
h_version = heketi_version.get_heketi_version(self.heketi_client_node)
if h_version < '9.0.0-9':
self.skipTest("heketi-client package {} does not support Heketi "
"LVM Wrapper functionality".format(h_version.v_str))
def _get_gluster_cmd(target, command):
if isinstance(command, six.string_types):
command = [command]
ocp_client_node = list(g.config['ocp_servers']['client'].keys())[0]
gluster_pods = get_ocp_gluster_pod_details(ocp_client_node)
if target == 'auto_get_gluster_endpoint':
if gluster_pods:
target = podcmd.Pod(ocp_client_node, gluster_pods[0]["pod_name"])
else:
target = list(g.config.get("gluster_servers", {}).keys())[0]
elif not isinstance(target, podcmd.Pod) and gluster_pods:
for g_pod in gluster_pods:
if target in (g_pod['pod_host_ip'], g_pod['pod_hostname']):
target = podcmd.Pod(ocp_client_node, g_pod['pod_name'])
break
if isinstance(target, podcmd.Pod):
return target.node, ' '.join(['oc', 'rsh', target.podname] + command)
return target, ' '.join(command)
def reboot_gluster_node_and_wait_for_services(self):
gluster_node_ip = (
g.config["gluster_servers"][self.gluster_servers[0]]["storage"])
gluster_pod = list(
filter(lambda pod: (pod["pod_host_ip"] == gluster_node_ip),
get_ocp_gluster_pod_details(self.oc_node)))
if not gluster_pod:
raise ExecutionError("Gluster pod Host IP '%s' not matched." %
gluster_node_ip)
gluster_pod = gluster_pod[0]["pod_name"]
self.addCleanup(wait_for_pod_be_ready, self.oc_node, gluster_pod)
node_reboot_by_command(gluster_node_ip, timeout=600, wait_step=10)
# wait for the gluster pod to be in 'Running' state
wait_for_pod_be_ready(self.oc_node, gluster_pod)
# glusterd and gluster-blockd service should be up and running
services = (("glusterd", "running"), ("gluster-blockd", "running"),
("tcmu-runner", "running"), ("gluster-block-target",
"exited"))
for service, state in services:
check_service_status_on_pod(self.oc_node, gluster_pod, service,
"active", state)
def test_restart_prometheus_glusterfs_pod(self):
"""Validate restarting glusterfs pod"""
# Add check for CRS version
openshift_ops.switch_oc_project(
self._master, self._registry_project_name)
if not self.is_containerized_gluster():
self.skipTest(
"Skipping this test case as CRS version check "
"can not be implemented")
# Get one of the prometheus pod name and respective pvc name
openshift_ops.switch_oc_project(
self._master, self._prometheus_project_name)
prometheus_pods = openshift_ops.oc_get_pods(
self._master, selector=self._prometheus_resources_selector)
if not prometheus_pods:
self.skipTest(
prometheus_pods, "Skipping test as prometheus"
" pod is not present")
prometheus_pod = list(prometheus_pods.keys())[0]
pvc_name = openshift_ops.oc_get_custom_resource(
self._master, "pod",
":.spec.volumes[*].persistentVolumeClaim.claimName",
prometheus_pod)[0]
self.assertTrue(
pvc_name,
"Failed to get pvc name from {} pod".format(prometheus_pod))
iqn, _, node = self.verify_iscsi_sessions_and_multipath(
pvc_name, prometheus_pod, rtype='pod',
heketi_server_url=self._registry_heketi_server_url,
is_registry_gluster=True)
# Get the ip of active path
devices = openshift_storage_libs.get_iscsi_block_devices_by_path(
node, iqn)
mpath = openshift_storage_libs.get_mpath_name_from_device_name(
node, list(devices.keys())[0])
mpath_dev = (
openshift_storage_libs.get_active_and_enabled_devices_from_mpath(
node, mpath))
node_ip = devices[mpath_dev['active'][0]]
# Get the name of gluster pod from the ip
openshift_ops.switch_oc_project(
self._master, self._registry_project_name)
gluster_pods = openshift_ops.get_ocp_gluster_pod_details(
self._master)
active_pod_name = list(
filter(lambda pod: (pod["pod_host_ip"] == node_ip), gluster_pods)
)[0]["pod_name"]
err_msg = "Failed to get the gluster pod name {} with active path"
self.assertTrue(active_pod_name, err_msg.format(active_pod_name))
g_pods = [pod['pod_name'] for pod in gluster_pods]
g_pods.remove(active_pod_name)
pod_list = [active_pod_name, g_pods[0]]
for pod_name in pod_list:
# Delete the glusterfs pods
openshift_ops.switch_oc_project(
self._master, self._prometheus_project_name)
self._fetch_metric_from_promtheus_pod(
metric='heketi_device_brick_count')
openshift_ops.switch_oc_project(
self._master, self._registry_project_name)
g_pod_list_before = [
pod["pod_name"]
for pod in openshift_ops.get_ocp_gluster_pod_details(
self._master)]
openshift_ops.oc_delete(self._master, 'pod', pod_name)
self.addCleanup(
self._guster_pod_delete, g_pod_list_before)
# Wait for gluster pod to be absent
openshift_ops.wait_for_resource_absence(
self._master, 'pod', pod_name)
# Try to fetch metric from prometheus pod
openshift_ops.switch_oc_project(
self._master, self._prometheus_project_name)
self._fetch_metric_from_promtheus_pod(
metric='heketi_device_brick_count')
# Wait for new pod to come up
openshift_ops.switch_oc_project(
self._master, self._registry_project_name)
self.assertTrue(self._get_newly_deployed_gluster_pod(
g_pod_list_before), "Failed to get new pod")
self._wait_for_gluster_pod_be_ready(g_pod_list_before)
# Validate iscsi and multipath
openshift_ops.switch_oc_project(
self._master, self._prometheus_project_name)
self.verify_iscsi_sessions_and_multipath(
pvc_name, prometheus_pod, rtype='pod',
heketi_server_url=self._registry_heketi_server_url,
is_registry_gluster=True)
# Try to fetch metric from prometheus pod
self._fetch_metric_from_promtheus_pod(
metric='heketi_device_brick_count')
def test_heketi_node_add_with_valid_cluster(self):
"""Test heketi node add operation with valid cluster id"""
if (openshift_storage_version.get_openshift_storage_version()
< "3.11.4"):
self.skipTest(
"This test case is not supported for < OCS 3.11.4 builds due "
"to bug BZ-1732831")
h_client, h_server = self.heketi_client_node, self.heketi_server_url
ocp_node = self.ocp_master_node[0]
# Get heketi endpoints before adding node
h_volume_ids = heketi_ops.heketi_volume_list(
h_client, h_server, json=True)
h_endpoints_before_new_node = heketi_ops.heketi_volume_endpoint_patch(
h_client, h_server, h_volume_ids["volumes"][0])
cluster_info = heketi_ops.heketi_cluster_list(
h_client, h_server, json=True)
storage_hostname, storage_ip = self.add_heketi_node_to_cluster(
cluster_info["clusters"][0])
# Get heketi nodes and validate for newly added node
h_node_ids = heketi_ops.heketi_node_list(h_client, h_server, json=True)
for h_node_id in h_node_ids:
node_hostname = heketi_ops.heketi_node_info(
h_client, h_server, h_node_id, json=True)
if node_hostname["hostnames"]["manage"][0] == storage_hostname:
break
node_hostname = None
err_msg = ("Newly added heketi node %s not found in heketi node "
"list %s" % (storage_hostname, h_node_ids))
self.assertTrue(node_hostname, err_msg)
# Check gluster peer status for newly added node
if self.is_containerized_gluster():
gluster_pods = openshift_ops.get_ocp_gluster_pod_details(ocp_node)
gluster_pod = [
gluster_pod["pod_name"]
for gluster_pod in gluster_pods
if gluster_pod["pod_hostname"] == storage_hostname][0]
gluster_peer_status = peer_ops.get_peer_status(
podcmd.Pod(ocp_node, gluster_pod))
else:
gluster_peer_status = peer_ops.get_peer_status(
storage_hostname)
self.assertEqual(
len(gluster_peer_status), len(self.gluster_servers))
err_msg = "Expected peer status is 1 and actual is %s"
for peer in gluster_peer_status:
peer_status = int(peer["connected"])
self.assertEqual(peer_status, 1, err_msg % peer_status)
# Get heketi endpoints after adding node
h_endpoints_after_new_node = heketi_ops.heketi_volume_endpoint_patch(
h_client, h_server, h_volume_ids["volumes"][0])
# Get openshift openshift endpoints and patch with heketi endpoints
heketi_db_endpoint = openshift_ops.oc_get_custom_resource(
ocp_node, "dc", name=self.heketi_dc_name,
custom=".:spec.template.spec.volumes[*].glusterfs.endpoints")[0]
openshift_ops.oc_patch(
ocp_node, "ep", heketi_db_endpoint, h_endpoints_after_new_node)
self.addCleanup(
openshift_ops.oc_patch, ocp_node, "ep", heketi_db_endpoint,
h_endpoints_before_new_node)
ep_addresses = openshift_ops.oc_get_custom_resource(
ocp_node, "ep", name=heketi_db_endpoint,
custom=".:subsets[*].addresses[*].ip")[0].split(",")
err_msg = "Hostname %s not present in endpoints %s" % (
storage_ip, ep_addresses)
self.assertIn(storage_ip, ep_addresses, err_msg)
