def validate_multipath_info(self, hacount):
"""validates multipath command on the pod node
Args:
hacount (int): hacount for which multipath to be checked
"""
# create pod using pvc created
dc_name = oc_create_app_dc_with_io(
self.ocp_master_node[0],
self.pvc_name,
image=self.io_container_image_cirros)
pod_name = get_pod_name_from_dc(self.ocp_master_node[0], dc_name)
self.addCleanup(oc_delete, self.ocp_master_node[0], "dc", dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, self.ocp_master_node[0],
dc_name, 0)
wait_for_pod_be_ready(self.ocp_master_node[0],
pod_name,
timeout=120,
wait_step=3)
# Get pod info
pod_info = oc_get_pods(self.ocp_master_node[0],
selector='deploymentconfig=%s' % dc_name)
node = pod_info[pod_name]['node']
# Find iqn from volume info
pv_name = get_pv_name_from_pvc(self.ocp_master_node[0], self.pvc_name)
custom = [r':.metadata.annotations."gluster\.org\/volume\-id"']
vol_id = oc_get_custom_resource(self.ocp_master_node[0], 'pv', custom,
pv_name)[0]
vol_info = heketi_blockvolume_info(self.heketi_client_node,
self.heketi_server_url,
vol_id,
json=True)
iqn = vol_info['blockvolume']['iqn']
# Get the paths info from the node
devices = get_iscsi_block_devices_by_path(node, iqn).keys()
self.assertEqual(hacount, len(devices))
# Validate mpath
mpaths = set()
for device in devices:
mpaths.add(get_mpath_name_from_device_name(node, device))
self.assertEqual(1, len(mpaths))
validate_multipath_pod(self.ocp_master_node[0], pod_name, hacount,
list(mpaths)[0])
def test_verify_metrics_data_during_gluster_pod_respin(self):
# Add check for CRS version
switch_oc_project(self.master, self.registry_project_name)
if not self.is_containerized_gluster():
self.skipTest("Skipping this test case as CRS version check "
"can not be implemented")
# Verify multipath and iscsi for cassandra pod
switch_oc_project(self.master, self.metrics_project_name)
hawkular_cassandra, pvc_name, iqn, _, node = (
self.verify_cassandra_pod_multipath_and_iscsi())
# Get the ip of active path
device_and_ip = get_iscsi_block_devices_by_path(node, iqn)
mpath = get_mpath_name_from_device_name(node,
list(device_and_ip.keys())[0])
active_passive_dict = get_active_and_enabled_devices_from_mpath(
node, mpath)
node_ip = device_and_ip[active_passive_dict['active'][0]]
# Get the name of gluster pod from the ip
switch_oc_project(self.master, self.registry_project_name)
gluster_pods = get_ocp_gluster_pod_details(self.master)
pod_name = list(
filter(lambda pod: (pod["pod_host_ip"] == node_ip),
gluster_pods))[0]["pod_name"]
err_msg = "Failed to get the gluster pod name {} with active path"
self.assertTrue(pod_name, err_msg.format(pod_name))
# Delete the pod
oc_delete(self.master, 'pod', pod_name)
wait_for_resource_absence(self.master, 'pod', pod_name)
# Wait for new pod to come up
pod_count = len(self.registry_servers_info.keys())
selector = "glusterfs-node=pod"
wait_for_pods_be_ready(self.master, pod_count, selector)
# Validate cassandra pod state, multipath and issci
switch_oc_project(self.master, self.metrics_project_name)
wait_for_pod_be_ready(self.master, hawkular_cassandra, timeout=2)
self.verify_iscsi_sessions_and_multipath(
pvc_name,
self.metrics_rc_hawkular_cassandra,
rtype='rc',
heketi_server_url=self.registry_heketi_server_url,
is_registry_gluster=True)
def test_block_vol_online_expand(self):
"""Test blockvol expansion while PVC is in use"""
node = self.ocp_master_node[0]
pvc_name, dc_name, bvol_info = (
self._block_vol_expand_common_offline_vs_online(True))
# get pod hostname
iqn, _, pod_hostname = self.verify_iscsi_sessions_and_multipath(
pvc_name, dc_name[0])
# Get the paths info from the node
device = list(
get_iscsi_block_devices_by_path(pod_hostname, iqn).keys())[0]
# Get mpath name
mpath = get_mpath_name_from_device_name(pod_hostname, device)
# rescan the devices on pod_hostname
cmd = "iscsiadm -m node -R -T {}".format(iqn)
self.cmd_run(cmd, pod_hostname)
# refresh multipath device size
cmd = "multipathd -k'resize map {}'".format(mpath)
self.cmd_run(cmd, pod_hostname)
# get mount point
cmd = "lsblk /dev/{} --output MOUNTPOINT --noheadings".format(device)
mount_point = self.cmd_run(cmd, pod_hostname)
cmd = "xfs_growfs {}".format(mount_point)
self.cmd_run(cmd, pod_hostname)
cmd = ("df -h {} | sed '/Filesystem/d' | awk '{{print $2}}' |"
" sed 's/G//'")
size = self.cmd_run(cmd.format(mount_point), pod_hostname)
self.assertEqual(
int(float(size)), bvol_info["size"], "new size is not "
"reflected at host mount point after block volume expand")
# verify expand size
pod_name = get_pod_name_from_dc(node, dc_name[0])
ret, size, _ = oc_rsh(node, pod_name, cmd.format("/mnt"))
self.assertFalse(ret, "Failed to get size from client side")
self.assertEqual(
int(float(size)), bvol_info["size"], "new size is not "
"reflected at mount point after block volume expand")
def validate_multipath_info(self, hacount):
"""validates multipath command on the pod node
Args:
hacount (int): hacount for which multipath to be checked
"""
# create pod using pvc created
dc_name = oc_create_app_dc_with_io(
self.ocp_master_node[0], self.pvc_name
)
pod_name = get_pod_name_from_dc(self.ocp_master_node[0], dc_name)
self.addCleanup(oc_delete, self.ocp_master_node[0], "dc", dc_name)
self.addCleanup(
scale_dc_pod_amount_and_wait, self.ocp_master_node[0], dc_name, 0
)
wait_for_pod_be_ready(
self.ocp_master_node[0], pod_name, timeout=120, wait_step=3
)
# Get pod info
pod_info = oc_get_pods(
self.ocp_master_node[0], selector='deploymentconfig=%s' % dc_name)
node = pod_info[pod_name]['node']
# Find iqn from volume info
pv_name = get_pv_name_from_pvc(self.ocp_master_node[0], self.pvc_name)
custom = [r':.metadata.annotations."gluster\.org\/volume\-id"']
vol_id = oc_get_custom_resource(
self.ocp_master_node[0], 'pv', custom, pv_name)[0]
vol_info = heketi_blockvolume_info(
self.heketi_client_node, self.heketi_server_url, vol_id, json=True)
iqn = vol_info['blockvolume']['iqn']
# Get the paths info from the node
devices = get_iscsi_block_devices_by_path(node, iqn).keys()
self.assertEqual(hacount, len(devices))
# Validate mpath
mpaths = set()
for device in devices:
mpaths.add(get_mpath_name_from_device_name(node, device))
self.assertEqual(1, len(mpaths))
validate_multipath_pod(
self.ocp_master_node[0], pod_name, hacount, list(mpaths)[0])
def initiator_side_failures(self):
# get storage ips of glusterfs pods
keys = self.gluster_servers
gluster_ips = []
for key in keys:
gluster_ips.append(self.gluster_servers_info[key]['storage'])
gluster_ips.sort()
self.create_storage_class()
self.create_and_wait_for_pvc()
# find iqn and hacount from volume info
pv_name = get_pv_name_from_pvc(self.node, self.pvc_name)
custom = [r':.metadata.annotations."gluster\.org\/volume\-id"']
vol_id = oc_get_custom_resource(self.node, 'pv', custom, pv_name)[0]
vol_info = heketi_blockvolume_info(
self.heketi_client_node, self.heketi_server_url, vol_id, json=True)
iqn = vol_info['blockvolume']['iqn']
hacount = int(self.sc['hacount'])
# create app pod
dc_name, pod_name = self.create_dc_with_pvc(self.pvc_name)
# When we have to verify iscsi login devices & mpaths, we run it twice
for i in range(2):
# get node hostname from pod info
pod_info = oc_get_pods(
self.node, selector='deploymentconfig=%s' % dc_name)
node = pod_info[pod_name]['node']
# get the iscsi sessions info from the node
iscsi = get_iscsi_session(node, iqn)
self.assertEqual(hacount, len(iscsi))
iscsi.sort()
self.assertEqual(set(iscsi), (set(gluster_ips) & set(iscsi)))
# get the paths info from the node
devices = get_iscsi_block_devices_by_path(node, iqn).keys()
self.assertEqual(hacount, len(devices))
# get mpath names and verify that only one mpath is there
mpaths = set()
for device in devices:
mpaths.add(get_mpath_name_from_device_name(node, device))
self.assertEqual(1, len(mpaths))
validate_multipath_pod(
self.node, pod_name, hacount, mpath=list(mpaths)[0])
# When we have to verify iscsi session logout, we run only once
if i == 1:
break
# make node unschedulabe where pod is running
oc_adm_manage_node(
self.node, '--schedulable=false', nodes=[node])
# make node schedulabe where pod is running
self.addCleanup(
oc_adm_manage_node, self.node, '--schedulable=true',
nodes=[node])
# delete pod so it get respun on any other node
oc_delete(self.node, 'pod', pod_name)
wait_for_resource_absence(self.node, 'pod', pod_name)
# wait for pod to come up
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
# get the iscsi session from the previous node to verify logout
iscsi = get_iscsi_session(node, iqn, raise_on_error=False)
self.assertFalse(iscsi)
def verify_iscsi_sessions_and_multipath(
self, pvc_name, rname, rtype='dc', heketi_server_url=None,
is_registry_gluster=False):
if not heketi_server_url:
heketi_server_url = self.heketi_server_url
# Get storage ips of glusterfs pods
keys = (list(g.config['gluster_registry_servers'].keys()) if
is_registry_gluster else self.gluster_servers)
servers_info = (g.config['gluster_registry_servers'] if
is_registry_gluster else self.gluster_servers_info)
gluster_ips = []
for key in keys:
gluster_ips.append(servers_info[key]['storage'])
gluster_ips.sort()
# Find iqn and hacount from volume info
pv_name = get_pv_name_from_pvc(self.ocp_client[0], pvc_name)
custom = [r':.metadata.annotations."gluster\.org\/volume\-id"']
vol_id = oc_get_custom_resource(
self.ocp_client[0], 'pv', custom, pv_name)[0]
vol_info = heketi_blockvolume_info(
self.heketi_client_node, heketi_server_url, vol_id, json=True)
iqn = vol_info['blockvolume']['iqn']
hacount = int(vol_info['hacount'])
# Find node on which pod is running
if rtype == 'dc':
pod_name = get_pod_name_from_dc(self.ocp_client[0], rname)
pod_info = oc_get_pods(
self.ocp_client[0], selector='deploymentconfig=%s' % rname)
elif rtype == 'pod':
pod_info = oc_get_pods(self.ocp_client[0], name=rname)
pod_name = rname
elif rtype == 'rc':
pod_name = get_pod_name_from_rc(self.ocp_client[0], rname)
pod_info = oc_get_pods(
self.ocp_client[0], selector='name=%s' % rname)
else:
raise NameError("Value of rtype should be either 'dc' or 'pod'")
node = pod_info[pod_name]['node']
# Get the iscsi sessions info from the node
iscsi = get_iscsi_session(node, iqn)
msg = ('Only %s iscsi sessions are present on node %s, expected %s.'
% (iscsi, node, hacount))
self.assertEqual(hacount, len(iscsi), msg)
iscsi.sort()
msg = ("Only gluster Nodes %s were expected in iscsi sessions, "
"but got other Nodes %s on Node %s" % (
gluster_ips, iscsi, node))
self.assertEqual(set(iscsi), (set(gluster_ips) & set(iscsi)), msg)
# Get the paths info from the node
devices = get_iscsi_block_devices_by_path(node, iqn)
msg = ("Only %s devices are present on Node %s, expected %s" % (
devices, node, hacount,))
self.assertEqual(hacount, len(devices), msg)
# Get mpath names and verify that only one mpath is there
mpaths = set()
for device in devices.keys():
mpaths.add(get_mpath_name_from_device_name(node, device))
msg = ("Only one mpath was expected on Node %s, but got %s" % (
node, mpaths))
self.assertEqual(1, len(mpaths), msg)
validate_multipath_pod(
self.ocp_client[0], pod_name, hacount, mpath=list(mpaths)[0])
return iqn, hacount, node
def initiator_side_failures(self):
# get storage ips of glusterfs pods
keys = self.gluster_servers
gluster_ips = []
for key in keys:
gluster_ips.append(self.gluster_servers_info[key]['storage'])
gluster_ips.sort()
self.create_storage_class()
self.create_and_wait_for_pvc()
# find iqn and hacount from volume info
pv_name = get_pv_name_from_pvc(self.node, self.pvc_name)
custom = [r':.metadata.annotations."gluster\.org\/volume\-id"']
vol_id = oc_get_custom_resource(self.node, 'pv', custom, pv_name)[0]
vol_info = heketi_blockvolume_info(self.heketi_client_node,
self.heketi_server_url,
vol_id,
json=True)
iqn = vol_info['blockvolume']['iqn']
hacount = int(self.sc['hacount'])
# create app pod
dc_name, pod_name = self.create_dc_with_pvc(self.pvc_name)
# When we have to verify iscsi login devices & mpaths, we run it twice
for i in range(2):
# get node hostname from pod info
pod_info = oc_get_pods(self.node,
selector='deploymentconfig=%s' % dc_name)
node = pod_info[pod_name]['node']
# get the iscsi sessions info from the node
iscsi = get_iscsi_session(node, iqn)
self.assertEqual(hacount, len(iscsi))
iscsi.sort()
self.assertEqual(set(iscsi), (set(gluster_ips) & set(iscsi)))
# get the paths info from the node
devices = get_iscsi_block_devices_by_path(node, iqn).keys()
self.assertEqual(hacount, len(devices))
# get mpath names and verify that only one mpath is there
mpaths = set()
for device in devices:
mpaths.add(get_mpath_name_from_device_name(node, device))
self.assertEqual(1, len(mpaths))
validate_multipath_pod(self.node,
pod_name,
hacount,
mpath=list(mpaths)[0])
# When we have to verify iscsi session logout, we run only once
if i == 1:
break
# make node unschedulabe where pod is running
oc_adm_manage_node(self.node, '--schedulable=false', nodes=[node])
# make node schedulabe where pod is running
self.addCleanup(oc_adm_manage_node,
self.node,
'--schedulable=true',
nodes=[node])
# delete pod so it get respun on any other node
oc_delete(self.node, 'pod', pod_name)
wait_for_resource_absence(self.node, 'pod', pod_name)
# wait for pod to come up
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
# get the iscsi session from the previous node to verify logout
iscsi = get_iscsi_session(node, iqn, raise_on_error=False)
self.assertFalse(iscsi)
def test_restart_prometheus_glusterfs_pod(self):
"""Validate restarting glusterfs pod"""
# Add check for CRS version
openshift_ops.switch_oc_project(
self._master, self._registry_project_name)
if not self.is_containerized_gluster():
self.skipTest(
"Skipping this test case as CRS version check "
"can not be implemented")
# Get one of the prometheus pod name and respective pvc name
openshift_ops.switch_oc_project(
self._master, self._prometheus_project_name)
prometheus_pods = openshift_ops.oc_get_pods(
self._master, selector=self._prometheus_resources_selector)
if not prometheus_pods:
self.skipTest(
prometheus_pods, "Skipping test as prometheus"
" pod is not present")
prometheus_pod = list(prometheus_pods.keys())[0]
pvc_name = openshift_ops.oc_get_custom_resource(
self._master, "pod",
":.spec.volumes[*].persistentVolumeClaim.claimName",
prometheus_pod)[0]
self.assertTrue(
pvc_name,
"Failed to get pvc name from {} pod".format(prometheus_pod))
iqn, _, node = self.verify_iscsi_sessions_and_multipath(
pvc_name, prometheus_pod, rtype='pod',
heketi_server_url=self._registry_heketi_server_url,
is_registry_gluster=True)
# Get the ip of active path
devices = openshift_storage_libs.get_iscsi_block_devices_by_path(
node, iqn)
mpath = openshift_storage_libs.get_mpath_name_from_device_name(
node, list(devices.keys())[0])
mpath_dev = (
openshift_storage_libs.get_active_and_enabled_devices_from_mpath(
node, mpath))
node_ip = devices[mpath_dev['active'][0]]
# Get the name of gluster pod from the ip
openshift_ops.switch_oc_project(
self._master, self._registry_project_name)
gluster_pods = openshift_ops.get_ocp_gluster_pod_details(
self._master)
active_pod_name = list(
filter(lambda pod: (pod["pod_host_ip"] == node_ip), gluster_pods)
)[0]["pod_name"]
err_msg = "Failed to get the gluster pod name {} with active path"
self.assertTrue(active_pod_name, err_msg.format(active_pod_name))
g_pods = [pod['pod_name'] for pod in gluster_pods]
g_pods.remove(active_pod_name)
pod_list = [active_pod_name, g_pods[0]]
for pod_name in pod_list:
# Delete the glusterfs pods
openshift_ops.switch_oc_project(
self._master, self._prometheus_project_name)
self._fetch_metric_from_promtheus_pod(
metric='heketi_device_brick_count')
openshift_ops.switch_oc_project(
self._master, self._registry_project_name)
g_pod_list_before = [
pod["pod_name"]
for pod in openshift_ops.get_ocp_gluster_pod_details(
self._master)]
openshift_ops.oc_delete(self._master, 'pod', pod_name)
self.addCleanup(
self._guster_pod_delete, g_pod_list_before)
# Wait for gluster pod to be absent
openshift_ops.wait_for_resource_absence(
self._master, 'pod', pod_name)
# Try to fetch metric from prometheus pod
openshift_ops.switch_oc_project(
self._master, self._prometheus_project_name)
self._fetch_metric_from_promtheus_pod(
metric='heketi_device_brick_count')
# Wait for new pod to come up
openshift_ops.switch_oc_project(
self._master, self._registry_project_name)
self.assertTrue(self._get_newly_deployed_gluster_pod(
g_pod_list_before), "Failed to get new pod")
self._wait_for_gluster_pod_be_ready(g_pod_list_before)
# Validate iscsi and multipath
openshift_ops.switch_oc_project(
self._master, self._prometheus_project_name)
self.verify_iscsi_sessions_and_multipath(
pvc_name, prometheus_pod, rtype='pod',
heketi_server_url=self._registry_heketi_server_url,
is_registry_gluster=True)
# Try to fetch metric from prometheus pod
self._fetch_metric_from_promtheus_pod(
metric='heketi_device_brick_count')