def validate_multipath_info(self, hacount):
"""validates multipath command on the pod node
Args:
hacount (int): hacount for which multipath to be checked
"""
# create pod using pvc created
dc_name = oc_create_app_dc_with_io(
self.ocp_master_node[0],
self.pvc_name,
image=self.io_container_image_cirros)
pod_name = get_pod_name_from_dc(self.ocp_master_node[0], dc_name)
self.addCleanup(oc_delete, self.ocp_master_node[0], "dc", dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, self.ocp_master_node[0],
dc_name, 0)
wait_for_pod_be_ready(self.ocp_master_node[0],
pod_name,
timeout=120,
wait_step=3)
# Get pod info
pod_info = oc_get_pods(self.ocp_master_node[0],
selector='deploymentconfig=%s' % dc_name)
node = pod_info[pod_name]['node']
# Find iqn from volume info
pv_name = get_pv_name_from_pvc(self.ocp_master_node[0], self.pvc_name)
custom = [r':.metadata.annotations."gluster\.org\/volume\-id"']
vol_id = oc_get_custom_resource(self.ocp_master_node[0], 'pv', custom,
pv_name)[0]
vol_info = heketi_blockvolume_info(self.heketi_client_node,
self.heketi_server_url,
vol_id,
json=True)
iqn = vol_info['blockvolume']['iqn']
# Get the paths info from the node
devices = get_iscsi_block_devices_by_path(node, iqn).keys()
self.assertEqual(hacount, len(devices))
# Validate mpath
mpaths = set()
for device in devices:
mpaths.add(get_mpath_name_from_device_name(node, device))
self.assertEqual(1, len(mpaths))
validate_multipath_pod(self.ocp_master_node[0], pod_name, hacount,
list(mpaths)[0])
def validate_multipath_info(self, hacount):
"""validates multipath command on the pod node
Args:
hacount (int): hacount for which multipath to be checked
"""
# create pod using pvc created
dc_name = oc_create_app_dc_with_io(
self.ocp_master_node[0], self.pvc_name
)
pod_name = get_pod_name_from_dc(self.ocp_master_node[0], dc_name)
self.addCleanup(oc_delete, self.ocp_master_node[0], "dc", dc_name)
self.addCleanup(
scale_dc_pod_amount_and_wait, self.ocp_master_node[0], dc_name, 0
)
wait_for_pod_be_ready(
self.ocp_master_node[0], pod_name, timeout=120, wait_step=3
)
# validates multipath for pod created with hacount
self.assertTrue(
validate_multipath_pod(self.ocp_master_node[0], pod_name, hacount),
"multipath validation failed"
)
def validate_multipath_info(self, hacount):
"""validates multipath command on the pod node
Args:
hacount (int): hacount for which multipath to be checked
"""
# create pod using pvc created
dc_name = oc_create_app_dc_with_io(
self.ocp_master_node[0], self.pvc_name
)
pod_name = get_pod_name_from_dc(self.ocp_master_node[0], dc_name)
self.addCleanup(oc_delete, self.ocp_master_node[0], "dc", dc_name)
self.addCleanup(
scale_dc_pod_amount_and_wait, self.ocp_master_node[0], dc_name, 0
)
wait_for_pod_be_ready(
self.ocp_master_node[0], pod_name, timeout=120, wait_step=3
)
# Get pod info
pod_info = oc_get_pods(
self.ocp_master_node[0], selector='deploymentconfig=%s' % dc_name)
node = pod_info[pod_name]['node']
# Find iqn from volume info
pv_name = get_pv_name_from_pvc(self.ocp_master_node[0], self.pvc_name)
custom = [r':.metadata.annotations."gluster\.org\/volume\-id"']
vol_id = oc_get_custom_resource(
self.ocp_master_node[0], 'pv', custom, pv_name)[0]
vol_info = heketi_blockvolume_info(
self.heketi_client_node, self.heketi_server_url, vol_id, json=True)
iqn = vol_info['blockvolume']['iqn']
# Get the paths info from the node
devices = get_iscsi_block_devices_by_path(node, iqn).keys()
self.assertEqual(hacount, len(devices))
# Validate mpath
mpaths = set()
for device in devices:
mpaths.add(get_mpath_name_from_device_name(node, device))
self.assertEqual(1, len(mpaths))
validate_multipath_pod(
self.ocp_master_node[0], pod_name, hacount, list(mpaths)[0])
def initiator_side_failures(self):
# get storage ips of glusterfs pods
keys = self.gluster_servers
gluster_ips = []
for key in keys:
gluster_ips.append(self.gluster_servers_info[key]['storage'])
gluster_ips.sort()
self.create_storage_class()
self.create_and_wait_for_pvc()
# find iqn and hacount from volume info
pv_name = get_pv_name_from_pvc(self.node, self.pvc_name)
custom = [r':.metadata.annotations."gluster\.org\/volume\-id"']
vol_id = oc_get_custom_resource(self.node, 'pv', custom, pv_name)[0]
vol_info = heketi_blockvolume_info(
self.heketi_client_node, self.heketi_server_url, vol_id, json=True)
iqn = vol_info['blockvolume']['iqn']
hacount = int(self.sc['hacount'])
# create app pod
dc_name, pod_name = self.create_dc_with_pvc(self.pvc_name)
# When we have to verify iscsi login devices & mpaths, we run it twice
for i in range(2):
# get node hostname from pod info
pod_info = oc_get_pods(
self.node, selector='deploymentconfig=%s' % dc_name)
node = pod_info[pod_name]['node']
# get the iscsi sessions info from the node
iscsi = get_iscsi_session(node, iqn)
self.assertEqual(hacount, len(iscsi))
iscsi.sort()
self.assertEqual(set(iscsi), (set(gluster_ips) & set(iscsi)))
# get the paths info from the node
devices = get_iscsi_block_devices_by_path(node, iqn).keys()
self.assertEqual(hacount, len(devices))
# get mpath names and verify that only one mpath is there
mpaths = set()
for device in devices:
mpaths.add(get_mpath_name_from_device_name(node, device))
self.assertEqual(1, len(mpaths))
validate_multipath_pod(
self.node, pod_name, hacount, mpath=list(mpaths)[0])
# When we have to verify iscsi session logout, we run only once
if i == 1:
break
# make node unschedulabe where pod is running
oc_adm_manage_node(
self.node, '--schedulable=false', nodes=[node])
# make node schedulabe where pod is running
self.addCleanup(
oc_adm_manage_node, self.node, '--schedulable=true',
nodes=[node])
# delete pod so it get respun on any other node
oc_delete(self.node, 'pod', pod_name)
wait_for_resource_absence(self.node, 'pod', pod_name)
# wait for pod to come up
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
# get the iscsi session from the previous node to verify logout
iscsi = get_iscsi_session(node, iqn, raise_on_error=False)
self.assertFalse(iscsi)
def verify_iscsi_sessions_and_multipath(
self, pvc_name, rname, rtype='dc', heketi_server_url=None,
is_registry_gluster=False):
if not heketi_server_url:
heketi_server_url = self.heketi_server_url
# Get storage ips of glusterfs pods
keys = (list(g.config['gluster_registry_servers'].keys()) if
is_registry_gluster else self.gluster_servers)
servers_info = (g.config['gluster_registry_servers'] if
is_registry_gluster else self.gluster_servers_info)
gluster_ips = []
for key in keys:
gluster_ips.append(servers_info[key]['storage'])
gluster_ips.sort()
# Find iqn and hacount from volume info
pv_name = get_pv_name_from_pvc(self.ocp_client[0], pvc_name)
custom = [r':.metadata.annotations."gluster\.org\/volume\-id"']
vol_id = oc_get_custom_resource(
self.ocp_client[0], 'pv', custom, pv_name)[0]
vol_info = heketi_blockvolume_info(
self.heketi_client_node, heketi_server_url, vol_id, json=True)
iqn = vol_info['blockvolume']['iqn']
hacount = int(vol_info['hacount'])
# Find node on which pod is running
if rtype == 'dc':
pod_name = get_pod_name_from_dc(self.ocp_client[0], rname)
pod_info = oc_get_pods(
self.ocp_client[0], selector='deploymentconfig=%s' % rname)
elif rtype == 'pod':
pod_info = oc_get_pods(self.ocp_client[0], name=rname)
pod_name = rname
elif rtype == 'rc':
pod_name = get_pod_name_from_rc(self.ocp_client[0], rname)
pod_info = oc_get_pods(
self.ocp_client[0], selector='name=%s' % rname)
else:
raise NameError("Value of rtype should be either 'dc' or 'pod'")
node = pod_info[pod_name]['node']
# Get the iscsi sessions info from the node
iscsi = get_iscsi_session(node, iqn)
msg = ('Only %s iscsi sessions are present on node %s, expected %s.'
% (iscsi, node, hacount))
self.assertEqual(hacount, len(iscsi), msg)
iscsi.sort()
msg = ("Only gluster Nodes %s were expected in iscsi sessions, "
"but got other Nodes %s on Node %s" % (
gluster_ips, iscsi, node))
self.assertEqual(set(iscsi), (set(gluster_ips) & set(iscsi)), msg)
# Get the paths info from the node
devices = get_iscsi_block_devices_by_path(node, iqn)
msg = ("Only %s devices are present on Node %s, expected %s" % (
devices, node, hacount,))
self.assertEqual(hacount, len(devices), msg)
# Get mpath names and verify that only one mpath is there
mpaths = set()
for device in devices.keys():
mpaths.add(get_mpath_name_from_device_name(node, device))
msg = ("Only one mpath was expected on Node %s, but got %s" % (
node, mpaths))
self.assertEqual(1, len(mpaths), msg)
validate_multipath_pod(
self.ocp_client[0], pod_name, hacount, mpath=list(mpaths)[0])
return iqn, hacount, node
def initiator_side_failures(self):
# get storage ips of glusterfs pods
keys = self.gluster_servers
gluster_ips = []
for key in keys:
gluster_ips.append(self.gluster_servers_info[key]['storage'])
gluster_ips.sort()
self.create_storage_class()
self.create_and_wait_for_pvc()
# find iqn and hacount from volume info
pv_name = get_pv_name_from_pvc(self.node, self.pvc_name)
custom = [r':.metadata.annotations."gluster\.org\/volume\-id"']
vol_id = oc_get_custom_resource(self.node, 'pv', custom, pv_name)[0]
vol_info = heketi_blockvolume_info(self.heketi_client_node,
self.heketi_server_url,
vol_id,
json=True)
iqn = vol_info['blockvolume']['iqn']
hacount = int(self.sc['hacount'])
# create app pod
dc_name, pod_name = self.create_dc_with_pvc(self.pvc_name)
# When we have to verify iscsi login devices & mpaths, we run it twice
for i in range(2):
# get node hostname from pod info
pod_info = oc_get_pods(self.node,
selector='deploymentconfig=%s' % dc_name)
node = pod_info[pod_name]['node']
# get the iscsi sessions info from the node
iscsi = get_iscsi_session(node, iqn)
self.assertEqual(hacount, len(iscsi))
iscsi.sort()
self.assertEqual(set(iscsi), (set(gluster_ips) & set(iscsi)))
# get the paths info from the node
devices = get_iscsi_block_devices_by_path(node, iqn).keys()
self.assertEqual(hacount, len(devices))
# get mpath names and verify that only one mpath is there
mpaths = set()
for device in devices:
mpaths.add(get_mpath_name_from_device_name(node, device))
self.assertEqual(1, len(mpaths))
validate_multipath_pod(self.node,
pod_name,
hacount,
mpath=list(mpaths)[0])
# When we have to verify iscsi session logout, we run only once
if i == 1:
break
# make node unschedulabe where pod is running
oc_adm_manage_node(self.node, '--schedulable=false', nodes=[node])
# make node schedulabe where pod is running
self.addCleanup(oc_adm_manage_node,
self.node,
'--schedulable=true',
nodes=[node])
# delete pod so it get respun on any other node
oc_delete(self.node, 'pod', pod_name)
wait_for_resource_absence(self.node, 'pod', pod_name)
# wait for pod to come up
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
# get the iscsi session from the previous node to verify logout
iscsi = get_iscsi_session(node, iqn, raise_on_error=False)
self.assertFalse(iscsi)