def test_heketi_device_delete(self):
"""Test Heketi device delete operation"""
# Get list of additional devices for one of the Gluster nodes
ip_with_devices = {}
for gluster_server in g.config["gluster_servers"].values():
if not gluster_server.get("additional_devices"):
continue
ip_with_devices = {
gluster_server['storage']: gluster_server['additional_devices']
}
break
# Skip test if no additional device is available
if not ip_with_devices:
self.skipTest(
"No additional devices attached to any of the gluster nodes")
# Select any additional device and get the node id of the gluster node
h_node, h_server = self.heketi_client_node, self.heketi_server_url
node_id, device_name = None, list(ip_with_devices.values())[0][0]
topology_info = heketi_topology_info(h_node, h_server, json=True)
for node in topology_info["clusters"][0]["nodes"]:
if list(ip_with_devices.keys())[0] == (
node['hostnames']["storage"][0]):
node_id = node["id"]
break
self.assertTrue(node_id)
# Add additional device to the cluster
heketi_device_add(h_node, h_server, device_name, node_id)
# Get the device id and number of bricks on the device
node_info_after_addition = heketi_node_info(h_node,
h_server,
node_id,
json=True)
device_id, bricks = None, None
for device in node_info_after_addition["devices"]:
if device["name"] == device_name:
device_id, bricks = device["id"], len(device['bricks'])
break
self.assertTrue(device_id, "Device not added in expected node")
# Delete heketi device
heketi_device_disable(h_node, h_server, device_id)
heketi_device_remove(h_node, h_server, device_id)
heketi_device_delete(h_node, h_server, device_id)
# Verify that there were no bricks on the newly added device
msg = (
"Number of bricks on the device %s of the node %s should be zero" %
(device_name, list(ip_with_devices.keys())[0]))
self.assertEqual(0, bricks, msg)
# Verify device deletion
node_info_after_deletion = heketi_node_info(h_node, h_server, node_id)
msg = ("Device %s should not be shown in node info of the node %s"
"after the device deletion" % (device_id, node_id))
self.assertNotIn(device_id, node_info_after_deletion, msg)
def _add_heketi_device(self, device_name, node_id, raise_on_error=True):
# Add device to heketi node
device_id, device_size = None, None
h_client, h_url = self.heketi_client_node, self.heketi_server_url
heketi_device_add(h_client, h_url, device_name, node_id)
node_info_addition = heketi_node_info(h_client,
h_url,
node_id,
json=True)
for device in node_info_addition.get("devices"):
if device.get("name") == device_name:
device_id = device.get("id")
device_size = device.get("storage", {}).get("total")
break
self.addCleanup(heketi_device_delete,
h_client,
h_url,
device_id,
raise_on_error=raise_on_error)
self.addCleanup(heketi_device_remove,
h_client,
h_url,
device_id,
raise_on_error=raise_on_error)
self.addCleanup(heketi_device_disable,
h_client,
h_url,
device_id,
raise_on_error=raise_on_error)
return device_id, device_size
def add_device_on_heketi_node(self, node_id, device_name):
# Add Devices on heketi node
heketi_ops.heketi_device_add(
self.h_node, self.h_url, device_name, node_id)
# Get device id of newly added device
node_info = heketi_ops.heketi_node_info(
self.h_node, self.h_url, node_id, json=True)
for device in node_info['devices']:
if device['name'] == device_name:
return device['id']
raise AssertionError('Device %s did not found on node %s' % (
device_name, node_id))
def test_heketi_device_remove(self, delete_device):
"""Validate remove/delete device using heketi-cli"""
gluster_server_0 = list(g.config["gluster_servers"].values())[0]
try:
device_name = gluster_server_0["additional_devices"][0]
except (KeyError, IndexError):
self.skipTest(
"Additional disk is not specified for node with following "
"hostnames and IP addresses: %s, %s." % (gluster_server_0.get(
'manage', '?'), gluster_server_0.get('storage', '?')))
manage_hostname = gluster_server_0["manage"]
# Get node ID of the Gluster hostname
topo_info = heketi_topology_info(self.heketi_client_node,
self.heketi_server_url,
json=True)
self.assertTrue(topo_info["clusters"][0]["nodes"],
"Cluster info command returned empty list of nodes.")
node_id = None
for node in topo_info["clusters"][0]["nodes"]:
if manage_hostname == node['hostnames']["manage"][0]:
node_id = node["id"]
break
self.assertNotEqual(
node_id, None,
"No information about node_id for %s" % manage_hostname)
# Iterate chosen node devices and pick the smallest online one.
lowest_device_size = lowest_device_id = None
online_hosts = self.get_online_nodes_disable_redundant()
for host in online_hosts[0:3]:
if node_id != host["id"]:
continue
for device in host["devices"]:
if device["state"].strip().lower() != "online":
continue
if (lowest_device_size is None
or device["storage"]["total"] < lowest_device_size):
lowest_device_size = device["storage"]["total"]
lowest_device_id = device["id"]
lowest_device_name = device["name"]
if lowest_device_id is None:
self.skipTest(
"Didn't find suitable device for disablement on '%s' node." %
(node_id))
# Create volume
vol_size = 1
vol_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
vol_size,
json=True)
self.addCleanup(heketi_volume_delete, self.heketi_client_node,
self.heketi_server_url, vol_info['id'])
# Add extra device, then remember it's ID and size
heketi_device_add(self.heketi_client_node, self.heketi_server_url,
device_name, node_id)
node_info_after_addition = heketi_node_info(self.heketi_client_node,
self.heketi_server_url,
node_id,
json=True)
for device in node_info_after_addition["devices"]:
if device["name"] != device_name:
continue
device_id_new = device["id"]
device_size_new = device["storage"]["total"]
self.addCleanup(heketi_device_delete, self.heketi_client_node,
self.heketi_server_url, device_id_new)
self.addCleanup(heketi_device_remove, self.heketi_client_node,
self.heketi_server_url, device_id_new)
self.addCleanup(heketi_device_disable, self.heketi_client_node,
self.heketi_server_url, device_id_new)
if lowest_device_size > device_size_new:
skip_msg = ("Skip test case, because newly added disk %s is "
"smaller than device which we want to remove %s." %
(device_size_new, lowest_device_size))
self.skipTest(skip_msg)
g.log.info("Removing device id %s" % lowest_device_id)
with self.assertRaises(AssertionError):
out = heketi_device_remove(self.heketi_client_node,
self.heketi_server_url,
lowest_device_id)
self.addCleanup(heketi_device_enable, self.heketi_client_node,
self.heketi_server_url, lowest_device_id)
self.addCleanup(heketi_device_disable, self.heketi_client_node,
self.heketi_server_url, lowest_device_id)
self.assertFalse(True, "Device removal didn't fail: %s" % out)
g.log.info("Device removal failed as expected")
# Need to disable device before removing
heketi_device_disable(self.heketi_client_node, self.heketi_server_url,
lowest_device_id)
if not delete_device:
self.addCleanup(heketi_device_enable, self.heketi_client_node,
self.heketi_server_url, lowest_device_id)
# Remove device from Heketi
try:
heketi_device_remove(self.heketi_client_node,
self.heketi_server_url, lowest_device_id)
except Exception:
if delete_device:
self.addCleanup(heketi_device_enable, self.heketi_client_node,
self.heketi_server_url, lowest_device_id)
raise
if not delete_device:
self.addCleanup(heketi_device_disable, self.heketi_client_node,
self.heketi_server_url, lowest_device_id)
if delete_device:
try:
heketi_device_delete(self.heketi_client_node,
self.heketi_server_url, lowest_device_id)
except Exception:
self.addCleanup(heketi_device_enable, self.heketi_client_node,
self.heketi_server_url, lowest_device_id)
self.addCleanup(heketi_device_disable, self.heketi_client_node,
self.heketi_server_url, lowest_device_id)
raise
self.addCleanup(heketi_device_add, self.heketi_client_node,
self.heketi_server_url, lowest_device_name,
node_id)
# Create volume
vol_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
vol_size,
json=True)
self.addCleanup(heketi_volume_delete, self.heketi_client_node,
self.heketi_server_url, vol_info['id'])
if delete_device:
return
# Check that none of volume's bricks is present on the device
present = self.check_any_of_bricks_present_in_device(
vol_info['bricks'], lowest_device_id)
self.assertFalse(
present,
"Some of the '%s' volume bricks is present of the removed "
"'%s' device." % (vol_info['id'], lowest_device_id))
def test_volume_expansion_no_free_space(self):
"""Validate volume expansion when there is no free space"""
vol_size, expand_size, additional_devices_attached = None, 10, {}
h_node, h_server_url = self.heketi_client_node, self.heketi_server_url
# Get nodes info
heketi_node_id_list = heketi_ops.heketi_node_list(h_node, h_server_url)
if len(heketi_node_id_list) < 3:
self.skipTest("3 Heketi nodes are required.")
# Disable 4th and other nodes
for node_id in heketi_node_id_list[3:]:
heketi_ops.heketi_node_disable(h_node, h_server_url, node_id)
self.addCleanup(
heketi_ops.heketi_node_enable, h_node, h_server_url, node_id)
# Prepare first 3 nodes
smallest_size = None
err_msg = ''
for node_id in heketi_node_id_list[0:3]:
node_info = heketi_ops.heketi_node_info(
h_node, h_server_url, node_id, json=True)
# Disable second and other devices
devices = node_info["devices"]
self.assertTrue(
devices, "Node '%s' does not have devices." % node_id)
if devices[0]["state"].strip().lower() != "online":
self.skipTest("Test expects first device to be enabled.")
if (smallest_size is None or
devices[0]["storage"]["free"] < smallest_size):
smallest_size = devices[0]["storage"]["free"]
for device in node_info["devices"][1:]:
heketi_ops.heketi_device_disable(
h_node, h_server_url, device["id"])
self.addCleanup(
heketi_ops.heketi_device_enable,
h_node, h_server_url, device["id"])
# Gather info about additional devices
additional_device_name = None
for gluster_server in self.gluster_servers:
gluster_server_data = self.gluster_servers_info[gluster_server]
g_manage = gluster_server_data["manage"]
g_storage = gluster_server_data["storage"]
if not (g_manage in node_info["hostnames"]["manage"] or
g_storage in node_info["hostnames"]["storage"]):
continue
additional_device_name = ((
gluster_server_data.get("additional_devices") or [''])[0])
break
if not additional_device_name:
err_msg += ("No 'additional_devices' are configured for "
"'%s' node, which has following hostnames and "
"IP addresses: %s.\n" % (
node_id,
', '.join(node_info["hostnames"]["manage"] +
node_info["hostnames"]["storage"])))
continue
heketi_ops.heketi_device_add(
h_node, h_server_url, additional_device_name, node_id)
additional_devices_attached.update(
{node_id: additional_device_name})
# Schedule cleanup of the added devices
for node_id in additional_devices_attached.keys():
node_info = heketi_ops.heketi_node_info(
h_node, h_server_url, node_id, json=True)
for device in node_info["devices"]:
if device["name"] != additional_devices_attached[node_id]:
continue
self.addCleanup(self.detach_devices_attached, device["id"])
break
else:
self.fail("Could not find ID for added device on "
"'%s' node." % node_id)
if err_msg:
self.skipTest(err_msg)
# Temporary disable new devices
self.disable_devices(additional_devices_attached)
# Create volume and save info about it
vol_size = int(smallest_size / (1024**2)) - 1
creation_info = heketi_ops.heketi_volume_create(
h_node, h_server_url, vol_size, json=True)
volume_name, volume_id = creation_info["name"], creation_info["id"]
self.addCleanup(
heketi_ops.heketi_volume_delete,
h_node, h_server_url, volume_id, raise_on_error=False)
volume_info_before_expansion = heketi_ops.heketi_volume_info(
h_node, h_server_url, volume_id, json=True)
num_of_bricks_before_expansion = self.get_num_of_bricks(volume_name)
self.get_brick_and_volume_status(volume_name)
free_space_before_expansion = self.get_devices_summary_free_space()
# Try to expand volume with not enough device space
self.assertRaises(
ExecutionError, heketi_ops.heketi_volume_expand,
h_node, h_server_url, volume_id, expand_size)
# Enable new devices to be able to expand our volume
self.enable_devices(additional_devices_attached)
# Expand volume and validate results
heketi_ops.heketi_volume_expand(
h_node, h_server_url, volume_id, expand_size, json=True)
free_space_after_expansion = self.get_devices_summary_free_space()
self.assertGreater(
free_space_before_expansion, free_space_after_expansion,
"Free space not consumed after expansion of %s" % volume_id)
num_of_bricks_after_expansion = self.get_num_of_bricks(volume_name)
self.get_brick_and_volume_status(volume_name)
volume_info_after_expansion = heketi_ops.heketi_volume_info(
h_node, h_server_url, volume_id, json=True)
self.assertGreater(
volume_info_after_expansion["size"],
volume_info_before_expansion["size"],
"Size of %s not increased" % volume_id)
self.assertGreater(
num_of_bricks_after_expansion, num_of_bricks_before_expansion)
self.assertEqual(
num_of_bricks_after_expansion % num_of_bricks_before_expansion, 0)
# Delete volume and validate release of the used space
heketi_ops.heketi_volume_delete(h_node, h_server_url, volume_id)
free_space_after_deletion = self.get_devices_summary_free_space()
self.assertGreater(
free_space_after_deletion, free_space_after_expansion,
"Free space not reclaimed after deletion of volume %s" % volume_id)
def test_volume_expansion_no_free_space(self):
"""Validate volume expansion when there is no free space"""
vol_size, expand_size, additional_devices_attached = None, 10, {}
h_node, h_server_url = self.heketi_client_node, self.heketi_server_url
# Get nodes info
heketi_node_id_list = heketi_ops.heketi_node_list(h_node, h_server_url)
if len(heketi_node_id_list) < 3:
self.skipTest("3 Heketi nodes are required.")
# Disable 4th and other nodes
for node_id in heketi_node_id_list[3:]:
heketi_ops.heketi_node_disable(h_node, h_server_url, node_id)
self.addCleanup(
heketi_ops.heketi_node_enable, h_node, h_server_url, node_id)
# Prepare first 3 nodes
smallest_size = None
err_msg = ''
for node_id in heketi_node_id_list[0:3]:
node_info = heketi_ops.heketi_node_info(
h_node, h_server_url, node_id, json=True)
# Disable second and other devices
devices = node_info["devices"]
self.assertTrue(
devices, "Node '%s' does not have devices." % node_id)
if devices[0]["state"].strip().lower() != "online":
self.skipTest("Test expects first device to be enabled.")
if (smallest_size is None
or devices[0]["storage"]["free"] < smallest_size):
smallest_size = devices[0]["storage"]["free"]
for device in node_info["devices"][1:]:
heketi_ops.heketi_device_disable(
h_node, h_server_url, device["id"])
self.addCleanup(
heketi_ops.heketi_device_enable,
h_node, h_server_url, device["id"])
# Gather info about additional devices
additional_device_name = None
for gluster_server in self.gluster_servers:
gluster_server_data = self.gluster_servers_info[gluster_server]
g_manage = gluster_server_data["manage"]
g_storage = gluster_server_data["storage"]
if not (g_manage in node_info["hostnames"]["manage"]
or g_storage in node_info["hostnames"]["storage"]):
continue
additional_device_name = ((
gluster_server_data.get("additional_devices") or [''])[0])
break
if not additional_device_name:
err_msg += ("No 'additional_devices' are configured for "
"'%s' node, which has following hostnames and "
"IP addresses: %s.\n" % (
node_id,
', '.join(
node_info["hostnames"]["manage"]
+ node_info["hostnames"]["storage"])))
continue
heketi_ops.heketi_device_add(
h_node, h_server_url, additional_device_name, node_id)
additional_devices_attached.update(
{node_id: additional_device_name})
# Schedule cleanup of the added devices
for node_id in additional_devices_attached.keys():
node_info = heketi_ops.heketi_node_info(
h_node, h_server_url, node_id, json=True)
for device in node_info["devices"]:
if device["name"] != additional_devices_attached[node_id]:
continue
self.addCleanup(self.detach_devices_attached, device["id"])
break
else:
self.fail("Could not find ID for added device on "
"'%s' node." % node_id)
if err_msg:
self.skipTest(err_msg)
# Temporary disable new devices
self.disable_devices(additional_devices_attached)
# Create volume and save info about it
vol_size = int(smallest_size / (1024**2)) - 1
creation_info = heketi_ops.heketi_volume_create(
h_node, h_server_url, vol_size, json=True)
volume_name, volume_id = creation_info["name"], creation_info["id"]
self.addCleanup(
heketi_ops.heketi_volume_delete,
h_node, h_server_url, volume_id, raise_on_error=False)
volume_info_before_expansion = heketi_ops.heketi_volume_info(
h_node, h_server_url, volume_id, json=True)
num_of_bricks_before_expansion = self.get_num_of_bricks(volume_name)
self.get_brick_and_volume_status(volume_name)
free_space_before_expansion = self.get_devices_summary_free_space()
# Try to expand volume with not enough device space
self.assertRaises(
AssertionError, heketi_ops.heketi_volume_expand,
h_node, h_server_url, volume_id, expand_size)
# Enable new devices to be able to expand our volume
self.enable_devices(additional_devices_attached)
# Expand volume and validate results
heketi_ops.heketi_volume_expand(
h_node, h_server_url, volume_id, expand_size, json=True)
free_space_after_expansion = self.get_devices_summary_free_space()
self.assertGreater(
free_space_before_expansion, free_space_after_expansion,
"Free space not consumed after expansion of %s" % volume_id)
num_of_bricks_after_expansion = self.get_num_of_bricks(volume_name)
self.get_brick_and_volume_status(volume_name)
volume_info_after_expansion = heketi_ops.heketi_volume_info(
h_node, h_server_url, volume_id, json=True)
self.assertGreater(
volume_info_after_expansion["size"],
volume_info_before_expansion["size"],
"Size of %s not increased" % volume_id)
self.assertGreater(
num_of_bricks_after_expansion, num_of_bricks_before_expansion)
self.assertEqual(
num_of_bricks_after_expansion % num_of_bricks_before_expansion, 0)
# Delete volume and validate release of the used space
heketi_ops.heketi_volume_delete(h_node, h_server_url, volume_id)
free_space_after_deletion = self.get_devices_summary_free_space()
self.assertGreater(
free_space_after_deletion, free_space_after_expansion,
"Free space not reclaimed after deletion of volume %s" % volume_id)
def test_heketi_prometheus_usedbytes_brickcount_on_device_delete(
self, operation):
"""Validate used bytes,device count on heketi and prometheus"""
h_node, h_server = self.heketi_client_node, self.heketi_server_url
# Get list of additional devices for one of the Gluster nodes
gluster_server_0 = list(self.gluster_servers_info.values())[0]
manage_hostname = gluster_server_0.get("manage")
self.assertTrue(
manage_hostname, "IP Address is not specified for "
"node {}".format(gluster_server_0))
device_name = gluster_server_0.get("additional_devices")[0]
self.assertTrue(
device_name, "Additional devices are not specified for "
"node {}".format(gluster_server_0))
# Get node ID of the Gluster hostname
node_list = heketi_ops.heketi_topology_info(
h_node, h_server, json=True).get("clusters")[0].get("nodes")
self.assertTrue(
node_list, "Cluster info command returned empty list of nodes")
node_id = [
node.get("id")
for node in node_list
if manage_hostname == node.get("hostnames").get("manage")[0]]
self.assertTrue(
node_id, "Failed to get node_id for {}".format(manage_hostname))
node_id = node_id[0]
# Adding heketi device
heketi_ops.heketi_device_add(h_node, h_server, device_name, node_id)
node_info_after_addition = heketi_ops.heketi_node_info(
h_node, h_server, node_id, json=True)
device_id, bricks = None, None
for device in node_info_after_addition.get("devices"):
if device.get("name") == device_name:
device_id, bricks = (
device.get("id"), len(device.get("bricks")))
break
# Verify zero bricks on the device
msg = (
"Number of bricks on the device {} of the nodes should be"
"zero".format(device_name))
self.assertFalse(bricks, msg)
self.addCleanup(
heketi_ops.heketi_device_delete, h_node, h_server, device_id,
raise_on_error=False)
self.addCleanup(
heketi_ops.heketi_device_remove, h_node, h_server, device_id,
raise_on_error=False)
self.addCleanup(
heketi_ops.heketi_device_disable, h_node, h_server, device_id,
raise_on_error=False)
# Disable,Remove and Delete heketi device
heketi_ops.heketi_device_disable(h_node, h_server, device_id)
heketi_ops.heketi_device_remove(h_node, h_server, device_id)
heketi_ops.heketi_device_delete(h_node, h_server, device_id)
# Verify device deletion
node_info_after_deletion = (
heketi_ops.heketi_node_info(h_node, h_server, node_id))
msg = ("Device {} should not be shown in node info of the node {}"
"after the device deletion".format(device_id, node_id))
self.assertNotIn(device_id, node_info_after_deletion, msg)
if operation == "usedbytes":
# Validate heketi and prometheus device used bytes
for w in waiter.Waiter(timeout=60, interval=10):
device_used_bytes_prometheus = 0
device_used_bytes_metrics = 0
openshift_ops.switch_oc_project(
self.ocp_master_node[0], 'openshift-monitoring')
metric_result = self._fetch_metric_from_promtheus_pod(
metric='heketi_device_used_bytes')
for result in metric_result:
if (node_id == result.get('cluster')
and device_name == result.get('device')):
device_used_bytes_prometheus += (
int(result.get('value')[1]))
openshift_ops.switch_oc_project(
self.ocp_master_node[0], 'glusterfs')
metrics = heketi_ops.get_heketi_metrics(h_node, h_server)
heketi_device_count_metric = (
metrics.get('heketi_device_used_bytes'))
for result in heketi_device_count_metric:
if (node_id == result.get('cluster')
and device_name == result.get('device')):
device_used_bytes_metrics = int(result.get('value'))
if device_used_bytes_prometheus == device_used_bytes_metrics:
break
if w.expired:
raise exceptions.ExecutionError(
"Failed to update device details in prometheus")
elif operation == "brickcount":
# Validate heketi and prometheus device brick count
for w in waiter.Waiter(timeout=60, interval=10):
device_brick_count_prometheus = 0
device_brick_count_metrics = 0
metrics = heketi_ops.get_heketi_metrics(h_node, h_server)
heketi_device_count_metric = metrics.get(
'heketi_device_brick_count')
for result in heketi_device_count_metric:
device_brick_count_metrics += int(result.get('value'))
openshift_ops.switch_oc_project(
self.ocp_master_node[0], 'openshift-monitoring')
metric_result = self._fetch_metric_from_promtheus_pod(
metric='heketi_device_brick_count')
for result in metric_result:
device_brick_count_prometheus += (
int(result.get('value')[1]))
if device_brick_count_prometheus == device_brick_count_metrics:
break
if w.expired:
raise exceptions.ExecutionError(
"Failed to update device details in prometheus")
def test_create_heketi_cluster_and_add_node(self):
"""Test heketi node add to a newly created cluster"""
storage_host_info = g.config.get("additional_gluster_servers")
if not storage_host_info:
self.skipTest(
"Skip test case as 'additional_gluster_servers' option is "
"not provided in config file")
storage_host_info = list(storage_host_info.values())[0]
try:
storage_hostname = storage_host_info["manage"]
storage_ip = storage_host_info["storage"]
storage_device = storage_host_info["devices"][0]
except KeyError:
msg = ("Config options 'additional_gluster_servers.manage' "
"'additional_gluster_servers.storage' and "
"'additional_gluster_servers.devices' "
"must be set.")
g.log.error(msg)
raise exceptions.ConfigError(msg)
h_client, h_server = self.heketi_client_node, self.heketi_server_url
storage_zone = 1
cluster_id = heketi_ops.heketi_cluster_create(self.heketi_client_node,
self.heketi_server_url,
json=True)["id"]
self.addCleanup(heketi_ops.heketi_cluster_delete,
self.heketi_client_node, self.heketi_server_url,
cluster_id)
self.configure_node_to_run_gluster(storage_hostname)
heketi_node_info = heketi_ops.heketi_node_add(h_client,
h_server,
storage_zone,
cluster_id,
storage_hostname,
storage_ip,
json=True)
heketi_node_id = heketi_node_info["id"]
self.addCleanup(heketi_ops.heketi_node_delete, h_client, h_server,
heketi_node_id)
self.addCleanup(heketi_ops.heketi_node_remove, h_client, h_server,
heketi_node_id)
self.addCleanup(heketi_ops.heketi_node_disable, h_client, h_server,
heketi_node_id)
self.assertEqual(
heketi_node_info["cluster"], cluster_id,
"Node got added in unexpected cluster exp: %s, act: %s" %
(cluster_id, heketi_node_info["cluster"]))
heketi_ops.heketi_device_add(h_client, h_server, storage_device,
heketi_node_id)
heketi_node_info = heketi_ops.heketi_node_info(h_client,
h_server,
heketi_node_id,
json=True)
device_id = None
for device in heketi_node_info["devices"]:
if device["name"] == storage_device:
device_id = device["id"]
break
err_msg = ("Failed to add device %s on node %s" %
(storage_device, heketi_node_id))
self.assertTrue(device_id, err_msg)
self.addCleanup(heketi_ops.heketi_device_delete, h_client, h_server,
device_id)
self.addCleanup(heketi_ops.heketi_device_remove, h_client, h_server,
device_id)
self.addCleanup(heketi_ops.heketi_device_disable, h_client, h_server,
device_id)
cluster_info = heketi_ops.heketi_cluster_info(h_client,
h_server,
cluster_id,
json=True)
self.assertIn(
heketi_node_info["id"], cluster_info["nodes"],
"Newly added node %s not found in cluster %s, cluster info %s" %
(heketi_node_info["id"], cluster_id, cluster_info))
topology_info = heketi_ops.heketi_topology_info(h_client,
h_server,
json=True)
cluster_details = [
cluster for cluster in topology_info["clusters"]
if cluster["id"] == cluster_id
]
err_msg = "Cluster details for id '%s' not found" % cluster_id
self.assertTrue(cluster_details, err_msg)
err_msg = ("Multiple clusters with same id '%s' found %s" %
(cluster_id, cluster_details))
self.assertEqual(len(cluster_details), 1, err_msg)
node_details = [
node for node in cluster_details[0]["nodes"]
if node["id"] == heketi_node_id
]
err_msg = "Node details for id '%s' not found" % heketi_node_id
self.assertTrue(node_details, err_msg)
err_msg = ("Multiple nodes with same id '%s' found %s" %
(heketi_node_id, node_details))
self.assertEqual(len(node_details), 1, err_msg)
err_msg = "Unexpected %s found '%s', expected '%s'"
exp_storage_hostname = node_details[0]["hostnames"]["manage"][0]
self.assertEqual(
exp_storage_hostname, storage_hostname, err_msg % (
"hostname",
exp_storage_hostname,
storage_hostname,
))
exp_storage_ip = node_details[0]["hostnames"]["storage"][0]
self.assertEqual(exp_storage_ip, storage_ip,
err_msg % ("IP address", exp_storage_ip, storage_ip))
zone = node_details[0]["zone"]
self.assertEqual(zone, storage_zone,
err_msg % ("zone", zone, storage_zone))
def _add_new_device_and_remove_existing_device(
self,
is_delete_device,
add_device_name,
node_id,
add_back_again=False,
skip_cleanup_addition=False):
"""Delete or remove device and also add one device on the same node.
"""
h_client, h_url = self.heketi_client_node, self.heketi_server_url
raise_on_error = False if add_back_again else True
# Iterate chosen node devices and pick the smallest online one.
lowest_device_size = lowest_device_id = None
online_hosts = self.get_online_nodes_disable_redundant()
for host in online_hosts[0:3]:
if node_id != host["id"]:
continue
for device in host["devices"]:
if device["state"].strip().lower() != "online":
continue
if (lowest_device_size is None
or device["storage"]["total"] < lowest_device_size):
lowest_device_size = device["storage"]["total"]
lowest_device_id = device["id"]
lowest_device_name = device["name"]
if lowest_device_id is None:
self.skipTest(
"Didn't find suitable device for disablement on '%s' node." %
(node_id))
# Create volume
vol_size = 1
vol_info = heketi_volume_create(h_client, h_url, vol_size, json=True)
self.addCleanup(heketi_volume_delete, self.heketi_client_node,
self.heketi_server_url, vol_info['id'])
# Add extra device, then remember it's ID and size
heketi_device_add(h_client, h_url, add_device_name, node_id)
node_info_after_addition = heketi_node_info(h_client,
h_url,
node_id,
json=True)
for device in node_info_after_addition["devices"]:
if device["name"] != add_device_name:
continue
device_id_new = device["id"]
device_size_new = device["storage"]["total"]
self.addCleanup(heketi_device_delete,
h_client,
h_url,
device_id_new,
raise_on_error=raise_on_error)
self.addCleanup(heketi_device_remove,
h_client,
h_url,
device_id_new,
raise_on_error=raise_on_error)
self.addCleanup(heketi_device_disable,
h_client,
h_url,
device_id_new,
raise_on_error=raise_on_error)
if lowest_device_size > device_size_new:
skip_msg = ("Skip test case, because newly added disk %s is "
"smaller than device which we want to remove %s." %
(device_size_new, lowest_device_size))
self.skipTest(skip_msg)
g.log.info("Removing device id %s" % lowest_device_id)
with self.assertRaises(AssertionError):
out = heketi_device_remove(h_client, h_url, lowest_device_id)
self.addCleanup(heketi_device_enable, h_client, h_url,
lowest_device_id)
self.addCleanup(heketi_device_disable, h_client, h_url,
lowest_device_id)
self.assertFalse(True, "Device removal didn't fail: %s" % out)
g.log.info("Device removal failed as expected")
# Need to disable device before removing
heketi_device_disable(h_client, h_url, lowest_device_id)
if not is_delete_device:
self.addCleanup(heketi_device_enable, h_client, h_url,
lowest_device_id)
# Remove device from Heketi
try:
heketi_device_remove(h_client, h_url, lowest_device_id)
except Exception:
if is_delete_device:
self.addCleanup(heketi_device_enable,
h_client,
h_url,
lowest_device_id,
raise_on_error=raise_on_error)
raise
if not is_delete_device:
self.addCleanup(heketi_device_disable, h_client, h_url,
lowest_device_id)
if is_delete_device:
try:
heketi_device_delete(h_client, h_url, lowest_device_id)
except Exception:
self.addCleanup(heketi_device_enable,
h_client,
h_url,
lowest_device_id,
raise_on_error=raise_on_error)
self.addCleanup(heketi_device_disable,
h_client,
h_url,
lowest_device_id,
raise_on_error=raise_on_error)
raise
if not skip_cleanup_addition:
# Do not add the additional device back, intially added
self.addCleanup(heketi_device_add,
h_client,
h_url,
lowest_device_name,
node_id,
raise_on_error=raise_on_error)
# Create volume
vol_info = heketi_volume_create(h_client, h_url, vol_size, json=True)
self.addCleanup(heketi_volume_delete, h_client, h_url, vol_info['id'])
if is_delete_device:
return lowest_device_name
# Check that none of volume's bricks is present on the device
present = self.check_any_of_bricks_present_in_device(
vol_info['bricks'], lowest_device_id)
self.assertFalse(
present,
"Some of the '%s' volume bricks is present of the removed "
"'%s' device." % (vol_info['id'], lowest_device_id))
def test_dev_path_mapping_heketi_node_delete(self):
"""Validate dev path mapping for heketi node deletion lifecycle"""
h_client, h_url = self.heketi_client_node, self.heketi_server_url
node_ids = heketi_ops.heketi_node_list(h_client, h_url)
self.assertTrue(node_ids, "Failed to get heketi node list")
# Fetch #4th node for the operations
h_disable_node = node_ids[3]
# Fetch bricks on the devices before volume create
h_node_details_before, h_node = self._get_bricks_and_device_details()
# Bricks count on the node before pvc creation
brick_count_before = [count[1] for count in h_node_details_before]
# Create file volume with app pod and verify IO's
# and compare path, UUID, vg_name
pod_name, dc_name, use_percent = self._create_app_pod_and_verify_pvs()
# Check if IO's are running
use_percent_after = self._get_space_use_percent_in_app_pod(pod_name)
self.assertNotEqual(
use_percent, use_percent_after,
"Failed to execute IO's in the app pod {} after respin".format(
pod_name))
# Fetch bricks on the devices after volume create
h_node_details_after, h_node = self._get_bricks_and_device_details()
# Bricks count on the node after pvc creation
brick_count_after = [count[1] for count in h_node_details_after]
self.assertGreater(
sum(brick_count_after), sum(brick_count_before),
"Failed to add bricks on the node {}".format(h_node))
self.addCleanup(heketi_ops.heketi_node_disable, h_client, h_url,
h_disable_node)
# Enable the #4th node
heketi_ops.heketi_node_enable(h_client, h_url, h_disable_node)
node_info = heketi_ops.heketi_node_info(h_client,
h_url,
h_disable_node,
json=True)
h_node_id = node_info['id']
self.assertEqual(node_info['state'], "online",
"Failed to enable node {}".format(h_disable_node))
# Disable the node and check for brick migrations
self.addCleanup(heketi_ops.heketi_node_enable,
h_client,
h_url,
h_node,
raise_on_error=False)
heketi_ops.heketi_node_disable(h_client, h_url, h_node)
node_info = heketi_ops.heketi_node_info(h_client,
h_url,
h_node,
json=True)
self.assertEqual(node_info['state'], "offline",
"Failed to disable node {}".format(h_node))
# Before bricks migration
h_node_info = heketi_ops.heketi_node_info(h_client,
h_url,
h_node,
json=True)
# Bricks before migration on the node i.e to be deleted
bricks_counts_before = 0
for device in h_node_info['devices']:
bricks_counts_before += (len(device['bricks']))
# Remove the node
heketi_ops.heketi_node_remove(h_client, h_url, h_node)
# After bricks migration
h_node_info_after = heketi_ops.heketi_node_info(h_client,
h_url,
h_node,
json=True)
# Bricks after migration on the node i.e to be delete
bricks_counts = 0
for device in h_node_info_after['devices']:
bricks_counts += (len(device['bricks']))
self.assertFalse(
bricks_counts,
"Failed to remove all the bricks from node {}".format(h_node))
# Old node which is to deleted, new node were bricks resides
old_node, new_node = h_node, h_node_id
# Node info for the new node were brick reside after migration
h_node_info_new = heketi_ops.heketi_node_info(h_client,
h_url,
new_node,
json=True)
bricks_counts_after = 0
for device in h_node_info_new['devices']:
bricks_counts_after += (len(device['bricks']))
self.assertEqual(
bricks_counts_before, bricks_counts_after,
"Failed to migrated bricks from {} node to {}".format(
old_node, new_node))
# Fetch device list i.e to be deleted
h_node_info = heketi_ops.heketi_node_info(h_client,
h_url,
h_node,
json=True)
devices_list = [[device['id'], device['name']]
for device in h_node_info['devices']]
for device in devices_list:
device_id = device[0]
device_name = device[1]
self.addCleanup(heketi_ops.heketi_device_add,
h_client,
h_url,
device_name,
h_node,
raise_on_error=False)
# Device deletion from heketi node
device_delete = heketi_ops.heketi_device_delete(
h_client, h_url, device_id)
self.assertTrue(device_delete,
"Failed to delete the device {}".format(device_id))
node_info = heketi_ops.heketi_node_info(h_client,
h_url,
h_node,
json=True)
cluster_id = node_info['cluster']
zone = node_info['zone']
storage_hostname = node_info['hostnames']['manage'][0]
storage_ip = node_info['hostnames']['storage'][0]
# Delete the node
self.addCleanup(heketi_ops.heketi_node_add,
h_client,
h_url,
zone,
cluster_id,
storage_hostname,
storage_ip,
raise_on_error=False)
heketi_ops.heketi_node_delete(h_client, h_url, h_node)
# Verify if the node is deleted
node_ids = heketi_ops.heketi_node_list(h_client, h_url)
self.assertNotIn(old_node, node_ids,
"Failed to delete the node {}".format(old_node))
# Check if IO's are running
use_percent_after = self._get_space_use_percent_in_app_pod(pod_name)
self.assertNotEqual(
use_percent, use_percent_after,
"Failed to execute IO's in the app pod {} after respin".format(
pod_name))
# Adding node back
h_node_info = heketi_ops.heketi_node_add(h_client,
h_url,
zone,
cluster_id,
storage_hostname,
storage_ip,
json=True)
self.assertTrue(
h_node_info,
"Failed to add the node in the cluster {}".format(cluster_id))
h_node_id = h_node_info["id"]
# Adding devices to the new node
for device in devices_list:
storage_device = device[1]
# Add device to the new heketi node
heketi_ops.heketi_device_add(h_client, h_url, storage_device,
h_node_id)
heketi_node_info = heketi_ops.heketi_node_info(h_client,
h_url,
h_node_id,
json=True)
device_id = None
for device in heketi_node_info["devices"]:
if device["name"] == storage_device:
device_id = device["id"]
break
self.assertTrue(
device_id, "Failed to add device {} on node {}".format(
storage_device, h_node_id))
# Create n pvc in order to verfiy if the bricks reside on the new node
pvc_amount, pvc_size = 5, 1
# Fetch bricks on the devices before volume create
h_node_details_before, h_node = self._get_bricks_and_device_details()
# Bricks count on the node before pvc creation
brick_count_before = [count[1] for count in h_node_details_before]
# Create file volumes
pvc_name = self.create_and_wait_for_pvcs(pvc_size=pvc_size,
pvc_amount=pvc_amount)
self.assertEqual(len(pvc_name), pvc_amount,
"Failed to create {} pvc".format(pvc_amount))
# Fetch bricks on the devices before volume create
h_node_details_after, h_node = self._get_bricks_and_device_details()
# Bricks count on the node after pvc creation
brick_count_after = [count[1] for count in h_node_details_after]
self.assertGreater(
sum(brick_count_after), sum(brick_count_before),
"Failed to add bricks on the new node {}".format(new_node))
# Check if IO's are running after new node is added
use_percent_after = self._get_space_use_percent_in_app_pod(pod_name)
self.assertNotEqual(
use_percent, use_percent_after,
"Failed to execute IO's in the app pod {} after respin".format(
pod_name))
def test_dev_path_mapping_heketi_device_delete(self):
"""Validate dev path mapping for heketi device delete lifecycle"""
h_client, h_url = self.heketi_client_node, self.heketi_server_url
node_ids = heketi_ops.heketi_node_list(h_client, h_url)
self.assertTrue(node_ids, "Failed to get heketi node list")
# Fetch #4th node for the operations
h_disable_node = node_ids[3]
# Fetch bricks on the devices before volume create
h_node_details_before, h_node = self._get_bricks_and_device_details()
# Bricks count on the node before pvc creation
brick_count_before = [count[1] for count in h_node_details_before]
# Create file volume with app pod and verify IO's
# and compare path, UUID, vg_name
pod_name, dc_name, use_percent = self._create_app_pod_and_verify_pvs()
# Check if IO's are running
use_percent_after = self._get_space_use_percent_in_app_pod(pod_name)
self.assertNotEqual(
use_percent, use_percent_after,
"Failed to execute IO's in the app pod {} after respin".format(
pod_name))
# Fetch bricks on the devices after volume create
h_node_details_after, h_node = self._get_bricks_and_device_details()
# Bricks count on the node after pvc creation
brick_count_after = [count[1] for count in h_node_details_after]
self.assertGreater(
sum(brick_count_after), sum(brick_count_before),
"Failed to add bricks on the node {}".format(h_node))
# Enable the #4th node
heketi_ops.heketi_node_enable(h_client, h_url, h_disable_node)
node_info = heketi_ops.heketi_node_info(h_client,
h_url,
h_disable_node,
json=True)
h_node_id = node_info['id']
self.assertEqual(node_info['state'], "online",
"Failed to enable node {}".format(h_disable_node))
# Fetch device list i.e to be deleted
h_node_info = heketi_ops.heketi_node_info(h_client,
h_url,
h_node,
json=True)
devices_list = [[device['id'], device['name']]
for device in h_node_info['devices']]
# Device deletion operation
for device in devices_list:
device_id, device_name = device[0], device[1]
self.addCleanup(heketi_ops.heketi_device_enable,
h_client,
h_url,
device_id,
raise_on_error=False)
# Disable device from heketi
device_disable = heketi_ops.heketi_device_disable(
h_client, h_url, device_id)
self.assertTrue(
device_disable,
"Device {} could not be disabled".format(device_id))
device_info = heketi_ops.heketi_device_info(h_client,
h_url,
device_id,
json=True)
self.assertEqual(device_info['state'], "offline",
"Failed to disable device {}".format(device_id))
# Remove device from heketi
device_remove = heketi_ops.heketi_device_remove(
h_client, h_url, device_id)
self.assertTrue(device_remove,
"Device {} could not be removed".format(device_id))
# Bricks after device removal
device_info = heketi_ops.heketi_device_info(h_client,
h_url,
device_id,
json=True)
bricks_count_after = len(device_info['bricks'])
self.assertFalse(
bricks_count_after,
"Failed to remove the bricks from the device {}".format(
device_id))
# Delete device from heketi
self.addCleanup(heketi_ops.heketi_device_add,
h_client,
h_url,
device_name,
h_node,
raise_on_error=False)
device_delete = heketi_ops.heketi_device_delete(
h_client, h_url, device_id)
self.assertTrue(device_delete,
"Device {} could not be deleted".format(device_id))
# Check if IO's are running after device is deleted
use_percent_after = self._get_space_use_percent_in_app_pod(pod_name)
self.assertNotEqual(
use_percent, use_percent_after,
"Failed to execute IO's in the app pod {} after respin".format(
pod_name))
# Add device operations
for device in devices_list:
device_name = device[1]
# Add device back to the node
heketi_ops.heketi_device_add(h_client, h_url, device_name, h_node)
# Fetch device info after device add
node_info = heketi_ops.heketi_node_info(h_client,
h_url,
h_node,
json=True)
device_id = None
for device in node_info["devices"]:
if device["name"] == device_name:
device_id = device["id"]
break
self.assertTrue(
device_id, "Failed to add device {} on node"
" {}".format(device_name, h_node))
# Disable the #4th node
heketi_ops.heketi_node_disable(h_client, h_url, h_node_id)
node_info = heketi_ops.heketi_node_info(h_client,
h_url,
h_node_id,
json=True)
self.assertEqual(node_info['state'], "offline",
"Failed to disable node {}".format(h_node_id))
pvc_amount, pvc_size = 5, 1
# Fetch bricks on the devices before volume create
h_node_details_before, h_node = self._get_bricks_and_device_details()
# Bricks count on the node before pvc creation
brick_count_before = [count[1] for count in h_node_details_before]
# Create file volumes
pvc_name = self.create_and_wait_for_pvcs(pvc_size=pvc_size,
pvc_amount=pvc_amount)
self.assertEqual(len(pvc_name), pvc_amount,
"Failed to create {} pvc".format(pvc_amount))
# Fetch bricks on the devices after volume create
h_node_details_after, h_node = self._get_bricks_and_device_details()
# Bricks count on the node after pvc creation
brick_count_after = [count[1] for count in h_node_details_after]
self.assertGreater(
sum(brick_count_after), sum(brick_count_before),
"Failed to add bricks on the node {}".format(h_node))
# Check if IO's are running after new device is added
use_percent_after = self._get_space_use_percent_in_app_pod(pod_name)
self.assertNotEqual(
use_percent, use_percent_after,
"Failed to execute IO's in the app pod {} after respin".format(
pod_name))
def test_udev_usage_in_container(self):
"""Validate LVM inside container does not use udev"""
# Skip the TC if independent mode deployment
if not self.is_containerized_gluster():
self.skipTest("Skipping this test case as it needs to run on "
"converged mode deployment")
h_client, h_url = self.heketi_client_node, self.heketi_server_url
server_info = list(g.config.get('gluster_servers').values())[0]
server_node = server_info.get('manage')
additional_device = server_info.get('additional_devices')[0]
# command to run pvscan
cmd_pvscan = "timeout 300 pvscan"
# Get pod name from on host
for pod_info in self.pod_name:
if pod_info.get('pod_hostname') == server_node:
pod_name = pod_info.get('pod_name')
break
# Create file volume
vol_info = heketi_ops.heketi_volume_create(h_client,
h_url,
self.volume_size,
json=True)
self.addCleanup(heketi_ops.heketi_volume_delete, h_client, h_url,
vol_info.get("id"))
# Create block volume
block_vol_info = heketi_ops.heketi_blockvolume_create(h_client,
h_url,
self.volume_size,
json=True)
self.addCleanup(heketi_ops.heketi_blockvolume_delete, h_client, h_url,
block_vol_info.get("id"))
# Check dmeventd service in container
err_msg = "dmeventd.service is running on setup"
with self.assertRaises(AssertionError, msg=err_msg):
openshift_ops.oc_rsh(self.oc_node, pod_name,
"systemctl is-active dmeventd.service")
# Service dmeventd should not be running in background
with self.assertRaises(AssertionError, msg=err_msg):
openshift_ops.oc_rsh(self.oc_node, pod_name,
"ps aux | grep dmeventd.service")
# Perform a pvscan in contaier
openshift_ops.oc_rsh(self.oc_node, pod_name, cmd_pvscan)
# Get heketi node to add new device
heketi_node_list = heketi_ops.heketi_node_list(h_client, h_url)
for h_node_id in heketi_node_list:
h_node_info = heketi_ops.heketi_node_info(h_client,
h_url,
h_node_id,
json=True)
h_node_host = h_node_info.get('hostnames', {}).get('manage')[0]
if h_node_host == server_node:
break
# Add new device to the node
heketi_ops.heketi_device_add(h_client, h_url, additional_device,
h_node_id)
h_node_info = heketi_ops.heketi_node_info(h_client,
h_url,
h_node_id,
json=True)
h_device_id = [
device.get('id') for device in h_node_info.get('devices')
if device.get('name') == additional_device
]
self.addCleanup(heketi_ops.heketi_device_delete, h_client, h_url,
h_device_id[0])
self.addCleanup(heketi_ops.heketi_device_remove, h_client, h_url,
h_device_id[0])
self.addCleanup(heketi_ops.heketi_device_disable, h_client, h_url,
h_device_id[0])
# Reboot the node on which device is added
self.addCleanup(self._check_heketi_and_gluster_pod_after_node_reboot,
server_node)
node_ops.node_reboot_by_command(server_node)
# Wait node to become NotReady
custom = r'":.status.conditions[?(@.type==\"Ready\")]".status'
for w in waiter.Waiter(300, 10):
status = openshift_ops.oc_get_custom_resource(
self.oc_node, 'node', custom, server_node)
if status[0] == 'False':
break
if w.expired:
raise exceptions.ExecutionError(
"Failed to bring node down {}".format(server_node))
# Wait for node to become ready
openshift_ops.wait_for_ocp_node_be_ready(self.oc_node, server_node)
# Wait for heketi and glusterfs pod to become ready
self._check_heketi_and_gluster_pod_after_node_reboot(server_node)
# Perform a pvscan in contaier
openshift_ops.oc_rsh(self.oc_node, pod_name, cmd_pvscan)
