def test_restart_heketi_pod(self):
"""Validate restarting heketi pod"""
# create heketi volume
vol_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
size=1,
json=True)
self.assertTrue(vol_info, "Failed to create heketi volume of size 1")
self.addCleanup(heketi_volume_delete,
self.heketi_client_node,
self.heketi_server_url,
vol_info['id'],
raise_on_error=False)
topo_info = heketi_topology_info(self.heketi_client_node,
self.heketi_server_url,
json=True)
# get heketi-pod name
heketi_pod_name = get_pod_name_from_dc(self.ocp_master_node[0],
self.heketi_dc_name)
# delete heketi-pod (it restarts the pod)
oc_delete(self.ocp_master_node[0],
'pod',
heketi_pod_name,
collect_logs=self.heketi_logs_before_delete)
wait_for_resource_absence(self.ocp_master_node[0], 'pod',
heketi_pod_name)
# get new heketi-pod name
heketi_pod_name = get_pod_name_from_dc(self.ocp_master_node[0],
self.heketi_dc_name)
wait_for_pod_be_ready(self.ocp_master_node[0], heketi_pod_name)
# check heketi server is running
self.assertTrue(
hello_heketi(self.heketi_client_node, self.heketi_server_url),
"Heketi server %s is not alive" % self.heketi_server_url)
# compare the topology
new_topo_info = heketi_topology_info(self.heketi_client_node,
self.heketi_server_url,
json=True)
self.assertEqual(
new_topo_info, topo_info, "topology info is not same,"
" difference - %s" % diff(topo_info, new_topo_info))
# create new volume
vol_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
size=2,
json=True)
self.assertTrue(vol_info, "Failed to create heketi volume of size 20")
heketi_volume_delete(self.heketi_client_node, self.heketi_server_url,
vol_info['id'])
def validate_file_volumes_count(self, h_node, h_server, node_ip):
# check volume count from heketi and gluster are same
heketi_topology_info(h_node, h_server, json=True)
h_volume_list = heketi_volume_list(h_node, h_server, json=True)
vol_list = get_volume_list(node_ip)
self.assertIsNotNone(
vol_list, "Failed to get volumes list")
self.assertEqual(
len(h_volume_list['volumes']), len(vol_list),
"Failed to verify volume count Expected:'{}', Actual:'{}'".format(
len(h_volume_list['volumes']), len(vol_list)))
def test_restart_heketi_pod(self):
"""Validate restarting heketi pod"""
# create heketi volume
vol_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
size=1, json=True)
self.assertTrue(vol_info, "Failed to create heketi volume of size 1")
self.addCleanup(
heketi_volume_delete, self.heketi_client_node,
self.heketi_server_url, vol_info['id'], raise_on_error=False)
topo_info = heketi_topology_info(self.heketi_client_node,
self.heketi_server_url,
json=True)
# get heketi-pod name
heketi_pod_name = get_pod_name_from_dc(self.ocp_master_node[0],
self.heketi_dc_name)
# delete heketi-pod (it restarts the pod)
oc_delete(self.ocp_master_node[0], 'pod', heketi_pod_name)
wait_for_resource_absence(self.ocp_master_node[0],
'pod', heketi_pod_name)
# get new heketi-pod name
heketi_pod_name = get_pod_name_from_dc(self.ocp_master_node[0],
self.heketi_dc_name)
wait_for_pod_be_ready(self.ocp_master_node[0],
heketi_pod_name)
# check heketi server is running
self.assertTrue(
hello_heketi(self.heketi_client_node, self.heketi_server_url),
"Heketi server %s is not alive" % self.heketi_server_url
)
# compare the topology
new_topo_info = heketi_topology_info(self.heketi_client_node,
self.heketi_server_url,
json=True)
self.assertEqual(new_topo_info, topo_info, "topology info is not same,"
" difference - %s" % diff(topo_info, new_topo_info))
# create new volume
vol_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
size=2, json=True)
self.assertTrue(vol_info, "Failed to create heketi volume of size 20")
heketi_volume_delete(
self.heketi_client_node, self.heketi_server_url, vol_info['id'])
def test_heketi_device_delete(self):
"""Test Heketi device delete operation"""
# Get list of additional devices for one of the Gluster nodes
ip_with_devices = {}
for gluster_server in g.config["gluster_servers"].values():
if not gluster_server.get("additional_devices"):
continue
ip_with_devices = {
gluster_server['storage']: gluster_server['additional_devices']
}
break
# Skip test if no additional device is available
if not ip_with_devices:
self.skipTest(
"No additional devices attached to any of the gluster nodes")
# Select any additional device and get the node id of the gluster node
h_node, h_server = self.heketi_client_node, self.heketi_server_url
node_id, device_name = None, list(ip_with_devices.values())[0][0]
topology_info = heketi_topology_info(h_node, h_server, json=True)
for node in topology_info["clusters"][0]["nodes"]:
if list(ip_with_devices.keys())[0] == (
node['hostnames']["storage"][0]):
node_id = node["id"]
break
self.assertTrue(node_id)
# Add additional device to the cluster
heketi_device_add(h_node, h_server, device_name, node_id)
# Get the device id and number of bricks on the device
node_info_after_addition = heketi_node_info(h_node,
h_server,
node_id,
json=True)
device_id, bricks = None, None
for device in node_info_after_addition["devices"]:
if device["name"] == device_name:
device_id, bricks = device["id"], len(device['bricks'])
break
self.assertTrue(device_id, "Device not added in expected node")
# Delete heketi device
heketi_device_disable(h_node, h_server, device_id)
heketi_device_remove(h_node, h_server, device_id)
heketi_device_delete(h_node, h_server, device_id)
# Verify that there were no bricks on the newly added device
msg = (
"Number of bricks on the device %s of the node %s should be zero" %
(device_name, list(ip_with_devices.keys())[0]))
self.assertEqual(0, bricks, msg)
# Verify device deletion
node_info_after_deletion = heketi_node_info(h_node, h_server, node_id)
msg = ("Device %s should not be shown in node info of the node %s"
"after the device deletion" % (device_id, node_id))
self.assertNotIn(device_id, node_info_after_deletion, msg)
def verify_node_is_present_or_not_in_heketi(
self, node_id, manage_hostname, storage_ip, state='present'):
topology = heketi_ops.heketi_topology_info(
self.h_node, self.h_url, json=True)
if state == 'present':
present = False
for node in topology['clusters'][0]['nodes']:
if node_id == node['id']:
self.assertEqual(
manage_hostname, node['hostnames']['manage'][0])
self.assertEqual(
storage_ip, node['hostnames']['storage'][0])
present = True
break
self.assertTrue(present, 'Node %s not found in heketi' % node_id)
elif state == 'absent':
for node in topology['clusters'][0]['nodes']:
self.assertNotEqual(
manage_hostname, node['hostnames']['manage'][0])
self.assertNotEqual(
storage_ip, node['hostnames']['storage'][0])
self.assertNotEqual(node_id, node['id'])
else:
msg = "State %s is other than present, absent" % state
raise AssertionError(msg)
def test_heketi_authentication_with_user_credentials(self):
"""Heketi command authentication with invalid and valid credentials"""
h_client, h_server = self.heketi_client_node, self.heketi_server_url
err_msg = "Error: Invalid JWT token: Token missing iss claim"
# Run heketi commands with invalid credentials
for each_cmd in ("volume list", "topology info"):
cmd = "timeout 120 heketi-cli -s {} {}".format(
self.heketi_server_url, each_cmd)
ret, _, err = g.run(h_client, cmd)
self.assertTrue(ret, "Command execution with invalid credentials"
" should not succeed")
self.assertEqual(
err_msg, err.strip(), "Error is different from the command"
" execution {}".format(err.strip()))
# Run heketi commands with valid credentials
kwar = {'json_arg': True, 'secret': self.heketi_cli_key,
'user': self.heketi_cli_user}
heketi_ops.heketi_volume_list(h_client, h_server, **kwar)
heketi_ops.heketi_topology_info(h_client, h_server, **kwar)
def _get_online_devices_and_nodes_with_zone(self):
"""
This function returns the list of nodes and devices associated to zone
Returns:
dict: dict with zone, devices and nodes values
e.g.,
{ zone_1: {
"nodes": [
node1, node2, node3],
"devices": [
device1, device2]
},
zone_2: {
"nodes": [
node1, node2, node3],
"devices": [
device1, device2]
}
}
"""
zone_devices_nodes = dict()
topology_info = heketi_ops.heketi_topology_info(self.h_client,
self.h_server,
json=True)
for cluster in topology_info['clusters']:
for node in cluster['nodes']:
if node['state'] == 'online':
if node['zone'] not in zone_devices_nodes:
zone_devices_nodes[node['zone']] = dict()
if 'nodes' not in zone_devices_nodes[node['zone']]:
zone_devices_nodes[node['zone']]['nodes'] = []
(zone_devices_nodes[node['zone']]['nodes'].append(
node['id']))
for device in node['devices']:
if device['state'] == 'online':
if node['zone'] not in zone_devices_nodes:
zone_devices_nodes[node['zone']] = dict()
if 'devices' not in zone_devices_nodes[node['zone']]:
zone_devices_nodes[node['zone']]['devices'] = []
(zone_devices_nodes[node['zone']]['devices'].append(
device['id']))
return zone_devices_nodes
def test_verify_db_check(self, count_type):
"""Validate the nodes, devices and bricks count in heketi db"""
# Get the total number of nodes, devices and bricks from db check
db_info = heketi_ops.heketi_db_check(self.heketi_client_node,
self.heketi_server_url)
db_devices_count = db_info["devices"]["total"]
db_nodes_count = db_info["nodes"]["total"]
db_bricks_count = db_info["bricks"]["total"]
# Get the total number of nodes, devices and bricks from topology info
topology_info = heketi_ops.heketi_topology_info(
self.heketi_client_node, self.heketi_server_url, json=True)
topology_devices_count, topology_nodes_count = 0, 0
topology_bricks_count = 0
for cluster in topology_info['clusters']:
topology_nodes_count += len(cluster['nodes'])
if count_type == 'bricks_count' or 'device_count':
for node in cluster['nodes']:
topology_devices_count += len(node['devices'])
if count_type == 'bricks_count':
for device in node['devices']:
topology_bricks_count += len(device['bricks'])
# Compare the device count
if count_type == 'device_count':
msg = ("Devices count in db check {} and in topology info {} is "
"not same".format(db_devices_count, topology_devices_count))
self.assertEqual(topology_devices_count, db_devices_count, msg)
# Compare the node count
elif count_type == 'node_count':
msg = (
"Nodes count in db check {} and nodes count in topology info "
"{} is not same".format(db_nodes_count, topology_nodes_count))
self.assertEqual(topology_nodes_count, db_nodes_count, msg)
# Compare the bricks count
elif count_type == 'bricks_count':
msg = ("Bricks count in db check {} and bricks count in topology "
"info {} is not same".format(db_bricks_count,
topology_bricks_count))
self.assertEqual(topology_bricks_count, db_bricks_count, msg)
def test_heketi_device_remove_delete(self, is_delete_device,
add_back_again):
"""Validate remove/delete device using heketi-cli"""
try:
gluster_server_0 = list(g.config["gluster_servers"].values())[0]
manage_hostname = gluster_server_0["manage"]
device_name = gluster_server_0["additional_devices"][0]
except (KeyError, IndexError):
self.skipTest(
"Additional disk is not specified for node with following "
"hostnames and IP addresses: {}, {}".format(
gluster_server_0.get('manage', '?'),
gluster_server_0.get('storage', '?')))
# Get node ID of the Gluster hostname
node_list = heketi_topology_info(self.heketi_client_node,
self.heketi_server_url,
json=True)["clusters"][0]["nodes"]
self.assertTrue(node_list,
"Cluster info command returned empty list of nodes")
node_id = None
for node in node_list:
if manage_hostname == node['hostnames']["manage"][0]:
node_id = node["id"]
break
self.assertTrue(
node_id, "No information about node_id"
" for {}".format(manage_hostname))
deleted_device = self._add_new_device_and_remove_existing_device(
is_delete_device, device_name, node_id, add_back_again)
if add_back_again:
self._add_new_device_and_remove_existing_device(
is_delete_device,
deleted_device,
node_id,
add_back_again,
skip_cleanup_addition=True)
def test_dev_path_mapping_basic_validation(self):
"""Validate dev_path of all the devices"""
node_with_devices = dict()
h_node, h_url = self.heketi_client_node, self.heketi_server_url
# Get the hostname and devices attached to each host
topology_info = heketi_topology_info(h_node, h_url, json=True)
for cluster in topology_info['clusters']:
for node in cluster['nodes']:
node_with_devices[node['hostnames']['manage'][0]] = [
device['id'] for device in node['devices']
]
# Validate dev_path of each device
for node, devices in node_with_devices.items():
for dev in list(devices):
is_true = validate_dev_path_vg_and_uuid(
h_node, h_url, node, dev)
self.assertTrue(
is_true, "Failed to verify dv_path for the "
"device {}".format(dev))
def test_heketi_device_replacement_in_node(self):
"""Validate device replacement operation on single node"""
h_client, h_server = self.heketi_client_node, self.heketi_server_url
try:
gluster_server_0 = list(g.config["gluster_servers"].values())[0]
manage_hostname = gluster_server_0["manage"]
add_device_name = gluster_server_0["additional_devices"][0]
except (KeyError, IndexError):
self.skipTest(
"Additional disk is not specified for node with following "
"hostnames and IP addresses: {}, {}".format(
gluster_server_0.get('manage', '?'),
gluster_server_0.get('storage', '?')))
# Get existing heketi volume list
existing_volumes = heketi_volume_list(h_client, h_server, json=True)
# Add cleanup function to clean stale volumes created during test
self.addCleanup(self._cleanup_heketi_volumes,
existing_volumes.get("volumes"))
# Get nodes info
node_id_list = heketi_node_list(h_client, h_server)
# Disable 4th and other nodes
if len(node_id_list) > 3:
for node_id in node_id_list[3:]:
heketi_node_disable(h_client, h_server, node_id)
self.addCleanup(heketi_node_enable, h_client, h_server,
node_id)
# Create volume when 3 nodes are online
vol_size, vol_count = 2, 4
for _ in range(vol_count):
vol_info = heketi_blockvolume_create(h_client,
h_server,
vol_size,
json=True)
self.addCleanup(heketi_blockvolume_delete, h_client, h_server,
vol_info['id'])
# Get node ID of the Gluster hostname
topology_info = heketi_topology_info(h_client, h_server, json=True)
self.assertIsNotNone(topology_info, "Failed to get topology info")
self.assertIn("clusters", topology_info.keys(),
"Failed to get cluster "
"details from topology info")
node_list = topology_info["clusters"][0]["nodes"]
self.assertTrue(node_list,
"Cluster info command returned empty list of nodes")
node_id = None
for node in node_list:
if manage_hostname == node['hostnames']["manage"][0]:
node_id = node["id"]
break
self.assertTrue(
node_id, "Failed to get node info for node id '{}'".format(
manage_hostname))
# Add extra device, then remember it's ID and size
device_id_new, device_size_new = self._add_heketi_device(
add_device_name, node_id)
# Remove one of the existing devices on node except new device
device_name, device_id = None, None
node_info_after_addition = heketi_node_info(h_client,
h_server,
node_id,
json=True)
for device in node_info_after_addition["devices"]:
if (device["name"] != add_device_name
and device["storage"]["total"] == device_size_new):
device_name = device["name"]
device_id = device["id"]
break
self.assertIsNotNone(device_name, "Failed to get device name")
self.assertIsNotNone(device_id, "Failed to get device id")
self.addCleanup(heketi_device_enable,
h_client,
h_server,
device_id,
raise_on_error=False)
self.addCleanup(heketi_device_add,
h_client,
h_server,
device_name,
node_id,
raise_on_error=False)
heketi_device_disable(h_client, h_server, device_id)
heketi_device_remove(h_client, h_server, device_id)
heketi_device_delete(h_client, h_server, device_id)
def test_heketi_device_info(self):
"""Validate whether device related information is displayed"""
# Get devices from topology info
devices_from_topology = {}
topology_info = heketi_topology_info(self.heketi_client_node,
self.heketi_server_url,
json=True)
self.assertTrue(topology_info)
self.assertIn('clusters', list(topology_info.keys()))
self.assertGreater(len(topology_info['clusters']), 0)
for cluster in topology_info['clusters']:
self.assertIn('nodes', list(cluster.keys()))
self.assertGreater(len(cluster['nodes']), 0)
for node in cluster['nodes']:
self.assertIn('devices', list(node.keys()))
self.assertGreater(len(node['devices']), 0)
for device in node['devices']:
# Expected keys are state, storage, id, name and bricks.
self.assertIn('id', list(device.keys()))
devices_from_topology[device['id']] = device
# Get devices info and make sure data are consistent and complete
for device_id, device_from_t_info in devices_from_topology.items():
device_info = heketi_device_info(self.heketi_client_node,
self.heketi_server_url,
device_id,
json=True)
self.assertTrue(device_info)
# Verify 'id', 'name', 'state' and 'storage' data
for key in ('id', 'name', 'state', 'storage', 'bricks'):
self.assertIn(key, list(device_from_t_info.keys()))
self.assertIn(key, list(device_info.keys()))
for key in ('id', 'name', 'state'):
self.assertEqual(device_info[key], device_from_t_info[key])
device_info_storage = device_info['storage']
device_from_t_info_storage = device_from_t_info['storage']
device_info_storage_keys = list(device_info_storage.keys())
device_from_t_info_storage_keys = list(
device_from_t_info_storage.keys())
for key in ('total', 'used', 'free'):
self.assertIn(key, device_info_storage_keys)
self.assertIn(key, device_from_t_info_storage_keys)
self.assertEqual(device_info_storage[key],
device_from_t_info_storage[key])
self.assertIsInstance(device_info_storage[key], int)
self.assertGreater(device_info_storage[key], -1)
# Verify 'bricks' data
self.assertEqual(len(device_info['bricks']),
len(device_from_t_info['bricks']))
brick_match_count = 0
for brick in device_info['bricks']:
for brick_from_t in device_from_t_info['bricks']:
if brick_from_t['id'] != brick['id']:
continue
brick_match_count += 1
brick_from_t_keys = list(brick_from_t.keys())
brick_keys = list(brick.keys())
for key in ('device', 'volume', 'size', 'path', 'id',
'node'):
self.assertIn(key, brick_from_t_keys)
self.assertIn(key, brick_keys)
self.assertEqual(brick[key], brick_from_t[key])
self.assertEqual(brick_match_count, len(device_info['bricks']))
def verify_heketi_metrics_with_topology_info(self):
topology = heketi_topology_info(
self.heketi_client_node, self.heketi_server_url, json=True)
metrics = get_heketi_metrics(
self.heketi_client_node, self.heketi_server_url)
self.assertTrue(topology)
self.assertIn('clusters', list(topology.keys()))
self.assertGreater(len(topology['clusters']), 0)
self.assertTrue(metrics)
self.assertGreater(len(metrics.keys()), 0)
self.assertEqual(
len(topology['clusters']), metrics['heketi_cluster_count'])
for cluster in topology['clusters']:
self.assertIn('nodes', list(cluster.keys()))
self.assertGreater(len(cluster['nodes']), 0)
cluster_id = cluster['id']
cluster_ids = ([obj['cluster']
for obj in metrics['heketi_nodes_count']])
self.assertIn(cluster_id, cluster_ids)
for node_count in metrics['heketi_nodes_count']:
if node_count['cluster'] == cluster_id:
self.assertEqual(
len(cluster['nodes']), node_count['value'])
cluster_ids = ([obj['cluster']
for obj in metrics['heketi_volumes_count']])
self.assertIn(cluster_id, cluster_ids)
for vol_count in metrics['heketi_volumes_count']:
if vol_count['cluster'] == cluster_id:
self.assertEqual(
len(cluster['volumes']), vol_count['value'])
for node in cluster['nodes']:
self.assertIn('devices', list(node.keys()))
self.assertGreater(len(node['devices']), 0)
hostname = node['hostnames']['manage'][0]
cluster_ids = ([obj['cluster']
for obj in metrics['heketi_device_count']])
self.assertIn(cluster_id, cluster_ids)
hostnames = ([obj['hostname']
for obj in metrics['heketi_device_count']])
self.assertIn(hostname, hostnames)
for device_count in metrics['heketi_device_count']:
if (device_count['cluster'] == cluster_id and
device_count['hostname'] == hostname):
self.assertEqual(
len(node['devices']), device_count['value'])
for device in node['devices']:
device_name = device['name']
device_size_t = device['storage']['total']
device_free_t = device['storage']['free']
device_used_t = device['storage']['used']
cluster_ids = ([obj['cluster']
for obj in
metrics['heketi_device_brick_count']])
self.assertIn(cluster_id, cluster_ids)
hostnames = ([obj['hostname']
for obj in
metrics['heketi_device_brick_count']])
self.assertIn(hostname, hostnames)
devices = ([obj['device']
for obj in
metrics['heketi_device_brick_count']])
self.assertIn(device_name, devices)
for brick_count in metrics['heketi_device_brick_count']:
if (brick_count['cluster'] == cluster_id and
brick_count['hostname'] == hostname and
brick_count['device'] == device_name):
self.assertEqual(
len(device['bricks']), brick_count['value'])
cluster_ids = ([obj['cluster']
for obj in metrics['heketi_device_size']])
self.assertIn(cluster_id, cluster_ids)
hostnames = ([obj['hostname']
for obj in metrics['heketi_device_size']])
self.assertIn(hostname, hostnames)
devices = ([obj['device']
for obj in metrics['heketi_device_size']])
self.assertIn(device_name, devices)
for device_size in metrics['heketi_device_size']:
if (device_size['cluster'] == cluster_id and
device_size['hostname'] == hostname and
device_size['device'] == device_name):
self.assertEqual(
device_size_t, device_size['value'])
cluster_ids = ([obj['cluster']
for obj in metrics['heketi_device_free']])
self.assertIn(cluster_id, cluster_ids)
hostnames = ([obj['hostname']
for obj in metrics['heketi_device_free']])
self.assertIn(hostname, hostnames)
devices = ([obj['device']
for obj in metrics['heketi_device_free']])
self.assertIn(device_name, devices)
for device_free in metrics['heketi_device_free']:
if (device_free['cluster'] == cluster_id and
device_free['hostname'] == hostname and
device_free['device'] == device_name):
self.assertEqual(
device_free_t, device_free['value'])
cluster_ids = ([obj['cluster']
for obj in metrics['heketi_device_used']])
self.assertIn(cluster_id, cluster_ids)
hostnames = ([obj['hostname']
for obj in metrics['heketi_device_used']])
self.assertIn(hostname, hostnames)
devices = ([obj['device']
for obj in metrics['heketi_device_used']])
self.assertIn(device_name, devices)
for device_used in metrics['heketi_device_used']:
if (device_used['cluster'] == cluster_id and
device_used['hostname'] == hostname and
device_used['device'] == device_name):
self.assertEqual(
device_used_t, device_used['value'])
def test_heketi_device_remove(self, delete_device):
"""Validate remove/delete device using heketi-cli"""
gluster_server_0 = list(g.config["gluster_servers"].values())[0]
try:
device_name = gluster_server_0["additional_devices"][0]
except (KeyError, IndexError):
self.skipTest(
"Additional disk is not specified for node with following "
"hostnames and IP addresses: %s, %s." % (gluster_server_0.get(
'manage', '?'), gluster_server_0.get('storage', '?')))
manage_hostname = gluster_server_0["manage"]
# Get node ID of the Gluster hostname
topo_info = heketi_topology_info(self.heketi_client_node,
self.heketi_server_url,
json=True)
self.assertTrue(topo_info["clusters"][0]["nodes"],
"Cluster info command returned empty list of nodes.")
node_id = None
for node in topo_info["clusters"][0]["nodes"]:
if manage_hostname == node['hostnames']["manage"][0]:
node_id = node["id"]
break
self.assertNotEqual(
node_id, None,
"No information about node_id for %s" % manage_hostname)
# Iterate chosen node devices and pick the smallest online one.
lowest_device_size = lowest_device_id = None
online_hosts = self.get_online_nodes_disable_redundant()
for host in online_hosts[0:3]:
if node_id != host["id"]:
continue
for device in host["devices"]:
if device["state"].strip().lower() != "online":
continue
if (lowest_device_size is None
or device["storage"]["total"] < lowest_device_size):
lowest_device_size = device["storage"]["total"]
lowest_device_id = device["id"]
lowest_device_name = device["name"]
if lowest_device_id is None:
self.skipTest(
"Didn't find suitable device for disablement on '%s' node." %
(node_id))
# Create volume
vol_size = 1
vol_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
vol_size,
json=True)
self.addCleanup(heketi_volume_delete, self.heketi_client_node,
self.heketi_server_url, vol_info['id'])
# Add extra device, then remember it's ID and size
heketi_device_add(self.heketi_client_node, self.heketi_server_url,
device_name, node_id)
node_info_after_addition = heketi_node_info(self.heketi_client_node,
self.heketi_server_url,
node_id,
json=True)
for device in node_info_after_addition["devices"]:
if device["name"] != device_name:
continue
device_id_new = device["id"]
device_size_new = device["storage"]["total"]
self.addCleanup(heketi_device_delete, self.heketi_client_node,
self.heketi_server_url, device_id_new)
self.addCleanup(heketi_device_remove, self.heketi_client_node,
self.heketi_server_url, device_id_new)
self.addCleanup(heketi_device_disable, self.heketi_client_node,
self.heketi_server_url, device_id_new)
if lowest_device_size > device_size_new:
skip_msg = ("Skip test case, because newly added disk %s is "
"smaller than device which we want to remove %s." %
(device_size_new, lowest_device_size))
self.skipTest(skip_msg)
g.log.info("Removing device id %s" % lowest_device_id)
with self.assertRaises(AssertionError):
out = heketi_device_remove(self.heketi_client_node,
self.heketi_server_url,
lowest_device_id)
self.addCleanup(heketi_device_enable, self.heketi_client_node,
self.heketi_server_url, lowest_device_id)
self.addCleanup(heketi_device_disable, self.heketi_client_node,
self.heketi_server_url, lowest_device_id)
self.assertFalse(True, "Device removal didn't fail: %s" % out)
g.log.info("Device removal failed as expected")
# Need to disable device before removing
heketi_device_disable(self.heketi_client_node, self.heketi_server_url,
lowest_device_id)
if not delete_device:
self.addCleanup(heketi_device_enable, self.heketi_client_node,
self.heketi_server_url, lowest_device_id)
# Remove device from Heketi
try:
heketi_device_remove(self.heketi_client_node,
self.heketi_server_url, lowest_device_id)
except Exception:
if delete_device:
self.addCleanup(heketi_device_enable, self.heketi_client_node,
self.heketi_server_url, lowest_device_id)
raise
if not delete_device:
self.addCleanup(heketi_device_disable, self.heketi_client_node,
self.heketi_server_url, lowest_device_id)
if delete_device:
try:
heketi_device_delete(self.heketi_client_node,
self.heketi_server_url, lowest_device_id)
except Exception:
self.addCleanup(heketi_device_enable, self.heketi_client_node,
self.heketi_server_url, lowest_device_id)
self.addCleanup(heketi_device_disable, self.heketi_client_node,
self.heketi_server_url, lowest_device_id)
raise
self.addCleanup(heketi_device_add, self.heketi_client_node,
self.heketi_server_url, lowest_device_name,
node_id)
# Create volume
vol_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
vol_size,
json=True)
self.addCleanup(heketi_volume_delete, self.heketi_client_node,
self.heketi_server_url, vol_info['id'])
if delete_device:
return
# Check that none of volume's bricks is present on the device
present = self.check_any_of_bricks_present_in_device(
vol_info['bricks'], lowest_device_id)
self.assertFalse(
present,
"Some of the '%s' volume bricks is present of the removed "
"'%s' device." % (vol_info['id'], lowest_device_id))
def verify_heketi_metrics_with_topology_info(self):
topology = heketi_topology_info(self.heketi_client_node,
self.heketi_server_url,
json=True)
metrics = get_heketi_metrics(self.heketi_client_node,
self.heketi_server_url)
self.assertTrue(topology)
self.assertIn('clusters', list(topology.keys()))
self.assertGreater(len(topology['clusters']), 0)
self.assertTrue(metrics)
self.assertGreater(len(metrics.keys()), 0)
self.assertEqual(len(topology['clusters']),
metrics['heketi_cluster_count'])
for cluster in topology['clusters']:
self.assertIn('nodes', list(cluster.keys()))
self.assertGreater(len(cluster['nodes']), 0)
cluster_id = cluster['id']
cluster_ids = ([
obj['cluster'] for obj in metrics['heketi_nodes_count']
])
self.assertIn(cluster_id, cluster_ids)
for node_count in metrics['heketi_nodes_count']:
if node_count['cluster'] == cluster_id:
self.assertEqual(len(cluster['nodes']),
node_count['value'])
cluster_ids = ([
obj['cluster'] for obj in metrics['heketi_volumes_count']
])
self.assertIn(cluster_id, cluster_ids)
for vol_count in metrics['heketi_volumes_count']:
if vol_count['cluster'] == cluster_id:
self.assertEqual(len(cluster['volumes']),
vol_count['value'])
for node in cluster['nodes']:
self.assertIn('devices', list(node.keys()))
self.assertGreater(len(node['devices']), 0)
hostname = node['hostnames']['manage'][0]
cluster_ids = ([
obj['cluster'] for obj in metrics['heketi_device_count']
])
self.assertIn(cluster_id, cluster_ids)
hostnames = ([
obj['hostname'] for obj in metrics['heketi_device_count']
])
self.assertIn(hostname, hostnames)
for device_count in metrics['heketi_device_count']:
if (device_count['cluster'] == cluster_id
and device_count['hostname'] == hostname):
self.assertEqual(len(node['devices']),
device_count['value'])
for device in node['devices']:
device_name = device['name']
device_size_t = device['storage']['total']
device_free_t = device['storage']['free']
device_used_t = device['storage']['used']
cluster_ids = ([
obj['cluster']
for obj in metrics['heketi_device_brick_count']
])
self.assertIn(cluster_id, cluster_ids)
hostnames = ([
obj['hostname']
for obj in metrics['heketi_device_brick_count']
])
self.assertIn(hostname, hostnames)
devices = ([
obj['device']
for obj in metrics['heketi_device_brick_count']
])
self.assertIn(device_name, devices)
for brick_count in metrics['heketi_device_brick_count']:
if (brick_count['cluster'] == cluster_id
and brick_count['hostname'] == hostname
and brick_count['device'] == device_name):
self.assertEqual(len(device['bricks']),
brick_count['value'])
cluster_ids = ([
obj['cluster'] for obj in metrics['heketi_device_size']
])
self.assertIn(cluster_id, cluster_ids)
hostnames = ([
obj['hostname']
for obj in metrics['heketi_device_size']
])
self.assertIn(hostname, hostnames)
devices = ([
obj['device'] for obj in metrics['heketi_device_size']
])
self.assertIn(device_name, devices)
for device_size in metrics['heketi_device_size']:
if (device_size['cluster'] == cluster_id
and device_size['hostname'] == hostname
and device_size['device'] == device_name):
self.assertEqual(device_size_t,
device_size['value'])
cluster_ids = ([
obj['cluster'] for obj in metrics['heketi_device_free']
])
self.assertIn(cluster_id, cluster_ids)
hostnames = ([
obj['hostname']
for obj in metrics['heketi_device_free']
])
self.assertIn(hostname, hostnames)
devices = ([
obj['device'] for obj in metrics['heketi_device_free']
])
self.assertIn(device_name, devices)
for device_free in metrics['heketi_device_free']:
if (device_free['cluster'] == cluster_id
and device_free['hostname'] == hostname
and device_free['device'] == device_name):
self.assertEqual(device_free_t,
device_free['value'])
cluster_ids = ([
obj['cluster'] for obj in metrics['heketi_device_used']
])
self.assertIn(cluster_id, cluster_ids)
hostnames = ([
obj['hostname']
for obj in metrics['heketi_device_used']
])
self.assertIn(hostname, hostnames)
devices = ([
obj['device'] for obj in metrics['heketi_device_used']
])
self.assertIn(device_name, devices)
for device_used in metrics['heketi_device_used']:
if (device_used['cluster'] == cluster_id
and device_used['hostname'] == hostname
and device_used['device'] == device_name):
self.assertEqual(device_used_t,
device_used['value'])
def test_create_heketi_cluster_and_add_node(self):
"""Test heketi node add to a newly created cluster"""
storage_host_info = g.config.get("additional_gluster_servers")
if not storage_host_info:
self.skipTest(
"Skip test case as 'additional_gluster_servers' option is "
"not provided in config file")
storage_host_info = list(storage_host_info.values())[0]
try:
storage_hostname = storage_host_info["manage"]
storage_ip = storage_host_info["storage"]
storage_device = storage_host_info["devices"][0]
except KeyError:
msg = ("Config options 'additional_gluster_servers.manage' "
"'additional_gluster_servers.storage' and "
"'additional_gluster_servers.devices' "
"must be set.")
g.log.error(msg)
raise exceptions.ConfigError(msg)
h_client, h_server = self.heketi_client_node, self.heketi_server_url
storage_zone = 1
cluster_id = heketi_ops.heketi_cluster_create(self.heketi_client_node,
self.heketi_server_url,
json=True)["id"]
self.addCleanup(heketi_ops.heketi_cluster_delete,
self.heketi_client_node, self.heketi_server_url,
cluster_id)
self.configure_node_to_run_gluster(storage_hostname)
heketi_node_info = heketi_ops.heketi_node_add(h_client,
h_server,
storage_zone,
cluster_id,
storage_hostname,
storage_ip,
json=True)
heketi_node_id = heketi_node_info["id"]
self.addCleanup(heketi_ops.heketi_node_delete, h_client, h_server,
heketi_node_id)
self.addCleanup(heketi_ops.heketi_node_remove, h_client, h_server,
heketi_node_id)
self.addCleanup(heketi_ops.heketi_node_disable, h_client, h_server,
heketi_node_id)
self.assertEqual(
heketi_node_info["cluster"], cluster_id,
"Node got added in unexpected cluster exp: %s, act: %s" %
(cluster_id, heketi_node_info["cluster"]))
heketi_ops.heketi_device_add(h_client, h_server, storage_device,
heketi_node_id)
heketi_node_info = heketi_ops.heketi_node_info(h_client,
h_server,
heketi_node_id,
json=True)
device_id = None
for device in heketi_node_info["devices"]:
if device["name"] == storage_device:
device_id = device["id"]
break
err_msg = ("Failed to add device %s on node %s" %
(storage_device, heketi_node_id))
self.assertTrue(device_id, err_msg)
self.addCleanup(heketi_ops.heketi_device_delete, h_client, h_server,
device_id)
self.addCleanup(heketi_ops.heketi_device_remove, h_client, h_server,
device_id)
self.addCleanup(heketi_ops.heketi_device_disable, h_client, h_server,
device_id)
cluster_info = heketi_ops.heketi_cluster_info(h_client,
h_server,
cluster_id,
json=True)
self.assertIn(
heketi_node_info["id"], cluster_info["nodes"],
"Newly added node %s not found in cluster %s, cluster info %s" %
(heketi_node_info["id"], cluster_id, cluster_info))
topology_info = heketi_ops.heketi_topology_info(h_client,
h_server,
json=True)
cluster_details = [
cluster for cluster in topology_info["clusters"]
if cluster["id"] == cluster_id
]
err_msg = "Cluster details for id '%s' not found" % cluster_id
self.assertTrue(cluster_details, err_msg)
err_msg = ("Multiple clusters with same id '%s' found %s" %
(cluster_id, cluster_details))
self.assertEqual(len(cluster_details), 1, err_msg)
node_details = [
node for node in cluster_details[0]["nodes"]
if node["id"] == heketi_node_id
]
err_msg = "Node details for id '%s' not found" % heketi_node_id
self.assertTrue(node_details, err_msg)
err_msg = ("Multiple nodes with same id '%s' found %s" %
(heketi_node_id, node_details))
self.assertEqual(len(node_details), 1, err_msg)
err_msg = "Unexpected %s found '%s', expected '%s'"
exp_storage_hostname = node_details[0]["hostnames"]["manage"][0]
self.assertEqual(
exp_storage_hostname, storage_hostname, err_msg % (
"hostname",
exp_storage_hostname,
storage_hostname,
))
exp_storage_ip = node_details[0]["hostnames"]["storage"][0]
self.assertEqual(exp_storage_ip, storage_ip,
err_msg % ("IP address", exp_storage_ip, storage_ip))
zone = node_details[0]["zone"]
self.assertEqual(zone, storage_zone,
err_msg % ("zone", zone, storage_zone))
def test_heketi_prometheus_usedbytes_brickcount_on_device_delete(
self, operation):
"""Validate used bytes,device count on heketi and prometheus"""
h_node, h_server = self.heketi_client_node, self.heketi_server_url
# Get list of additional devices for one of the Gluster nodes
gluster_server_0 = list(self.gluster_servers_info.values())[0]
manage_hostname = gluster_server_0.get("manage")
self.assertTrue(
manage_hostname, "IP Address is not specified for "
"node {}".format(gluster_server_0))
device_name = gluster_server_0.get("additional_devices")[0]
self.assertTrue(
device_name, "Additional devices are not specified for "
"node {}".format(gluster_server_0))
# Get node ID of the Gluster hostname
node_list = heketi_ops.heketi_topology_info(
h_node, h_server, json=True).get("clusters")[0].get("nodes")
self.assertTrue(
node_list, "Cluster info command returned empty list of nodes")
node_id = [
node.get("id")
for node in node_list
if manage_hostname == node.get("hostnames").get("manage")[0]]
self.assertTrue(
node_id, "Failed to get node_id for {}".format(manage_hostname))
node_id = node_id[0]
# Adding heketi device
heketi_ops.heketi_device_add(h_node, h_server, device_name, node_id)
node_info_after_addition = heketi_ops.heketi_node_info(
h_node, h_server, node_id, json=True)
device_id, bricks = None, None
for device in node_info_after_addition.get("devices"):
if device.get("name") == device_name:
device_id, bricks = (
device.get("id"), len(device.get("bricks")))
break
# Verify zero bricks on the device
msg = (
"Number of bricks on the device {} of the nodes should be"
"zero".format(device_name))
self.assertFalse(bricks, msg)
self.addCleanup(
heketi_ops.heketi_device_delete, h_node, h_server, device_id,
raise_on_error=False)
self.addCleanup(
heketi_ops.heketi_device_remove, h_node, h_server, device_id,
raise_on_error=False)
self.addCleanup(
heketi_ops.heketi_device_disable, h_node, h_server, device_id,
raise_on_error=False)
# Disable,Remove and Delete heketi device
heketi_ops.heketi_device_disable(h_node, h_server, device_id)
heketi_ops.heketi_device_remove(h_node, h_server, device_id)
heketi_ops.heketi_device_delete(h_node, h_server, device_id)
# Verify device deletion
node_info_after_deletion = (
heketi_ops.heketi_node_info(h_node, h_server, node_id))
msg = ("Device {} should not be shown in node info of the node {}"
"after the device deletion".format(device_id, node_id))
self.assertNotIn(device_id, node_info_after_deletion, msg)
if operation == "usedbytes":
# Validate heketi and prometheus device used bytes
for w in waiter.Waiter(timeout=60, interval=10):
device_used_bytes_prometheus = 0
device_used_bytes_metrics = 0
openshift_ops.switch_oc_project(
self.ocp_master_node[0], 'openshift-monitoring')
metric_result = self._fetch_metric_from_promtheus_pod(
metric='heketi_device_used_bytes')
for result in metric_result:
if (node_id == result.get('cluster')
and device_name == result.get('device')):
device_used_bytes_prometheus += (
int(result.get('value')[1]))
openshift_ops.switch_oc_project(
self.ocp_master_node[0], 'glusterfs')
metrics = heketi_ops.get_heketi_metrics(h_node, h_server)
heketi_device_count_metric = (
metrics.get('heketi_device_used_bytes'))
for result in heketi_device_count_metric:
if (node_id == result.get('cluster')
and device_name == result.get('device')):
device_used_bytes_metrics = int(result.get('value'))
if device_used_bytes_prometheus == device_used_bytes_metrics:
break
if w.expired:
raise exceptions.ExecutionError(
"Failed to update device details in prometheus")
elif operation == "brickcount":
# Validate heketi and prometheus device brick count
for w in waiter.Waiter(timeout=60, interval=10):
device_brick_count_prometheus = 0
device_brick_count_metrics = 0
metrics = heketi_ops.get_heketi_metrics(h_node, h_server)
heketi_device_count_metric = metrics.get(
'heketi_device_brick_count')
for result in heketi_device_count_metric:
device_brick_count_metrics += int(result.get('value'))
openshift_ops.switch_oc_project(
self.ocp_master_node[0], 'openshift-monitoring')
metric_result = self._fetch_metric_from_promtheus_pod(
metric='heketi_device_brick_count')
for result in metric_result:
device_brick_count_prometheus += (
int(result.get('value')[1]))
if device_brick_count_prometheus == device_brick_count_metrics:
break
if w.expired:
raise exceptions.ExecutionError(
"Failed to update device details in prometheus")
def test_volume_creation_of_size_greater_than_the_device_size(self):
"""Validate creation of a volume of size greater than the size of a
device.
"""
h_node, h_url = self.heketi_client_node, self.heketi_server_url
# Remove existing BHV to calculate freespace
bhv_list = heketi_ops.get_block_hosting_volume_list(h_node, h_url)
if bhv_list:
for bhv in bhv_list:
bhv_info = heketi_ops.heketi_volume_info(h_node,
h_url,
bhv,
json=True)
if bhv_info['blockinfo'].get('blockvolume') is None:
heketi_ops.heketi_volume_delete(h_node, h_url, bhv)
topology = heketi_ops.heketi_topology_info(h_node, h_url, json=True)
nodes_free_space, nodes_ips = [], []
selected_nodes, selected_devices = [], []
cluster = topology['clusters'][0]
node_count = len(cluster['nodes'])
msg = ("At least 3 Nodes are required in cluster. "
"But only %s Nodes are present." % node_count)
if node_count < 3:
self.skipTest(msg)
online_nodes_count = 0
for node in cluster['nodes']:
nodes_ips.append(node['hostnames']['storage'][0])
if node['state'] != 'online':
continue
online_nodes_count += 1
# Disable nodes after 3rd online nodes
if online_nodes_count > 3:
heketi_ops.heketi_node_disable(h_node, h_url, node['id'])
self.addCleanup(heketi_ops.heketi_node_enable, h_node, h_url,
node['id'])
continue
selected_nodes.append(node['id'])
device_count = len(node['devices'])
msg = ("At least 2 Devices are required on each Node."
"But only %s Devices are present." % device_count)
if device_count < 2:
self.skipTest(msg)
sel_devices, online_devices_count, free_space = [], 0, 0
for device in node['devices']:
if device['state'] != 'online':
continue
online_devices_count += 1
# Disable devices after 2nd online devices
if online_devices_count > 2:
heketi_ops.heketi_device_disable(h_node, h_url,
device['id'])
self.addCleanup(heketi_ops.heketi_device_enable, h_node,
h_url, device['id'])
continue
sel_devices.append(device['id'])
free_space += int(device['storage']['free'] / (1024**2))
selected_devices.append(sel_devices)
nodes_free_space.append(free_space)
msg = ("At least 2 online Devices are required on each Node. "
"But only %s Devices are online on Node: %s." %
(online_devices_count, node['id']))
if online_devices_count < 2:
self.skipTest(msg)
msg = ("At least 3 online Nodes are required in cluster. "
"But only %s Nodes are online in Cluster: %s." %
(online_nodes_count, cluster['id']))
if online_nodes_count < 3:
self.skipTest(msg)
# Select node with minimum free space
min_free_size = min(nodes_free_space)
index = nodes_free_space.index(min_free_size)
# Get max device size from selected node
device_size = 0
for device in selected_devices[index]:
device_info = heketi_ops.heketi_device_info(h_node,
h_url,
device,
json=True)
device_size = max(device_size,
(int(device_info['storage']['total'] /
(1024**2))))
vol_size = device_size + 1
if vol_size >= min_free_size:
self.skipTest('Required free space %s is not available' % vol_size)
# Create heketi volume with device size + 1
vol_info = self.create_heketi_volume_with_name_and_wait(
name="volume_size_greater_than_device_size",
size=vol_size,
json=True)
# Get gluster server IP's from heketi volume info
glusterfs_servers = heketi_ops.get_vol_file_servers_and_hosts(
h_node, h_url, vol_info['id'])
# Verify gluster server IP's in heketi volume info
msg = ("gluster IP's '%s' does not match with IP's '%s' found in "
"heketi volume info" %
(nodes_ips, glusterfs_servers['vol_servers']))
self.assertEqual(set(glusterfs_servers['vol_servers']), set(nodes_ips),
msg)
vol_name = vol_info['name']
gluster_v_info = self.get_gluster_vol_info(vol_name)
# Verify replica count in gluster v info
msg = "Volume %s is replica %s instead of replica 3" % (
vol_name, gluster_v_info['replicaCount'])
self.assertEqual('3', gluster_v_info['replicaCount'])
# Verify distCount in gluster v info
msg = "Volume %s distCount is %s instead of distCount as 3" % (
vol_name, int(gluster_v_info['distCount']))
self.assertEqual(
int(gluster_v_info['brickCount']) // 3,
int(gluster_v_info['distCount']), msg)
# Verify bricks count in gluster v info
msg = (
"Volume %s does not have bricks count multiple of 3. It has %s" %
(vol_name, gluster_v_info['brickCount']))
self.assertFalse(int(gluster_v_info['brickCount']) % 3, msg)
