def test_restart_heketi_pod(self):
"""Validate restarting heketi pod"""
# create heketi volume
vol_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
size=1, json=True)
self.assertTrue(vol_info, "Failed to create heketi volume of size 1")
self.addCleanup(
heketi_volume_delete, self.heketi_client_node,
self.heketi_server_url, vol_info['id'], raise_on_error=False)
topo_info = heketi_topology_info(self.heketi_client_node,
self.heketi_server_url,
json=True)
# get heketi-pod name
heketi_pod_name = get_pod_name_from_dc(self.ocp_master_node[0],
self.heketi_dc_name)
# delete heketi-pod (it restarts the pod)
oc_delete(self.ocp_master_node[0], 'pod', heketi_pod_name)
wait_for_resource_absence(self.ocp_master_node[0],
'pod', heketi_pod_name)
# get new heketi-pod name
heketi_pod_name = get_pod_name_from_dc(self.ocp_master_node[0],
self.heketi_dc_name)
wait_for_pod_be_ready(self.ocp_master_node[0],
heketi_pod_name)
# check heketi server is running
self.assertTrue(
hello_heketi(self.heketi_client_node, self.heketi_server_url),
"Heketi server %s is not alive" % self.heketi_server_url
)
# compare the topology
new_topo_info = heketi_topology_info(self.heketi_client_node,
self.heketi_server_url,
json=True)
self.assertEqual(new_topo_info, topo_info, "topology info is not same,"
" difference - %s" % diff(topo_info, new_topo_info))
# create new volume
vol_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
size=2, json=True)
self.assertTrue(vol_info, "Failed to create heketi volume of size 20")
heketi_volume_delete(
self.heketi_client_node, self.heketi_server_url, vol_info['id'])
def test_device_remove_operation(self, delete_device):
"""Validate remove/delete device using heketi-cli"""
gluster_server_0 = g.config["gluster_servers"].values()[0]
try:
device_name = gluster_server_0["additional_devices"][0]
except (KeyError, IndexError):
self.skipTest(
"Additional disk is not specified for node with following "
"hostnames and IP addresses: %s, %s." % (
gluster_server_0.get('manage', '?'),
gluster_server_0.get('storage', '?')))
manage_hostname = gluster_server_0["manage"]
# Get node ID of the Gluster hostname
topo_info = heketi_topology_info(self.heketi_client_node,
self.heketi_server_url, json=True)
self.assertTrue(
topo_info["clusters"][0]["nodes"],
"Cluster info command returned empty list of nodes.")
node_id = None
for node in topo_info["clusters"][0]["nodes"]:
if manage_hostname == node['hostnames']["manage"][0]:
node_id = node["id"]
break
self.assertNotEqual(
node_id, None,
"No information about node_id for %s" % manage_hostname)
# Iterate chosen node devices and pick the smallest online one.
lowest_device_size = lowest_device_id = None
online_hosts = self.get_online_nodes_disable_redundant()
for host in online_hosts[0:3]:
if node_id != host["id"]:
continue
for device in host["devices"]:
if device["state"].strip().lower() != "online":
continue
if (lowest_device_size is None or
device["storage"]["total"] < lowest_device_size):
lowest_device_size = device["storage"]["total"]
lowest_device_id = device["id"]
lowest_device_name = device["name"]
if lowest_device_id is None:
self.skipTest(
"Didn't find suitable device for disablement on '%s' node." % (
node_id))
# Create volume
vol_size = 1
vol_info = heketi_volume_create(
self.heketi_client_node, self.heketi_server_url, vol_size,
json=True)
self.assertTrue(vol_info, (
"Failed to create heketi volume of size %d" % vol_size))
self.addCleanup(self.delete_volumes, vol_info['id'])
# Add extra device, then remember it's ID and size
heketi_device_add(self.heketi_client_node, self.heketi_server_url,
device_name, node_id)
node_info_after_addition = heketi_node_info(
self.heketi_client_node, self.heketi_server_url, node_id,
json=True)
for device in node_info_after_addition["devices"]:
if device["name"] != device_name:
continue
device_id_new = device["id"]
device_size_new = device["storage"]["total"]
self.addCleanup(heketi_device_delete, self.heketi_client_node,
self.heketi_server_url, device_id_new)
self.addCleanup(heketi_device_remove, self.heketi_client_node,
self.heketi_server_url, device_id_new)
self.addCleanup(heketi_device_disable, self.heketi_client_node,
self.heketi_server_url, device_id_new)
if lowest_device_size > device_size_new:
skip_msg = ("Skip test case, because newly added disk %s is "
"smaller than device which we want to remove %s." % (
device_size_new, lowest_device_size))
self.skipTest(skip_msg)
g.log.info("Removing device id %s" % lowest_device_id)
ret, out, err = heketi_device_remove(
self.heketi_client_node, self.heketi_server_url,
lowest_device_id, raw_cli_output=True)
if ret == 0:
self.addCleanup(heketi_device_enable, self.heketi_client_node,
self.heketi_server_url, lowest_device_id)
self.addCleanup(heketi_device_disable, self.heketi_client_node,
self.heketi_server_url, lowest_device_id)
self.assertNotEqual(ret, 0, (
"Device removal did not fail. ret: %s, out: %s, err: %s." % (
ret, out, err)))
g.log.info("Device removal failed as expected, err- %s", err)
# Need to disable device before removing
heketi_device_disable(
self.heketi_client_node, self.heketi_server_url,
lowest_device_id)
if not delete_device:
self.addCleanup(heketi_device_enable, self.heketi_client_node,
self.heketi_server_url, lowest_device_id)
# Remove device from Heketi
try:
heketi_device_remove(
self.heketi_client_node, self.heketi_server_url,
lowest_device_id)
except Exception:
if delete_device:
self.addCleanup(heketi_device_enable, self.heketi_client_node,
self.heketi_server_url, lowest_device_id)
raise
if not delete_device:
self.addCleanup(heketi_device_disable, self.heketi_client_node,
self.heketi_server_url, lowest_device_id)
if delete_device:
try:
heketi_device_delete(
self.heketi_client_node, self.heketi_server_url,
lowest_device_id)
except Exception:
self.addCleanup(heketi_device_enable, self.heketi_client_node,
self.heketi_server_url, lowest_device_id)
self.addCleanup(heketi_device_disable, self.heketi_client_node,
self.heketi_server_url, lowest_device_id)
raise
self.addCleanup(
heketi_device_add,
self.heketi_client_node, self.heketi_server_url,
lowest_device_name, node_id)
# Create volume
vol_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url, vol_size,
json=True)
self.assertTrue(vol_info, (
"Failed to create heketi volume of size %d" % vol_size))
self.addCleanup(self.delete_volumes, vol_info['id'])
if delete_device:
return
# Check that none of volume's bricks is present on the device
present = self.check_any_of_bricks_present_in_device(
vol_info['bricks'], lowest_device_id)
self.assertFalse(
present,
"Some of the '%s' volume bricks is present of the removed "
"'%s' device." % (vol_info['id'], lowest_device_id))
def test_heketi_devices_info_verification(self):
"""Validate whether device related information is displayed"""
# Get devices from topology info
devices_from_topology = {}
topology_info = heketi_ops.heketi_topology_info(
self.heketi_client_node, self.heketi_server_url, json=True)
self.assertTrue(topology_info)
self.assertIn('clusters', list(topology_info.keys()))
self.assertGreater(len(topology_info['clusters']), 0)
for cluster in topology_info['clusters']:
self.assertIn('nodes', list(cluster.keys()))
self.assertGreater(len(cluster['nodes']), 0)
for node in cluster['nodes']:
self.assertIn('devices', list(node.keys()))
self.assertGreater(len(node['devices']), 0)
for device in node['devices']:
# Expected keys are state, storage, id, name and bricks.
self.assertIn('id', list(device.keys()))
devices_from_topology[device['id']] = device
# Get devices info and make sure data are consistent and complete
for device_id, device_from_t_info in devices_from_topology.items():
device_info = heketi_ops.heketi_device_info(
self.heketi_client_node,
self.heketi_server_url,
device_id,
json=True)
self.assertTrue(device_info)
# Verify 'id', 'name', 'state' and 'storage' data
for key in ('id', 'name', 'state', 'storage', 'bricks'):
self.assertIn(key, list(device_from_t_info.keys()))
self.assertIn(key, list(device_info.keys()))
self.assertEqual(device_info['id'], device_from_t_info['id'])
self.assertEqual(device_info['name'], device_from_t_info['name'])
self.assertEqual(device_info['state'], device_from_t_info['state'])
device_info_storage = device_info['storage']
device_from_t_info_storage = device_from_t_info['storage']
device_info_storage_keys = list(device_info_storage.keys())
device_from_t_info_storage_keys = list(
device_from_t_info_storage.keys())
for key in ('total', 'used', 'free'):
self.assertIn(key, device_info_storage_keys)
self.assertIn(key, device_from_t_info_storage_keys)
self.assertEqual(device_info_storage[key],
device_from_t_info_storage[key])
self.assertIsInstance(device_info_storage[key], int)
self.assertGreater(device_info_storage[key], -1)
# Verify 'bricks' data
self.assertEqual(len(device_info['bricks']),
len(device_from_t_info['bricks']))
brick_match_count = 0
for brick in device_info['bricks']:
for brick_from_t in device_from_t_info['bricks']:
if brick_from_t['id'] != brick['id']:
continue
brick_match_count += 1
brick_from_t_keys = list(brick_from_t.keys())
brick_keys = list(brick.keys())
for key in ('device', 'volume', 'size', 'path', 'id',
'node'):
self.assertIn(key, brick_from_t_keys)
self.assertIn(key, brick_keys)
self.assertEqual(brick[key], brick_from_t[key])
self.assertEqual(brick_match_count, len(device_info['bricks']))
def verify_heketi_metrics_with_topology_info(self):
topology = heketi_topology_info(self.heketi_client_node,
self.heketi_server_url,
json=True)
metrics = get_heketi_metrics(self.heketi_client_node,
self.heketi_server_url)
self.assertTrue(topology)
self.assertIn('clusters', list(topology.keys()))
self.assertGreater(len(topology['clusters']), 0)
self.assertTrue(metrics)
self.assertGreater(len(metrics.keys()), 0)
self.assertEqual(len(topology['clusters']),
metrics['heketi_cluster_count'])
for cluster in topology['clusters']:
self.assertIn('nodes', list(cluster.keys()))
self.assertGreater(len(cluster['nodes']), 0)
cluster_id = cluster['id']
cluster_ids = ([
obj['cluster'] for obj in metrics['heketi_nodes_count']
])
self.assertIn(cluster_id, cluster_ids)
for node_count in metrics['heketi_nodes_count']:
if node_count['cluster'] == cluster_id:
self.assertEqual(len(cluster['nodes']),
node_count['value'])
cluster_ids = ([
obj['cluster'] for obj in metrics['heketi_volumes_count']
])
self.assertIn(cluster_id, cluster_ids)
for vol_count in metrics['heketi_volumes_count']:
if vol_count['cluster'] == cluster_id:
self.assertEqual(len(cluster['volumes']),
vol_count['value'])
for node in cluster['nodes']:
self.assertIn('devices', list(node.keys()))
self.assertGreater(len(node['devices']), 0)
hostname = node['hostnames']['manage'][0]
cluster_ids = ([
obj['cluster'] for obj in metrics['heketi_device_count']
])
self.assertIn(cluster_id, cluster_ids)
hostnames = ([
obj['hostname'] for obj in metrics['heketi_device_count']
])
self.assertIn(hostname, hostnames)
for device_count in metrics['heketi_device_count']:
if (device_count['cluster'] == cluster_id
and device_count['hostname'] == hostname):
self.assertEqual(len(node['devices']),
device_count['value'])
for device in node['devices']:
device_name = device['name']
device_size_t = device['storage']['total']
device_free_t = device['storage']['free']
device_used_t = device['storage']['used']
cluster_ids = ([
obj['cluster']
for obj in metrics['heketi_device_brick_count']
])
self.assertIn(cluster_id, cluster_ids)
hostnames = ([
obj['hostname']
for obj in metrics['heketi_device_brick_count']
])
self.assertIn(hostname, hostnames)
devices = ([
obj['device']
for obj in metrics['heketi_device_brick_count']
])
self.assertIn(device_name, devices)
for brick_count in metrics['heketi_device_brick_count']:
if (brick_count['cluster'] == cluster_id
and brick_count['hostname'] == hostname
and brick_count['device'] == device_name):
self.assertEqual(len(device['bricks']),
brick_count['value'])
cluster_ids = ([
obj['cluster'] for obj in metrics['heketi_device_size']
])
self.assertIn(cluster_id, cluster_ids)
hostnames = ([
obj['hostname']
for obj in metrics['heketi_device_size']
])
self.assertIn(hostname, hostnames)
devices = ([
obj['device'] for obj in metrics['heketi_device_size']
])
self.assertIn(device_name, devices)
for device_size in metrics['heketi_device_size']:
if (device_size['cluster'] == cluster_id
and device_size['hostname'] == hostname
and device_size['device'] == device_name):
self.assertEqual(device_size_t,
device_size['value'])
cluster_ids = ([
obj['cluster'] for obj in metrics['heketi_device_free']
])
self.assertIn(cluster_id, cluster_ids)
hostnames = ([
obj['hostname']
for obj in metrics['heketi_device_free']
])
self.assertIn(hostname, hostnames)
devices = ([
obj['device'] for obj in metrics['heketi_device_free']
])
self.assertIn(device_name, devices)
for device_free in metrics['heketi_device_free']:
if (device_free['cluster'] == cluster_id
and device_free['hostname'] == hostname
and device_free['device'] == device_name):
self.assertEqual(device_free_t,
device_free['value'])
cluster_ids = ([
obj['cluster'] for obj in metrics['heketi_device_used']
])
self.assertIn(cluster_id, cluster_ids)
hostnames = ([
obj['hostname']
for obj in metrics['heketi_device_used']
])
self.assertIn(hostname, hostnames)
devices = ([
obj['device'] for obj in metrics['heketi_device_used']
])
self.assertIn(device_name, devices)
for device_used in metrics['heketi_device_used']:
if (device_used['cluster'] == cluster_id
and device_used['hostname'] == hostname
and device_used['device'] == device_name):
self.assertEqual(device_used_t,
device_used['value'])