def test_restart_heketi_pod(self):
"""Validate restarting heketi pod"""
# create heketi volume
vol_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
size=1, json=True)
self.assertTrue(vol_info, "Failed to create heketi volume of size 1")
self.addCleanup(
heketi_volume_delete, self.heketi_client_node,
self.heketi_server_url, vol_info['id'], raise_on_error=False)
topo_info = heketi_topology_info(self.heketi_client_node,
self.heketi_server_url,
json=True)
# get heketi-pod name
heketi_pod_name = get_pod_name_from_dc(self.ocp_master_node[0],
self.heketi_dc_name)
# delete heketi-pod (it restarts the pod)
oc_delete(self.ocp_master_node[0], 'pod', heketi_pod_name)
wait_for_resource_absence(self.ocp_master_node[0],
'pod', heketi_pod_name)
# get new heketi-pod name
heketi_pod_name = get_pod_name_from_dc(self.ocp_master_node[0],
self.heketi_dc_name)
wait_for_pod_be_ready(self.ocp_master_node[0],
heketi_pod_name)
# check heketi server is running
self.assertTrue(
hello_heketi(self.heketi_client_node, self.heketi_server_url),
"Heketi server %s is not alive" % self.heketi_server_url
)
# compare the topology
new_topo_info = heketi_topology_info(self.heketi_client_node,
self.heketi_server_url,
json=True)
self.assertEqual(new_topo_info, topo_info, "topology info is not same,"
" difference - %s" % diff(topo_info, new_topo_info))
# create new volume
vol_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
size=2, json=True)
self.assertTrue(vol_info, "Failed to create heketi volume of size 20")
heketi_volume_delete(
self.heketi_client_node, self.heketi_server_url, vol_info['id'])
def test_create_vol_and_retrieve_vol_info(self):
"""Validate heketi and gluster volume info"""
g.log.info("Create a heketi volume")
out = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
self.volume_size,
json=True)
self.assertTrue(out, ("Failed to create heketi "
"volume of size %s" % self.volume_size))
g.log.info("Heketi volume successfully created" % out)
volume_id = out["bricks"][0]["volume"]
self.addCleanup(heketi_volume_delete, self.heketi_client_node,
self.heketi_server_url, volume_id)
g.log.info("Retrieving heketi volume info")
out = heketi_volume_info(self.heketi_client_node,
self.heketi_server_url,
volume_id,
json=True)
self.assertTrue(out, ("Failed to get heketi volume info"))
g.log.info("Successfully got the heketi volume info")
name = out["name"]
vol_info = get_volume_info('auto_get_gluster_endpoint', volname=name)
self.assertTrue(vol_info, "Failed to get volume info %s" % name)
g.log.info("Successfully got the volume info %s" % name)
def test_volume_inconsistencies(self):
# Examine Gluster cluster and Heketi that there is no inconsistencies
out = heketi_ops.heketi_examine_gluster(self.heketi_client_node,
self.heketi_server_url)
if ("heketi volume list matches with volume list of all nodes"
not in out['report']):
self.skipTest("heketi and Gluster are inconsistent to each other")
# create volume
vol = heketi_ops.heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
1,
json=True)
self.addCleanup(heketi_ops.heketi_volume_delete,
self.heketi_client_node, self.heketi_server_url,
vol['id'])
# delete volume from gluster cluster directly
openshift_ops.cmd_run_on_gluster_pod_or_node(
self.node, "gluster vol stop %s force --mode=script" % vol['name'])
openshift_ops.cmd_run_on_gluster_pod_or_node(
self.node, "gluster vol delete %s --mode=script" % vol['name'])
# verify that heketi is reporting inconsistencies
out = heketi_ops.heketi_examine_gluster(self.heketi_client_node,
self.heketi_server_url)
self.assertNotIn(
"heketi volume list matches with volume list of all nodes",
out['report'])
def test_to_check_entry_in_fstab_file(self):
"""Validate /etc/fstab entries after creation/deletion of volume"""
# Create heketi volume
vol = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
size=1,
json=True)
self.assertTrue(vol, "Failed to create 1Gb heketi volume")
vol_id = vol["bricks"][0]["volume"]
self.addCleanup(heketi_volume_delete,
self.heketi_client_node,
self.heketi_server_url,
vol_id,
raise_on_error=False)
# Gather brick paths
brick_paths = [p['path'].rstrip("/brick") for p in vol["bricks"]]
# Make sure that volume's brick paths exist in the fstab files
self._find_bricks_in_fstab_files(brick_paths, present=True)
# Delete heketi volume
out = heketi_volume_delete(self.heketi_client_node,
self.heketi_server_url, vol_id)
self.assertTrue(out, "Failed to delete heketi volume %s" % vol_id)
# Make sure that volume's brick paths are absent in the fstab file
self._find_bricks_in_fstab_files(brick_paths, present=False)
def test_delete_heketi_volume(self):
"""
Method to test heketi volume deletion and whether it
frees up used space after deletion
"""
creation_output_dict = heketi_ops.heketi_volume_create(
self.heketi_client_node, self.heketi_server_url, 10, json=True)
self.assertNotEqual(creation_output_dict, False,
"Volume creation failed")
volume_id = creation_output_dict["name"].strip().split("_")[1]
free_space_after_creation = self.get_free_space_summary_devices()
deletion_output = heketi_ops.heketi_volume_delete(
self.heketi_client_node, self.heketi_server_url, volume_id)
self.assertNotEqual(deletion_output, False,
"Deletion of volume failed, id: %s" % volume_id)
free_space_after_deletion = self.get_free_space_summary_devices()
self.assertTrue(
free_space_after_deletion > free_space_after_creation,
"Free space is not reclaimed after deletion of %s" % volume_id)
def test_heketi_metrics_validating_vol_count_on_vol_deletion(self):
"""Validate heketi metrics VolumeCount after volume deletion"""
vol_list = []
for i in range(3):
# Create volume
vol = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
1,
json=True)
self.assertTrue(vol)
self.addCleanup(heketi_volume_delete,
self.heketi_client_node,
self.heketi_server_url,
vol['id'],
raise_on_error=False)
volume_list = heketi_volume_list(self.heketi_client_node,
self.heketi_server_url)
self.assertIn(vol['id'], volume_list)
vol_list.append(vol)
for vol in vol_list:
# delete volume
heketi_volume_delete(self.heketi_client_node,
self.heketi_server_url, vol['id'])
volume_list = heketi_volume_list(self.heketi_client_node,
self.heketi_server_url)
self.assertNotIn(vol['id'], volume_list)
self.verify_volume_count()
def test_heketi_with_expand_volume(self):
"""
Test volume expand and size if updated correctly in heketi-cli info
"""
vol_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
self.volume_size,
json=True)
self.assertTrue(
vol_info,
("Failed to create heketi volume of size %s" % self.volume_size))
self.addCleanup(heketi_volume_delete, self.heketi_client_node,
self.heketi_server_url, vol_info['id'])
self.assertEqual(vol_info['size'], self.volume_size,
("Failed to create volume."
"Expected Size: %s, Actual Size: %s" %
(self.volume_size, vol_info['size'])))
volume_id = vol_info["id"]
expand_size = 2
ret = heketi_volume_expand(self.heketi_client_node,
self.heketi_server_url, volume_id,
expand_size)
self.assertTrue(
ret, ("Failed to expand heketi volume of id %s" % volume_id))
volume_info = heketi_volume_info(self.heketi_client_node,
self.heketi_server_url,
volume_id,
json=True)
expected_size = self.volume_size + expand_size
self.assertEqual(volume_info['size'], expected_size,
("Volume Expansion failed Expected Size: %s, Actual "
"Size: %s" %
(str(expected_size), str(volume_info['size']))))
def test_validate_brick_paths_on_gluster_pods_or_nodes(self):
"""Validate brick paths after creation and deletion of a volume."""
# Create heketi volume
vol = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
size=1,
json=True)
self.assertTrue(vol, "Failed to create 1Gb heketi volume")
vol_id = vol["bricks"][0]["volume"]
self.addCleanup(heketi_volume_delete,
self.heketi_client_node,
self.heketi_server_url,
vol_id,
raise_on_error=False)
# Gather brick paths
brick_paths = [p['path'] for p in vol["bricks"]]
# Make sure that volume's brick paths exist in the fstab files
self._find_bricks(brick_paths, present=True)
# Delete heketi volume
out = heketi_volume_delete(self.heketi_client_node,
self.heketi_server_url, vol_id)
self.assertTrue(out, "Failed to delete heketi volume %s" % vol_id)
# Make sure that volume's brick paths are absent in the fstab file
self._find_bricks(brick_paths, present=False)
def test_to_check_deletion_of_node(self):
"""Validate deletion of a node which contains devices"""
# Create Heketi volume to make sure we have devices with usages
heketi_url = self.heketi_server_url
vol = heketi_volume_create(self.heketi_client_node,
heketi_url,
1,
json=True)
self.assertTrue(vol, "Failed to create heketi volume.")
g.log.info("Heketi volume successfully created")
volume_id = vol["bricks"][0]["volume"]
self.addCleanup(heketi_volume_delete, self.heketi_client_node,
self.heketi_server_url, volume_id)
# Pick up suitable node
node_ids = heketi_node_list(self.heketi_client_node, heketi_url)
self.assertTrue(node_ids)
for node_id in node_ids:
node_info = heketi_node_info(self.heketi_client_node,
heketi_url,
node_id,
json=True)
if (node_info['state'].lower() != 'online'
or not node_info['devices']):
continue
for device in node_info['devices']:
if device['state'].lower() != 'online':
continue
if device['storage']['used']:
node_id = node_info['id']
break
else:
self.assertTrue(
node_id, "Failed to find online node with online device which "
"has some usages.")
# Try to delete the node by its ID
g.log.info("Trying to delete the node which contains devices in it. "
"Expecting failure.")
self.assertRaises(ExecutionError, heketi_node_delete,
self.heketi_client_node, heketi_url, node_id)
# Make sure our node hasn't been deleted
g.log.info("Listing heketi node list")
node_list = heketi_node_list(self.heketi_client_node, heketi_url)
self.assertTrue(node_list, ("Failed to list heketi nodes"))
self.assertIn(node_id, node_list)
node_info = heketi_node_info(self.heketi_client_node,
heketi_url,
node_id,
json=True)
self.assertEqual(node_info['state'].lower(), 'online')
def test_to_check_deletion_of_cluster(self):
"""Validate deletion of cluster with volumes"""
# List heketi volumes
g.log.info("List heketi volumes")
volumes = heketi_volume_list(self.heketi_client_node,
self.heketi_server_url,
json=True)
if (len(volumes["volumes"]) == 0):
g.log.info("Creating heketi volume")
out = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
self.volume_size,
json=True)
self.assertTrue(out, ("Failed to create heketi "
"volume of size %s" % self.volume_size))
g.log.info("Heketi volume successfully created" % out)
volume_id = out["bricks"][0]["volume"]
self.addCleanup(heketi_volume_delete, self.heketi_client_node,
self.heketi_server_url, volume_id)
# List heketi cluster's
g.log.info("Listing heketi cluster list")
out = heketi_cluster_list(self.heketi_client_node,
self.heketi_server_url,
json=True)
self.assertTrue(out, ("Failed to list heketi cluster"))
g.log.info("All heketi cluster successfully listed")
cluster_id = out["clusters"][0]
# Deleting a heketi cluster
g.log.info("Trying to delete a heketi cluster"
" which contains volumes and/or nodes:"
" Expected to fail")
self.assertRaises(
ExecutionError,
heketi_cluster_delete,
self.heketi_client_node,
self.heketi_server_url,
cluster_id,
)
g.log.info("Expected result: Unable to delete cluster %s"
" because it contains volumes "
" and/or nodes" % cluster_id)
# To confirm deletion failed, check heketi cluster list
g.log.info("Listing heketi cluster list")
out = heketi_cluster_list(self.heketi_client_node,
self.heketi_server_url,
json=True)
self.assertTrue(out, ("Failed to list heketi cluster"))
g.log.info("All heketi cluster successfully listed")
def test_delete_heketidb_volume(self):
"""
Method to test heketidb volume deletion via heketi-cli
"""
volume_id_list = []
heketidbexists = False
msg = "Error: Cannot delete volume containing the Heketi database"
for i in range(0, 2):
volume_info = heketi_ops.heketi_volume_create(
self.heketi_client_node, self.heketi_server_url, 10, json=True)
self.assertNotEqual(volume_info, False, "Volume creation failed")
volume_id_list.append(volume_info["id"])
self.addCleanup(self.delete_volumes, volume_id_list)
volume_list_info = heketi_ops.heketi_volume_list(
self.heketi_client_node, self.heketi_server_url, json=True)
self.assertNotEqual(volume_list_info, False,
"Heketi volume list command failed")
if volume_list_info["volumes"] == []:
raise ExecutionError("Heketi volume list empty")
for volume_id in volume_list_info["volumes"]:
volume_info = heketi_ops.heketi_volume_info(
self.heketi_client_node,
self.heketi_server_url,
volume_id,
json=True)
if volume_info["name"] == "heketidbstorage":
heketidbexists = True
delete_ret, delete_output, delete_error = (
heketi_ops.heketi_volume_delete(self.heketi_client_node,
self.heketi_server_url,
volume_id,
raw_cli_output=True))
self.assertNotEqual(delete_ret, 0, "Return code not 0")
self.assertEqual(
delete_error.strip(), msg,
"Invalid reason for heketidb deletion failure")
if not heketidbexists:
raise ExecutionError(
"Warning: heketidbstorage doesn't exist in list of volumes")
def test_heketi_metrics_heketipod_failure(self):
"""Validate heketi metrics after heketi pod failure"""
scale_dc_pod_amount_and_wait(self.ocp_master_node,
self.heketi_dc_name,
pod_amount=0)
self.addCleanup(scale_dc_pod_amount_and_wait,
self.ocp_master_node,
self.heketi_dc_name,
pod_amount=1)
# verify that metrics is not accessable when heketi pod is down
with self.assertRaises(exceptions.ExecutionError):
get_heketi_metrics(self.heketi_client_node,
self.heketi_server_url,
prometheus_format=True)
scale_dc_pod_amount_and_wait(self.ocp_master_node,
self.heketi_dc_name,
pod_amount=1)
pod_name = get_pod_name_from_dc(self.ocp_master_node,
self.heketi_dc_name,
self.heketi_dc_name)
wait_for_pod_be_ready(self.ocp_master_node, pod_name, wait_step=5)
for i in range(3):
vol = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
1,
json=True)
self.assertTrue(vol)
self.addCleanup(heketi_volume_delete,
self.heketi_client_node,
self.heketi_server_url,
vol['id'],
raise_on_error=False)
vol_list = heketi_volume_list(self.heketi_client_node,
self.heketi_server_url)
self.assertIn(vol['id'], vol_list)
self.verify_heketi_metrics_with_topology_info()
def test_create_block_vol_after_host_vol_creation(self):
"""Validate block-device after manual block hosting volume creation
using heketi
"""
block_host_create_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
5,
json=True,
block=True)
self.addCleanup(heketi_volume_delete, self.heketi_client_node,
self.heketi_server_url, block_host_create_info["id"])
block_vol = heketi_blockvolume_create(self.heketi_client_node,
self.heketi_server_url,
1,
json=True)
self.addCleanup(heketi_blockvolume_delete, self.heketi_client_node,
self.heketi_server_url, block_vol["id"])
def test_volume_create_and_list_volume(self):
"""Validate heketi and gluster volume list"""
g.log.info("List gluster volumes before Heketi volume creation")
existing_g_vol_list = get_volume_list('auto_get_gluster_endpoint')
self.assertTrue(existing_g_vol_list, ("Unable to get volumes list"))
g.log.info("List heketi volumes before volume creation")
existing_h_vol_list = heketi_volume_list(self.heketi_client_node,
self.heketi_server_url,
json=True)["volumes"]
g.log.info("Heketi volumes successfully listed")
g.log.info("Create a heketi volume")
out = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
self.volume_size,
json=True)
g.log.info("Heketi volume successfully created" % out)
volume_id = out["bricks"][0]["volume"]
self.addCleanup(heketi_volume_delete, self.heketi_client_node,
self.heketi_server_url, volume_id)
g.log.info("List heketi volumes after volume creation")
h_vol_list = heketi_volume_list(self.heketi_client_node,
self.heketi_server_url,
json=True)["volumes"]
g.log.info("Heketi volumes successfully listed")
g.log.info("List gluster volumes after Heketi volume creation")
g_vol_list = get_volume_list('auto_get_gluster_endpoint')
self.assertTrue(g_vol_list, ("Unable to get volumes list"))
g.log.info("Successfully got the volumes list")
# Perform checks
self.assertEqual(
len(existing_g_vol_list) + 1, len(g_vol_list),
"Expected creation of only one volume in Gluster creating "
"Heketi volume. Here is lists before and after volume creation: "
"%s \n%s" % (existing_g_vol_list, g_vol_list))
self.assertEqual(
len(existing_h_vol_list) + 1, len(h_vol_list),
"Expected creation of only one volume in Heketi. Here is lists "
"of Heketi volumes before and after volume creation: %s\n%s" %
(existing_h_vol_list, h_vol_list))
def test_create_block_vol_after_host_vol_creation(self):
"""Validate block-device after manual block hosting volume creation
using heketi
"""
block_host_create_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
5,
json=True,
block=True)
self.assertNotEqual(block_host_create_info, False,
"Block host volume creation failed")
block_hosting_vol_id = block_host_create_info["id"]
self.addCleanup(self.delete_volumes, block_hosting_vol_id)
block_vol = heketi_blockvolume_create(self.heketi_client_node,
self.heketi_server_url,
1,
json=True)
self.assertNotEqual(block_vol, False, "Block volume creation failed")
self.addCleanup(self.delete_block_volumes, block_vol["id"])
def test_heketi_with_default_options(self):
"""
Test to create volume with default options.
"""
vol_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
self.volume_size,
json=True)
self.assertTrue(
vol_info,
("Failed to create heketi volume of size %s" % self.volume_size))
self.addCleanup(heketi_volume_delete, self.heketi_client_node,
self.heketi_server_url, vol_info['id'])
self.assertEqual(vol_info['size'], self.volume_size,
("Failed to create volume with default options."
"Expected Size: %s, Actual Size: %s" %
(self.volume_size, vol_info['size'])))
def test_block_host_volume_delete_without_block_volumes(self):
"""Validate deletion of empty block hosting volume"""
block_host_create_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
1,
json=True,
block=True)
block_hosting_vol_id = block_host_create_info["id"]
self.addCleanup(heketi_volume_delete,
self.heketi_client_node,
self.heketi_server_url,
block_hosting_vol_id,
raise_on_error=False)
heketi_volume_delete(self.heketi_client_node,
self.heketi_server_url,
block_hosting_vol_id,
json=True)
def test_block_host_volume_delete_without_block_volumes(self):
"""Validate deletion of empty block hosting volume"""
block_host_create_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
1,
json=True,
block=True)
self.assertNotEqual(block_host_create_info, False,
"Block host volume creation failed")
block_hosting_vol_id = block_host_create_info["id"]
self.addCleanup(heketi_volume_delete,
self.heketi_client_node,
self.heketi_server_url,
block_hosting_vol_id,
raise_on_error=False)
block_host_delete_output = heketi_volume_delete(
self.heketi_client_node,
self.heketi_server_url,
block_hosting_vol_id,
json=True)
self.assertNotEqual(
block_host_delete_output, False,
"Block host volume delete failed, ID: %s" % block_hosting_vol_id)
def test_device_enable_disable(self):
"""Validate device enable and disable functionality"""
# Disable all but one device on the first online node
online_hosts = self.get_online_nodes_disable_redundant()
online_device_id = ""
for device in online_hosts[0]["devices"]:
if device["state"].strip().lower() != "online":
continue
device_id = device["id"]
if online_device_id == "":
online_device_id = device_id
else:
g.log.info("going to disable device %s", device_id)
heketi_device_disable(
self.heketi_client_node, self.heketi_server_url, device_id)
self.addCleanup(
heketi_device_enable,
self.heketi_client_node, self.heketi_server_url, device_id)
if online_device_id == "":
self.skipTest(
"No device online on node %s" % online_hosts[0]["id"])
# Create volume when only 1 device is online
vol_size = 1
vol_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url, vol_size,
json=True)
self.assertTrue(vol_info, (
"Failed to create heketi volume of size %d" % vol_size))
self.addCleanup(self.delete_volumes, vol_info['id'])
# Check that one of volume's bricks is present on the device
present = self.check_any_of_bricks_present_in_device(
vol_info['bricks'], online_device_id)
self.assertTrue(
present,
"None of '%s' volume bricks is present on the '%s' device." % (
vol_info['id'], online_device_id))
g.log.info("Going to disable device id %s", online_device_id)
heketi_device_disable(
self.heketi_client_node, self.heketi_server_url, online_device_id)
self.addCleanup(heketi_device_enable, self.heketi_client_node,
self.heketi_server_url, online_device_id)
ret, out, err = heketi_volume_create(
self.heketi_client_node, self.heketi_server_url,
vol_size, json=True, raw_cli_output=True)
if ret == 0:
self.addCleanup(self.delete_volumes, json.loads(out)["id"])
self.assertNotEqual(ret, 0,
("Volume creation did not fail. ret- %s "
"out- %s err- %s" % (ret, out, err)))
g.log.info("Volume creation failed as expected, err- %s", err)
# Enable back the device which was previously disabled
g.log.info("Going to enable device id %s", online_device_id)
heketi_device_enable(
self.heketi_client_node, self.heketi_server_url, online_device_id)
# Create volume when device is enabled
vol_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url, vol_size,
json=True)
self.assertTrue(vol_info, (
"Failed to create heketi volume of size %d" % vol_size))
self.addCleanup(self.delete_volumes, vol_info['id'])
# Check that one of volume's bricks is present on the device
present = self.check_any_of_bricks_present_in_device(
vol_info['bricks'], online_device_id)
self.assertTrue(
present,
"None of '%s' volume bricks is present on the '%s' device." % (
vol_info['id'], online_device_id))
def test_heketi_with_device_removal_insuff_space(self):
"""Validate heketi with device removal insufficient space"""
# Disable 4+ nodes and 3+ devices on the first 3 nodes
min_free_space_gb = 5
min_free_space = min_free_space_gb * 1024**2
heketi_url = self.heketi_server_url
heketi_node = self.heketi_client_node
nodes = {}
node_ids = heketi_node_list(heketi_node, heketi_url)
self.assertTrue(node_ids)
for node_id in node_ids:
node_info = heketi_node_info(
heketi_node, heketi_url, node_id, json=True)
if (node_info["state"].lower() != "online" or
not node_info["devices"]):
continue
if len(nodes) > 2:
heketi_node_disable(heketi_node, heketi_url, node_id)
self.addCleanup(
heketi_node_enable, heketi_node, heketi_url, node_id)
continue
for device in node_info["devices"]:
if device["state"].lower() != "online":
continue
free_space = device["storage"]["free"]
if node_id not in nodes:
nodes[node_id] = []
if (free_space < min_free_space or len(nodes[node_id]) > 1):
heketi_device_disable(
heketi_node, heketi_url, device["id"])
self.addCleanup(
heketi_device_enable,
heketi_node, heketi_url, device["id"])
continue
nodes[node_id].append({
"device_id": device["id"], "free": free_space})
# Skip test if nodes requirements are not met
if (len(nodes) < 3 or
not all(map((lambda _list: len(_list) > 1), nodes.values()))):
raise self.skipTest(
"Could not find 3 online nodes with 2 online devices "
"having free space bigger than %dGb." % min_free_space_gb)
# Calculate size of a potential distributed vol
if nodes[node_ids[0]][0]["free"] > nodes[node_ids[0]][1]["free"]:
index = 0
else:
index = 1
vol_size_gb = int(nodes[node_ids[0]][index]["free"] / (1024 ** 2)) + 1
device_id = nodes[node_ids[0]][index]["device_id"]
# Create volume with such size that we consume space more than
# size of smaller disks
try:
heketi_vol = heketi_volume_create(
heketi_node, heketi_url, vol_size_gb, json=True)
except Exception as e:
g.log.warning(
"Got following error trying to create '%s'Gb vol: %s" % (
vol_size_gb, e))
vol_size_gb -= 1
heketi_vol = heketi_volume_create(
heketi_node, heketi_url, vol_size_gb, json=True)
self.addCleanup(self.delete_volumes, heketi_vol["bricks"][0]["volume"])
# Try to 'remove' bigger Heketi disk expecting error,
# because there is no space on smaller disk to relocate bricks to
heketi_device_disable(heketi_node, heketi_url, device_id)
self.addCleanup(
heketi_device_enable, heketi_node, heketi_url, device_id)
try:
self.assertRaises(
ExecutionError, heketi_device_remove,
heketi_node, heketi_url, device_id)
except Exception:
self.addCleanup(
heketi_device_disable, heketi_node, heketi_url, device_id)
raise
def test_device_remove_operation(self, delete_device):
"""Validate remove/delete device using heketi-cli"""
gluster_server_0 = g.config["gluster_servers"].values()[0]
try:
device_name = gluster_server_0["additional_devices"][0]
except (KeyError, IndexError):
self.skipTest(
"Additional disk is not specified for node with following "
"hostnames and IP addresses: %s, %s." % (
gluster_server_0.get('manage', '?'),
gluster_server_0.get('storage', '?')))
manage_hostname = gluster_server_0["manage"]
# Get node ID of the Gluster hostname
topo_info = heketi_topology_info(self.heketi_client_node,
self.heketi_server_url, json=True)
self.assertTrue(
topo_info["clusters"][0]["nodes"],
"Cluster info command returned empty list of nodes.")
node_id = None
for node in topo_info["clusters"][0]["nodes"]:
if manage_hostname == node['hostnames']["manage"][0]:
node_id = node["id"]
break
self.assertNotEqual(
node_id, None,
"No information about node_id for %s" % manage_hostname)
# Iterate chosen node devices and pick the smallest online one.
lowest_device_size = lowest_device_id = None
online_hosts = self.get_online_nodes_disable_redundant()
for host in online_hosts[0:3]:
if node_id != host["id"]:
continue
for device in host["devices"]:
if device["state"].strip().lower() != "online":
continue
if (lowest_device_size is None or
device["storage"]["total"] < lowest_device_size):
lowest_device_size = device["storage"]["total"]
lowest_device_id = device["id"]
lowest_device_name = device["name"]
if lowest_device_id is None:
self.skipTest(
"Didn't find suitable device for disablement on '%s' node." % (
node_id))
# Create volume
vol_size = 1
vol_info = heketi_volume_create(
self.heketi_client_node, self.heketi_server_url, vol_size,
json=True)
self.assertTrue(vol_info, (
"Failed to create heketi volume of size %d" % vol_size))
self.addCleanup(self.delete_volumes, vol_info['id'])
# Add extra device, then remember it's ID and size
heketi_device_add(self.heketi_client_node, self.heketi_server_url,
device_name, node_id)
node_info_after_addition = heketi_node_info(
self.heketi_client_node, self.heketi_server_url, node_id,
json=True)
for device in node_info_after_addition["devices"]:
if device["name"] != device_name:
continue
device_id_new = device["id"]
device_size_new = device["storage"]["total"]
self.addCleanup(heketi_device_delete, self.heketi_client_node,
self.heketi_server_url, device_id_new)
self.addCleanup(heketi_device_remove, self.heketi_client_node,
self.heketi_server_url, device_id_new)
self.addCleanup(heketi_device_disable, self.heketi_client_node,
self.heketi_server_url, device_id_new)
if lowest_device_size > device_size_new:
skip_msg = ("Skip test case, because newly added disk %s is "
"smaller than device which we want to remove %s." % (
device_size_new, lowest_device_size))
self.skipTest(skip_msg)
g.log.info("Removing device id %s" % lowest_device_id)
ret, out, err = heketi_device_remove(
self.heketi_client_node, self.heketi_server_url,
lowest_device_id, raw_cli_output=True)
if ret == 0:
self.addCleanup(heketi_device_enable, self.heketi_client_node,
self.heketi_server_url, lowest_device_id)
self.addCleanup(heketi_device_disable, self.heketi_client_node,
self.heketi_server_url, lowest_device_id)
self.assertNotEqual(ret, 0, (
"Device removal did not fail. ret: %s, out: %s, err: %s." % (
ret, out, err)))
g.log.info("Device removal failed as expected, err- %s", err)
# Need to disable device before removing
heketi_device_disable(
self.heketi_client_node, self.heketi_server_url,
lowest_device_id)
if not delete_device:
self.addCleanup(heketi_device_enable, self.heketi_client_node,
self.heketi_server_url, lowest_device_id)
# Remove device from Heketi
try:
heketi_device_remove(
self.heketi_client_node, self.heketi_server_url,
lowest_device_id)
except Exception:
if delete_device:
self.addCleanup(heketi_device_enable, self.heketi_client_node,
self.heketi_server_url, lowest_device_id)
raise
if not delete_device:
self.addCleanup(heketi_device_disable, self.heketi_client_node,
self.heketi_server_url, lowest_device_id)
if delete_device:
try:
heketi_device_delete(
self.heketi_client_node, self.heketi_server_url,
lowest_device_id)
except Exception:
self.addCleanup(heketi_device_enable, self.heketi_client_node,
self.heketi_server_url, lowest_device_id)
self.addCleanup(heketi_device_disable, self.heketi_client_node,
self.heketi_server_url, lowest_device_id)
raise
self.addCleanup(
heketi_device_add,
self.heketi_client_node, self.heketi_server_url,
lowest_device_name, node_id)
# Create volume
vol_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url, vol_size,
json=True)
self.assertTrue(vol_info, (
"Failed to create heketi volume of size %d" % vol_size))
self.addCleanup(self.delete_volumes, vol_info['id'])
if delete_device:
return
# Check that none of volume's bricks is present on the device
present = self.check_any_of_bricks_present_in_device(
vol_info['bricks'], lowest_device_id)
self.assertFalse(
present,
"Some of the '%s' volume bricks is present of the removed "
"'%s' device." % (vol_info['id'], lowest_device_id))
def test_node_state(self):
"""
Test node enable and disable functionality.
If we have 4 gluster servers, if we disable 1/4 nodes from heketi
and create a volume, the volume creation should be successful.
If we disable 2/4 nodes from heketi-cli and create a volume
the volume creation should fail.
If we enable back one gluster server and create a volume
the volume creation should be successful.
"""
g.log.info("Disable node in heketi")
node_list = heketi_node_list(self.heketi_client_node,
self.heketi_server_url)
self.assertTrue(node_list, "Failed to list heketi nodes")
g.log.info("Successfully got the list of nodes")
online_hosts = self.get_online_nodes(node_list)
if len(online_hosts) < 3:
raise self.skipTest(
"This test can run only if online hosts are more "
"than 2")
# if we have n nodes, disable n-3 nodes
for node_info in online_hosts[3:]:
node_id = node_info["id"]
g.log.info("going to disable node id %s", node_id)
self.disable_node(node_id)
self.addCleanup(self.enable_node, node_id)
vol_size = 1
# create volume when 3 nodes are online
vol_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
vol_size,
json=True)
self.addCleanup(heketi_volume_delete, self.heketi_client_node,
self.heketi_server_url, vol_info['id'])
node_id = online_hosts[0]['id']
g.log.info("going to disable node id %s", node_id)
self.disable_node(node_id)
self.addCleanup(self.enable_node, node_id)
# try to create a volume, volume creation should fail
ret, out, err = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
vol_size,
raw_cli_output=True)
if ret == 0:
out_json = json.loads(out)
self.addCleanup(heketi_volume_delete, self.heketi_client_node,
self.heketi_server_url, out_json["id"])
self.assertNotEqual(ret, 0, ("Volume creation did not fail ret- %s "
"out- %s err- %s" % (ret, out, err)))
g.log.info("Volume creation failed as expected, err- %s", err)
# enable node
self.enable_node(node_id)
# create volume when node is enabled
vol_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
vol_size,
json=True)
self.addCleanup(heketi_volume_delete, self.heketi_client_node,
self.heketi_server_url, vol_info['id'])
def test_volume_expansion_no_free_space(self):
"""Validate volume expansion when there is no free space"""
vol_size, expand_size, additional_devices_attached = None, 10, {}
h_node, h_server_url = self.heketi_client_node, self.heketi_server_url
# Get nodes info
heketi_node_id_list = heketi_ops.heketi_node_list(h_node, h_server_url)
if len(heketi_node_id_list) < 3:
self.skipTest("3 Heketi nodes are required.")
# Disable 4th and other nodes
for node_id in heketi_node_id_list[3:]:
heketi_ops.heketi_node_disable(h_node, h_server_url, node_id)
self.addCleanup(heketi_ops.heketi_node_enable, h_node,
h_server_url, node_id)
# Prepare first 3 nodes
smallest_size = None
err_msg = ''
for node_id in heketi_node_id_list[0:3]:
node_info = heketi_ops.heketi_node_info(h_node,
h_server_url,
node_id,
json=True)
# Disable second and other devices
devices = node_info["devices"]
self.assertTrue(devices,
"Node '%s' does not have devices." % node_id)
if devices[0]["state"].strip().lower() != "online":
self.skipTest("Test expects first device to be enabled.")
if (smallest_size is None
or devices[0]["storage"]["free"] < smallest_size):
smallest_size = devices[0]["storage"]["free"]
for device in node_info["devices"][1:]:
heketi_ops.heketi_device_disable(h_node, h_server_url,
device["id"])
self.addCleanup(heketi_ops.heketi_device_enable, h_node,
h_server_url, device["id"])
# Gather info about additional devices
additional_device_name = None
for gluster_server in self.gluster_servers:
gluster_server_data = self.gluster_servers_info[gluster_server]
g_manage = gluster_server_data["manage"]
g_storage = gluster_server_data["storage"]
if not (g_manage in node_info["hostnames"]["manage"]
or g_storage in node_info["hostnames"]["storage"]):
continue
additional_device_name = ((
gluster_server_data.get("additional_devices") or [''])[0])
break
if not additional_device_name:
err_msg += (
"No 'additional_devices' are configured for "
"'%s' node, which has following hostnames and "
"IP addresses: %s.\n" %
(node_id, ', '.join(node_info["hostnames"]["manage"] +
node_info["hostnames"]["storage"])))
continue
heketi_ops.heketi_device_add(h_node, h_server_url,
additional_device_name, node_id)
additional_devices_attached.update(
{node_id: additional_device_name})
# Schedule cleanup of the added devices
for node_id in additional_devices_attached.keys():
node_info = heketi_ops.heketi_node_info(h_node,
h_server_url,
node_id,
json=True)
for device in node_info["devices"]:
if device["name"] != additional_devices_attached[node_id]:
continue
self.addCleanup(self.detach_devices_attached, device["id"])
break
else:
self.fail("Could not find ID for added device on "
"'%s' node." % node_id)
if err_msg:
self.skipTest(err_msg)
# Temporary disable new devices
self.disable_devices(additional_devices_attached)
# Create volume and save info about it
vol_size = int(smallest_size / (1024**2)) - 1
creation_info = heketi_ops.heketi_volume_create(h_node,
h_server_url,
vol_size,
json=True)
volume_name, volume_id = creation_info["name"], creation_info["id"]
self.addCleanup(heketi_ops.heketi_volume_delete,
h_node,
h_server_url,
volume_id,
raise_on_error=False)
volume_info_before_expansion = heketi_ops.heketi_volume_info(
h_node, h_server_url, volume_id, json=True)
num_of_bricks_before_expansion = self.get_num_of_bricks(volume_name)
self.get_brick_and_volume_status(volume_name)
free_space_before_expansion = self.get_devices_summary_free_space()
# Try to expand volume with not enough device space
self.assertRaises(ExecutionError, heketi_ops.heketi_volume_expand,
h_node, h_server_url, volume_id, expand_size)
# Enable new devices to be able to expand our volume
self.enable_devices(additional_devices_attached)
# Expand volume and validate results
heketi_ops.heketi_volume_expand(h_node,
h_server_url,
volume_id,
expand_size,
json=True)
free_space_after_expansion = self.get_devices_summary_free_space()
self.assertGreater(
free_space_before_expansion, free_space_after_expansion,
"Free space not consumed after expansion of %s" % volume_id)
num_of_bricks_after_expansion = self.get_num_of_bricks(volume_name)
self.get_brick_and_volume_status(volume_name)
volume_info_after_expansion = heketi_ops.heketi_volume_info(
h_node, h_server_url, volume_id, json=True)
self.assertGreater(volume_info_after_expansion["size"],
volume_info_before_expansion["size"],
"Size of %s not increased" % volume_id)
self.assertGreater(num_of_bricks_after_expansion,
num_of_bricks_before_expansion)
self.assertEqual(
num_of_bricks_after_expansion % num_of_bricks_before_expansion, 0)
# Delete volume and validate release of the used space
heketi_ops.heketi_volume_delete(h_node, h_server_url, volume_id)
free_space_after_deletion = self.get_devices_summary_free_space()
self.assertGreater(
free_space_after_deletion, free_space_after_expansion,
"Free space not reclaimed after deletion of volume %s" % volume_id)
def test_volume_expansion_rebalance_brick(self):
"""Validate volume expansion with brick and check rebalance"""
creation_info = heketi_ops.heketi_volume_create(
self.heketi_client_node, self.heketi_server_url, 10, json=True)
self.assertNotEqual(creation_info, False, "Volume creation failed")
volume_name = creation_info["name"]
volume_id = creation_info["id"]
free_space_after_creation = self.get_devices_summary_free_space()
volume_info_before_expansion = heketi_ops.heketi_volume_info(
self.heketi_client_node,
self.heketi_server_url,
volume_id,
json=True)
self.assertNotEqual(volume_info_before_expansion, False,
"Volume info for %s failed" % volume_id)
heketi_vol_info_size_before_expansion = (
volume_info_before_expansion["size"])
self.get_brick_and_volume_status(volume_name)
num_of_bricks_before_expansion = self.get_num_of_bricks(volume_name)
expansion_info = heketi_ops.heketi_volume_expand(
self.heketi_client_node, self.heketi_server_url, volume_id, 5)
self.assertNotEqual(expansion_info, False,
"Volume expansion of %s failed" % volume_id)
free_space_after_expansion = self.get_devices_summary_free_space()
self.assertTrue(
free_space_after_creation > free_space_after_expansion,
"Free space not consumed after expansion of %s" % volume_id)
volume_info_after_expansion = heketi_ops.heketi_volume_info(
self.heketi_client_node,
self.heketi_server_url,
volume_id,
json=True)
self.assertNotEqual(volume_info_after_expansion, False,
"Volume info failed for %s" % volume_id)
heketi_vol_info_size_after_expansion = (
volume_info_after_expansion["size"])
difference_size = (heketi_vol_info_size_after_expansion -
heketi_vol_info_size_before_expansion)
self.assertTrue(difference_size > 0,
"Size not increased after expansion of %s" % volume_id)
self.get_brick_and_volume_status(volume_name)
num_of_bricks_after_expansion = self.get_num_of_bricks(volume_name)
num_of_bricks_added = (num_of_bricks_after_expansion -
num_of_bricks_before_expansion)
self.assertEqual(num_of_bricks_added, 3,
"Number of bricks added is not 3 for %s" % volume_id)
self.get_rebalance_status(volume_name)
deletion_info = heketi_ops.heketi_volume_delete(
self.heketi_client_node,
self.heketi_server_url,
volume_id,
json=True)
self.assertNotEqual(deletion_info, False,
"Deletion of volume %s failed" % volume_id)
free_space_after_deletion = self.get_devices_summary_free_space()
self.assertTrue(
free_space_after_deletion > free_space_after_expansion,
"Free space is not reclaimed after volume deletion of %s" %
volume_id)
def test_volume_creation_no_free_devices(self):
"""Validate heketi error is returned when no free devices available"""
node, server_url = self.heketi_client_node, self.heketi_server_url
# Get nodes info
node_id_list = heketi_ops.heketi_node_list(node, server_url)
node_info_list = []
for node_id in node_id_list[0:3]:
node_info = heketi_ops.heketi_node_info(node,
server_url,
node_id,
json=True)
node_info_list.append(node_info)
# Disable 4th and other nodes
for node_id in node_id_list[3:]:
heketi_ops.heketi_node_disable(node, server_url, node_id)
self.addCleanup(heketi_ops.heketi_node_enable, node, server_url,
node_id)
# Disable second and other devices on the first 3 nodes
for node_info in node_info_list[0:3]:
devices = node_info["devices"]
self.assertTrue(
devices, "Node '%s' does not have devices." % node_info["id"])
if devices[0]["state"].strip().lower() != "online":
self.skipTest("Test expects first device to be enabled.")
if len(devices) < 2:
continue
for device in node_info["devices"][1:]:
out = heketi_ops.heketi_device_disable(node, server_url,
device["id"])
self.assertTrue(
out, "Failed to disable the device %s" % device["id"])
self.addCleanup(heketi_ops.heketi_device_enable, node,
server_url, device["id"])
# Calculate common available space
available_spaces = [
int(node_info["devices"][0]["storage"]["free"])
for n in node_info_list[0:3]
]
min_space_gb = int(min(available_spaces) / 1024**2)
self.assertGreater(min_space_gb, 3, "Not enough available free space.")
# Create first small volume
vol = heketi_ops.heketi_volume_create(node, server_url, 1, json=True)
self.addCleanup(heketi_ops.heketi_volume_delete,
self.heketi_client_node, self.heketi_server_url,
vol["id"])
# Try to create second volume getting "no free space" error
try:
vol_fail = heketi_ops.heketi_volume_create(node,
server_url,
min_space_gb,
json=True)
except exceptions.ExecutionError:
g.log.info("Volume was not created as expected.")
else:
self.addCleanup(heketi_ops.heketi_volume_delete,
self.heketi_client_node, self.heketi_server_url,
vol_fail["bricks"][0]["volume"])
self.assertFalse(
vol_fail,
"Volume should have not been created. Out: %s" % vol_fail)
def test_create_heketi_volume(self):
"""Test heketi volume creation and background gluster validation"""
hosts = []
gluster_servers = []
brick_info = []
output_dict = heketi_ops.heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
10,
json=True)
self.assertNotEqual(output_dict, False, "Volume could not be created")
volume_name = output_dict["name"]
volume_id = output_dict["id"]
self.addCleanup(heketi_ops.heketi_volume_delete,
self.heketi_client_node, self.heketi_server_url,
volume_id)
self.assertEqual(output_dict["durability"]["replicate"]["replica"], 3,
"Volume %s is not replica 3" % volume_id)
self.assertEqual(output_dict["size"], 10,
"Volume %s is not of intended size" % volume_id)
mount_node = (
output_dict["mount"]["glusterfs"]["device"].strip().split(":")[0])
hosts.append(mount_node)
for backup_volfile_server in (
output_dict["mount"]["glusterfs"]["options"]
["backup-volfile-servers"].strip().split(",")):
hosts.append(backup_volfile_server)
for gluster_server in self.gluster_servers:
gluster_servers.append(
g.config["gluster_servers"][gluster_server]["storage"])
self.assertEqual(
set(hosts), set(gluster_servers),
"Hosts and gluster servers not matching for %s" % volume_id)
volume_info = volume_ops.get_volume_info('auto_get_gluster_endpoint',
volume_name)
self.assertIsNotNone(volume_info, "get_volume_info returned None")
volume_status = volume_ops.get_volume_status(
'auto_get_gluster_endpoint', volume_name)
self.assertIsNotNone(volume_status, "get_volume_status returned None")
self.assertEqual(int(volume_info[volume_name]["status"]), 1,
"Volume %s status down" % volume_id)
for brick_details in volume_info[volume_name]["bricks"]["brick"]:
brick_info.append(brick_details["name"])
self.assertNotEqual(brick_info, [],
"Brick details are empty for %s" % volume_name)
for brick in brick_info:
brick_data = brick.strip().split(":")
brick_ip = brick_data[0]
brick_name = brick_data[1]
self.assertEqual(
int(volume_status[volume_name][brick_ip][brick_name]
["status"]), 1, "Brick %s is not up" % brick_name)
def _create_distributed_replica_vol(self, validate_cleanup):
# Create distributed vol
vol_size_gb = self._get_vol_size()
heketi_url = self.heketi_server_url
try:
g.log.info(
"Trying to create distributed '%s'Gb volume." % vol_size_gb)
heketi_vol = heketi_volume_create(
self.heketi_client_node, heketi_url, vol_size_gb, json=True)
except exceptions.ExecutionError as e:
# NOTE: rare situation when we need to decrease size of a volume.
# and we expect this vol to be distributed.
g.log.info("Failed to create distributed '%s'Gb volume. "
"Trying to create another one, smaller for 1Gb.")
if ('more required' in str(e)
and ('Insufficient suitable allocatable extents for '
'logical volume' in str(e))):
vol_size_gb -= 1
heketi_vol = heketi_volume_create(
self.heketi_client_node, heketi_url, vol_size_gb,
json=True)
else:
raise
g.log.info("Successfully created distributed volume.")
vol_name = heketi_vol['name']
vol_id = heketi_vol["bricks"][0]["volume"]
self.addCleanup(
heketi_volume_delete, self.heketi_client_node, heketi_url,
vol_id, raise_on_error=(not validate_cleanup))
# Get gluster volume info
g.log.info("Get gluster volume '%s' info" % vol_name)
gluster_vol = get_volume_info(
'auto_get_gluster_endpoint', volname=vol_name)
self.assertTrue(
gluster_vol, "Failed to get volume '%s' info" % vol_name)
g.log.info("Successfully got volume '%s' info" % vol_name)
gluster_vol = gluster_vol[vol_name]
self.assertEqual(
gluster_vol["typeStr"], "Distributed-Replicate",
"'%s' gluster vol isn't a Distributed-Replicate volume" % vol_name)
# Check amount of bricks
brick_amount = len(gluster_vol['bricks']['brick'])
self.assertEqual(brick_amount % 3, 0,
"Brick amount is expected to be divisible by 3. "
"Actual amount is '%s'" % brick_amount)
self.assertGreater(brick_amount, 3,
"Brick amount is expected to be bigger than 3. "
"Actual amount is '%s'." % brick_amount)
# Run unique actions to Validate whether deleting a dist-rep
# volume is handled by heketi else return
if not validate_cleanup:
return
# Get the free space after creating heketi volume
free_space_after_creating_vol = self._get_free_space()
# Delete heketi volume
g.log.info("Deleting heketi volume '%s'" % vol_id)
volume_deleted = heketi_volume_delete(
self.heketi_client_node, heketi_url, vol_id)
self.assertTrue(
volume_deleted, "Failed to delete heketi volume '%s'" % vol_id)
g.log.info("Heketi volume '%s' has successfully been deleted" % vol_id)
# Check the heketi volume list
g.log.info("List heketi volumes")
heketi_volumes = heketi_volume_list(
self.heketi_client_node, self.heketi_server_url, json=True)
self.assertTrue(heketi_volumes, "Failed to list heketi volumes")
g.log.info("Heketi volumes have successfully been listed")
heketi_volumes = heketi_volumes.get('volumes', heketi_volumes)
self.assertNotIn(vol_id, heketi_volumes)
self.assertNotIn(vol_name, heketi_volumes)
# Check the gluster volume list
g.log.info("Get the gluster volume list")
gluster_volumes = get_volume_list('auto_get_gluster_endpoint')
self.assertTrue(gluster_volumes, "Unable to get Gluster volume list")
g.log.info("Successfully got Gluster volume list" % gluster_volumes)
self.assertNotIn(vol_id, gluster_volumes)
self.assertNotIn(vol_name, gluster_volumes)
# Get the used space after deleting heketi volume
free_space_after_deleting_vol = self._get_free_space()
# Compare the free space before and after deleting the volume
g.log.info("Comparing the free space before and after deleting volume")
self.assertLessEqual(
free_space_after_creating_vol + (3 * vol_size_gb),
free_space_after_deleting_vol)
g.log.info("Volume successfully deleted and space is reallocated. "
"Free space after creating volume %s. "
"Free space after deleting volume %s." % (
free_space_after_creating_vol,
free_space_after_deleting_vol))
def test_create_volumes_enabling_and_disabling_heketi_devices(self):
"""Validate enable/disable of heketi device"""
# Get nodes info
node_id_list = heketi_ops.heketi_node_list(self.heketi_client_node,
self.heketi_server_url)
node_info_list = []
for node_id in node_id_list[0:3]:
node_info = heketi_ops.heketi_node_info(self.heketi_client_node,
self.heketi_server_url,
node_id,
json=True)
node_info_list.append(node_info)
# Disable 4th and other nodes
if len(node_id_list) > 3:
for node in node_id_list[3:]:
heketi_ops.heketi_node_disable(self.heketi_client_node,
self.heketi_server_url, node_id)
self.addCleanup(heketi_ops.heketi_node_enable,
self.heketi_client_node,
self.heketi_server_url, node_id)
# Disable second and other devices on the first 3 nodes
for node_info in node_info_list[0:3]:
devices = node_info["devices"]
self.assertTrue(
devices, "Node '%s' does not have devices." % node_info["id"])
if devices[0]["state"].strip().lower() != "online":
self.skipTest("Test expects first device to be enabled.")
if len(devices) < 2:
continue
for device in node_info["devices"][1:]:
out = heketi_ops.heketi_device_disable(self.heketi_client_node,
self.heketi_server_url,
device["id"])
self.assertTrue(
out, "Failed to disable the device %s" % device["id"])
self.addCleanup(heketi_ops.heketi_device_enable,
self.heketi_client_node,
self.heketi_server_url, device["id"])
# Create heketi volume
out = heketi_ops.heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
1,
json=True)
self.assertTrue(out, "Failed to create heketi volume of size 1")
g.log.info("Successfully created heketi volume of size 1")
device_id = out["bricks"][0]["device"]
self.addCleanup(heketi_ops.heketi_volume_delete,
self.heketi_client_node, self.heketi_server_url,
out["bricks"][0]["volume"])
# Disable device
g.log.info("Disabling '%s' device" % device_id)
out = heketi_ops.heketi_device_disable(self.heketi_client_node,
self.heketi_server_url,
device_id)
self.assertTrue(out, "Failed to disable the device %s" % device_id)
g.log.info("Successfully disabled device %s" % device_id)
try:
# Get device info
g.log.info("Retrieving '%s' device info" % device_id)
out = heketi_ops.heketi_device_info(self.heketi_client_node,
self.heketi_server_url,
device_id,
json=True)
self.assertTrue(out, "Failed to get device info %s" % device_id)
g.log.info("Successfully retrieved device info %s" % device_id)
name = out["name"]
if out["state"].lower().strip() != "offline":
raise exceptions.ExecutionError(
"Device %s is not in offline state." % name)
g.log.info("Device %s is now offine" % name)
# Try to create heketi volume
g.log.info("Creating heketi volume: Expected to fail.")
try:
out = heketi_ops.heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
1,
json=True)
except exceptions.ExecutionError:
g.log.info("Volume was not created as expected.")
else:
self.addCleanup(heketi_ops.heketi_volume_delete,
self.heketi_client_node,
self.heketi_server_url,
out["bricks"][0]["volume"])
msg = "Volume unexpectedly created. Out: %s" % out
assert False, msg
finally:
# Enable the device back
g.log.info("Enable '%s' device back." % device_id)
out = heketi_ops.heketi_device_enable(self.heketi_client_node,
self.heketi_server_url,
device_id)
self.assertTrue(out, "Failed to enable the device %s" % device_id)
g.log.info("Successfully enabled device %s" % device_id)
# Get device info
out = heketi_ops.heketi_device_info(self.heketi_client_node,
self.heketi_server_url,
device_id,
json=True)
self.assertTrue(out, ("Failed to get device info %s" % device_id))
g.log.info("Successfully retrieved device info %s" % device_id)
name = out["name"]
if out["state"] != "online":
raise exceptions.ExecutionError(
"Device %s is not in online state." % name)
# Create heketi volume of size
out = heketi_ops.heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
1,
json=True)
self.assertTrue(out, "Failed to create volume of size 1")
self.addCleanup(heketi_ops.heketi_volume_delete,
self.heketi_client_node, self.heketi_server_url,
out["bricks"][0]["volume"])
g.log.info("Successfully created volume of size 1")
name = out["name"]
# Get gluster volume info
vol_info = get_volume_info('auto_get_gluster_endpoint', volname=name)
self.assertTrue(vol_info, "Failed to get '%s' volume info." % name)
g.log.info("Successfully got the '%s' volume info." % name)
