def test_delete_heketi_volume(self):
"""
Method to test heketi volume deletion and whether it
frees up used space after deletion
"""
volume_info = heketi_ops.heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
10,
json=True)
self.addCleanup(heketi_ops.heketi_volume_delete,
self.heketi_client_node,
self.heketi_server_url,
volume_info["id"],
raise_on_error=False)
free_space_after_creation = self.get_free_space_summary_devices()
heketi_ops.heketi_volume_delete(self.heketi_client_node,
self.heketi_server_url,
volume_info["id"])
free_space_after_deletion = self.get_free_space_summary_devices()
self.assertTrue(
free_space_after_deletion > free_space_after_creation,
"Free space is not reclaimed after deletion "
"of %s" % volume_info["id"])
def test_heketi_metrics_validating_vol_count_on_vol_deletion(self):
"""Validate heketi metrics VolumeCount after volume deletion"""
vol_list = []
for i in range(3):
# Create volume
vol = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
1,
json=True)
self.assertTrue(vol)
self.addCleanup(heketi_volume_delete,
self.heketi_client_node,
self.heketi_server_url,
vol['id'],
raise_on_error=False)
volume_list = heketi_volume_list(self.heketi_client_node,
self.heketi_server_url)
self.assertIn(vol['id'], volume_list)
vol_list.append(vol)
for vol in vol_list:
# delete volume
heketi_volume_delete(self.heketi_client_node,
self.heketi_server_url, vol['id'])
volume_list = heketi_volume_list(self.heketi_client_node,
self.heketi_server_url)
self.assertNotIn(vol['id'], volume_list)
self.verify_volume_count()
def test_create_vol_and_retrieve_topology_info(self):
volume_names = []
volume_ids = []
# Create 3 volumes and make 3rd volume of type distributed replica
g.log.info("Creating 3 volumes")
for i in range(3):
out = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
self.volume_size,
json=True)
g.log.info("Heketi volume %s successfully created" % out)
volume_names.append(out["name"])
volume_ids.append(out["bricks"][0]["volume"])
self.addCleanup(heketi_volume_delete,
self.heketi_client_node,
self.heketi_server_url,
volume_ids[i],
raise_on_error=(i == 2))
heketi_volume_expand(self.heketi_client_node, self.heketi_server_url,
volume_ids[1], 1)
# Check if volume is shown in the heketi topology
topology_volumes = get_heketi_volume_and_brick_count_list(
self.heketi_client_node, self.heketi_server_url)
existing_volumes = [v for v, _ in topology_volumes]
for v in volume_names:
self.assertIn(v, existing_volumes)
for v, b_count in topology_volumes:
expected_bricks_count = 6 if v == volume_names[1] else 3
self.assertGreaterEqual(
b_count, expected_bricks_count,
'Bricks number of the %s volume is %s and it is expected '
'to be greater or equal to %s' %
(v, b_count, expected_bricks_count))
# Delete first 2 volumes and verify their deletion in the topology
for vol_id in volume_ids[:2]:
g.log.info("Deleting volume %s" % vol_id)
heketi_volume_delete(self.heketi_client_node,
self.heketi_server_url, vol_id)
topology_volumes = get_heketi_volume_and_brick_count_list(
self.heketi_client_node, self.heketi_server_url)
existing_volumes = [v for v, _ in topology_volumes]
for vol_name in volume_names[:2]:
self.assertNotIn(
vol_name, existing_volumes,
("volume %s shown in the heketi topology after deletion"
"\nTopology info:\n%s" % (vol_name, existing_volumes)))
# Check the existence of third volume
self.assertIn(
volume_names[2], existing_volumes, "volume %s not "
"shown in the heketi topology\nTopology info"
"\n%s" % (volume_ids[2], existing_volumes))
g.log.info("Sucessfully verified the topology info")
def test_restart_heketi_pod(self):
"""Validate restarting heketi pod"""
# create heketi volume
vol_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
size=1,
json=True)
self.assertTrue(vol_info, "Failed to create heketi volume of size 1")
self.addCleanup(heketi_volume_delete,
self.heketi_client_node,
self.heketi_server_url,
vol_info['id'],
raise_on_error=False)
topo_info = heketi_topology_info(self.heketi_client_node,
self.heketi_server_url,
json=True)
# get heketi-pod name
heketi_pod_name = get_pod_name_from_dc(self.ocp_master_node[0],
self.heketi_dc_name)
# delete heketi-pod (it restarts the pod)
oc_delete(self.ocp_master_node[0],
'pod',
heketi_pod_name,
collect_logs=self.heketi_logs_before_delete)
wait_for_resource_absence(self.ocp_master_node[0], 'pod',
heketi_pod_name)
# get new heketi-pod name
heketi_pod_name = get_pod_name_from_dc(self.ocp_master_node[0],
self.heketi_dc_name)
wait_for_pod_be_ready(self.ocp_master_node[0], heketi_pod_name)
# check heketi server is running
self.assertTrue(
hello_heketi(self.heketi_client_node, self.heketi_server_url),
"Heketi server %s is not alive" % self.heketi_server_url)
# compare the topology
new_topo_info = heketi_topology_info(self.heketi_client_node,
self.heketi_server_url,
json=True)
self.assertEqual(
new_topo_info, topo_info, "topology info is not same,"
" difference - %s" % diff(topo_info, new_topo_info))
# create new volume
vol_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
size=2,
json=True)
self.assertTrue(vol_info, "Failed to create heketi volume of size 20")
heketi_volume_delete(self.heketi_client_node, self.heketi_server_url,
vol_info['id'])
def test_heketi_node_states_enable_disable(self):
"""Test node enable and disable functionality
"""
h_client, h_server = self.heketi_client_node, self.heketi_server_url
node_list = heketi_ops.heketi_node_list(h_client, h_server)
online_hosts = []
for node_id in node_list:
node_info = heketi_ops.heketi_node_info(h_client,
h_server,
node_id,
json=True)
if node_info["state"] == "online":
online_hosts.append(node_info)
if len(online_hosts) < 3:
raise self.skipTest(
"This test can run only if online hosts are more than 2")
# Disable n-3 nodes, in case we have n nodes
for node_info in online_hosts[3:]:
node_id = node_info["id"]
heketi_ops.heketi_node_disable(h_client, h_server, node_id)
self.addCleanup(heketi_ops.heketi_node_enable, h_client, h_server,
node_id)
# Create volume when 3 nodes are online
vol_size = 1
vol_info = heketi_ops.heketi_volume_create(h_client,
h_server,
vol_size,
json=True)
self.addCleanup(heketi_ops.heketi_volume_delete, h_client, h_server,
vol_info['id'])
node_id = online_hosts[0]['id']
try:
heketi_ops.heketi_node_disable(h_client, h_server, node_id)
# Try to create a volume, volume creation should fail
with self.assertRaises(AssertionError):
heketi_volume = heketi_ops.heketi_volume_create(
h_client, h_server, vol_size)
self.addCleanup(heketi_ops.heketi_volume_delete, h_client,
h_server, heketi_volume["id"])
finally:
# Enable heketi node
heketi_ops.heketi_node_enable(h_client, h_server, node_id)
# Create volume when heketi node is enabled
vol_info = heketi_ops.heketi_volume_create(h_client,
h_server,
vol_size,
json=True)
heketi_ops.heketi_volume_delete(h_client, h_server, vol_info['id'])
def _cleanup_heketi_volumes(self, existing_volumes):
h_node, h_url = self.heketi_client_node, self.heketi_server_url
volumes = heketi_volume_list(h_node, h_url, json=True).get("volumes")
new_volumes = list(set(volumes) - set(existing_volumes))
for volume in new_volumes:
h_vol_info = heketi_volume_info(h_node, h_url, volume, json=True)
if h_vol_info.get("block"):
for block_vol in (
h_vol_info.get("blockinfo").get("blockvolume")):
heketi_blockvolume_delete(h_node, h_url, block_vol)
heketi_volume_delete(h_node, h_url, volume, raise_on_error=False)
def test_dynamic_provisioning_glusterfile_reclaim_policy_retain(self):
"""Validate retain policy for glusterfs after deletion of pvc"""
if get_openshift_version() < "3.9":
self.skipTest(
"'Reclaim' feature is not supported in OCP older than 3.9")
self.create_storage_class(reclaim_policy='Retain')
self.create_and_wait_for_pvc()
# get the name of the volume
pv_name = get_pv_name_from_pvc(self.node, self.pvc_name)
custom = [
r':.metadata.annotations.'
r'"gluster\.kubernetes\.io\/heketi\-volume\-id"',
r':.spec.persistentVolumeReclaimPolicy'
]
vol_id, reclaim_policy = oc_get_custom_resource(
self.node, 'pv', custom, pv_name)
self.assertEqual(reclaim_policy, 'Retain')
# Create DC with POD and attached PVC to it.
try:
dc_name = oc_create_app_dc_with_io(
self.node, self.pvc_name, image=self.io_container_image_cirros)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
finally:
scale_dc_pod_amount_and_wait(self.node, dc_name, 0)
oc_delete(self.node, 'dc', dc_name)
wait_for_resource_absence(self.node, 'pod', pod_name)
oc_delete(self.node, 'pvc', self.pvc_name)
with self.assertRaises(ExecutionError):
wait_for_resource_absence(self.node,
'pvc',
self.pvc_name,
interval=3,
timeout=30)
heketi_volume_delete(self.heketi_client_node, self.heketi_server_url,
vol_id)
vol_list = heketi_volume_list(self.heketi_client_node,
self.heketi_server_url)
self.assertNotIn(vol_id, vol_list)
oc_delete(self.node, 'pv', pv_name)
wait_for_resource_absence(self.node, 'pv', pv_name)
def test_block_host_volume_delete_without_block_volumes(self):
"""Validate deletion of empty block hosting volume"""
block_host_create_info = heketi_volume_create(
self.heketi_client_node, self.heketi_server_url, 1, json=True,
block=True)
block_hosting_vol_id = block_host_create_info["id"]
self.addCleanup(
heketi_volume_delete, self.heketi_client_node,
self.heketi_server_url, block_hosting_vol_id, raise_on_error=False)
heketi_volume_delete(
self.heketi_client_node, self.heketi_server_url,
block_hosting_vol_id, json=True)
def test_restart_heketi_pod(self):
"""Validate restarting heketi pod"""
# create heketi volume
vol_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
size=1, json=True)
self.assertTrue(vol_info, "Failed to create heketi volume of size 1")
self.addCleanup(
heketi_volume_delete, self.heketi_client_node,
self.heketi_server_url, vol_info['id'], raise_on_error=False)
topo_info = heketi_topology_info(self.heketi_client_node,
self.heketi_server_url,
json=True)
# get heketi-pod name
heketi_pod_name = get_pod_name_from_dc(self.ocp_master_node[0],
self.heketi_dc_name)
# delete heketi-pod (it restarts the pod)
oc_delete(self.ocp_master_node[0], 'pod', heketi_pod_name)
wait_for_resource_absence(self.ocp_master_node[0],
'pod', heketi_pod_name)
# get new heketi-pod name
heketi_pod_name = get_pod_name_from_dc(self.ocp_master_node[0],
self.heketi_dc_name)
wait_for_pod_be_ready(self.ocp_master_node[0],
heketi_pod_name)
# check heketi server is running
self.assertTrue(
hello_heketi(self.heketi_client_node, self.heketi_server_url),
"Heketi server %s is not alive" % self.heketi_server_url
)
# compare the topology
new_topo_info = heketi_topology_info(self.heketi_client_node,
self.heketi_server_url,
json=True)
self.assertEqual(new_topo_info, topo_info, "topology info is not same,"
" difference - %s" % diff(topo_info, new_topo_info))
# create new volume
vol_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
size=2, json=True)
self.assertTrue(vol_info, "Failed to create heketi volume of size 20")
heketi_volume_delete(
self.heketi_client_node, self.heketi_server_url, vol_info['id'])
def test_validate_brick_paths_on_gluster_pods_or_nodes(self):
"""Validate brick paths after creation and deletion of a volume."""
# Create heketi volume
vol = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
size=1,
json=True)
self.assertTrue(vol, "Failed to create 1Gb heketi volume")
vol_id = vol["bricks"][0]["volume"]
self.addCleanup(heketi_volume_delete,
self.heketi_client_node,
self.heketi_server_url,
vol_id,
raise_on_error=False)
# Gather brick paths
brick_paths = [p['path'] for p in vol["bricks"]]
# Make sure that volume's brick paths exist in the fstab files
self._find_bricks(brick_paths, present=True)
# Delete heketi volume
out = heketi_volume_delete(self.heketi_client_node,
self.heketi_server_url, vol_id)
self.assertTrue(out, "Failed to delete heketi volume %s" % vol_id)
# Make sure that volume's brick paths are absent in the fstab file
self._find_bricks(brick_paths, present=False)
def test_validate_brick_paths_on_gluster_pods_or_nodes(self):
"""Validate brick paths after creation and deletion of a volume."""
# Create heketi volume
vol = heketi_volume_create(
self.heketi_client_node, self.heketi_server_url, size=1, json=True)
self.assertTrue(vol, "Failed to create 1Gb heketi volume")
vol_id = vol["bricks"][0]["volume"]
self.addCleanup(
heketi_volume_delete,
self.heketi_client_node, self.heketi_server_url, vol_id,
raise_on_error=False)
# Gather brick paths
brick_paths = [p['path'] for p in vol["bricks"]]
# Make sure that volume's brick paths exist in the fstab files
self._find_bricks(brick_paths, present=True)
# Delete heketi volume
out = heketi_volume_delete(
self.heketi_client_node, self.heketi_server_url, vol_id)
self.assertTrue(out, "Failed to delete heketi volume %s" % vol_id)
# Make sure that volume's brick paths are absent in the fstab file
self._find_bricks(brick_paths, present=False)
def test_dynamic_provisioning_glusterfile_reclaim_policy_retain(self):
"""Validate retain policy for glusterfs after deletion of pvc"""
if get_openshift_version() < "3.9":
self.skipTest(
"'Reclaim' feature is not supported in OCP older than 3.9")
self.create_storage_class(reclaim_policy='Retain')
self.create_and_wait_for_pvc()
# get the name of the volume
pv_name = get_pv_name_from_pvc(self.node, self.pvc_name)
custom = [r':.metadata.annotations.'
r'"gluster\.kubernetes\.io\/heketi\-volume\-id"',
r':.spec.persistentVolumeReclaimPolicy']
vol_id, reclaim_policy = oc_get_custom_resource(
self.node, 'pv', custom, pv_name)
self.assertEqual(reclaim_policy, 'Retain')
# Create DC with POD and attached PVC to it.
try:
dc_name = oc_create_app_dc_with_io(self.node, self.pvc_name)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
finally:
scale_dc_pod_amount_and_wait(self.node, dc_name, 0)
oc_delete(self.node, 'dc', dc_name)
wait_for_resource_absence(self.node, 'pod', pod_name)
oc_delete(self.node, 'pvc', self.pvc_name)
with self.assertRaises(ExecutionError):
wait_for_resource_absence(
self.node, 'pvc', self.pvc_name, interval=3, timeout=30)
heketi_volume_delete(self.heketi_client_node,
self.heketi_server_url, vol_id)
vol_list = heketi_volume_list(self.heketi_client_node,
self.heketi_server_url)
self.assertNotIn(vol_id, vol_list)
oc_delete(self.node, 'pv', pv_name)
wait_for_resource_absence(self.node, 'pv', pv_name)
def test_block_host_volume_delete_without_block_volumes(self):
"""Validate deletion of empty block hosting volume"""
block_host_create_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
1,
json=True,
block=True)
block_hosting_vol_id = block_host_create_info["id"]
self.addCleanup(heketi_volume_delete,
self.heketi_client_node,
self.heketi_server_url,
block_hosting_vol_id,
raise_on_error=False)
heketi_volume_delete(self.heketi_client_node,
self.heketi_server_url,
block_hosting_vol_id,
json=True)
def test_delete_heketi_volume(self):
"""
Method to test heketi volume deletion and whether it
frees up used space after deletion
"""
creation_output_dict = heketi_ops.heketi_volume_create(
self.heketi_client_node, self.heketi_server_url, 10, json=True)
volume_id = creation_output_dict["name"].strip().split("_")[1]
free_space_after_creation = self.get_free_space_summary_devices()
heketi_ops.heketi_volume_delete(self.heketi_client_node,
self.heketi_server_url, volume_id)
free_space_after_deletion = self.get_free_space_summary_devices()
self.assertTrue(
free_space_after_deletion > free_space_after_creation,
"Free space is not reclaimed after deletion of %s" % volume_id)
def test_delete_heketi_volume(self):
"""
Method to test heketi volume deletion and whether it
frees up used space after deletion
"""
creation_output_dict = heketi_ops.heketi_volume_create(
self.heketi_client_node,
self.heketi_server_url, 10, json=True)
volume_id = creation_output_dict["name"].strip().split("_")[1]
free_space_after_creation = self.get_free_space_summary_devices()
heketi_ops.heketi_volume_delete(
self.heketi_client_node, self.heketi_server_url, volume_id)
free_space_after_deletion = self.get_free_space_summary_devices()
self.assertTrue(
free_space_after_deletion > free_space_after_creation,
"Free space is not reclaimed after deletion of %s" % volume_id)
def test_heketi_metrics_validating_vol_count_on_vol_deletion(self):
"""Validate heketi metrics VolumeCount after volume deletion"""
vol_list = []
for i in range(3):
# Create volume
vol = heketi_volume_create(
self.heketi_client_node,
self.heketi_server_url, 1, json=True)
self.assertTrue(vol)
self.addCleanup(
heketi_volume_delete,
self.heketi_client_node,
self.heketi_server_url,
vol['id'],
raise_on_error=False)
volume_list = heketi_volume_list(
self.heketi_client_node,
self.heketi_server_url)
self.assertIn(vol['id'], volume_list)
vol_list.append(vol)
for vol in vol_list:
# delete volume
heketi_volume_delete(
self.heketi_client_node,
self.heketi_server_url,
vol['id'])
volume_list = heketi_volume_list(
self.heketi_client_node,
self.heketi_server_url)
self.assertNotIn(vol['id'], volume_list)
self.verify_volume_count()
def _cleanup_heketi_volumes(self, existing_volumes):
"""Cleanup created BHV and BV"""
volumes = heketi_ops.heketi_volume_list(self.h_node,
self.h_server,
json=True).get("volumes")
new_volumes = list(set(volumes) - set(existing_volumes))
for volume in new_volumes:
h_vol_info = heketi_ops.heketi_volume_info(self.h_node,
self.h_server,
volume,
json=True)
if h_vol_info.get("block"):
for block_vol in (
h_vol_info.get("blockinfo").get("blockvolume")):
heketi_ops.heketi_blockvolume_delete(self.h_node,
self.h_server,
block_vol,
raise_on_error=False)
heketi_ops.heketi_volume_delete(self.h_node,
self.h_server,
volume,
raise_on_error=False)
def test_delete_heketidb_volume(self):
"""
Method to test heketidb volume deletion via heketi-cli
"""
heketidbexists = False
msg = "Error: Cannot delete volume containing the Heketi database"
for i in range(0, 2):
volume_info = heketi_ops.heketi_volume_create(
self.heketi_client_node, self.heketi_server_url,
10, json=True)
self.addCleanup(
heketi_ops.heketi_volume_delete, self.heketi_client_node,
self.heketi_server_url, volume_info["id"])
volume_list_info = heketi_ops.heketi_volume_list(
self.heketi_client_node,
self.heketi_server_url, json=True)
if volume_list_info["volumes"] == []:
raise ExecutionError("Heketi volume list empty")
for volume_id in volume_list_info["volumes"]:
volume_info = heketi_ops.heketi_volume_info(
self.heketi_client_node, self.heketi_server_url,
volume_id, json=True)
if volume_info["name"] == "heketidbstorage":
heketidbexists = True
delete_ret, delete_output, delete_error = (
heketi_ops.heketi_volume_delete(
self.heketi_client_node,
self.heketi_server_url, volume_id,
raw_cli_output=True))
self.assertNotEqual(delete_ret, 0, "Return code not 0")
self.assertEqual(
delete_error.strip(), msg,
"Invalid reason for heketidb deletion failure")
if not heketidbexists:
raise ExecutionError(
"Warning: heketidbstorage doesn't exist in list of volumes")
def test_volume_expansion_no_free_space(self):
"""Validate volume expansion when there is no free space"""
vol_size, expand_size, additional_devices_attached = None, 10, {}
h_node, h_server_url = self.heketi_client_node, self.heketi_server_url
# Get nodes info
heketi_node_id_list = heketi_ops.heketi_node_list(h_node, h_server_url)
if len(heketi_node_id_list) < 3:
self.skipTest("3 Heketi nodes are required.")
# Disable 4th and other nodes
for node_id in heketi_node_id_list[3:]:
heketi_ops.heketi_node_disable(h_node, h_server_url, node_id)
self.addCleanup(
heketi_ops.heketi_node_enable, h_node, h_server_url, node_id)
# Prepare first 3 nodes
smallest_size = None
err_msg = ''
for node_id in heketi_node_id_list[0:3]:
node_info = heketi_ops.heketi_node_info(
h_node, h_server_url, node_id, json=True)
# Disable second and other devices
devices = node_info["devices"]
self.assertTrue(
devices, "Node '%s' does not have devices." % node_id)
if devices[0]["state"].strip().lower() != "online":
self.skipTest("Test expects first device to be enabled.")
if (smallest_size is None or
devices[0]["storage"]["free"] < smallest_size):
smallest_size = devices[0]["storage"]["free"]
for device in node_info["devices"][1:]:
heketi_ops.heketi_device_disable(
h_node, h_server_url, device["id"])
self.addCleanup(
heketi_ops.heketi_device_enable,
h_node, h_server_url, device["id"])
# Gather info about additional devices
additional_device_name = None
for gluster_server in self.gluster_servers:
gluster_server_data = self.gluster_servers_info[gluster_server]
g_manage = gluster_server_data["manage"]
g_storage = gluster_server_data["storage"]
if not (g_manage in node_info["hostnames"]["manage"] or
g_storage in node_info["hostnames"]["storage"]):
continue
additional_device_name = ((
gluster_server_data.get("additional_devices") or [''])[0])
break
if not additional_device_name:
err_msg += ("No 'additional_devices' are configured for "
"'%s' node, which has following hostnames and "
"IP addresses: %s.\n" % (
node_id,
', '.join(node_info["hostnames"]["manage"] +
node_info["hostnames"]["storage"])))
continue
heketi_ops.heketi_device_add(
h_node, h_server_url, additional_device_name, node_id)
additional_devices_attached.update(
{node_id: additional_device_name})
# Schedule cleanup of the added devices
for node_id in additional_devices_attached.keys():
node_info = heketi_ops.heketi_node_info(
h_node, h_server_url, node_id, json=True)
for device in node_info["devices"]:
if device["name"] != additional_devices_attached[node_id]:
continue
self.addCleanup(self.detach_devices_attached, device["id"])
break
else:
self.fail("Could not find ID for added device on "
"'%s' node." % node_id)
if err_msg:
self.skipTest(err_msg)
# Temporary disable new devices
self.disable_devices(additional_devices_attached)
# Create volume and save info about it
vol_size = int(smallest_size / (1024**2)) - 1
creation_info = heketi_ops.heketi_volume_create(
h_node, h_server_url, vol_size, json=True)
volume_name, volume_id = creation_info["name"], creation_info["id"]
self.addCleanup(
heketi_ops.heketi_volume_delete,
h_node, h_server_url, volume_id, raise_on_error=False)
volume_info_before_expansion = heketi_ops.heketi_volume_info(
h_node, h_server_url, volume_id, json=True)
num_of_bricks_before_expansion = self.get_num_of_bricks(volume_name)
self.get_brick_and_volume_status(volume_name)
free_space_before_expansion = self.get_devices_summary_free_space()
# Try to expand volume with not enough device space
self.assertRaises(
ExecutionError, heketi_ops.heketi_volume_expand,
h_node, h_server_url, volume_id, expand_size)
# Enable new devices to be able to expand our volume
self.enable_devices(additional_devices_attached)
# Expand volume and validate results
heketi_ops.heketi_volume_expand(
h_node, h_server_url, volume_id, expand_size, json=True)
free_space_after_expansion = self.get_devices_summary_free_space()
self.assertGreater(
free_space_before_expansion, free_space_after_expansion,
"Free space not consumed after expansion of %s" % volume_id)
num_of_bricks_after_expansion = self.get_num_of_bricks(volume_name)
self.get_brick_and_volume_status(volume_name)
volume_info_after_expansion = heketi_ops.heketi_volume_info(
h_node, h_server_url, volume_id, json=True)
self.assertGreater(
volume_info_after_expansion["size"],
volume_info_before_expansion["size"],
"Size of %s not increased" % volume_id)
self.assertGreater(
num_of_bricks_after_expansion, num_of_bricks_before_expansion)
self.assertEqual(
num_of_bricks_after_expansion % num_of_bricks_before_expansion, 0)
# Delete volume and validate release of the used space
heketi_ops.heketi_volume_delete(h_node, h_server_url, volume_id)
free_space_after_deletion = self.get_devices_summary_free_space()
self.assertGreater(
free_space_after_deletion, free_space_after_expansion,
"Free space not reclaimed after deletion of volume %s" % volume_id)
def test_verify_delete_heketi_volumes_pending_entries_in_db(
self, vol_type):
"""Verify pending entries of blockvolumes/volumes and bricks in heketi
db during blockvolume/volume delete operation.
"""
# Create a large volumes to observe the pending operation
vol_count, volume_ids, async_obj = 10, [], []
h_node, h_url = self.heketi_client_node, self.heketi_server_url
# Verify file/block volumes pending operation before creation,
h_db_check_before = heketi_db_check(h_node, h_url)
h_db_check_bricks_before = h_db_check_before.get("bricks")
h_db_check_vol_before = (h_db_check_before.get(
"{}volumes".format(vol_type)))
# Get existing heketi volume list
existing_volumes = heketi_volume_list(h_node, h_url, json=True)
# Add cleanup function to clean stale volumes created during test
self.addCleanup(self._cleanup_heketi_volumes,
existing_volumes.get("volumes"))
# Delete heketi pod to clean db operations
if (h_db_check_bricks_before.get("pending")
or h_db_check_vol_before.get("pending")):
self._respin_heketi_pod()
# Calculate heketi volume size
free_space, nodenum = get_total_free_space(h_node, h_url)
free_space_available = int(free_space / nodenum)
if free_space_available > vol_count:
h_volume_size = int(free_space_available / vol_count)
if h_volume_size > 50:
h_volume_size = 50
else:
h_volume_size, vol_count = 1, free_space_available
# Create BHV in case blockvolume size is greater than default BHV size
if vol_type:
default_bhv_size = get_default_block_hosting_volume_size(
h_node, self.heketi_dc_name)
if default_bhv_size < h_volume_size:
h_volume_name = "autotest-{}".format(utils.get_random_str())
bhv_info = self.create_heketi_volume_with_name_and_wait(
h_volume_name,
free_space_available,
raise_on_cleanup_error=False,
block=True,
json=True)
free_space_available -= (
int(bhv_info.get("blockinfo").get("reservedsize")) + 1)
h_volume_size = int(free_space_available / vol_count)
# Create file/block volumes
for _ in range(vol_count):
vol_id = eval("heketi_{}volume_create".format(vol_type))(
h_node, h_url, h_volume_size, json=True).get("id")
volume_ids.append(vol_id)
self.addCleanup(eval("heketi_{}volume_delete".format(vol_type)),
h_node,
h_url,
vol_id,
raise_on_error=False)
def run_async(cmd, hostname, raise_on_error=True):
async_op = g.run_async(host=hostname, command=cmd)
async_obj.append(async_op)
return async_op
bhv_list = []
for vol_id in volume_ids:
# Get BHV ids to delete in case of block volumes
if vol_type:
vol_info = (heketi_blockvolume_info(h_node,
h_url,
vol_id,
json=True))
if not vol_info.get("blockhostingvolume") in bhv_list:
bhv_list.append(vol_info.get("blockhostingvolume"))
# Temporary replace g.run with g.async_run in heketi_volume_delete
# and heketi_blockvolume_delete func to be able to run it in
# background.
with mock.patch.object(command, 'cmd_run', side_effect=run_async):
eval("heketi_{}volume_delete".format(vol_type))(h_node, h_url,
vol_id)
# Wait for pending operations to get generate
for w in waiter.Waiter(timeout=30, interval=3):
h_db_check = heketi_db_check(h_node, h_url)
h_db_check_vol = h_db_check.get("{}volumes".format(vol_type))
if h_db_check_vol.get("pending"):
h_db_check_bricks = h_db_check.get("bricks")
break
if w.expired:
raise exceptions.ExecutionError(
"No any pending operations found during {}volumes deletion "
"{}".format(vol_type, h_db_check_vol.get("pending")))
# Verify bricks pending operation during creation
if not vol_type:
self.assertTrue(h_db_check_bricks.get("pending"),
"Expecting at least one bricks pending count")
self.assertFalse(
h_db_check_bricks.get("pending") % 3,
"Expecting bricks pending count to be multiple of 3 but "
"found {}".format(h_db_check_bricks.get("pending")))
# Verify file/block volume pending operation during delete
for w in waiter.Waiter(timeout=120, interval=10):
h_db_check = heketi_db_check(h_node, h_url)
h_db_check_vol = h_db_check.get("{}volumes".format(vol_type))
h_db_check_bricks = h_db_check.get("bricks")
if ((not h_db_check_bricks.get("pending"))
and (not h_db_check_vol.get("pending"))):
break
if w.expired:
raise AssertionError(
"Failed to delete {}volumes after 120 secs".format(vol_type))
# Check that all background processes got exited
for obj in async_obj:
ret, out, err = obj.async_communicate()
self.assertFalse(
ret, "Failed to delete {}volume due to error: {}".format(
vol_type, err))
# Delete BHV created during block volume creation
if vol_type:
for bhv_id in bhv_list:
heketi_volume_delete(h_node, h_url, bhv_id)
# Verify bricks and volume pending operations
h_db_check_after = heketi_db_check(h_node, h_url)
h_db_check_bricks_after = h_db_check_after.get("bricks")
h_db_check_vol_after = (h_db_check_after.get(
"{}volumes".format(vol_type)))
act_brick_count = h_db_check_bricks_after.get("pending")
act_vol_count = h_db_check_vol_after.get("pending")
# Verify bricks pending operation after delete
err_msg = "{} operations are pending for {} after {}volume deletion"
if not vol_type:
self.assertFalse(
act_brick_count,
err_msg.format(act_brick_count, "brick", vol_type))
# Verify file/bock volumes pending operation after delete
self.assertFalse(act_vol_count,
err_msg.format(act_vol_count, "volume", vol_type))
act_brick_count = h_db_check_bricks_after.get("total")
act_vol_count = h_db_check_vol_after.get("total")
exp_brick_count = h_db_check_bricks_before.get("total")
exp_vol_count = h_db_check_vol_before.get("total")
err_msg = "Actual {} and expected {} {} counts are not matched"
# Verify if initial and final file/block volumes are same
self.assertEqual(
act_vol_count, exp_vol_count,
err_msg.format(act_vol_count, exp_vol_count, "volume"))
# Verify if initial and final bricks are same
self.assertEqual(
act_brick_count, exp_brick_count,
err_msg.format(act_brick_count, exp_brick_count, "brick"))
def test_heketi_server_stale_operations_during_heketi_pod_reboot(self):
"""
Validate failed/stale entries in db and performs a cleanup
of those entries
"""
volume_id_list, async_obj, ocp_node = [], [], self.ocp_master_node[0]
h_node, h_server = self.heketi_client_node, self.heketi_server_url
for i in range(0, 8):
volume_info = heketi_ops.heketi_volume_create(h_node,
h_server,
1,
json=True)
volume_id_list.append(volume_info["id"])
self.addCleanup(heketi_ops.heketi_volume_delete,
h_node,
h_server,
volume_info["id"],
raise_on_error=False)
def run_async(cmd, hostname, raise_on_error=True):
async_op = g.run_async(host=hostname, command=cmd)
async_obj.append(async_op)
return async_op
# Temporary replace g.run with g.async_run in heketi_volume_delete
# to be able to run it in background.
for vol_id in volume_id_list:
with mock.patch.object(command, 'cmd_run', side_effect=run_async):
heketi_ops.heketi_volume_delete(h_node, h_server, vol_id)
# Restart heketi pod and check pod is running
heketi_pod_name = openshift_ops.get_pod_name_from_dc(
ocp_node, self.heketi_dc_name)
openshift_ops.oc_delete(ocp_node,
'pod',
heketi_pod_name,
collect_logs=self.heketi_logs_before_delete)
self.addCleanup(self._heketi_pod_delete_cleanup, ocp_node)
openshift_ops.wait_for_resource_absence(ocp_node, 'pod',
heketi_pod_name)
heketi_pod_name = openshift_ops.get_pod_name_from_dc(
ocp_node, self.heketi_dc_name)
openshift_ops.wait_for_pod_be_ready(ocp_node, heketi_pod_name)
self.assertTrue(heketi_ops.hello_heketi(h_node, h_server),
"Heketi server {} is not alive".format(h_server))
# Wait for pending operations to get generate
for w in waiter.Waiter(timeout=30, interval=3):
h_db_check = heketi_ops.heketi_db_check(h_node, h_server)
h_db_check_vol = h_db_check.get("volumes")
h_db_check_bricks = h_db_check.get("bricks")
if ((h_db_check_vol.get("pending"))
and (h_db_check_bricks.get("pending"))):
break
if w.expired:
raise exceptions.ExecutionError(
"No any pending operations found during volumes deletion "
"volumes:{}, Bricks:{} ".format(
h_db_check_vol.get("pending"),
h_db_check_bricks.get("pending")))
# Verify pending bricks are multiples of 3
self.assertFalse(
h_db_check_bricks.get("pending") % 3,
"Expecting bricks pending count to be multiple of 3 but "
"found {}".format(h_db_check_bricks.get("pending")))
# Verify and Wait for pending operations to complete
for w in waiter.Waiter(timeout=120, interval=10):
h_db_check = heketi_ops.heketi_db_check(h_node, h_server)
h_db_check_vol = h_db_check.get("volumes")
h_db_check_bricks = h_db_check.get("bricks")
if ((not h_db_check_bricks.get("pending"))
and (not h_db_check_vol.get("pending"))):
break
if w.expired:
raise AssertionError("Failed to delete volumes after 120 secs")
def test_targetcli_when_block_hosting_volume_down(self):
"""Validate no inconsistencies occur in targetcli when block volumes
are created with one block hosting volume down."""
h_node, h_server = self.heketi_client_node, self.heketi_server_url
cmd = ("targetcli ls | egrep '%s' || echo unavailable")
error_msg = ("targetcli has inconsistencies when block devices are "
"created with one block hosting volume %s is down")
# Delete BHV which has no BV or fill it completely
bhv_list = get_block_hosting_volume_list(h_node, h_server).keys()
for bhv in bhv_list:
bhv_info = heketi_volume_info(h_node, h_server, bhv, json=True)
if not bhv_info["blockinfo"].get("blockvolume", []):
heketi_volume_delete(h_node, h_server, bhv)
continue
free_size = bhv_info["blockinfo"].get("freesize", 0)
if free_size:
bv = heketi_volume_create(h_node,
h_server,
free_size,
json=True)
self.addCleanup(heketi_volume_delete, h_node, h_server,
bv["id"])
# Create BV
bv = heketi_blockvolume_create(h_node, h_server, 2, json=True)
self.addCleanup(heketi_blockvolume_delete, h_node, h_server, bv["id"])
# Bring down BHV
bhv_name = get_block_hosting_volume_name(h_node, h_server, bv["id"])
ret, out, err = volume_stop("auto_get_gluster_endpoint", bhv_name)
if ret != 0:
err_msg = "Failed to stop gluster volume %s. error: %s" % (
bhv_name, err)
g.log.error(err_msg)
raise AssertionError(err_msg)
self.addCleanup(podcmd.GlustoPod()(volume_start),
"auto_get_gluster_endpoint", bhv_name)
ocp_node = self.ocp_master_node[0]
gluster_block_svc = "gluster-block-target"
self.addCleanup(wait_for_service_status_on_gluster_pod_or_node,
ocp_node,
gluster_block_svc,
"active",
"exited",
gluster_node=self.gluster_servers[0])
self.addCleanup(restart_service_on_gluster_pod_or_node, ocp_node,
gluster_block_svc, self.gluster_servers[0])
for condition in ("continue", "break"):
restart_service_on_gluster_pod_or_node(
ocp_node,
gluster_block_svc,
gluster_node=self.gluster_servers[0])
wait_for_service_status_on_gluster_pod_or_node(
ocp_node,
gluster_block_svc,
"active",
"exited",
gluster_node=self.gluster_servers[0])
targetcli = cmd_run_on_gluster_pod_or_node(ocp_node,
cmd % bv["id"],
self.gluster_servers[0])
if condition == "continue":
self.assertEqual(targetcli, "unavailable",
error_msg % bhv_name)
else:
self.assertNotEqual(targetcli, "unavailable",
error_msg % bhv_name)
break
# Bring up the same BHV
ret, out, err = volume_start("auto_get_gluster_endpoint", bhv_name)
if ret != 0:
err = "Failed to start gluster volume %s on %s. error: %s" % (
bhv_name, h_node, err)
raise exceptions.ExecutionError(err)
def test_create_max_num_blockhostingvolumes(self):
num_of_bv = 10
new_bhv_list, bv_list, g_nodes = [], [], []
free_space, nodenum = get_total_free_space(self.heketi_client_node,
self.heketi_server_url)
if nodenum < 3:
self.skipTest("Skip the test case since number of"
"online nodes is less than 3.")
free_space_available = int(free_space / nodenum)
default_bhv_size = get_default_block_hosting_volume_size(
self.heketi_client_node, self.heketi_dc_name)
# Get existing list of BHV's
existing_bhv_list = get_block_hosting_volume_list(
self.heketi_client_node, self.heketi_server_url)
# Skip the test if available space is less than default_bhv_size
if free_space_available < default_bhv_size:
self.skipTest("Skip the test case since free_space_available %s"
"is less than space_required_for_bhv %s ." %
(free_space_available, default_bhv_size))
# Create BHV's
while free_space_available > default_bhv_size:
block_host_create_info = heketi_volume_create(
self.heketi_client_node,
self.heketi_server_url,
default_bhv_size,
json=True,
block=True)
if block_host_create_info["id"] not in existing_bhv_list.keys():
new_bhv_list.append(block_host_create_info["id"])
self.addCleanup(heketi_volume_delete,
self.heketi_client_node,
self.heketi_server_url,
block_host_create_info["id"],
raise_on_error=False)
block_vol_size = int(
block_host_create_info["blockinfo"]["freesize"] / num_of_bv)
# Create specified number of BV's in BHV's created
for i in range(0, num_of_bv):
block_vol = heketi_blockvolume_create(self.heketi_client_node,
self.heketi_server_url,
block_vol_size,
json=True,
ha=3,
auth=True)
self.addCleanup(heketi_blockvolume_delete,
self.heketi_client_node,
self.heketi_server_url,
block_vol["id"],
raise_on_error=False)
bv_list.append(block_vol["id"])
free_space_available = int(free_space_available - default_bhv_size)
# Get gluster node ips
h_nodes_ids = heketi_node_list(self.heketi_client_node,
self.heketi_server_url)
for h_node in h_nodes_ids[:2]:
g_node = heketi_node_info(self.heketi_client_node,
self.heketi_server_url,
h_node,
json=True)
g_nodes.append(g_node['hostnames']['manage'][0])
# Check if there is no crash in gluster related services & heketi
services = (("glusterd", "running"), ("gluster-blockd", "running"),
("tcmu-runner", "running"), ("gluster-block-target",
"exited"))
for g_node in g_nodes:
for service, state in services:
wait_for_service_status_on_gluster_pod_or_node(
self.ocp_client[0],
service,
'active',
state,
g_node,
raise_on_error=False)
out = hello_heketi(self.heketi_client_node, self.heketi_server_url)
self.assertTrue(
out, "Heketi server %s is not alive" % self.heketi_server_url)
# Delete all the BHV's and BV's created
for bv_volume in bv_list:
heketi_blockvolume_delete(self.heketi_client_node,
self.heketi_server_url, bv_volume)
# Check if any blockvolume exist in heketi & gluster
for bhv_volume in new_bhv_list[:]:
heketi_vol_info = heketi_volume_info(self.heketi_client_node,
self.heketi_server_url,
bhv_volume,
json=True)
self.assertNotIn("blockvolume",
heketi_vol_info["blockinfo"].keys())
gluster_vol_info = get_block_list('auto_get_gluster_endpoint',
volname="vol_%s" % bhv_volume)
self.assertIsNotNone(gluster_vol_info,
"Failed to get volume info %s" % bhv_volume)
new_bhv_list.remove(bhv_volume)
for blockvol in gluster_vol_info:
self.assertNotIn("blockvol_", blockvol)
heketi_volume_delete(self.heketi_client_node,
self.heketi_server_url, bhv_volume)
# Check if all blockhosting volumes are deleted from heketi
self.assertFalse(new_bhv_list)
def test_volume_expansion_no_free_space(self):
"""Validate volume expansion when there is no free space"""
vol_size, expand_size, additional_devices_attached = None, 10, {}
h_node, h_server_url = self.heketi_client_node, self.heketi_server_url
# Get nodes info
heketi_node_id_list = heketi_ops.heketi_node_list(h_node, h_server_url)
if len(heketi_node_id_list) < 3:
self.skipTest("3 Heketi nodes are required.")
# Disable 4th and other nodes
for node_id in heketi_node_id_list[3:]:
heketi_ops.heketi_node_disable(h_node, h_server_url, node_id)
self.addCleanup(
heketi_ops.heketi_node_enable, h_node, h_server_url, node_id)
# Prepare first 3 nodes
smallest_size = None
err_msg = ''
for node_id in heketi_node_id_list[0:3]:
node_info = heketi_ops.heketi_node_info(
h_node, h_server_url, node_id, json=True)
# Disable second and other devices
devices = node_info["devices"]
self.assertTrue(
devices, "Node '%s' does not have devices." % node_id)
if devices[0]["state"].strip().lower() != "online":
self.skipTest("Test expects first device to be enabled.")
if (smallest_size is None
or devices[0]["storage"]["free"] < smallest_size):
smallest_size = devices[0]["storage"]["free"]
for device in node_info["devices"][1:]:
heketi_ops.heketi_device_disable(
h_node, h_server_url, device["id"])
self.addCleanup(
heketi_ops.heketi_device_enable,
h_node, h_server_url, device["id"])
# Gather info about additional devices
additional_device_name = None
for gluster_server in self.gluster_servers:
gluster_server_data = self.gluster_servers_info[gluster_server]
g_manage = gluster_server_data["manage"]
g_storage = gluster_server_data["storage"]
if not (g_manage in node_info["hostnames"]["manage"]
or g_storage in node_info["hostnames"]["storage"]):
continue
additional_device_name = ((
gluster_server_data.get("additional_devices") or [''])[0])
break
if not additional_device_name:
err_msg += ("No 'additional_devices' are configured for "
"'%s' node, which has following hostnames and "
"IP addresses: %s.\n" % (
node_id,
', '.join(
node_info["hostnames"]["manage"]
+ node_info["hostnames"]["storage"])))
continue
heketi_ops.heketi_device_add(
h_node, h_server_url, additional_device_name, node_id)
additional_devices_attached.update(
{node_id: additional_device_name})
# Schedule cleanup of the added devices
for node_id in additional_devices_attached.keys():
node_info = heketi_ops.heketi_node_info(
h_node, h_server_url, node_id, json=True)
for device in node_info["devices"]:
if device["name"] != additional_devices_attached[node_id]:
continue
self.addCleanup(self.detach_devices_attached, device["id"])
break
else:
self.fail("Could not find ID for added device on "
"'%s' node." % node_id)
if err_msg:
self.skipTest(err_msg)
# Temporary disable new devices
self.disable_devices(additional_devices_attached)
# Create volume and save info about it
vol_size = int(smallest_size / (1024**2)) - 1
creation_info = heketi_ops.heketi_volume_create(
h_node, h_server_url, vol_size, json=True)
volume_name, volume_id = creation_info["name"], creation_info["id"]
self.addCleanup(
heketi_ops.heketi_volume_delete,
h_node, h_server_url, volume_id, raise_on_error=False)
volume_info_before_expansion = heketi_ops.heketi_volume_info(
h_node, h_server_url, volume_id, json=True)
num_of_bricks_before_expansion = self.get_num_of_bricks(volume_name)
self.get_brick_and_volume_status(volume_name)
free_space_before_expansion = self.get_devices_summary_free_space()
# Try to expand volume with not enough device space
self.assertRaises(
AssertionError, heketi_ops.heketi_volume_expand,
h_node, h_server_url, volume_id, expand_size)
# Enable new devices to be able to expand our volume
self.enable_devices(additional_devices_attached)
# Expand volume and validate results
heketi_ops.heketi_volume_expand(
h_node, h_server_url, volume_id, expand_size, json=True)
free_space_after_expansion = self.get_devices_summary_free_space()
self.assertGreater(
free_space_before_expansion, free_space_after_expansion,
"Free space not consumed after expansion of %s" % volume_id)
num_of_bricks_after_expansion = self.get_num_of_bricks(volume_name)
self.get_brick_and_volume_status(volume_name)
volume_info_after_expansion = heketi_ops.heketi_volume_info(
h_node, h_server_url, volume_id, json=True)
self.assertGreater(
volume_info_after_expansion["size"],
volume_info_before_expansion["size"],
"Size of %s not increased" % volume_id)
self.assertGreater(
num_of_bricks_after_expansion, num_of_bricks_before_expansion)
self.assertEqual(
num_of_bricks_after_expansion % num_of_bricks_before_expansion, 0)
# Delete volume and validate release of the used space
heketi_ops.heketi_volume_delete(h_node, h_server_url, volume_id)
free_space_after_deletion = self.get_devices_summary_free_space()
self.assertGreater(
free_space_after_deletion, free_space_after_expansion,
"Free space not reclaimed after deletion of volume %s" % volume_id)
def test_volume_expansion_rebalance_brick(self):
"""Validate volume expansion with brick and check rebalance"""
creation_info = heketi_ops.heketi_volume_create(
self.heketi_client_node, self.heketi_server_url, 10, json=True)
self.assertNotEqual(creation_info, False, "Volume creation failed")
volume_name = creation_info["name"]
volume_id = creation_info["id"]
free_space_after_creation = self.get_devices_summary_free_space()
volume_info_before_expansion = heketi_ops.heketi_volume_info(
self.heketi_client_node,
self.heketi_server_url,
volume_id, json=True)
self.assertNotEqual(volume_info_before_expansion, False,
"Volume info for %s failed" % volume_id)
heketi_vol_info_size_before_expansion = (
volume_info_before_expansion["size"])
self.get_brick_and_volume_status(volume_name)
num_of_bricks_before_expansion = self.get_num_of_bricks(volume_name)
expansion_info = heketi_ops.heketi_volume_expand(
self.heketi_client_node,
self.heketi_server_url,
volume_id, 5)
self.assertNotEqual(expansion_info, False,
"Volume expansion of %s failed" % volume_id)
free_space_after_expansion = self.get_devices_summary_free_space()
self.assertTrue(
free_space_after_creation > free_space_after_expansion,
"Free space not consumed after expansion of %s" % volume_id)
volume_info_after_expansion = heketi_ops.heketi_volume_info(
self.heketi_client_node,
self.heketi_server_url,
volume_id, json=True)
self.assertNotEqual(volume_info_after_expansion, False,
"Volume info failed for %s" % volume_id)
heketi_vol_info_size_after_expansion = (
volume_info_after_expansion["size"])
difference_size = (heketi_vol_info_size_after_expansion -
heketi_vol_info_size_before_expansion)
self.assertTrue(
difference_size > 0,
"Size not increased after expansion of %s" % volume_id)
self.get_brick_and_volume_status(volume_name)
num_of_bricks_after_expansion = self.get_num_of_bricks(volume_name)
num_of_bricks_added = (num_of_bricks_after_expansion -
num_of_bricks_before_expansion)
self.assertEqual(
num_of_bricks_added, 3,
"Number of bricks added is not 3 for %s" % volume_id)
self.get_rebalance_status(volume_name)
deletion_info = heketi_ops.heketi_volume_delete(
self.heketi_client_node, self.heketi_server_url,
volume_id, json=True)
self.assertNotEqual(deletion_info, False,
"Deletion of volume %s failed" % volume_id)
free_space_after_deletion = self.get_devices_summary_free_space()
self.assertTrue(
free_space_after_deletion > free_space_after_expansion,
"Free space is not reclaimed after volume deletion of %s"
% volume_id)
def test_set_heketi_vol_size_and_brick_amount_limits(self):
# Get Heketi secret name
cmd_get_heketi_secret_name = (
"oc get dc -n %s %s -o jsonpath='{.spec.template.spec.volumes"
"[?(@.name==\"config\")].secret.secretName}'" %
(self.storage_project_name, self.heketi_dc_name))
heketi_secret_name = self.cmd_run(cmd_get_heketi_secret_name)
# Read Heketi secret data
self.node = self.ocp_master_node[0]
heketi_secret_data_str_base64 = oc_get_custom_resource(
self.node,
"secret",
":.data.'heketi\.json'", # noqa
name=heketi_secret_name)[0]
heketi_secret_data_str = self.cmd_run("echo %s | base64 -d" %
heketi_secret_data_str_base64)
heketi_secret_data = json.loads(heketi_secret_data_str)
# Update Heketi secret data
brick_min_size_gb, brick_max_size_gb = 2, 4
heketi_secret_data["glusterfs"].update({
"brick_min_size_gb": brick_min_size_gb,
"brick_max_size_gb": brick_max_size_gb,
"max_bricks_per_volume": 3,
})
heketi_secret_data_patched = json.dumps(heketi_secret_data)
heketi_secret_data_str_encoded = self.cmd_run(
"echo '%s' |base64" % heketi_secret_data_patched).replace(
'\n', '')
h_client, h_server = self.heketi_client_node, self.heketi_server_url
try:
# Patch Heketi secret
cmd_patch_heketi_secret = (
'oc patch secret -n %s %s -p '
'"{\\"data\\": {\\"heketi.json\\": \\"%s\\"}}"') % (
self.storage_project_name, heketi_secret_name, "%s")
self.cmd_run(cmd_patch_heketi_secret %
heketi_secret_data_str_encoded)
# Recreate the Heketi pod to make it reuse updated configuration
scale_dc_pod_amount_and_wait(self.node, self.heketi_dc_name, 0)
scale_dc_pod_amount_and_wait(self.node, self.heketi_dc_name, 1)
# Try to create too small and too big volumes
# It must fail because allowed range is not satisfied
for gb in (brick_min_size_gb - 1, brick_max_size_gb + 1):
try:
vol_1 = heketi_volume_create(h_client,
h_server,
size=gb,
json=True)
except AssertionError:
pass
else:
self.addCleanup(heketi_volume_delete, h_client, h_server,
vol_1['id'])
self.assertFalse(
vol_1,
"Volume '%s' got unexpectedly created. Heketi server "
"configuration haven't made required effect." %
(vol_1.get('id', 'failed_to_get_heketi_vol_id')))
# Create the smallest allowed volume
vol_2 = heketi_volume_create(h_client,
h_server,
size=brick_min_size_gb,
json=True)
self.addCleanup(heketi_volume_delete, h_client, h_server,
vol_2['id'])
# Try to expand volume, it must fail due to the brick amount limit
self.assertRaises(AssertionError, heketi_volume_expand, h_client,
h_server, vol_2['id'], 2)
# Create the largest allowed volume
vol_3 = heketi_volume_create(h_client,
h_server,
size=brick_max_size_gb,
json=True)
heketi_volume_delete(h_client, h_server, vol_3['id'])
finally:
# Revert the Heketi configuration back
self.cmd_run(cmd_patch_heketi_secret %
heketi_secret_data_str_base64)
scale_dc_pod_amount_and_wait(self.node, self.heketi_dc_name, 0)
scale_dc_pod_amount_and_wait(self.node, self.heketi_dc_name, 1)
# Create volume less than the old minimum limit
vol_4 = heketi_volume_create(h_client,
h_server,
size=(brick_min_size_gb - 1),
json=True)
self.addCleanup(heketi_volume_delete, h_client, h_server, vol_4['id'])
# Create volume bigger than the old maximum limit and expand it
vol_5 = heketi_volume_create(h_client,
h_server,
size=(brick_max_size_gb + 1),
json=True)
self.addCleanup(heketi_volume_delete, h_client, h_server, vol_5['id'])
heketi_volume_expand(h_client, h_server, vol_5['id'], 2)
def _create_distributed_replica_vol(self, validate_cleanup, block=False):
# Create distributed vol
vol_size_gb = self._get_vol_size()
heketi_url = self.heketi_server_url
h_volume_name = "autotests-heketi-volume-%s" % utils.get_random_str()
try:
heketi_vol = self.create_heketi_volume_with_name_and_wait(
h_volume_name,
vol_size_gb,
json=True,
raise_on_cleanup_error=False,
block=block)
except AssertionError as e:
# NOTE: rare situation when we need to decrease size of a volume.
# and we expect this vol to be distributed.
g.log.info("Failed to create distributed '%s'Gb volume. "
"Trying to create another one, smaller for 1Gb.")
if not ('more required' in str(e) and
('Insufficient suitable allocatable extents for '
'logical volume' in str(e))):
raise
vol_size_gb -= 1
heketi_vol = self.create_heketi_volume_with_name_and_wait(
h_volume_name,
vol_size_gb,
json=True,
raise_on_cleanup_error=False,
block=block)
g.log.info("Successfully created distributed volume.")
vol_name = heketi_vol['name']
vol_id = heketi_vol["bricks"][0]["volume"]
# Get gluster volume info
g.log.info("Get gluster volume '%s' info" % vol_name)
gluster_vol = get_volume_info('auto_get_gluster_endpoint',
volname=vol_name)
self.assertTrue(gluster_vol,
"Failed to get volume '%s' info" % vol_name)
g.log.info("Successfully got volume '%s' info" % vol_name)
gluster_vol = gluster_vol[vol_name]
self.assertEqual(
gluster_vol["typeStr"], "Distributed-Replicate",
"'%s' gluster vol isn't a Distributed-Replicate volume" % vol_name)
# Check amount of bricks
brick_amount = len(gluster_vol['bricks']['brick'])
self.assertEqual(
brick_amount % 3, 0,
"Brick amount is expected to be divisible by 3. "
"Actual amount is '%s'" % brick_amount)
self.assertGreater(
brick_amount, 3, "Brick amount is expected to be bigger than 3. "
"Actual amount is '%s'." % brick_amount)
# Run unique actions to Validate whether deleting a dist-rep
# volume is handled by heketi else return
if not validate_cleanup:
return vol_id
# Get the free space after creating heketi volume
free_space_after_creating_vol = self._get_free_space()
# Delete heketi volume
g.log.info("Deleting heketi volume '%s'" % vol_id)
volume_deleted = heketi_volume_delete(self.heketi_client_node,
heketi_url, vol_id)
self.assertTrue(volume_deleted,
"Failed to delete heketi volume '%s'" % vol_id)
g.log.info("Heketi volume '%s' has successfully been deleted" % vol_id)
# Check the heketi volume list
g.log.info("List heketi volumes")
heketi_volumes = heketi_volume_list(self.heketi_client_node,
self.heketi_server_url,
json=True)
self.assertTrue(heketi_volumes, "Failed to list heketi volumes")
g.log.info("Heketi volumes have successfully been listed")
heketi_volumes = heketi_volumes.get('volumes', heketi_volumes)
self.assertNotIn(vol_id, heketi_volumes)
self.assertNotIn(vol_name, heketi_volumes)
# Check the gluster volume list
g.log.info("Get the gluster volume list")
gluster_volumes = get_volume_list('auto_get_gluster_endpoint')
self.assertTrue(gluster_volumes, "Unable to get Gluster volume list")
g.log.info("Successfully got Gluster volume list" % gluster_volumes)
self.assertNotIn(vol_id, gluster_volumes)
self.assertNotIn(vol_name, gluster_volumes)
# Get the used space after deleting heketi volume
free_space_after_deleting_vol = self._get_free_space()
# Compare the free space before and after deleting the volume
g.log.info("Comparing the free space before and after deleting volume")
self.assertLessEqual(free_space_after_creating_vol + (3 * vol_size_gb),
free_space_after_deleting_vol)
g.log.info(
"Volume successfully deleted and space is reallocated. "
"Free space after creating volume %s. "
"Free space after deleting volume %s." %
(free_space_after_creating_vol, free_space_after_deleting_vol))
def test_volume_expansion_rebalance_brick(self):
"""Validate volume expansion with brick and check rebalance"""
creation_info = heketi_ops.heketi_volume_create(
self.heketi_client_node, self.heketi_server_url, 10, json=True)
self.assertNotEqual(creation_info, False, "Volume creation failed")
volume_name = creation_info["name"]
volume_id = creation_info["id"]
free_space_after_creation = self.get_devices_summary_free_space()
volume_info_before_expansion = heketi_ops.heketi_volume_info(
self.heketi_client_node,
self.heketi_server_url,
volume_id, json=True)
self.assertNotEqual(volume_info_before_expansion, False,
"Volume info for %s failed" % volume_id)
heketi_vol_info_size_before_expansion = (
volume_info_before_expansion["size"])
self.get_brick_and_volume_status(volume_name)
num_of_bricks_before_expansion = self.get_num_of_bricks(volume_name)
expansion_info = heketi_ops.heketi_volume_expand(
self.heketi_client_node,
self.heketi_server_url,
volume_id, 5)
self.assertNotEqual(expansion_info, False,
"Volume expansion of %s failed" % volume_id)
free_space_after_expansion = self.get_devices_summary_free_space()
self.assertTrue(
free_space_after_creation > free_space_after_expansion,
"Free space not consumed after expansion of %s" % volume_id)
volume_info_after_expansion = heketi_ops.heketi_volume_info(
self.heketi_client_node,
self.heketi_server_url,
volume_id, json=True)
self.assertNotEqual(volume_info_after_expansion, False,
"Volume info failed for %s" % volume_id)
heketi_vol_info_size_after_expansion = (
volume_info_after_expansion["size"])
difference_size = (heketi_vol_info_size_after_expansion
- heketi_vol_info_size_before_expansion)
self.assertTrue(
difference_size > 0,
"Size not increased after expansion of %s" % volume_id)
self.get_brick_and_volume_status(volume_name)
num_of_bricks_after_expansion = self.get_num_of_bricks(volume_name)
num_of_bricks_added = (
num_of_bricks_after_expansion - num_of_bricks_before_expansion)
self.assertEqual(
num_of_bricks_added, 3,
"Number of bricks added is not 3 for %s" % volume_id)
self.get_rebalance_status(volume_name)
deletion_info = heketi_ops.heketi_volume_delete(
self.heketi_client_node, self.heketi_server_url,
volume_id, json=True)
self.assertNotEqual(deletion_info, False,
"Deletion of volume %s failed" % volume_id)
free_space_after_deletion = self.get_devices_summary_free_space()
self.assertTrue(
free_space_after_deletion > free_space_after_expansion,
"Free space is not reclaimed after volume deletion of %s"
% volume_id)
def test_volume_creation_of_size_greater_than_the_device_size(self):
"""Validate creation of a volume of size greater than the size of a
device.
"""
h_node, h_url = self.heketi_client_node, self.heketi_server_url
# Remove existing BHV to calculate freespace
bhv_list = heketi_ops.get_block_hosting_volume_list(h_node, h_url)
if bhv_list:
for bhv in bhv_list:
bhv_info = heketi_ops.heketi_volume_info(h_node,
h_url,
bhv,
json=True)
if bhv_info['blockinfo'].get('blockvolume') is None:
heketi_ops.heketi_volume_delete(h_node, h_url, bhv)
topology = heketi_ops.heketi_topology_info(h_node, h_url, json=True)
nodes_free_space, nodes_ips = [], []
selected_nodes, selected_devices = [], []
cluster = topology['clusters'][0]
node_count = len(cluster['nodes'])
msg = ("At least 3 Nodes are required in cluster. "
"But only %s Nodes are present." % node_count)
if node_count < 3:
self.skipTest(msg)
online_nodes_count = 0
for node in cluster['nodes']:
nodes_ips.append(node['hostnames']['storage'][0])
if node['state'] != 'online':
continue
online_nodes_count += 1
# Disable nodes after 3rd online nodes
if online_nodes_count > 3:
heketi_ops.heketi_node_disable(h_node, h_url, node['id'])
self.addCleanup(heketi_ops.heketi_node_enable, h_node, h_url,
node['id'])
continue
selected_nodes.append(node['id'])
device_count = len(node['devices'])
msg = ("At least 2 Devices are required on each Node."
"But only %s Devices are present." % device_count)
if device_count < 2:
self.skipTest(msg)
sel_devices, online_devices_count, free_space = [], 0, 0
for device in node['devices']:
if device['state'] != 'online':
continue
online_devices_count += 1
# Disable devices after 2nd online devices
if online_devices_count > 2:
heketi_ops.heketi_device_disable(h_node, h_url,
device['id'])
self.addCleanup(heketi_ops.heketi_device_enable, h_node,
h_url, device['id'])
continue
sel_devices.append(device['id'])
free_space += int(device['storage']['free'] / (1024**2))
selected_devices.append(sel_devices)
nodes_free_space.append(free_space)
msg = ("At least 2 online Devices are required on each Node. "
"But only %s Devices are online on Node: %s." %
(online_devices_count, node['id']))
if online_devices_count < 2:
self.skipTest(msg)
msg = ("At least 3 online Nodes are required in cluster. "
"But only %s Nodes are online in Cluster: %s." %
(online_nodes_count, cluster['id']))
if online_nodes_count < 3:
self.skipTest(msg)
# Select node with minimum free space
min_free_size = min(nodes_free_space)
index = nodes_free_space.index(min_free_size)
# Get max device size from selected node
device_size = 0
for device in selected_devices[index]:
device_info = heketi_ops.heketi_device_info(h_node,
h_url,
device,
json=True)
device_size = max(device_size,
(int(device_info['storage']['total'] /
(1024**2))))
vol_size = device_size + 1
if vol_size >= min_free_size:
self.skipTest('Required free space %s is not available' % vol_size)
# Create heketi volume with device size + 1
vol_info = self.create_heketi_volume_with_name_and_wait(
name="volume_size_greater_than_device_size",
size=vol_size,
json=True)
# Get gluster server IP's from heketi volume info
glusterfs_servers = heketi_ops.get_vol_file_servers_and_hosts(
h_node, h_url, vol_info['id'])
# Verify gluster server IP's in heketi volume info
msg = ("gluster IP's '%s' does not match with IP's '%s' found in "
"heketi volume info" %
(nodes_ips, glusterfs_servers['vol_servers']))
self.assertEqual(set(glusterfs_servers['vol_servers']), set(nodes_ips),
msg)
vol_name = vol_info['name']
gluster_v_info = self.get_gluster_vol_info(vol_name)
# Verify replica count in gluster v info
msg = "Volume %s is replica %s instead of replica 3" % (
vol_name, gluster_v_info['replicaCount'])
self.assertEqual('3', gluster_v_info['replicaCount'])
# Verify distCount in gluster v info
msg = "Volume %s distCount is %s instead of distCount as 3" % (
vol_name, int(gluster_v_info['distCount']))
self.assertEqual(
int(gluster_v_info['brickCount']) // 3,
int(gluster_v_info['distCount']), msg)
# Verify bricks count in gluster v info
msg = (
"Volume %s does not have bricks count multiple of 3. It has %s" %
(vol_name, gluster_v_info['brickCount']))
self.assertFalse(int(gluster_v_info['brickCount']) % 3, msg)
