def test_heketi_metrics_validating_vol_count_on_vol_deletion(self):
"""Validate heketi metrics VolumeCount after volume deletion"""
vol_list = []
for i in range(3):
# Create volume
vol = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
1,
json=True)
self.assertTrue(vol)
self.addCleanup(heketi_volume_delete,
self.heketi_client_node,
self.heketi_server_url,
vol['id'],
raise_on_error=False)
volume_list = heketi_volume_list(self.heketi_client_node,
self.heketi_server_url)
self.assertIn(vol['id'], volume_list)
vol_list.append(vol)
for vol in vol_list:
# delete volume
heketi_volume_delete(self.heketi_client_node,
self.heketi_server_url, vol['id'])
volume_list = heketi_volume_list(self.heketi_client_node,
self.heketi_server_url)
self.assertNotIn(vol['id'], volume_list)
self.verify_volume_count()
def test_volume_create_and_list_volume(self):
"""Validate heketi and gluster volume list"""
g.log.info("List gluster volumes before Heketi volume creation")
existing_g_vol_list = get_volume_list('auto_get_gluster_endpoint')
self.assertTrue(existing_g_vol_list, ("Unable to get volumes list"))
g.log.info("List heketi volumes before volume creation")
existing_h_vol_list = heketi_volume_list(self.heketi_client_node,
self.heketi_server_url,
json=True)["volumes"]
g.log.info("Heketi volumes successfully listed")
g.log.info("Create a heketi volume")
out = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
self.volume_size,
json=True)
g.log.info("Heketi volume successfully created" % out)
volume_id = out["bricks"][0]["volume"]
self.addCleanup(heketi_volume_delete, self.heketi_client_node,
self.heketi_server_url, volume_id)
g.log.info("List heketi volumes after volume creation")
h_vol_list = heketi_volume_list(self.heketi_client_node,
self.heketi_server_url,
json=True)["volumes"]
g.log.info("Heketi volumes successfully listed")
g.log.info("List gluster volumes after Heketi volume creation")
g_vol_list = get_volume_list('auto_get_gluster_endpoint')
self.assertTrue(g_vol_list, ("Unable to get volumes list"))
g.log.info("Successfully got the volumes list")
# Perform checks
self.assertEqual(
len(existing_g_vol_list) + 1, len(g_vol_list),
"Expected creation of only one volume in Gluster creating "
"Heketi volume. Here is lists before and after volume creation: "
"%s \n%s" % (existing_g_vol_list, g_vol_list))
self.assertEqual(
len(existing_h_vol_list) + 1, len(h_vol_list),
"Expected creation of only one volume in Heketi. Here is lists "
"of Heketi volumes before and after volume creation: %s\n%s" %
(existing_h_vol_list, h_vol_list))
def test_to_check_deletion_of_cluster(self):
"""Validate deletion of cluster with volumes"""
# List heketi volumes
g.log.info("List heketi volumes")
volumes = heketi_volume_list(self.heketi_client_node,
self.heketi_server_url,
json=True)
if (len(volumes["volumes"]) == 0):
g.log.info("Creating heketi volume")
out = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
self.volume_size,
json=True)
self.assertTrue(out, ("Failed to create heketi "
"volume of size %s" % self.volume_size))
g.log.info("Heketi volume successfully created" % out)
volume_id = out["bricks"][0]["volume"]
self.addCleanup(heketi_volume_delete, self.heketi_client_node,
self.heketi_server_url, volume_id)
# List heketi cluster's
g.log.info("Listing heketi cluster list")
out = heketi_cluster_list(self.heketi_client_node,
self.heketi_server_url,
json=True)
self.assertTrue(out, ("Failed to list heketi cluster"))
g.log.info("All heketi cluster successfully listed")
cluster_id = out["clusters"][0]
# Deleting a heketi cluster
g.log.info("Trying to delete a heketi cluster"
" which contains volumes and/or nodes:"
" Expected to fail")
self.assertRaises(
ExecutionError,
heketi_cluster_delete,
self.heketi_client_node,
self.heketi_server_url,
cluster_id,
)
g.log.info("Expected result: Unable to delete cluster %s"
" because it contains volumes "
" and/or nodes" % cluster_id)
# To confirm deletion failed, check heketi cluster list
g.log.info("Listing heketi cluster list")
out = heketi_cluster_list(self.heketi_client_node,
self.heketi_server_url,
json=True)
self.assertTrue(out, ("Failed to list heketi cluster"))
g.log.info("All heketi cluster successfully listed")
def test_delete_heketidb_volume(self):
"""
Method to test heketidb volume deletion via heketi-cli
"""
volume_id_list = []
heketidbexists = False
msg = "Error: Cannot delete volume containing the Heketi database"
for i in range(0, 2):
volume_info = heketi_ops.heketi_volume_create(
self.heketi_client_node, self.heketi_server_url, 10, json=True)
self.assertNotEqual(volume_info, False, "Volume creation failed")
volume_id_list.append(volume_info["id"])
self.addCleanup(self.delete_volumes, volume_id_list)
volume_list_info = heketi_ops.heketi_volume_list(
self.heketi_client_node, self.heketi_server_url, json=True)
self.assertNotEqual(volume_list_info, False,
"Heketi volume list command failed")
if volume_list_info["volumes"] == []:
raise ExecutionError("Heketi volume list empty")
for volume_id in volume_list_info["volumes"]:
volume_info = heketi_ops.heketi_volume_info(
self.heketi_client_node,
self.heketi_server_url,
volume_id,
json=True)
if volume_info["name"] == "heketidbstorage":
heketidbexists = True
delete_ret, delete_output, delete_error = (
heketi_ops.heketi_volume_delete(self.heketi_client_node,
self.heketi_server_url,
volume_id,
raw_cli_output=True))
self.assertNotEqual(delete_ret, 0, "Return code not 0")
self.assertEqual(
delete_error.strip(), msg,
"Invalid reason for heketidb deletion failure")
if not heketidbexists:
raise ExecutionError(
"Warning: heketidbstorage doesn't exist in list of volumes")
def test_dynamic_provisioning_glusterfile_reclaim_policy_retain(self):
"""Validate retain policy for glusterfs after deletion of pvc"""
self.create_storage_class(reclaim_policy='Retain')
self.create_and_wait_for_pvc()
# get the name of the volume
pv_name = get_pv_name_from_pvc(self.node, self.pvc_name)
custom = [
r':.metadata.annotations.'
r'"gluster\.kubernetes\.io\/heketi\-volume\-id"',
r':.spec.persistentVolumeReclaimPolicy'
]
vol_id, reclaim_policy = oc_get_custom_resource(
self.node, 'pv', custom, pv_name)
self.assertEqual(reclaim_policy, 'Retain')
# Create DC with POD and attached PVC to it.
try:
dc_name = oc_create_app_dc_with_io(self.node, self.pvc_name)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
finally:
scale_dc_pod_amount_and_wait(self.node, dc_name, 0)
oc_delete(self.node, 'dc', dc_name)
wait_for_resource_absence(self.node, 'pod', pod_name)
oc_delete(self.node, 'pvc', self.pvc_name)
with self.assertRaises(ExecutionError):
wait_for_resource_absence(self.node,
'pvc',
self.pvc_name,
interval=3,
timeout=30)
heketi_volume_delete(self.heketi_client_node, self.heketi_server_url,
vol_id)
vol_list = heketi_volume_list(self.heketi_client_node,
self.heketi_server_url)
self.assertNotIn(vol_id, vol_list)
oc_delete(self.node, 'pv', pv_name)
wait_for_resource_absence(self.node, 'pv', pv_name)
def test_heketi_metrics_heketipod_failure(self):
"""Validate heketi metrics after heketi pod failure"""
scale_dc_pod_amount_and_wait(self.ocp_master_node,
self.heketi_dc_name,
pod_amount=0)
self.addCleanup(scale_dc_pod_amount_and_wait,
self.ocp_master_node,
self.heketi_dc_name,
pod_amount=1)
# verify that metrics is not accessable when heketi pod is down
with self.assertRaises(exceptions.ExecutionError):
get_heketi_metrics(self.heketi_client_node,
self.heketi_server_url,
prometheus_format=True)
scale_dc_pod_amount_and_wait(self.ocp_master_node,
self.heketi_dc_name,
pod_amount=1)
pod_name = get_pod_name_from_dc(self.ocp_master_node,
self.heketi_dc_name,
self.heketi_dc_name)
wait_for_pod_be_ready(self.ocp_master_node, pod_name, wait_step=5)
for i in range(3):
vol = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
1,
json=True)
self.assertTrue(vol)
self.addCleanup(heketi_volume_delete,
self.heketi_client_node,
self.heketi_server_url,
vol['id'],
raise_on_error=False)
vol_list = heketi_volume_list(self.heketi_client_node,
self.heketi_server_url)
self.assertIn(vol['id'], vol_list)
self.verify_heketi_metrics_with_topology_info()
def _heketi_vols(ocp_node, url):
# Unfortunately, getting json from heketi-cli only gets the ids
# To get a mapping of ids & volume names without a lot of
# back and forth between the test and the ocp_node we end up having
# to scrape the output of 'volume list'
# TODO: This probably should be made into a utility function
out = heketi_volume_list(ocp_node, url, json=False)
res = []
for line in out.splitlines():
if not line.startswith('Id:'):
continue
row = {}
for section in line.split():
if ':' in section:
key, value = section.split(':', 1)
row[key.lower()] = value.strip()
res.append(row)
return res
def test_blockvolume_create_no_free_space(self):
"""Validate error is returned when free capacity is exhausted"""
# Create first small blockvolume
blockvol1 = heketi_blockvolume_create(self.heketi_client_node,
self.heketi_server_url,
1,
json=True)
self.assertTrue(blockvol1, "Failed to create block volume.")
self.addCleanup(heketi_blockvolume_delete, self.heketi_client_node,
self.heketi_server_url, blockvol1['id'])
# Get info about block hosting volumes
file_volumes = heketi_volume_list(self.heketi_client_node,
self.heketi_server_url,
json=True)
self.assertTrue(file_volumes)
self.assertIn("volumes", file_volumes)
self.assertTrue(file_volumes["volumes"])
max_block_hosting_vol_size, file_volumes_debug_info = 0, []
for vol_id in file_volumes["volumes"]:
vol = heketi_volume_info(self.heketi_client_node,
self.heketi_server_url,
vol_id,
json=True)
current_block_hosting_vol_size = vol.get('size', 0)
if current_block_hosting_vol_size > max_block_hosting_vol_size:
max_block_hosting_vol_size = current_block_hosting_vol_size
if current_block_hosting_vol_size:
file_volumes_debug_info.append(
six.text_type({
'id': vol.get('id', '?'),
'name': vol.get('name', '?'),
'size': current_block_hosting_vol_size,
'blockinfo': vol.get('blockinfo', '?'),
}))
self.assertGreater(max_block_hosting_vol_size, 0)
# Try to create blockvolume with size bigger than available
too_big_vol_size = max_block_hosting_vol_size + 1
try:
blockvol2 = heketi_blockvolume_create(self.heketi_client_node,
self.heketi_server_url,
too_big_vol_size,
json=True)
except ExecutionError:
return
if blockvol2 and blockvol2.get('id'):
self.addCleanup(heketi_blockvolume_delete, self.heketi_client_node,
self.heketi_server_url, blockvol2['id'])
block_hosting_vol = heketi_volume_info(
self.heketi_client_node,
self.heketi_server_url,
blockvol2.get('blockhostingvolume'),
json=True)
self.assertGreater(
block_hosting_vol.get('size', -2), blockvol2.get('size', -1),
("Block volume unexpectedly was created. "
"Calculated 'max free size' is '%s'.\nBlock volume info is: %s \n"
"File volume info, which hosts block volume: \n%s,"
"Block hosting volumes which were considered: \n%s" %
(max_block_hosting_vol_size, blockvol2, block_hosting_vol,
'\n'.join(file_volumes_debug_info))))
def _create_distributed_replica_vol(self, validate_cleanup):
# Create distributed vol
vol_size_gb = self._get_vol_size()
heketi_url = self.heketi_server_url
try:
g.log.info(
"Trying to create distributed '%s'Gb volume." % vol_size_gb)
heketi_vol = heketi_volume_create(
self.heketi_client_node, heketi_url, vol_size_gb, json=True)
except exceptions.ExecutionError as e:
# NOTE: rare situation when we need to decrease size of a volume.
# and we expect this vol to be distributed.
g.log.info("Failed to create distributed '%s'Gb volume. "
"Trying to create another one, smaller for 1Gb.")
if ('more required' in str(e)
and ('Insufficient suitable allocatable extents for '
'logical volume' in str(e))):
vol_size_gb -= 1
heketi_vol = heketi_volume_create(
self.heketi_client_node, heketi_url, vol_size_gb,
json=True)
else:
raise
g.log.info("Successfully created distributed volume.")
vol_name = heketi_vol['name']
vol_id = heketi_vol["bricks"][0]["volume"]
self.addCleanup(
heketi_volume_delete, self.heketi_client_node, heketi_url,
vol_id, raise_on_error=(not validate_cleanup))
# Get gluster volume info
g.log.info("Get gluster volume '%s' info" % vol_name)
gluster_vol = get_volume_info(
'auto_get_gluster_endpoint', volname=vol_name)
self.assertTrue(
gluster_vol, "Failed to get volume '%s' info" % vol_name)
g.log.info("Successfully got volume '%s' info" % vol_name)
gluster_vol = gluster_vol[vol_name]
self.assertEqual(
gluster_vol["typeStr"], "Distributed-Replicate",
"'%s' gluster vol isn't a Distributed-Replicate volume" % vol_name)
# Check amount of bricks
brick_amount = len(gluster_vol['bricks']['brick'])
self.assertEqual(brick_amount % 3, 0,
"Brick amount is expected to be divisible by 3. "
"Actual amount is '%s'" % brick_amount)
self.assertGreater(brick_amount, 3,
"Brick amount is expected to be bigger than 3. "
"Actual amount is '%s'." % brick_amount)
# Run unique actions to Validate whether deleting a dist-rep
# volume is handled by heketi else return
if not validate_cleanup:
return
# Get the free space after creating heketi volume
free_space_after_creating_vol = self._get_free_space()
# Delete heketi volume
g.log.info("Deleting heketi volume '%s'" % vol_id)
volume_deleted = heketi_volume_delete(
self.heketi_client_node, heketi_url, vol_id)
self.assertTrue(
volume_deleted, "Failed to delete heketi volume '%s'" % vol_id)
g.log.info("Heketi volume '%s' has successfully been deleted" % vol_id)
# Check the heketi volume list
g.log.info("List heketi volumes")
heketi_volumes = heketi_volume_list(
self.heketi_client_node, self.heketi_server_url, json=True)
self.assertTrue(heketi_volumes, "Failed to list heketi volumes")
g.log.info("Heketi volumes have successfully been listed")
heketi_volumes = heketi_volumes.get('volumes', heketi_volumes)
self.assertNotIn(vol_id, heketi_volumes)
self.assertNotIn(vol_name, heketi_volumes)
# Check the gluster volume list
g.log.info("Get the gluster volume list")
gluster_volumes = get_volume_list('auto_get_gluster_endpoint')
self.assertTrue(gluster_volumes, "Unable to get Gluster volume list")
g.log.info("Successfully got Gluster volume list" % gluster_volumes)
self.assertNotIn(vol_id, gluster_volumes)
self.assertNotIn(vol_name, gluster_volumes)
# Get the used space after deleting heketi volume
free_space_after_deleting_vol = self._get_free_space()
# Compare the free space before and after deleting the volume
g.log.info("Comparing the free space before and after deleting volume")
self.assertLessEqual(
free_space_after_creating_vol + (3 * vol_size_gb),
free_space_after_deleting_vol)
g.log.info("Volume successfully deleted and space is reallocated. "
"Free space after creating volume %s. "
"Free space after deleting volume %s." % (
free_space_after_creating_vol,
free_space_after_deleting_vol))