def test_blockvolume_create_no_free_space(self):
"""Validate error is returned when free capacity is exhausted"""
# Create first small blockvolume
blockvol1 = heketi_blockvolume_create(
self.heketi_client_node, self.heketi_server_url, 1, json=True)
self.assertTrue(blockvol1, "Failed to create block volume.")
self.addCleanup(
heketi_blockvolume_delete, self.heketi_client_node,
self.heketi_server_url, blockvol1['id'])
# Get info about block hosting volumes
file_volumes = heketi_volume_list(
self.heketi_client_node, self.heketi_server_url, json=True)
self.assertTrue(file_volumes)
self.assertIn("volumes", file_volumes)
self.assertTrue(file_volumes["volumes"])
max_block_hosting_vol_size, file_volumes_debug_info = 0, []
for vol_id in file_volumes["volumes"]:
vol = heketi_volume_info(
self.heketi_client_node, self.heketi_server_url,
vol_id, json=True)
current_block_hosting_vol_size = vol.get('size', 0)
if current_block_hosting_vol_size > max_block_hosting_vol_size:
max_block_hosting_vol_size = current_block_hosting_vol_size
if current_block_hosting_vol_size:
file_volumes_debug_info.append(six.text_type({
'id': vol.get('id', '?'),
'name': vol.get('name', '?'),
'size': current_block_hosting_vol_size,
'blockinfo': vol.get('blockinfo', '?'),
}))
self.assertGreater(max_block_hosting_vol_size, 0)
# Try to create blockvolume with size bigger than available
too_big_vol_size = max_block_hosting_vol_size + 1
try:
blockvol2 = heketi_blockvolume_create(
self.heketi_client_node, self.heketi_server_url,
too_big_vol_size, json=True)
except ExecutionError:
return
if blockvol2 and blockvol2.get('id'):
self.addCleanup(
heketi_blockvolume_delete, self.heketi_client_node,
self.heketi_server_url, blockvol2['id'])
block_hosting_vol = heketi_volume_info(
self.heketi_client_node, self.heketi_server_url,
blockvol2.get('blockhostingvolume'), json=True)
self.assertGreater(
block_hosting_vol.get('size', -2), blockvol2.get('size', -1),
("Block volume unexpectedly was created. "
"Calculated 'max free size' is '%s'.\nBlock volume info is: %s \n"
"File volume info, which hosts block volume: \n%s,"
"Block hosting volumes which were considered: \n%s" % (
max_block_hosting_vol_size, blockvol2, block_hosting_vol,
'\n'.join(file_volumes_debug_info))))
def test_blockvolume_create_no_free_space(self):
"""Validate error is returned when free capacity is exhausted"""
# Create first small blockvolume
blockvol1 = heketi_blockvolume_create(
self.heketi_client_node, self.heketi_server_url, 1, json=True)
self.assertTrue(blockvol1, "Failed to create block volume.")
self.addCleanup(
heketi_blockvolume_delete, self.heketi_client_node,
self.heketi_server_url, blockvol1['id'])
# Get info about block hosting volumes
file_volumes = heketi_volume_list(
self.heketi_client_node, self.heketi_server_url, json=True)
self.assertTrue(file_volumes)
self.assertIn("volumes", file_volumes)
self.assertTrue(file_volumes["volumes"])
max_block_hosting_vol_size, file_volumes_debug_info = 0, []
for vol_id in file_volumes["volumes"]:
vol = heketi_volume_info(
self.heketi_client_node, self.heketi_server_url,
vol_id, json=True)
current_block_hosting_vol_size = vol.get('size', 0)
if current_block_hosting_vol_size > max_block_hosting_vol_size:
max_block_hosting_vol_size = current_block_hosting_vol_size
if current_block_hosting_vol_size:
file_volumes_debug_info.append(six.text_type({
'id': vol.get('id', '?'),
'name': vol.get('name', '?'),
'size': current_block_hosting_vol_size,
'blockinfo': vol.get('blockinfo', '?'),
}))
self.assertGreater(max_block_hosting_vol_size, 0)
# Try to create blockvolume with size bigger than available
too_big_vol_size = max_block_hosting_vol_size + 1
try:
blockvol2 = heketi_blockvolume_create(
self.heketi_client_node, self.heketi_server_url,
too_big_vol_size, json=True)
except AssertionError:
return
if blockvol2 and blockvol2.get('id'):
self.addCleanup(
heketi_blockvolume_delete, self.heketi_client_node,
self.heketi_server_url, blockvol2['id'])
block_hosting_vol = heketi_volume_info(
self.heketi_client_node, self.heketi_server_url,
blockvol2.get('blockhostingvolume'), json=True)
self.assertGreater(
block_hosting_vol.get('size', -2), blockvol2.get('size', -1),
("Block volume unexpectedly was created. "
"Calculated 'max free size' is '%s'.\nBlock volume info is: %s \n"
"File volume info, which hosts block volume: \n%s,"
"Block hosting volumes which were considered: \n%s" % (
max_block_hosting_vol_size, blockvol2, block_hosting_vol,
'\n'.join(file_volumes_debug_info))))
def test_sc_create_with_clusterid(self):
"""Create storage class with 'cluster id'"""
h_cluster_list = heketi_cluster_list(self.heketi_client_node,
self.heketi_server_url,
json=True)
self.assertTrue(h_cluster_list, "Failed to list heketi cluster")
cluster_id = h_cluster_list["clusters"][0]
sc = self.create_storage_class(clusterid=cluster_id)
pvc_name = self.create_and_wait_for_pvc(sc_name=sc)
# Validate if cluster id is correct in the heketi volume info
pv_name = get_pv_name_from_pvc(self.ocp_master_node[0], pvc_name)
volume_id = oc_get_custom_resource(
self.ocp_master_node[0],
'pv', r':metadata.annotations."gluster\.kubernetes\.io'
r'\/heketi-volume-id"',
name=pv_name)[0]
volume_info = heketi_volume_info(self.heketi_client_node,
self.heketi_server_url,
volume_id,
json=True)
self.assertEqual(
cluster_id, volume_info["cluster"],
"Cluster ID %s has NOT been used to"
"create the PVC %s. Found %s" %
(cluster_id, pvc_name, volume_info["cluster"]))
def test_create_vol_and_retrieve_vol_info(self):
"""Validate heketi and gluster volume info"""
g.log.info("Create a heketi volume")
out = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
self.volume_size,
json=True)
self.assertTrue(out, ("Failed to create heketi "
"volume of size %s" % self.volume_size))
g.log.info("Heketi volume successfully created" % out)
volume_id = out["bricks"][0]["volume"]
self.addCleanup(heketi_volume_delete, self.heketi_client_node,
self.heketi_server_url, volume_id)
g.log.info("Retrieving heketi volume info")
out = heketi_volume_info(self.heketi_client_node,
self.heketi_server_url,
volume_id,
json=True)
self.assertTrue(out, ("Failed to get heketi volume info"))
g.log.info("Successfully got the heketi volume info")
name = out["name"]
vol_info = get_volume_info('auto_get_gluster_endpoint', volname=name)
self.assertTrue(vol_info, "Failed to get volume info %s" % name)
g.log.info("Successfully got the volume info %s" % name)
def dynamic_provisioning_glusterfile(self, create_vol_name_prefix):
# Create secret and storage class
self.create_storage_class(
create_vol_name_prefix=create_vol_name_prefix)
# Create PVC
pvc_name = self.create_and_wait_for_pvc()
# Create DC with POD and attached PVC to it.
dc_name = oc_create_app_dc_with_io(self.node, pvc_name)
self.addCleanup(oc_delete, self.node, 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, self.node, dc_name, 0)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
# Verify Heketi volume name for prefix presence if provided
if create_vol_name_prefix:
ret = verify_volume_name_prefix(self.node,
self.sc['volumenameprefix'],
self.sc['secretnamespace'],
pvc_name, self.sc['resturl'])
self.assertTrue(ret, "verify volnameprefix failed")
else:
# Get the volume name and volume id from PV
pv_name = get_pv_name_from_pvc(self.ocp_client[0], pvc_name)
custom = [
r':spec.glusterfs.path',
r':metadata.annotations.'
r'"gluster\.kubernetes\.io\/heketi-volume-id"'
]
pv_vol_name, vol_id = oc_get_custom_resource(
self.ocp_client[0], 'pv', custom, pv_name)
# check if the pv_volume_name is present in heketi
# Check if volume name is "vol_"+volumeid or not
heketi_vol_name = heketi_volume_info(
self.ocp_client[0], self.heketi_server_url, vol_id,
json=True)['name']
self.assertEqual(pv_vol_name, heketi_vol_name,
'Volume with vol_id = %s not found'
'in heketidb' % vol_id)
self.assertEqual(heketi_vol_name, 'vol_' + vol_id,
'Volume with vol_id = %s have a'
'custom perfix' % vol_id)
out = cmd_run_on_gluster_pod_or_node(self.ocp_master_node[0],
"gluster volume list")
self.assertIn(pv_vol_name, out,
"Volume with id %s does not exist" % vol_id)
# Make sure we are able to work with files on the mounted volume
filepath = "/mnt/file_for_testing_io.log"
for cmd in ("dd if=/dev/urandom of=%s bs=1K count=100",
"ls -lrt %s",
"rm -rf %s"):
cmd = cmd % filepath
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0,
"Failed to execute '%s' command on %s" % (cmd, self.node))
def test_heketi_volume_create_with_clusterid(self):
"""Validate creation of heketi volume with clusters argument"""
h_node, h_url = self.heketi_client_node, self.heketi_server_url
# Get one of the cluster id from heketi cluster list
creation_cluster_id = heketi_cluster_list(h_node, h_url,
json=True)['clusters'][0]
# Create a heketi volume specific to cluster list
volume_id = heketi_volume_create(h_node,
h_url,
self.volume_size,
clusters=creation_cluster_id,
json=True)["bricks"][0]["volume"]
self.addCleanup(heketi_volume_delete, self.heketi_client_node,
self.heketi_server_url, volume_id)
# Get the cluster id from heketi volume info
info_cluster_id = heketi_volume_info(h_node,
h_url,
volume_id,
json=True)['cluster']
# Match the creation cluster id with the info cluster id
self.assertEqual(
info_cluster_id, creation_cluster_id,
"Volume creation cluster id {} not matching the info cluster id "
"{}".format(creation_cluster_id, info_cluster_id))
def test_heketi_brick_evict(self):
"""Test brick evict basic functionality and verify it replace a brick
properly
"""
h_node, h_server = self.heketi_client_node, self.heketi_server_url
size = 1
vol_info_old = heketi_ops.heketi_volume_create(
h_node, h_server, size, json=True)
self.addCleanup(
heketi_ops.heketi_volume_delete, h_node, h_server,
vol_info_old['id'])
heketi_ops.heketi_brick_evict(
h_node, h_server, vol_info_old["bricks"][0]['id'])
vol_info_new = heketi_ops.heketi_volume_info(
h_node, h_server, vol_info_old['id'], json=True)
bricks_old = set({brick['path'] for brick in vol_info_old["bricks"]})
bricks_new = set({brick['path'] for brick in vol_info_new["bricks"]})
self.assertEqual(
len(bricks_new - bricks_old), 1,
"Brick was not replaced with brick evict for vol \n {}".format(
vol_info_new))
gvol_info = self._get_gluster_vol_info(vol_info_new['name'])
gbricks = set(
{brick['name'].split(":")[1]
for brick in gvol_info["bricks"]["brick"]})
self.assertEqual(
bricks_new, gbricks, "gluster vol info and heketi vol info "
"mismatched after brick evict {} \n {}".format(
gvol_info, vol_info_new))
def test_heketi_with_expand_volume(self):
"""
Test volume expand and size if updated correctly in heketi-cli info
"""
vol_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
self.volume_size, json=True)
self.assertTrue(vol_info, ("Failed to create heketi volume of size %s"
% self.volume_size))
self.addCleanup(
heketi_volume_delete,
self.heketi_client_node, self.heketi_server_url, vol_info['id'])
self.assertEqual(vol_info['size'], self.volume_size,
("Failed to create volume."
"Expected Size: %s, Actual Size: %s"
% (self.volume_size, vol_info['size'])))
volume_id = vol_info["id"]
expand_size = 2
ret = heketi_volume_expand(self.heketi_client_node,
self.heketi_server_url, volume_id,
expand_size)
self.assertTrue(ret, ("Failed to expand heketi volume of id %s"
% volume_id))
volume_info = heketi_volume_info(self.heketi_client_node,
self.heketi_server_url,
volume_id, json=True)
expected_size = self.volume_size + expand_size
self.assertEqual(volume_info['size'], expected_size,
("Volume Expansion failed Expected Size: %s, Actual "
"Size: %s" % (str(expected_size),
str(volume_info['size']))))
def dynamic_provisioning_glusterfile(self, create_vol_name_prefix):
# Create secret and storage class
self.create_storage_class(
create_vol_name_prefix=create_vol_name_prefix)
# Create PVC
pvc_name = self.create_and_wait_for_pvc()
# Create DC with POD and attached PVC to it.
dc_name = oc_create_app_dc_with_io(self.node, pvc_name)
self.addCleanup(oc_delete, self.node, 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, self.node, dc_name, 0)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
# Verify Heketi volume name for prefix presence if provided
if create_vol_name_prefix:
ret = verify_volume_name_prefix(self.node,
self.sc['volumenameprefix'],
self.sc['secretnamespace'],
pvc_name, self.sc['resturl'])
self.assertTrue(ret, "verify volnameprefix failed")
else:
# Get the volume name and volume id from PV
pv_name = get_pv_name_from_pvc(self.ocp_client[0], pvc_name)
custom = [
r':spec.glusterfs.path',
r':metadata.annotations.'
r'"gluster\.kubernetes\.io\/heketi-volume-id"'
]
pv_vol_name, vol_id = oc_get_custom_resource(
self.ocp_client[0], 'pv', custom, pv_name)
# check if the pv_volume_name is present in heketi
# Check if volume name is "vol_"+volumeid or not
heketi_vol_name = heketi_volume_info(
self.ocp_client[0], self.heketi_server_url, vol_id,
json=True)['name']
self.assertEqual(pv_vol_name, heketi_vol_name,
'Volume with vol_id = %s not found'
'in heketidb' % vol_id)
self.assertEqual(heketi_vol_name, 'vol_' + vol_id,
'Volume with vol_id = %s have a'
'custom perfix' % vol_id)
out = cmd_run_on_gluster_pod_or_node(self.ocp_master_node[0],
"gluster volume list")
self.assertIn(pv_vol_name, out,
"Volume with id %s does not exist" % vol_id)
# Make sure we are able to work with files on the mounted volume
filepath = "/mnt/file_for_testing_io.log"
for cmd in ("dd if=/dev/urandom of=%s bs=1K count=100",
"ls -lrt %s",
"rm -rf %s"):
cmd = cmd % filepath
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0,
"Failed to execute '%s' command on %s" % (cmd, self.node))
def test_delete_heketidb_volume(self):
"""Method to test heketidb volume deletion via heketi-cli."""
for i in range(0, 2):
volume_info = heketi_ops.heketi_volume_create(
self.heketi_client_node, self.heketi_server_url, 10, json=True)
self.addCleanup(heketi_ops.heketi_volume_delete,
self.heketi_client_node, self.heketi_server_url,
volume_info["id"])
volume_list_info = heketi_ops.heketi_volume_list(
self.heketi_client_node, self.heketi_server_url, json=True)
self.assertTrue(volume_list_info["volumes"],
"Heketi volume list empty.")
for volume_id in volume_list_info["volumes"]:
volume_info = heketi_ops.heketi_volume_info(
self.heketi_client_node,
self.heketi_server_url,
volume_id,
json=True)
if volume_info["name"] == "heketidbstorage":
self.assertRaises(AssertionError,
heketi_ops.heketi_volume_delete,
self.heketi_client_node,
self.heketi_server_url, volume_id)
return
raise ExecutionError(
"Warning: heketidbstorage doesn't exist in list of volumes")
def test_heketi_with_expand_volume(self):
"""
Test volume expand and size if updated correctly in heketi-cli info
"""
vol_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
self.volume_size,
json=True)
self.assertTrue(
vol_info,
("Failed to create heketi volume of size %s" % self.volume_size))
self.addCleanup(heketi_volume_delete, self.heketi_client_node,
self.heketi_server_url, vol_info['id'])
self.assertEqual(vol_info['size'], self.volume_size,
("Failed to create volume."
"Expected Size: %s, Actual Size: %s" %
(self.volume_size, vol_info['size'])))
volume_id = vol_info["id"]
expand_size = 2
ret = heketi_volume_expand(self.heketi_client_node,
self.heketi_server_url, volume_id,
expand_size)
self.assertTrue(
ret, ("Failed to expand heketi volume of id %s" % volume_id))
volume_info = heketi_volume_info(self.heketi_client_node,
self.heketi_server_url,
volume_id,
json=True)
expected_size = self.volume_size + expand_size
self.assertEqual(volume_info['size'], expected_size,
("Volume Expansion failed Expected Size: %s, Actual "
"Size: %s" %
(str(expected_size), str(volume_info['size']))))
def verify_volume_bricks_are_present_or_not_on_heketi_node(
self, vol_ids, node_id, state='present'):
for vol_id in vol_ids:
vol_info = heketi_ops.heketi_volume_info(
self.h_node, self.h_url, vol_id, json=True)
bricks = vol_info['bricks']
self.assertFalse((len(bricks) % 3))
if state == 'present':
found = False
for brick in bricks:
if brick['node'] == node_id:
found = True
break
self.assertTrue(
found, 'Bricks of vol %s does not present on node %s'
% (vol_id, node_id))
elif state == 'absent':
for brick in bricks:
self.assertNotEqual(
brick['node'], node_id, 'Bricks of vol %s is present '
'on node %s' % (vol_id, node_id))
else:
msg = "State %s is other than present, absent" % state
raise AssertionError(msg)
def get_vol_names_from_vol_ids(self, vol_ids):
vol_names = []
for vol_id in vol_ids:
vol_info = heketi_ops.heketi_volume_info(
self.h_node, self.h_url, vol_id, json=True)
vol_names.append(vol_info['name'])
return vol_names
def test_create_vol_and_retrieve_vol_info(self):
"""Validate heketi and gluster volume info"""
g.log.info("Create a heketi volume")
out = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
self.volume_size, json=True)
self.assertTrue(out, ("Failed to create heketi "
"volume of size %s" % self.volume_size))
g.log.info("Heketi volume successfully created" % out)
volume_id = out["bricks"][0]["volume"]
self.addCleanup(
heketi_volume_delete, self.heketi_client_node,
self.heketi_server_url, volume_id)
g.log.info("Retrieving heketi volume info")
out = heketi_volume_info(
self.heketi_client_node, self.heketi_server_url, volume_id,
json=True)
self.assertTrue(out, ("Failed to get heketi volume info"))
g.log.info("Successfully got the heketi volume info")
name = out["name"]
vol_info = get_volume_info('auto_get_gluster_endpoint', volname=name)
self.assertTrue(vol_info, "Failed to get volume info %s" % name)
g.log.info("Successfully got the volume info %s" % name)
def test_brick_evict_on_more_than_three_node_with_one_down(self):
"""Test brick evict basic functionality and verify brick evict
will success after one node down out of more than three nodes"""
h_node, h_server = self.heketi_client_node, self.heketi_server_url
# Create heketi volume
vol_info = heketi_ops.heketi_volume_create(
h_node, h_server, 1, json=True)
self.addCleanup(
heketi_ops.heketi_volume_delete,
h_node, h_server, vol_info.get('id'))
# Get node on which heketi pod is scheduled
heketi_pod = openshift_ops.get_pod_name_from_dc(
self.ocp_client, self.heketi_dc_name)
heketi_node = openshift_ops.oc_get_custom_resource(
self.ocp_client, 'pod', '.:spec.nodeName', heketi_pod)[0]
# Get brick id and glusterfs node which is not heketi node
for node in vol_info.get('bricks', {}):
node_info = heketi_ops.heketi_node_info(
h_node, h_server, node.get('node'), json=True)
hostname = node_info.get('hostnames').get('manage')[0]
if hostname != heketi_node:
brick_id = node.get('id')
break
self._power_off_node_and_wait_node_to_be_not_ready(hostname)
# Perform brick evict operation
heketi_ops.heketi_brick_evict(h_node, h_server, brick_id)
# Get volume info after brick evict operation
vol_info_new = heketi_ops.heketi_volume_info(
h_node, h_server, vol_info.get('id'), json=True)
# Get previous and new bricks from volume
bricks_old = set(
{brick.get('path') for brick in vol_info.get("bricks")})
bricks_new = set(
{brick.get('path') for brick in vol_info_new.get("bricks")})
self.assertEqual(
len(bricks_new - bricks_old), 1,
"Brick was not replaced with brick evict for vol \n {}".format(
vol_info_new))
# Get gluster volume info
g_vol_info = self._get_gluster_vol_info(vol_info_new.get('name'))
# Validate bricks on gluster volume and heketi volume
g_bricks = set(
{brick.get('name').split(":")[1]
for brick in g_vol_info.get("bricks", {}).get("brick")})
self.assertEqual(
bricks_new, g_bricks, "gluster vol info and heketi vol info "
"mismatched after brick evict {} \n {}".format(
g_bricks, g_vol_info))
def test_replica_volume_expand(self):
"""
Test expansion of a replica volume
"""
h_node, h_server = self.heketi_client_node, self.heketi_server_url
volume_name = ("autotests-heketi-volume-{}".format(
utils.get_random_str()))
volume_size = 10
creation_info = self.create_heketi_volume_with_name_and_wait(
volume_name, volume_size, json=True, raise_on_cleanup_error=False)
volume_id = creation_info["id"]
volume_info = heketi_ops.heketi_volume_info(h_node,
h_server,
volume_id,
json=True)
# Get gluster volume info
gluster_vol = volume_ops.get_volume_info('auto_get_gluster_endpoint',
volname=volume_name)
self.assertTrue(gluster_vol,
"Failed to get volume {} info".format(volume_name))
vol_name = gluster_vol[volume_name]
self.assertEqual(
vol_name['replicaCount'], "3",
"Replica count is different for volume {} Actual:{} "
"Expected : 3".format(vol_name, vol_name['replicaCount']))
expand_size = 5
heketi_ops.heketi_volume_expand(h_node, h_server, volume_id,
expand_size)
volume_info = heketi_ops.heketi_volume_info(h_node,
h_server,
volume_id,
json=True)
expected_size = volume_size + expand_size
self.assertEqual(
volume_info['size'], expected_size,
"Volume Expansion failed, Expected Size: {}, Actual "
"Size: {}".format(str(expected_size), str(volume_info['size'])))
self.get_brick_and_volume_status(volume_name)
self.get_rebalance_status(volume_name)
def _cleanup_heketi_volumes(self, existing_volumes):
h_node, h_url = self.heketi_client_node, self.heketi_server_url
volumes = heketi_volume_list(h_node, h_url, json=True).get("volumes")
new_volumes = list(set(volumes) - set(existing_volumes))
for volume in new_volumes:
h_vol_info = heketi_volume_info(h_node, h_url, volume, json=True)
if h_vol_info.get("block"):
for block_vol in (
h_vol_info.get("blockinfo").get("blockvolume")):
heketi_blockvolume_delete(h_node, h_url, block_vol)
heketi_volume_delete(h_node, h_url, volume, raise_on_error=False)
def _block_vol_expand_common_offline_vs_online(self, is_online_expand):
node = self.ocp_master_node[0]
h_node, h_server = self.heketi_client_node, self.heketi_server_url
version = heketi_version.get_heketi_version(h_node)
if version < '9.0.0-14':
self.skipTest("heketi-client package {} does not support "
"blockvolume expand".format(version.v_str))
pvc_name = self.create_and_wait_for_pvc()
dc_name = self.create_dc_with_pvc(pvc_name)
pv_name = get_pv_name_from_pvc(node, pvc_name)
# get block volume id
custom = r":.metadata.annotations.'gluster\.org\/volume-id'"
bvol_id = oc_get_custom_resource(node, 'pv', custom, pv_name)
self.assertNotEqual(bvol_id[0], "<none>",
"volume name not found from pv {}".format(pv_name))
bvol_info = heketi_blockvolume_info(h_node,
h_server,
bvol_id[0],
json=True)
# verify required blockhostingvolume free size
bhv_id = bvol_info["blockhostingvolume"]
bhv_info = heketi_volume_info(h_node, h_server, bhv_id, json=True)
if bhv_info["blockinfo"]["freesize"] < 1:
self.skipTest("blockhostingvolume doesn't have required freespace")
if not is_online_expand:
scale_dc_pod_amount_and_wait(node, dc_name[0], pod_amount=0)
# expand block volume and verify usable size
bvol_info = heketi_blockvolume_expand(h_node,
h_server,
bvol_id[0],
2,
json=True)
self.assertEqual(bvol_info["size"], 2,
"Block volume expand does not works")
self.assertEqual(
bvol_info["size"], bvol_info["usablesize"],
"block volume size is not equal to the usablesize: {}".format(
bvol_info))
return pvc_name, dc_name, bvol_info
def test_delete_heketidb_volume(self):
"""
Method to test heketidb volume deletion via heketi-cli
"""
heketidbexists = False
msg = "Error: Cannot delete volume containing the Heketi database"
for i in range(0, 2):
volume_info = heketi_ops.heketi_volume_create(
self.heketi_client_node, self.heketi_server_url,
10, json=True)
self.addCleanup(
heketi_ops.heketi_volume_delete, self.heketi_client_node,
self.heketi_server_url, volume_info["id"])
volume_list_info = heketi_ops.heketi_volume_list(
self.heketi_client_node,
self.heketi_server_url, json=True)
if volume_list_info["volumes"] == []:
raise ExecutionError("Heketi volume list empty")
for volume_id in volume_list_info["volumes"]:
volume_info = heketi_ops.heketi_volume_info(
self.heketi_client_node, self.heketi_server_url,
volume_id, json=True)
if volume_info["name"] == "heketidbstorage":
heketidbexists = True
delete_ret, delete_output, delete_error = (
heketi_ops.heketi_volume_delete(
self.heketi_client_node,
self.heketi_server_url, volume_id,
raw_cli_output=True))
self.assertNotEqual(delete_ret, 0, "Return code not 0")
self.assertEqual(
delete_error.strip(), msg,
"Invalid reason for heketidb deletion failure")
if not heketidbexists:
raise ExecutionError(
"Warning: heketidbstorage doesn't exist in list of volumes")
def create_heketi_volume_with_name_and_wait(self,
name,
size,
raise_on_cleanup_error=True,
timeout=600,
wait_step=10,
**kwargs):
json = kwargs.get("json", False)
try:
h_volume_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
size,
name=name,
**kwargs)
except Exception as e:
if ('more required' in six.text_type(e)
or ('Failed to allocate new volume' in six.text_type(e))):
raise
for w in Waiter(timeout, wait_step):
h_volumes = heketi_volume_list(self.heketi_client_node,
self.heketi_server_url)
h_volume_match = re.search(HEKETI_VOLUME_REGEX % name,
h_volumes)
if h_volume_match:
h_volume_info = heketi_volume_info(self.heketi_client_node,
self.heketi_server_url,
h_volume_match.group(1),
json=json)
break
if w.expired:
g.log.info(
"Heketi volume with name %s not created in 600 sec" % name)
raise
self.addCleanup(heketi_volume_delete,
self.heketi_client_node,
self.heketi_server_url,
h_volume_info["id"],
raise_on_error=raise_on_cleanup_error)
return h_volume_info
def test_block_host_volume_delete_block_volume_delete(self):
"""Validate block volume and BHV removal using heketi"""
free_space, nodenum = get_total_free_space(self.heketi_client_node,
self.heketi_server_url)
if nodenum < 3:
self.skipTest("Skipping the test since number of nodes"
"online are less than 3")
free_space_available = int(free_space / nodenum)
default_bhv_size = get_default_block_hosting_volume_size(
self.heketi_client_node, self.heketi_dc_name)
if free_space_available < default_bhv_size:
self.skipTest("Skipping the test since free_space_available %s"
"is less than the default_bhv_size %s" %
(free_space_available, default_bhv_size))
h_volume_name = ("autotests-heketi-volume-%s" % utils.get_random_str())
block_host_create_info = self.create_heketi_volume_with_name_and_wait(
h_volume_name, default_bhv_size, json=True, block=True)
block_vol_size = block_host_create_info["blockinfo"]["freesize"]
block_hosting_vol_id = block_host_create_info["id"]
block_vol_info = {"blockhostingvolume": "init_value"}
while (block_vol_info['blockhostingvolume'] != block_hosting_vol_id):
block_vol = heketi_blockvolume_create(self.heketi_client_node,
self.heketi_server_url,
block_vol_size,
json=True,
ha=3,
auth=True)
self.addCleanup(heketi_blockvolume_delete,
self.heketi_client_node,
self.heketi_server_url,
block_vol["id"],
raise_on_error=True)
block_vol_info = heketi_blockvolume_info(self.heketi_client_node,
self.heketi_server_url,
block_vol["id"],
json=True)
bhv_info = heketi_volume_info(self.heketi_client_node,
self.heketi_server_url,
block_hosting_vol_id,
json=True)
self.assertIn(block_vol_info["id"],
bhv_info["blockinfo"]["blockvolume"])
def validate_block_volumes_count(self, h_node, h_server, node_ip):
# get list of block volumes using heketi
h_blockvol_list = heketi_blockvolume_list(
h_node, h_server, json=True)
# Get existing BHV list
bhv_list = list(
get_block_hosting_volume_list(h_node, h_server).keys())
for vol in bhv_list:
bhv_info = heketi_volume_info(h_node, h_server, vol, json=True)
bhv_name = bhv_info['name']
gluster_block_list = get_block_list(node_ip, volname=bhv_name)
self.assertIsNotNone(
gluster_block_list, "Failed to get gluster block list")
self.assertEqual(
len(h_blockvol_list['blockvolumes']), len(gluster_block_list),
"Failed to verify blockvolume count Expected:'{}', "
"Actual:'{}'".format(
len(h_blockvol_list['blockvolumes']), len(gluster_block_list)))
def _cleanup_heketi_volumes(self, existing_volumes):
"""Cleanup created BHV and BV"""
volumes = heketi_ops.heketi_volume_list(self.h_node,
self.h_server,
json=True).get("volumes")
new_volumes = list(set(volumes) - set(existing_volumes))
for volume in new_volumes:
h_vol_info = heketi_ops.heketi_volume_info(self.h_node,
self.h_server,
volume,
json=True)
if h_vol_info.get("block"):
for block_vol in (
h_vol_info.get("blockinfo").get("blockvolume")):
heketi_ops.heketi_blockvolume_delete(self.h_node,
self.h_server,
block_vol,
raise_on_error=False)
heketi_ops.heketi_volume_delete(self.h_node,
self.h_server,
volume,
raise_on_error=False)
def test_heketi_block_volume_create_with_size_more_than_bhv_free_space(
self):
""" Test to create heketi block volume of size greater than
free space in BHV so that it will create a new BHV.
"""
h_node, h_url = self.heketi_client_node, self.heketi_server_url
default_bhv_size = get_default_block_hosting_volume_size(
self.node, self.heketi_dc_name)
reserve_size = math.ceil(default_bhv_size * 0.02)
bhv_list, pvc_size = [], (default_bhv_size - reserve_size)
# Get existing BHV list
bhv_list = list(get_block_hosting_volume_list(h_node, h_url).keys())
for vol in bhv_list:
info = heketi_volume_info(h_node, h_url, vol, json=True)
if info['blockinfo']['freesize'] >= pvc_size:
self.skipTest(
"Skip test case since there is atleast one BHV with free"
" space {} greater than the default value {}".format(
info['blockinfo']['freesize'], pvc_size))
# To verify if there is enough free space for two BHVs
self.verify_free_space(2 * (default_bhv_size + 1))
sc_name = self.create_storage_class()
self._dynamic_provisioning_block_with_bhv_cleanup(
sc_name, pvc_size, bhv_list)
self._dynamic_provisioning_block_with_bhv_cleanup(
sc_name, pvc_size, bhv_list)
bhv_post = len(get_block_hosting_volume_list(h_node, h_url))
err_msg = ("New BHVs were not created to satisfy the block PV requests"
" No. of BHV before the test : {} \n"
" No. of BHV after the test : {}".format(
len(bhv_list), bhv_post))
self.assertEqual(bhv_post, (len(bhv_list) + 2), err_msg)
def test_volume_expansion_rebalance_brick(self):
"""Validate volume expansion with brick and check rebalance"""
creation_info = heketi_ops.heketi_volume_create(
self.heketi_client_node, self.heketi_server_url, 10, json=True)
self.assertNotEqual(creation_info, False, "Volume creation failed")
volume_name = creation_info["name"]
volume_id = creation_info["id"]
free_space_after_creation = self.get_devices_summary_free_space()
volume_info_before_expansion = heketi_ops.heketi_volume_info(
self.heketi_client_node,
self.heketi_server_url,
volume_id, json=True)
self.assertNotEqual(volume_info_before_expansion, False,
"Volume info for %s failed" % volume_id)
heketi_vol_info_size_before_expansion = (
volume_info_before_expansion["size"])
self.get_brick_and_volume_status(volume_name)
num_of_bricks_before_expansion = self.get_num_of_bricks(volume_name)
expansion_info = heketi_ops.heketi_volume_expand(
self.heketi_client_node,
self.heketi_server_url,
volume_id, 5)
self.assertNotEqual(expansion_info, False,
"Volume expansion of %s failed" % volume_id)
free_space_after_expansion = self.get_devices_summary_free_space()
self.assertTrue(
free_space_after_creation > free_space_after_expansion,
"Free space not consumed after expansion of %s" % volume_id)
volume_info_after_expansion = heketi_ops.heketi_volume_info(
self.heketi_client_node,
self.heketi_server_url,
volume_id, json=True)
self.assertNotEqual(volume_info_after_expansion, False,
"Volume info failed for %s" % volume_id)
heketi_vol_info_size_after_expansion = (
volume_info_after_expansion["size"])
difference_size = (heketi_vol_info_size_after_expansion -
heketi_vol_info_size_before_expansion)
self.assertTrue(
difference_size > 0,
"Size not increased after expansion of %s" % volume_id)
self.get_brick_and_volume_status(volume_name)
num_of_bricks_after_expansion = self.get_num_of_bricks(volume_name)
num_of_bricks_added = (num_of_bricks_after_expansion -
num_of_bricks_before_expansion)
self.assertEqual(
num_of_bricks_added, 3,
"Number of bricks added is not 3 for %s" % volume_id)
self.get_rebalance_status(volume_name)
deletion_info = heketi_ops.heketi_volume_delete(
self.heketi_client_node, self.heketi_server_url,
volume_id, json=True)
self.assertNotEqual(deletion_info, False,
"Deletion of volume %s failed" % volume_id)
free_space_after_deletion = self.get_devices_summary_free_space()
self.assertTrue(
free_space_after_deletion > free_space_after_expansion,
"Free space is not reclaimed after volume deletion of %s"
% volume_id)
def test_volume_expansion_no_free_space(self):
"""Validate volume expansion when there is no free space"""
vol_size, expand_size, additional_devices_attached = None, 10, {}
h_node, h_server_url = self.heketi_client_node, self.heketi_server_url
# Get nodes info
heketi_node_id_list = heketi_ops.heketi_node_list(h_node, h_server_url)
if len(heketi_node_id_list) < 3:
self.skipTest("3 Heketi nodes are required.")
# Disable 4th and other nodes
for node_id in heketi_node_id_list[3:]:
heketi_ops.heketi_node_disable(h_node, h_server_url, node_id)
self.addCleanup(
heketi_ops.heketi_node_enable, h_node, h_server_url, node_id)
# Prepare first 3 nodes
smallest_size = None
err_msg = ''
for node_id in heketi_node_id_list[0:3]:
node_info = heketi_ops.heketi_node_info(
h_node, h_server_url, node_id, json=True)
# Disable second and other devices
devices = node_info["devices"]
self.assertTrue(
devices, "Node '%s' does not have devices." % node_id)
if devices[0]["state"].strip().lower() != "online":
self.skipTest("Test expects first device to be enabled.")
if (smallest_size is None or
devices[0]["storage"]["free"] < smallest_size):
smallest_size = devices[0]["storage"]["free"]
for device in node_info["devices"][1:]:
heketi_ops.heketi_device_disable(
h_node, h_server_url, device["id"])
self.addCleanup(
heketi_ops.heketi_device_enable,
h_node, h_server_url, device["id"])
# Gather info about additional devices
additional_device_name = None
for gluster_server in self.gluster_servers:
gluster_server_data = self.gluster_servers_info[gluster_server]
g_manage = gluster_server_data["manage"]
g_storage = gluster_server_data["storage"]
if not (g_manage in node_info["hostnames"]["manage"] or
g_storage in node_info["hostnames"]["storage"]):
continue
additional_device_name = ((
gluster_server_data.get("additional_devices") or [''])[0])
break
if not additional_device_name:
err_msg += ("No 'additional_devices' are configured for "
"'%s' node, which has following hostnames and "
"IP addresses: %s.\n" % (
node_id,
', '.join(node_info["hostnames"]["manage"] +
node_info["hostnames"]["storage"])))
continue
heketi_ops.heketi_device_add(
h_node, h_server_url, additional_device_name, node_id)
additional_devices_attached.update(
{node_id: additional_device_name})
# Schedule cleanup of the added devices
for node_id in additional_devices_attached.keys():
node_info = heketi_ops.heketi_node_info(
h_node, h_server_url, node_id, json=True)
for device in node_info["devices"]:
if device["name"] != additional_devices_attached[node_id]:
continue
self.addCleanup(self.detach_devices_attached, device["id"])
break
else:
self.fail("Could not find ID for added device on "
"'%s' node." % node_id)
if err_msg:
self.skipTest(err_msg)
# Temporary disable new devices
self.disable_devices(additional_devices_attached)
# Create volume and save info about it
vol_size = int(smallest_size / (1024**2)) - 1
creation_info = heketi_ops.heketi_volume_create(
h_node, h_server_url, vol_size, json=True)
volume_name, volume_id = creation_info["name"], creation_info["id"]
self.addCleanup(
heketi_ops.heketi_volume_delete,
h_node, h_server_url, volume_id, raise_on_error=False)
volume_info_before_expansion = heketi_ops.heketi_volume_info(
h_node, h_server_url, volume_id, json=True)
num_of_bricks_before_expansion = self.get_num_of_bricks(volume_name)
self.get_brick_and_volume_status(volume_name)
free_space_before_expansion = self.get_devices_summary_free_space()
# Try to expand volume with not enough device space
self.assertRaises(
ExecutionError, heketi_ops.heketi_volume_expand,
h_node, h_server_url, volume_id, expand_size)
# Enable new devices to be able to expand our volume
self.enable_devices(additional_devices_attached)
# Expand volume and validate results
heketi_ops.heketi_volume_expand(
h_node, h_server_url, volume_id, expand_size, json=True)
free_space_after_expansion = self.get_devices_summary_free_space()
self.assertGreater(
free_space_before_expansion, free_space_after_expansion,
"Free space not consumed after expansion of %s" % volume_id)
num_of_bricks_after_expansion = self.get_num_of_bricks(volume_name)
self.get_brick_and_volume_status(volume_name)
volume_info_after_expansion = heketi_ops.heketi_volume_info(
h_node, h_server_url, volume_id, json=True)
self.assertGreater(
volume_info_after_expansion["size"],
volume_info_before_expansion["size"],
"Size of %s not increased" % volume_id)
self.assertGreater(
num_of_bricks_after_expansion, num_of_bricks_before_expansion)
self.assertEqual(
num_of_bricks_after_expansion % num_of_bricks_before_expansion, 0)
# Delete volume and validate release of the used space
heketi_ops.heketi_volume_delete(h_node, h_server_url, volume_id)
free_space_after_deletion = self.get_devices_summary_free_space()
self.assertGreater(
free_space_after_deletion, free_space_after_expansion,
"Free space not reclaimed after deletion of volume %s" % volume_id)
def test_heketi_manual_cleanup_operation_in_bhv(self):
"""Validate heketi db cleanup will resolve the mismatch
in the free size of the block hosting volume with failed
block device create operations.
"""
bhv_size_before, bhv_size_after, vol_count = [], [], 5
ocp_node, g_node = self.ocp_master_node[0], self.gluster_servers[0]
h_node, h_url = self.heketi_client_node, self.heketi_server_url
# Get existing heketi volume list
existing_volumes = heketi_volume_list(h_node, h_url, json=True)
# Add function to clean stale volumes created during test
self.addCleanup(self._cleanup_heketi_volumes,
existing_volumes.get("volumes"))
# Get nodes id list
node_id_list = heketi_node_list(h_node, h_url)
# Disable 4th and other nodes
for node_id in node_id_list[3:]:
heketi_node_disable(h_node, h_url, node_id)
self.addCleanup(heketi_node_enable, h_node, h_url, node_id)
# Calculate heketi volume size
free_space, nodenum = get_total_free_space(h_node, h_url)
free_space_available = int(free_space / nodenum)
if free_space_available > vol_count:
h_volume_size = int(free_space_available / vol_count)
if h_volume_size > 50:
h_volume_size = 50
else:
h_volume_size, vol_count = 1, free_space_available
# Create BHV in case blockvolume size is greater than default BHV size
default_bhv_size = get_default_block_hosting_volume_size(
h_node, self.heketi_dc_name)
if default_bhv_size < h_volume_size:
h_volume_name = "autotest-{}".format(utils.get_random_str())
bhv_info = self.create_heketi_volume_with_name_and_wait(
h_volume_name,
free_space_available,
raise_on_cleanup_error=False,
block=True,
json=True)
free_space_available -= (
int(bhv_info.get("blockinfo").get("reservedsize")) + 1)
h_volume_size = int(free_space_available / vol_count)
# Get BHV list
h_bhv_list = get_block_hosting_volume_list(h_node, h_url).keys()
self.assertTrue(h_bhv_list, "Failed to get the BHV list")
# Get BHV size
for bhv in h_bhv_list:
vol_info = heketi_volume_info(h_node, h_url, bhv, json=True)
bhv_vol_size_before = vol_info.get("freesize")
bhv_size_before.append(bhv_vol_size_before)
# Kill Tcmu-runner service
services = ("tcmu-runner", "gluster-block-target", "gluster-blockd")
kill_service_on_gluster_pod_or_node(ocp_node, "tcmu-runner", g_node)
# Restart the services
for service in services:
state = ('exited'
if service == 'gluster-block-target' else 'running')
self.addCleanup(wait_for_service_status_on_gluster_pod_or_node,
ocp_node, service, 'active', state, g_node)
self.addCleanup(restart_service_on_gluster_pod_or_node, ocp_node,
service, g_node)
def run_async(cmd, hostname, raise_on_error=True):
return g.run_async(host=hostname, command=cmd)
# Create stale block volumes in async
for count in range(vol_count):
with mock.patch.object(json, 'loads', side_effect=(lambda j: j)):
with mock.patch.object(command,
'cmd_run',
side_effect=run_async):
heketi_blockvolume_create(h_node,
h_url,
h_volume_size,
json=True)
# Wait for pending operation to get generated
self._check_for_pending_operations(h_node, h_url)
# Restart the services
for service in services:
state = ('exited'
if service == 'gluster-block-target' else 'running')
restart_service_on_gluster_pod_or_node(ocp_node, service, g_node)
wait_for_service_status_on_gluster_pod_or_node(
ocp_node, service, 'active', state, g_node)
# Cleanup pending operation
heketi_server_operation_cleanup(h_node, h_url)
# wait for pending operation to get cleaned up
for w in waiter.Waiter(timeout=120, interval=10):
# Get BHV size
for bhv in h_bhv_list:
vol_info = heketi_volume_info(h_node, h_url, bhv, json=True)
bhv_vol_size_after = vol_info.get("freesize")
bhv_size_after.append(bhv_vol_size_after)
if (set(bhv_size_before) == set(bhv_size_after)):
break
if w.expired:
raise exceptions.ExecutionError(
"Failed to Validate volume size Actual:{},"
" Expected:{}".format(set(bhv_size_before),
set(bhv_size_after)))
def test_volume_creation_of_size_greater_than_the_device_size(self):
"""Validate creation of a volume of size greater than the size of a
device.
"""
h_node, h_url = self.heketi_client_node, self.heketi_server_url
# Remove existing BHV to calculate freespace
bhv_list = heketi_ops.get_block_hosting_volume_list(h_node, h_url)
if bhv_list:
for bhv in bhv_list:
bhv_info = heketi_ops.heketi_volume_info(h_node,
h_url,
bhv,
json=True)
if bhv_info['blockinfo'].get('blockvolume') is None:
heketi_ops.heketi_volume_delete(h_node, h_url, bhv)
topology = heketi_ops.heketi_topology_info(h_node, h_url, json=True)
nodes_free_space, nodes_ips = [], []
selected_nodes, selected_devices = [], []
cluster = topology['clusters'][0]
node_count = len(cluster['nodes'])
msg = ("At least 3 Nodes are required in cluster. "
"But only %s Nodes are present." % node_count)
if node_count < 3:
self.skipTest(msg)
online_nodes_count = 0
for node in cluster['nodes']:
nodes_ips.append(node['hostnames']['storage'][0])
if node['state'] != 'online':
continue
online_nodes_count += 1
# Disable nodes after 3rd online nodes
if online_nodes_count > 3:
heketi_ops.heketi_node_disable(h_node, h_url, node['id'])
self.addCleanup(heketi_ops.heketi_node_enable, h_node, h_url,
node['id'])
continue
selected_nodes.append(node['id'])
device_count = len(node['devices'])
msg = ("At least 2 Devices are required on each Node."
"But only %s Devices are present." % device_count)
if device_count < 2:
self.skipTest(msg)
sel_devices, online_devices_count, free_space = [], 0, 0
for device in node['devices']:
if device['state'] != 'online':
continue
online_devices_count += 1
# Disable devices after 2nd online devices
if online_devices_count > 2:
heketi_ops.heketi_device_disable(h_node, h_url,
device['id'])
self.addCleanup(heketi_ops.heketi_device_enable, h_node,
h_url, device['id'])
continue
sel_devices.append(device['id'])
free_space += int(device['storage']['free'] / (1024**2))
selected_devices.append(sel_devices)
nodes_free_space.append(free_space)
msg = ("At least 2 online Devices are required on each Node. "
"But only %s Devices are online on Node: %s." %
(online_devices_count, node['id']))
if online_devices_count < 2:
self.skipTest(msg)
msg = ("At least 3 online Nodes are required in cluster. "
"But only %s Nodes are online in Cluster: %s." %
(online_nodes_count, cluster['id']))
if online_nodes_count < 3:
self.skipTest(msg)
# Select node with minimum free space
min_free_size = min(nodes_free_space)
index = nodes_free_space.index(min_free_size)
# Get max device size from selected node
device_size = 0
for device in selected_devices[index]:
device_info = heketi_ops.heketi_device_info(h_node,
h_url,
device,
json=True)
device_size = max(device_size,
(int(device_info['storage']['total'] /
(1024**2))))
vol_size = device_size + 1
if vol_size >= min_free_size:
self.skipTest('Required free space %s is not available' % vol_size)
# Create heketi volume with device size + 1
vol_info = self.create_heketi_volume_with_name_and_wait(
name="volume_size_greater_than_device_size",
size=vol_size,
json=True)
# Get gluster server IP's from heketi volume info
glusterfs_servers = heketi_ops.get_vol_file_servers_and_hosts(
h_node, h_url, vol_info['id'])
# Verify gluster server IP's in heketi volume info
msg = ("gluster IP's '%s' does not match with IP's '%s' found in "
"heketi volume info" %
(nodes_ips, glusterfs_servers['vol_servers']))
self.assertEqual(set(glusterfs_servers['vol_servers']), set(nodes_ips),
msg)
vol_name = vol_info['name']
gluster_v_info = self.get_gluster_vol_info(vol_name)
# Verify replica count in gluster v info
msg = "Volume %s is replica %s instead of replica 3" % (
vol_name, gluster_v_info['replicaCount'])
self.assertEqual('3', gluster_v_info['replicaCount'])
# Verify distCount in gluster v info
msg = "Volume %s distCount is %s instead of distCount as 3" % (
vol_name, int(gluster_v_info['distCount']))
self.assertEqual(
int(gluster_v_info['brickCount']) // 3,
int(gluster_v_info['distCount']), msg)
# Verify bricks count in gluster v info
msg = (
"Volume %s does not have bricks count multiple of 3. It has %s" %
(vol_name, gluster_v_info['brickCount']))
self.assertFalse(int(gluster_v_info['brickCount']) % 3, msg)
def test_heket_block_volume_info_with_gluster_block_volume_info(self):
"""Verify heketi block volume info with the backend gluster
block volume info
"""
h_node, h_server = self.heketi_client_node, self.heketi_server_url
vol_size = 1
h_block_vol = heketi_blockvolume_create(h_node,
h_server,
vol_size,
auth=True,
json=True)
self.addCleanup(heketi_blockvolume_delete, h_node, h_server,
h_block_vol["id"])
h_block_vol["blockvolume"]["hosts"].sort()
h_block_vol_gbid = h_block_vol["blockvolume"]["username"]
h_block_vol_name = h_block_vol["name"]
h_block_vol_paswd = h_block_vol["blockvolume"]["password"]
h_block_vol_hosts = h_block_vol["blockvolume"]["hosts"]
h_block_host_vol_id = h_block_vol["blockhostingvolume"]
h_block_vol_ha = h_block_vol["hacount"]
# Fetch heketi blockhostingvolume info
h_bhv_info = heketi_volume_info(h_node,
h_server,
h_block_host_vol_id,
json=True)
err_msg = "Failed to get heketi blockhostingvolume info for {}"
self.assertTrue(h_bhv_info, err_msg.format(h_block_host_vol_id))
h_bhv_name = h_bhv_info['name']
err_msg = "Failed to get heketi BHV name for heketi blockvolume Id {}"
self.assertTrue(h_bhv_name, err_msg.format(h_block_host_vol_id))
# Get gluster blockvolume list
g_block_vol_list = get_block_list('auto_get_gluster_endpoint',
h_bhv_name)
err_msg = ("Failed to get gluter blockvolume list {}".format(
g_block_vol_list))
self.assertTrue(g_block_vol_list, err_msg)
err_msg = (
"Heketi block volume {} not present in gluster blockvolumes {}".
format(h_block_vol_name, g_block_vol_list))
self.assertIn(h_block_vol_name, g_block_vol_list, err_msg)
g_block_info = get_block_info('auto_get_gluster_endpoint', h_bhv_name,
h_block_vol_name)
g_block_info["EXPORTED ON"].sort()
g_block_vol_hosts = g_block_info["EXPORTED ON"]
g_block_vol_gbid = g_block_info["GBID"]
g_block_vol_name = g_block_info["NAME"]
g_block_vol_paswd = g_block_info["PASSWORD"]
g_block_vol_id = g_block_info["VOLUME"][4:]
g_block_vol_ha = g_block_info["HA"]
# verfiy block device info and glusterblock volume info
err_msg = ("Did not match {} from heketi {} and gluster {} side")
self.assertEqual(
h_block_vol_gbid, g_block_vol_gbid,
err_msg.format("GBID", h_block_vol_gbid, g_block_vol_gbid,
err_msg))
self.assertEqual(
h_block_vol_name, g_block_vol_name,
err_msg.format("blockvolume", h_block_vol_name, g_block_vol_name,
err_msg))
self.assertEqual(
h_block_vol_paswd, g_block_vol_paswd,
err_msg.format("password", h_block_vol_paswd, g_block_vol_paswd,
err_msg))
self.assertEqual(
h_block_vol_hosts, g_block_vol_hosts,
err_msg.format("hosts", h_block_vol_hosts, g_block_vol_hosts,
err_msg))
self.assertEqual(
h_block_host_vol_id, g_block_vol_id,
err_msg.format("blockhost vol id", h_block_host_vol_id,
g_block_vol_id, err_msg))
self.assertEqual(
h_block_vol_ha, g_block_vol_ha,
err_msg.format("ha", h_block_vol_ha, g_block_vol_ha, err_msg))
def test_volume_expansion_no_free_space(self):
"""Validate volume expansion when there is no free space"""
vol_size, expand_size, additional_devices_attached = None, 10, {}
h_node, h_server_url = self.heketi_client_node, self.heketi_server_url
# Get nodes info
heketi_node_id_list = heketi_ops.heketi_node_list(h_node, h_server_url)
if len(heketi_node_id_list) < 3:
self.skipTest("3 Heketi nodes are required.")
# Disable 4th and other nodes
for node_id in heketi_node_id_list[3:]:
heketi_ops.heketi_node_disable(h_node, h_server_url, node_id)
self.addCleanup(
heketi_ops.heketi_node_enable, h_node, h_server_url, node_id)
# Prepare first 3 nodes
smallest_size = None
err_msg = ''
for node_id in heketi_node_id_list[0:3]:
node_info = heketi_ops.heketi_node_info(
h_node, h_server_url, node_id, json=True)
# Disable second and other devices
devices = node_info["devices"]
self.assertTrue(
devices, "Node '%s' does not have devices." % node_id)
if devices[0]["state"].strip().lower() != "online":
self.skipTest("Test expects first device to be enabled.")
if (smallest_size is None
or devices[0]["storage"]["free"] < smallest_size):
smallest_size = devices[0]["storage"]["free"]
for device in node_info["devices"][1:]:
heketi_ops.heketi_device_disable(
h_node, h_server_url, device["id"])
self.addCleanup(
heketi_ops.heketi_device_enable,
h_node, h_server_url, device["id"])
# Gather info about additional devices
additional_device_name = None
for gluster_server in self.gluster_servers:
gluster_server_data = self.gluster_servers_info[gluster_server]
g_manage = gluster_server_data["manage"]
g_storage = gluster_server_data["storage"]
if not (g_manage in node_info["hostnames"]["manage"]
or g_storage in node_info["hostnames"]["storage"]):
continue
additional_device_name = ((
gluster_server_data.get("additional_devices") or [''])[0])
break
if not additional_device_name:
err_msg += ("No 'additional_devices' are configured for "
"'%s' node, which has following hostnames and "
"IP addresses: %s.\n" % (
node_id,
', '.join(
node_info["hostnames"]["manage"]
+ node_info["hostnames"]["storage"])))
continue
heketi_ops.heketi_device_add(
h_node, h_server_url, additional_device_name, node_id)
additional_devices_attached.update(
{node_id: additional_device_name})
# Schedule cleanup of the added devices
for node_id in additional_devices_attached.keys():
node_info = heketi_ops.heketi_node_info(
h_node, h_server_url, node_id, json=True)
for device in node_info["devices"]:
if device["name"] != additional_devices_attached[node_id]:
continue
self.addCleanup(self.detach_devices_attached, device["id"])
break
else:
self.fail("Could not find ID for added device on "
"'%s' node." % node_id)
if err_msg:
self.skipTest(err_msg)
# Temporary disable new devices
self.disable_devices(additional_devices_attached)
# Create volume and save info about it
vol_size = int(smallest_size / (1024**2)) - 1
creation_info = heketi_ops.heketi_volume_create(
h_node, h_server_url, vol_size, json=True)
volume_name, volume_id = creation_info["name"], creation_info["id"]
self.addCleanup(
heketi_ops.heketi_volume_delete,
h_node, h_server_url, volume_id, raise_on_error=False)
volume_info_before_expansion = heketi_ops.heketi_volume_info(
h_node, h_server_url, volume_id, json=True)
num_of_bricks_before_expansion = self.get_num_of_bricks(volume_name)
self.get_brick_and_volume_status(volume_name)
free_space_before_expansion = self.get_devices_summary_free_space()
# Try to expand volume with not enough device space
self.assertRaises(
AssertionError, heketi_ops.heketi_volume_expand,
h_node, h_server_url, volume_id, expand_size)
# Enable new devices to be able to expand our volume
self.enable_devices(additional_devices_attached)
# Expand volume and validate results
heketi_ops.heketi_volume_expand(
h_node, h_server_url, volume_id, expand_size, json=True)
free_space_after_expansion = self.get_devices_summary_free_space()
self.assertGreater(
free_space_before_expansion, free_space_after_expansion,
"Free space not consumed after expansion of %s" % volume_id)
num_of_bricks_after_expansion = self.get_num_of_bricks(volume_name)
self.get_brick_and_volume_status(volume_name)
volume_info_after_expansion = heketi_ops.heketi_volume_info(
h_node, h_server_url, volume_id, json=True)
self.assertGreater(
volume_info_after_expansion["size"],
volume_info_before_expansion["size"],
"Size of %s not increased" % volume_id)
self.assertGreater(
num_of_bricks_after_expansion, num_of_bricks_before_expansion)
self.assertEqual(
num_of_bricks_after_expansion % num_of_bricks_before_expansion, 0)
# Delete volume and validate release of the used space
heketi_ops.heketi_volume_delete(h_node, h_server_url, volume_id)
free_space_after_deletion = self.get_devices_summary_free_space()
self.assertGreater(
free_space_after_deletion, free_space_after_expansion,
"Free space not reclaimed after deletion of volume %s" % volume_id)
def test_expansion_of_block_hosting_volume_using_heketi(self):
"""Verify that after expanding block hosting volume we are able to
consume the expanded space"""
h_node = self.heketi_client_node
h_url = self.heketi_server_url
bvols_in_bhv = set([])
bvols_pv = set([])
BHVS = get_block_hosting_volume_list(h_node, h_url)
free_BHVS_count = 0
for vol in BHVS.keys():
info = heketi_volume_info(h_node, h_url, vol, json=True)
if info['blockinfo']['freesize'] > 0:
free_BHVS_count += 1
if free_BHVS_count > 1:
self.skipTest("Skip test case because there is more than one"
" Block Hosting Volume with free space")
# create block volume of 1gb
bvol_info = heketi_blockvolume_create(h_node, h_url, 1, json=True)
expand_size = 20
try:
self.verify_free_space(expand_size)
bhv = bvol_info['blockhostingvolume']
vol_info = heketi_volume_info(h_node, h_url, bhv, json=True)
bvols_in_bhv.update(vol_info['blockinfo']['blockvolume'])
finally:
# cleanup BHV if there is only one block volume inside it
if len(bvols_in_bhv) == 1:
self.addCleanup(heketi_volume_delete,
h_node,
h_url,
bhv,
json=True)
self.addCleanup(heketi_blockvolume_delete, h_node, h_url,
bvol_info['id'])
size = vol_info['size']
free_size = vol_info['blockinfo']['freesize']
bvol_count = int(free_size / expand_size)
bricks = vol_info['bricks']
# create pvs to fill the BHV
pvcs = self.create_and_wait_for_pvcs(
pvc_size=(expand_size if bvol_count else free_size),
pvc_amount=(bvol_count or 1),
timeout=300)
vol_expand = True
for i in range(2):
# get the vol ids from pvcs
for pvc in pvcs:
pv = get_pv_name_from_pvc(self.node, pvc)
custom = r':.metadata.annotations."gluster\.org\/volume-id"'
bvol_id = oc_get_custom_resource(self.node, 'pv', custom, pv)
bvols_pv.add(bvol_id[0])
vol_info = heketi_volume_info(h_node, h_url, bhv, json=True)
bvols = vol_info['blockinfo']['blockvolume']
bvols_in_bhv.update(bvols)
self.assertEqual(bvols_pv, (bvols_in_bhv & bvols_pv))
# Expand BHV and verify bricks and size of BHV
if vol_expand:
vol_expand = False
heketi_volume_expand(h_node,
h_url,
bhv,
expand_size,
json=True)
vol_info = heketi_volume_info(h_node, h_url, bhv, json=True)
self.assertEqual(size + expand_size, vol_info['size'])
self.assertFalse(len(vol_info['bricks']) % 3)
self.assertLess(len(bricks), len(vol_info['bricks']))
# create more PVCs in expanded BHV
pvcs = self.create_and_wait_for_pvcs(pvc_size=(expand_size -
1),
pvc_amount=1)
def test_pv_resize_when_heketi_down(self):
"""Create a PVC and try to expand it when heketi is down, It should
fail. After heketi is up, expand PVC should work.
"""
self.create_storage_class(allow_volume_expansion=True)
pvc_name = self.create_and_wait_for_pvc()
dc_name, pod_name = self.create_dc_with_pvc(pvc_name)
pv_name = get_pv_name_from_pvc(self.node, pvc_name)
custom = (r':metadata.annotations.'
r'"gluster\.kubernetes\.io\/heketi-volume-id"')
vol_id = oc_get_custom_resource(self.node, 'pv', custom, pv_name)[0]
h_vol_info = heketi_ops.heketi_volume_info(
self.heketi_client_node, self.heketi_server_url, vol_id, json=True)
# Bring the heketi POD down
scale_dc_pod_amount_and_wait(
self.node, self.heketi_dc_name, pod_amount=0)
self.addCleanup(
scale_dc_pod_amount_and_wait, self.node,
self.heketi_dc_name, pod_amount=1)
cmd = 'dd if=/dev/urandom of=/mnt/%s bs=614400k count=1'
ret, out, err = oc_rsh(self.node, pod_name, cmd % 'file1')
self.assertFalse(ret, 'Not able to write file with err: %s' % err)
wait_for_pod_be_ready(self.node, pod_name, 10, 5)
resize_pvc(self.node, pvc_name, 2)
wait_for_events(
self.node, pvc_name, obj_type='PersistentVolumeClaim',
event_type='Warning', event_reason='VolumeResizeFailed')
# Verify volume was not expanded
vol_info = get_gluster_vol_info_by_pvc_name(self.node, pvc_name)
self.assertEqual(vol_info['gluster_vol_id'], h_vol_info['name'])
self.assertEqual(
len(vol_info['bricks']['brick']), len(h_vol_info['bricks']))
# Bring the heketi POD up
scale_dc_pod_amount_and_wait(
self.node, self.heketi_dc_name, pod_amount=1)
# Verify volume expansion
verify_pvc_size(self.node, pvc_name, 2)
vol_info = get_gluster_vol_info_by_pvc_name(self.node, pvc_name)
self.assertFalse(len(vol_info['bricks']['brick']) % 3)
self.assertLess(
len(h_vol_info['bricks']), len(vol_info['bricks']['brick']))
# Wait for remount after expansion
for w in waiter.Waiter(timeout=30, interval=5):
ret, out, err = oc_rsh(
self.node, pod_name,
"df -Ph /mnt | awk '{print $2}' | tail -1")
self.assertFalse(ret, 'Failed with err: %s and Output: %s' % (
err, out))
if out.strip() == '2.0G':
break
# Write data making sure we have more space than it was
ret, out, err = oc_rsh(self.node, pod_name, cmd % 'file2')
self.assertFalse(ret, 'Not able to write file with err: %s' % err)
# Verify pod is running
wait_for_pod_be_ready(self.node, pod_name, 10, 5)
def test_volume_expansion_rebalance_brick(self):
"""Validate volume expansion with brick and check rebalance"""
creation_info = heketi_ops.heketi_volume_create(
self.heketi_client_node, self.heketi_server_url, 10, json=True)
self.assertNotEqual(creation_info, False, "Volume creation failed")
volume_name = creation_info["name"]
volume_id = creation_info["id"]
free_space_after_creation = self.get_devices_summary_free_space()
volume_info_before_expansion = heketi_ops.heketi_volume_info(
self.heketi_client_node,
self.heketi_server_url,
volume_id, json=True)
self.assertNotEqual(volume_info_before_expansion, False,
"Volume info for %s failed" % volume_id)
heketi_vol_info_size_before_expansion = (
volume_info_before_expansion["size"])
self.get_brick_and_volume_status(volume_name)
num_of_bricks_before_expansion = self.get_num_of_bricks(volume_name)
expansion_info = heketi_ops.heketi_volume_expand(
self.heketi_client_node,
self.heketi_server_url,
volume_id, 5)
self.assertNotEqual(expansion_info, False,
"Volume expansion of %s failed" % volume_id)
free_space_after_expansion = self.get_devices_summary_free_space()
self.assertTrue(
free_space_after_creation > free_space_after_expansion,
"Free space not consumed after expansion of %s" % volume_id)
volume_info_after_expansion = heketi_ops.heketi_volume_info(
self.heketi_client_node,
self.heketi_server_url,
volume_id, json=True)
self.assertNotEqual(volume_info_after_expansion, False,
"Volume info failed for %s" % volume_id)
heketi_vol_info_size_after_expansion = (
volume_info_after_expansion["size"])
difference_size = (heketi_vol_info_size_after_expansion
- heketi_vol_info_size_before_expansion)
self.assertTrue(
difference_size > 0,
"Size not increased after expansion of %s" % volume_id)
self.get_brick_and_volume_status(volume_name)
num_of_bricks_after_expansion = self.get_num_of_bricks(volume_name)
num_of_bricks_added = (
num_of_bricks_after_expansion - num_of_bricks_before_expansion)
self.assertEqual(
num_of_bricks_added, 3,
"Number of bricks added is not 3 for %s" % volume_id)
self.get_rebalance_status(volume_name)
deletion_info = heketi_ops.heketi_volume_delete(
self.heketi_client_node, self.heketi_server_url,
volume_id, json=True)
self.assertNotEqual(deletion_info, False,
"Deletion of volume %s failed" % volume_id)
free_space_after_deletion = self.get_devices_summary_free_space()
self.assertTrue(
free_space_after_deletion > free_space_after_expansion,
"Free space is not reclaimed after volume deletion of %s"
% volume_id)
def test_create_max_num_blockhostingvolumes(self):
num_of_bv = 10
new_bhv_list, bv_list, g_nodes = [], [], []
free_space, nodenum = get_total_free_space(self.heketi_client_node,
self.heketi_server_url)
if nodenum < 3:
self.skipTest("Skip the test case since number of"
"online nodes is less than 3.")
free_space_available = int(free_space / nodenum)
default_bhv_size = get_default_block_hosting_volume_size(
self.heketi_client_node, self.heketi_dc_name)
# Get existing list of BHV's
existing_bhv_list = get_block_hosting_volume_list(
self.heketi_client_node, self.heketi_server_url)
# Skip the test if available space is less than default_bhv_size
if free_space_available < default_bhv_size:
self.skipTest("Skip the test case since free_space_available %s"
"is less than space_required_for_bhv %s ." %
(free_space_available, default_bhv_size))
# Create BHV's
while free_space_available > default_bhv_size:
block_host_create_info = heketi_volume_create(
self.heketi_client_node,
self.heketi_server_url,
default_bhv_size,
json=True,
block=True)
if block_host_create_info["id"] not in existing_bhv_list.keys():
new_bhv_list.append(block_host_create_info["id"])
self.addCleanup(heketi_volume_delete,
self.heketi_client_node,
self.heketi_server_url,
block_host_create_info["id"],
raise_on_error=False)
block_vol_size = int(
block_host_create_info["blockinfo"]["freesize"] / num_of_bv)
# Create specified number of BV's in BHV's created
for i in range(0, num_of_bv):
block_vol = heketi_blockvolume_create(self.heketi_client_node,
self.heketi_server_url,
block_vol_size,
json=True,
ha=3,
auth=True)
self.addCleanup(heketi_blockvolume_delete,
self.heketi_client_node,
self.heketi_server_url,
block_vol["id"],
raise_on_error=False)
bv_list.append(block_vol["id"])
free_space_available = int(free_space_available - default_bhv_size)
# Get gluster node ips
h_nodes_ids = heketi_node_list(self.heketi_client_node,
self.heketi_server_url)
for h_node in h_nodes_ids[:2]:
g_node = heketi_node_info(self.heketi_client_node,
self.heketi_server_url,
h_node,
json=True)
g_nodes.append(g_node['hostnames']['manage'][0])
# Check if there is no crash in gluster related services & heketi
services = (("glusterd", "running"), ("gluster-blockd", "running"),
("tcmu-runner", "running"), ("gluster-block-target",
"exited"))
for g_node in g_nodes:
for service, state in services:
wait_for_service_status_on_gluster_pod_or_node(
self.ocp_client[0],
service,
'active',
state,
g_node,
raise_on_error=False)
out = hello_heketi(self.heketi_client_node, self.heketi_server_url)
self.assertTrue(
out, "Heketi server %s is not alive" % self.heketi_server_url)
# Delete all the BHV's and BV's created
for bv_volume in bv_list:
heketi_blockvolume_delete(self.heketi_client_node,
self.heketi_server_url, bv_volume)
# Check if any blockvolume exist in heketi & gluster
for bhv_volume in new_bhv_list[:]:
heketi_vol_info = heketi_volume_info(self.heketi_client_node,
self.heketi_server_url,
bhv_volume,
json=True)
self.assertNotIn("blockvolume",
heketi_vol_info["blockinfo"].keys())
gluster_vol_info = get_block_list('auto_get_gluster_endpoint',
volname="vol_%s" % bhv_volume)
self.assertIsNotNone(gluster_vol_info,
"Failed to get volume info %s" % bhv_volume)
new_bhv_list.remove(bhv_volume)
for blockvol in gluster_vol_info:
self.assertNotIn("blockvol_", blockvol)
heketi_volume_delete(self.heketi_client_node,
self.heketi_server_url, bhv_volume)
# Check if all blockhosting volumes are deleted from heketi
self.assertFalse(new_bhv_list)
def test_expansion_of_block_hosting_volume_using_heketi(self):
"""Verify that after expanding block hosting volume we are able to
consume the expanded space"""
h_node = self.heketi_client_node
h_url = self.heketi_server_url
bvols_in_bhv = set([])
bvols_pv = set([])
BHVS = get_block_hosting_volume_list(h_node, h_url)
free_BHVS_count = 0
for vol in BHVS.keys():
info = heketi_volume_info(h_node, h_url, vol, json=True)
if info['blockinfo']['freesize'] > 0:
free_BHVS_count += 1
if free_BHVS_count > 1:
self.skipTest("Skip test case because there is more than one"
" Block Hosting Volume with free space")
# create block volume of 1gb
bvol_info = heketi_blockvolume_create(h_node, h_url, 1, json=True)
expand_size = 20
try:
self.verify_free_space(expand_size)
bhv = bvol_info['blockhostingvolume']
vol_info = heketi_volume_info(h_node, h_url, bhv, json=True)
bvols_in_bhv.update(vol_info['blockinfo']['blockvolume'])
finally:
# cleanup BHV if there is only one block volume inside it
if len(bvols_in_bhv) == 1:
self.addCleanup(
heketi_volume_delete, h_node, h_url, bhv, json=True)
self.addCleanup(
heketi_blockvolume_delete, h_node, h_url, bvol_info['id'])
size = vol_info['size']
free_size = vol_info['blockinfo']['freesize']
bvol_count = int(free_size / expand_size)
bricks = vol_info['bricks']
# create pvs to fill the BHV
pvcs = self.create_and_wait_for_pvcs(
pvc_size=(expand_size if bvol_count else free_size),
pvc_amount=(bvol_count or 1), timeout=300)
vol_expand = True
for i in range(2):
# get the vol ids from pvcs
for pvc in pvcs:
pv = get_pv_name_from_pvc(self.node, pvc)
custom = r':.metadata.annotations."gluster\.org\/volume-id"'
bvol_id = oc_get_custom_resource(self.node, 'pv', custom, pv)
bvols_pv.add(bvol_id[0])
vol_info = heketi_volume_info(h_node, h_url, bhv, json=True)
bvols = vol_info['blockinfo']['blockvolume']
bvols_in_bhv.update(bvols)
self.assertEqual(bvols_pv, (bvols_in_bhv & bvols_pv))
# Expand BHV and verify bricks and size of BHV
if vol_expand:
vol_expand = False
heketi_volume_expand(
h_node, h_url, bhv, expand_size, json=True)
vol_info = heketi_volume_info(h_node, h_url, bhv, json=True)
self.assertEqual(size + expand_size, vol_info['size'])
self.assertFalse(len(vol_info['bricks']) % 3)
self.assertLess(len(bricks), len(vol_info['bricks']))
# create more PVCs in expanded BHV
pvcs = self.create_and_wait_for_pvcs(
pvc_size=(expand_size - 1), pvc_amount=1)
def test_verify_create_heketi_volumes_pending_entries_in_db(
self, vol_type):
"""Verify pending entries of file/block volumes in db during
volumes creation from heketi side
"""
# Create large volumes to observe the pending operations
vol_count, h_vol_creation_async_op = 3, []
h_node, h_url = self.heketi_client_node, self.heketi_server_url
# Verify file/block volumes pending operation before creation,
h_db_check_before = heketi_db_check(h_node, h_url)
h_db_check_vol_before = (h_db_check_before.get(
"{}volumes".format(vol_type)))
# Delete heketi pod to clean db operations
if (h_db_check_vol_before.get("pending")
or h_db_check_before.get("bricks").get("pending")):
self._respin_heketi_pod()
# Calculate heketi volume size
free_space, nodenum = get_total_free_space(h_node, h_url)
free_space_available = int(free_space / nodenum)
if free_space_available > vol_count:
h_volume_size = int(free_space_available / vol_count)
if h_volume_size > 30:
h_volume_size = 30
else:
h_volume_size, vol_count = 1, free_space_available
# Get existing heketi volume list
existing_volumes = heketi_volume_list(h_node, h_url, json=True)
# Add cleanup function to clean stale volumes created during test
self.addCleanup(self._cleanup_heketi_volumes,
existing_volumes.get("volumes"))
# Create BHV in case blockvolume size is greater than default BHV size
if vol_type:
default_bhv_size = get_default_block_hosting_volume_size(
h_node, self.heketi_dc_name)
if default_bhv_size < h_volume_size:
h_volume_name = "autotest-{}".format(utils.get_random_str())
bhv_info = self.create_heketi_volume_with_name_and_wait(
h_volume_name,
free_space_available,
raise_on_cleanup_error=False,
block=True,
json=True)
free_space_available -= (
int(bhv_info.get("blockinfo").get("reservedsize")) + 1)
h_volume_size = int(free_space_available / vol_count)
# Temporary replace g.run with g.async_run in heketi_blockvolume_create
# func to be able to run it in background.Also, avoid parsing the
# output as it won't be json at that moment. Parse it after reading
# the async operation results.
def run_async(cmd, hostname, raise_on_error=True):
return g.run_async(host=hostname, command=cmd)
for count in range(vol_count):
with mock.patch.object(json, 'loads', side_effect=(lambda j: j)):
with mock.patch.object(command,
'cmd_run',
side_effect=run_async):
h_vol_creation_async_op.append(
eval("heketi_{}volume_create".format(vol_type))(
h_node, h_url, h_volume_size, json=True))
# Check for pending operations
for w in waiter.Waiter(timeout=120, interval=10):
h_db_check = heketi_db_check(h_node, h_url)
h_db_check_vol = h_db_check.get("{}volumes".format(vol_type))
if h_db_check_vol.get("pending"):
h_db_check_bricks = h_db_check.get("bricks")
break
if w.expired:
raise exceptions.ExecutionError(
"No any pending operations found during {}volumes creation "
"{}".format(vol_type, h_db_check_vol.get("pending")))
# Verify bricks pending operation during creation
if not vol_type:
self.assertTrue(h_db_check_bricks.get("pending"),
"Expecting at least one bricks pending count")
self.assertFalse(
h_db_check_bricks.get("pending") % 3,
"Expecting bricks pending count to be multiple of 3 but "
"found {}".format(h_db_check_bricks.get("pending")))
# Wait for all counts of pending operations to be zero
for w in waiter.Waiter(timeout=300, interval=10):
h_db_check = heketi_db_check(h_node, h_url)
h_db_check_vol = h_db_check.get("{}volumes".format(vol_type))
if not h_db_check_vol.get("pending"):
break
if w.expired:
raise exceptions.ExecutionError(
"Expecting no pending operations after 300 sec but "
"found {} operation".format(h_db_check_vol.get("pending")))
# Get heketi server DB details
h_db_check_after = heketi_db_check(h_node, h_url)
h_db_check_vol_after = (h_db_check_after.get(
"{}volumes".format(vol_type)))
h_db_check_bricks_after = h_db_check_after.get("bricks")
# Verify if initial and final file/block volumes are same
act_vol_count = h_db_check_vol_after.get("total")
exp_vol_count = h_db_check_vol_before.get("total") + vol_count
err_msg = (
"Actual {} and expected {} {}volume counts are not matched".format(
act_vol_count, exp_vol_count, vol_type))
self.assertEqual(act_vol_count, exp_vol_count, err_msg)
# Verify if initial and final bricks are same for file volume
volumes = heketi_volume_list(h_node, h_url, json=True).get("volumes")
new_volumes = list(set(volumes) - set(existing_volumes))
exp_brick_count = 0
for volume in new_volumes:
vol_info = heketi_volume_info(h_node, h_url, volume, json=True)
exp_brick_count += len(vol_info.get("bricks"))
err_msg = "Actual {} and expected {} bricks counts are not matched"
act_brick_count = h_db_check_bricks_after.get("total")
self.assertEqual(act_brick_count, exp_brick_count,
err_msg.format(act_brick_count, exp_brick_count))
def test_pv_resize_when_heketi_down(self):
"""Create a PVC and try to expand it when heketi is down, It should
fail. After heketi is up, expand PVC should work.
"""
self.create_storage_class(allow_volume_expansion=True)
pvc_name = self.create_and_wait_for_pvc()
dc_name, pod_name = self.create_dc_with_pvc(pvc_name)
pv_name = get_pv_name_from_pvc(self.node, pvc_name)
custom = (r':metadata.annotations.'
r'"gluster\.kubernetes\.io\/heketi-volume-id"')
vol_id = oc_get_custom_resource(self.node, 'pv', custom, pv_name)[0]
h_vol_info = heketi_ops.heketi_volume_info(self.heketi_client_node,
self.heketi_server_url,
vol_id,
json=True)
# Bring the heketi POD down
scale_dc_pod_amount_and_wait(self.node,
self.heketi_dc_name,
pod_amount=0)
self.addCleanup(scale_dc_pod_amount_and_wait,
self.node,
self.heketi_dc_name,
pod_amount=1)
cmd = 'dd if=/dev/urandom of=/mnt/%s bs=614400k count=1'
ret, out, err = oc_rsh(self.node, pod_name, cmd % 'file1')
self.assertFalse(ret, 'Not able to write file with err: %s' % err)
wait_for_pod_be_ready(self.node, pod_name, 10, 5)
resize_pvc(self.node, pvc_name, 2)
wait_for_events(self.node,
pvc_name,
obj_type='PersistentVolumeClaim',
event_type='Warning',
event_reason='VolumeResizeFailed')
# Verify volume was not expanded
vol_info = get_gluster_vol_info_by_pvc_name(self.node, pvc_name)
self.assertEqual(vol_info['gluster_vol_id'], h_vol_info['name'])
self.assertEqual(len(vol_info['bricks']['brick']),
len(h_vol_info['bricks']))
# Bring the heketi POD up
scale_dc_pod_amount_and_wait(self.node,
self.heketi_dc_name,
pod_amount=1)
# Verify volume expansion
verify_pvc_size(self.node, pvc_name, 2)
vol_info = get_gluster_vol_info_by_pvc_name(self.node, pvc_name)
self.assertFalse(len(vol_info['bricks']['brick']) % 3)
self.assertLess(len(h_vol_info['bricks']),
len(vol_info['bricks']['brick']))
# Wait for remount after expansion
for w in waiter.Waiter(timeout=30, interval=5):
ret, out, err = oc_rsh(self.node, pod_name,
"df -Ph /mnt | awk '{print $2}' | tail -1")
self.assertFalse(ret,
'Failed with err: %s and Output: %s' % (err, out))
if out.strip() == '2.0G':
break
# Write data making sure we have more space than it was
ret, out, err = oc_rsh(self.node, pod_name, cmd % 'file2')
self.assertFalse(ret, 'Not able to write file with err: %s' % err)
# Verify pod is running
wait_for_pod_be_ready(self.node, pod_name, 10, 5)
def test_targetcli_when_block_hosting_volume_down(self):
"""Validate no inconsistencies occur in targetcli when block volumes
are created with one block hosting volume down."""
h_node, h_server = self.heketi_client_node, self.heketi_server_url
cmd = ("targetcli ls | egrep '%s' || echo unavailable")
error_msg = ("targetcli has inconsistencies when block devices are "
"created with one block hosting volume %s is down")
# Delete BHV which has no BV or fill it completely
bhv_list = get_block_hosting_volume_list(h_node, h_server).keys()
for bhv in bhv_list:
bhv_info = heketi_volume_info(h_node, h_server, bhv, json=True)
if not bhv_info["blockinfo"].get("blockvolume", []):
heketi_volume_delete(h_node, h_server, bhv)
continue
free_size = bhv_info["blockinfo"].get("freesize", 0)
if free_size:
bv = heketi_volume_create(h_node,
h_server,
free_size,
json=True)
self.addCleanup(heketi_volume_delete, h_node, h_server,
bv["id"])
# Create BV
bv = heketi_blockvolume_create(h_node, h_server, 2, json=True)
self.addCleanup(heketi_blockvolume_delete, h_node, h_server, bv["id"])
# Bring down BHV
bhv_name = get_block_hosting_volume_name(h_node, h_server, bv["id"])
ret, out, err = volume_stop("auto_get_gluster_endpoint", bhv_name)
if ret != 0:
err_msg = "Failed to stop gluster volume %s. error: %s" % (
bhv_name, err)
g.log.error(err_msg)
raise AssertionError(err_msg)
self.addCleanup(podcmd.GlustoPod()(volume_start),
"auto_get_gluster_endpoint", bhv_name)
ocp_node = self.ocp_master_node[0]
gluster_block_svc = "gluster-block-target"
self.addCleanup(wait_for_service_status_on_gluster_pod_or_node,
ocp_node,
gluster_block_svc,
"active",
"exited",
gluster_node=self.gluster_servers[0])
self.addCleanup(restart_service_on_gluster_pod_or_node, ocp_node,
gluster_block_svc, self.gluster_servers[0])
for condition in ("continue", "break"):
restart_service_on_gluster_pod_or_node(
ocp_node,
gluster_block_svc,
gluster_node=self.gluster_servers[0])
wait_for_service_status_on_gluster_pod_or_node(
ocp_node,
gluster_block_svc,
"active",
"exited",
gluster_node=self.gluster_servers[0])
targetcli = cmd_run_on_gluster_pod_or_node(ocp_node,
cmd % bv["id"],
self.gluster_servers[0])
if condition == "continue":
self.assertEqual(targetcli, "unavailable",
error_msg % bhv_name)
else:
self.assertNotEqual(targetcli, "unavailable",
error_msg % bhv_name)
break
# Bring up the same BHV
ret, out, err = volume_start("auto_get_gluster_endpoint", bhv_name)
if ret != 0:
err = "Failed to start gluster volume %s on %s. error: %s" % (
bhv_name, h_node, err)
raise exceptions.ExecutionError(err)
