def test_create_vol_and_retrieve_topology_info(self):
volume_names = []
volume_ids = []
# Create 3 volumes and make 3rd volume of type distributed replica
g.log.info("Creating 3 volumes")
for i in range(3):
out = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
self.volume_size,
json=True)
g.log.info("Heketi volume %s successfully created" % out)
volume_names.append(out["name"])
volume_ids.append(out["bricks"][0]["volume"])
self.addCleanup(heketi_volume_delete,
self.heketi_client_node,
self.heketi_server_url,
volume_ids[i],
raise_on_error=(i == 2))
heketi_volume_expand(self.heketi_client_node, self.heketi_server_url,
volume_ids[1], 1)
# Check if volume is shown in the heketi topology
topology_volumes = get_heketi_volume_and_brick_count_list(
self.heketi_client_node, self.heketi_server_url)
existing_volumes = [v for v, _ in topology_volumes]
for v in volume_names:
self.assertIn(v, existing_volumes)
for v, b_count in topology_volumes:
expected_bricks_count = 6 if v == volume_names[1] else 3
self.assertGreaterEqual(
b_count, expected_bricks_count,
'Bricks number of the %s volume is %s and it is expected '
'to be greater or equal to %s' %
(v, b_count, expected_bricks_count))
# Delete first 2 volumes and verify their deletion in the topology
for vol_id in volume_ids[:2]:
g.log.info("Deleting volume %s" % vol_id)
heketi_volume_delete(self.heketi_client_node,
self.heketi_server_url, vol_id)
topology_volumes = get_heketi_volume_and_brick_count_list(
self.heketi_client_node, self.heketi_server_url)
existing_volumes = [v for v, _ in topology_volumes]
for vol_name in volume_names[:2]:
self.assertNotIn(
vol_name, existing_volumes,
("volume %s shown in the heketi topology after deletion"
"\nTopology info:\n%s" % (vol_name, existing_volumes)))
# Check the existence of third volume
self.assertIn(
volume_names[2], existing_volumes, "volume %s not "
"shown in the heketi topology\nTopology info"
"\n%s" % (volume_ids[2], existing_volumes))
g.log.info("Sucessfully verified the topology info")
def test_heketi_with_expand_volume(self):
"""
Test volume expand and size if updated correctly in heketi-cli info
"""
vol_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
self.volume_size, json=True)
self.assertTrue(vol_info, ("Failed to create heketi volume of size %s"
% self.volume_size))
self.addCleanup(
heketi_volume_delete,
self.heketi_client_node, self.heketi_server_url, vol_info['id'])
self.assertEqual(vol_info['size'], self.volume_size,
("Failed to create volume."
"Expected Size: %s, Actual Size: %s"
% (self.volume_size, vol_info['size'])))
volume_id = vol_info["id"]
expand_size = 2
ret = heketi_volume_expand(self.heketi_client_node,
self.heketi_server_url, volume_id,
expand_size)
self.assertTrue(ret, ("Failed to expand heketi volume of id %s"
% volume_id))
volume_info = heketi_volume_info(self.heketi_client_node,
self.heketi_server_url,
volume_id, json=True)
expected_size = self.volume_size + expand_size
self.assertEqual(volume_info['size'], expected_size,
("Volume Expansion failed Expected Size: %s, Actual "
"Size: %s" % (str(expected_size),
str(volume_info['size']))))
def test_heketi_with_expand_volume(self):
"""
Test volume expand and size if updated correctly in heketi-cli info
"""
vol_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
self.volume_size,
json=True)
self.assertTrue(
vol_info,
("Failed to create heketi volume of size %s" % self.volume_size))
self.addCleanup(heketi_volume_delete, self.heketi_client_node,
self.heketi_server_url, vol_info['id'])
self.assertEqual(vol_info['size'], self.volume_size,
("Failed to create volume."
"Expected Size: %s, Actual Size: %s" %
(self.volume_size, vol_info['size'])))
volume_id = vol_info["id"]
expand_size = 2
ret = heketi_volume_expand(self.heketi_client_node,
self.heketi_server_url, volume_id,
expand_size)
self.assertTrue(
ret, ("Failed to expand heketi volume of id %s" % volume_id))
volume_info = heketi_volume_info(self.heketi_client_node,
self.heketi_server_url,
volume_id,
json=True)
expected_size = self.volume_size + expand_size
self.assertEqual(volume_info['size'], expected_size,
("Volume Expansion failed Expected Size: %s, Actual "
"Size: %s" %
(str(expected_size), str(volume_info['size']))))
def test_promethoues_validation_while_creation_or_expansion(self, motive):
"""Validate mertics data after volume creation or expansion"""
# Define the variables to perform validations
metrics = [
'heketi_device_size_bytes', 'heketi_device_free_bytes',
'heketi_device_used_bytes', 'heketi_device_brick_count'
]
h_client, h_server = self.heketi_client_node, self.heketi_server_url
vol_size = 1
# Collect the metrics data from prometheus pod
if motive == 'creation':
initial_result = self._get_and_manipulate_metric_data(metrics)
# Create a volume
volume_id = heketi_ops.heketi_volume_create(
h_client, h_server, vol_size, json=True)["bricks"][0]["volume"]
self.addCleanup(heketi_ops.heketi_volume_delete, h_client, h_server,
volume_id)
# Expand the volume
if motive == 'expansion':
initial_result = self._get_and_manipulate_metric_data(metrics)
heketi_ops.heketi_volume_expand(h_client, h_server, volume_id,
vol_size)
# Fetch the latest metrics data form prometheus pod
final_result = self._get_and_manipulate_metric_data(metrics)
# Validate the data variation
for metric in metrics:
msg = (
"intial {} and final value {} of metric '{} should be".format(
initial_result[metric], final_result[metric], metric))
if metric == 'heketi_device_size_bytes':
self.assertEqual(initial_result[metric], final_result[metric],
msg + " same")
if metric == 'heketi_device_free_bytes':
self.assertGreater(initial_result[metric],
final_result[metric], msg + " differnt")
if metric == ('heketi_device_used_bytes'
or 'heketi_device_brick_count'):
self.assertLess(initial_result[metric], final_result[metric],
msg + " differnt")
def test_replica_volume_expand(self):
"""
Test expansion of a replica volume
"""
h_node, h_server = self.heketi_client_node, self.heketi_server_url
volume_name = ("autotests-heketi-volume-{}".format(
utils.get_random_str()))
volume_size = 10
creation_info = self.create_heketi_volume_with_name_and_wait(
volume_name, volume_size, json=True, raise_on_cleanup_error=False)
volume_id = creation_info["id"]
volume_info = heketi_ops.heketi_volume_info(h_node,
h_server,
volume_id,
json=True)
# Get gluster volume info
gluster_vol = volume_ops.get_volume_info('auto_get_gluster_endpoint',
volname=volume_name)
self.assertTrue(gluster_vol,
"Failed to get volume {} info".format(volume_name))
vol_name = gluster_vol[volume_name]
self.assertEqual(
vol_name['replicaCount'], "3",
"Replica count is different for volume {} Actual:{} "
"Expected : 3".format(vol_name, vol_name['replicaCount']))
expand_size = 5
heketi_ops.heketi_volume_expand(h_node, h_server, volume_id,
expand_size)
volume_info = heketi_ops.heketi_volume_info(h_node,
h_server,
volume_id,
json=True)
expected_size = volume_size + expand_size
self.assertEqual(
volume_info['size'], expected_size,
"Volume Expansion failed, Expected Size: {}, Actual "
"Size: {}".format(str(expected_size), str(volume_info['size'])))
self.get_brick_and_volume_status(volume_name)
self.get_rebalance_status(volume_name)
def test_volume_expansion_rebalance_brick(self):
"""Validate volume expansion with brick and check rebalance"""
creation_info = heketi_ops.heketi_volume_create(
self.heketi_client_node, self.heketi_server_url, 10, json=True)
self.assertNotEqual(creation_info, False, "Volume creation failed")
volume_name = creation_info["name"]
volume_id = creation_info["id"]
free_space_after_creation = self.get_devices_summary_free_space()
volume_info_before_expansion = heketi_ops.heketi_volume_info(
self.heketi_client_node,
self.heketi_server_url,
volume_id, json=True)
self.assertNotEqual(volume_info_before_expansion, False,
"Volume info for %s failed" % volume_id)
heketi_vol_info_size_before_expansion = (
volume_info_before_expansion["size"])
self.get_brick_and_volume_status(volume_name)
num_of_bricks_before_expansion = self.get_num_of_bricks(volume_name)
expansion_info = heketi_ops.heketi_volume_expand(
self.heketi_client_node,
self.heketi_server_url,
volume_id, 5)
self.assertNotEqual(expansion_info, False,
"Volume expansion of %s failed" % volume_id)
free_space_after_expansion = self.get_devices_summary_free_space()
self.assertTrue(
free_space_after_creation > free_space_after_expansion,
"Free space not consumed after expansion of %s" % volume_id)
volume_info_after_expansion = heketi_ops.heketi_volume_info(
self.heketi_client_node,
self.heketi_server_url,
volume_id, json=True)
self.assertNotEqual(volume_info_after_expansion, False,
"Volume info failed for %s" % volume_id)
heketi_vol_info_size_after_expansion = (
volume_info_after_expansion["size"])
difference_size = (heketi_vol_info_size_after_expansion -
heketi_vol_info_size_before_expansion)
self.assertTrue(
difference_size > 0,
"Size not increased after expansion of %s" % volume_id)
self.get_brick_and_volume_status(volume_name)
num_of_bricks_after_expansion = self.get_num_of_bricks(volume_name)
num_of_bricks_added = (num_of_bricks_after_expansion -
num_of_bricks_before_expansion)
self.assertEqual(
num_of_bricks_added, 3,
"Number of bricks added is not 3 for %s" % volume_id)
self.get_rebalance_status(volume_name)
deletion_info = heketi_ops.heketi_volume_delete(
self.heketi_client_node, self.heketi_server_url,
volume_id, json=True)
self.assertNotEqual(deletion_info, False,
"Deletion of volume %s failed" % volume_id)
free_space_after_deletion = self.get_devices_summary_free_space()
self.assertTrue(
free_space_after_deletion > free_space_after_expansion,
"Free space is not reclaimed after volume deletion of %s"
% volume_id)
def test_volume_expansion_no_free_space(self):
"""Validate volume expansion when there is no free space"""
vol_size, expand_size, additional_devices_attached = None, 10, {}
h_node, h_server_url = self.heketi_client_node, self.heketi_server_url
# Get nodes info
heketi_node_id_list = heketi_ops.heketi_node_list(h_node, h_server_url)
if len(heketi_node_id_list) < 3:
self.skipTest("3 Heketi nodes are required.")
# Disable 4th and other nodes
for node_id in heketi_node_id_list[3:]:
heketi_ops.heketi_node_disable(h_node, h_server_url, node_id)
self.addCleanup(
heketi_ops.heketi_node_enable, h_node, h_server_url, node_id)
# Prepare first 3 nodes
smallest_size = None
err_msg = ''
for node_id in heketi_node_id_list[0:3]:
node_info = heketi_ops.heketi_node_info(
h_node, h_server_url, node_id, json=True)
# Disable second and other devices
devices = node_info["devices"]
self.assertTrue(
devices, "Node '%s' does not have devices." % node_id)
if devices[0]["state"].strip().lower() != "online":
self.skipTest("Test expects first device to be enabled.")
if (smallest_size is None or
devices[0]["storage"]["free"] < smallest_size):
smallest_size = devices[0]["storage"]["free"]
for device in node_info["devices"][1:]:
heketi_ops.heketi_device_disable(
h_node, h_server_url, device["id"])
self.addCleanup(
heketi_ops.heketi_device_enable,
h_node, h_server_url, device["id"])
# Gather info about additional devices
additional_device_name = None
for gluster_server in self.gluster_servers:
gluster_server_data = self.gluster_servers_info[gluster_server]
g_manage = gluster_server_data["manage"]
g_storage = gluster_server_data["storage"]
if not (g_manage in node_info["hostnames"]["manage"] or
g_storage in node_info["hostnames"]["storage"]):
continue
additional_device_name = ((
gluster_server_data.get("additional_devices") or [''])[0])
break
if not additional_device_name:
err_msg += ("No 'additional_devices' are configured for "
"'%s' node, which has following hostnames and "
"IP addresses: %s.\n" % (
node_id,
', '.join(node_info["hostnames"]["manage"] +
node_info["hostnames"]["storage"])))
continue
heketi_ops.heketi_device_add(
h_node, h_server_url, additional_device_name, node_id)
additional_devices_attached.update(
{node_id: additional_device_name})
# Schedule cleanup of the added devices
for node_id in additional_devices_attached.keys():
node_info = heketi_ops.heketi_node_info(
h_node, h_server_url, node_id, json=True)
for device in node_info["devices"]:
if device["name"] != additional_devices_attached[node_id]:
continue
self.addCleanup(self.detach_devices_attached, device["id"])
break
else:
self.fail("Could not find ID for added device on "
"'%s' node." % node_id)
if err_msg:
self.skipTest(err_msg)
# Temporary disable new devices
self.disable_devices(additional_devices_attached)
# Create volume and save info about it
vol_size = int(smallest_size / (1024**2)) - 1
creation_info = heketi_ops.heketi_volume_create(
h_node, h_server_url, vol_size, json=True)
volume_name, volume_id = creation_info["name"], creation_info["id"]
self.addCleanup(
heketi_ops.heketi_volume_delete,
h_node, h_server_url, volume_id, raise_on_error=False)
volume_info_before_expansion = heketi_ops.heketi_volume_info(
h_node, h_server_url, volume_id, json=True)
num_of_bricks_before_expansion = self.get_num_of_bricks(volume_name)
self.get_brick_and_volume_status(volume_name)
free_space_before_expansion = self.get_devices_summary_free_space()
# Try to expand volume with not enough device space
self.assertRaises(
ExecutionError, heketi_ops.heketi_volume_expand,
h_node, h_server_url, volume_id, expand_size)
# Enable new devices to be able to expand our volume
self.enable_devices(additional_devices_attached)
# Expand volume and validate results
heketi_ops.heketi_volume_expand(
h_node, h_server_url, volume_id, expand_size, json=True)
free_space_after_expansion = self.get_devices_summary_free_space()
self.assertGreater(
free_space_before_expansion, free_space_after_expansion,
"Free space not consumed after expansion of %s" % volume_id)
num_of_bricks_after_expansion = self.get_num_of_bricks(volume_name)
self.get_brick_and_volume_status(volume_name)
volume_info_after_expansion = heketi_ops.heketi_volume_info(
h_node, h_server_url, volume_id, json=True)
self.assertGreater(
volume_info_after_expansion["size"],
volume_info_before_expansion["size"],
"Size of %s not increased" % volume_id)
self.assertGreater(
num_of_bricks_after_expansion, num_of_bricks_before_expansion)
self.assertEqual(
num_of_bricks_after_expansion % num_of_bricks_before_expansion, 0)
# Delete volume and validate release of the used space
heketi_ops.heketi_volume_delete(h_node, h_server_url, volume_id)
free_space_after_deletion = self.get_devices_summary_free_space()
self.assertGreater(
free_space_after_deletion, free_space_after_expansion,
"Free space not reclaimed after deletion of volume %s" % volume_id)
def test_expansion_of_block_hosting_volume_using_heketi(self):
"""Verify that after expanding block hosting volume we are able to
consume the expanded space"""
h_node = self.heketi_client_node
h_url = self.heketi_server_url
bvols_in_bhv = set([])
bvols_pv = set([])
BHVS = get_block_hosting_volume_list(h_node, h_url)
free_BHVS_count = 0
for vol in BHVS.keys():
info = heketi_volume_info(h_node, h_url, vol, json=True)
if info['blockinfo']['freesize'] > 0:
free_BHVS_count += 1
if free_BHVS_count > 1:
self.skipTest("Skip test case because there is more than one"
" Block Hosting Volume with free space")
# create block volume of 1gb
bvol_info = heketi_blockvolume_create(h_node, h_url, 1, json=True)
expand_size = 20
try:
self.verify_free_space(expand_size)
bhv = bvol_info['blockhostingvolume']
vol_info = heketi_volume_info(h_node, h_url, bhv, json=True)
bvols_in_bhv.update(vol_info['blockinfo']['blockvolume'])
finally:
# cleanup BHV if there is only one block volume inside it
if len(bvols_in_bhv) == 1:
self.addCleanup(heketi_volume_delete,
h_node,
h_url,
bhv,
json=True)
self.addCleanup(heketi_blockvolume_delete, h_node, h_url,
bvol_info['id'])
size = vol_info['size']
free_size = vol_info['blockinfo']['freesize']
bvol_count = int(free_size / expand_size)
bricks = vol_info['bricks']
# create pvs to fill the BHV
pvcs = self.create_and_wait_for_pvcs(
pvc_size=(expand_size if bvol_count else free_size),
pvc_amount=(bvol_count or 1),
timeout=300)
vol_expand = True
for i in range(2):
# get the vol ids from pvcs
for pvc in pvcs:
pv = get_pv_name_from_pvc(self.node, pvc)
custom = r':.metadata.annotations."gluster\.org\/volume-id"'
bvol_id = oc_get_custom_resource(self.node, 'pv', custom, pv)
bvols_pv.add(bvol_id[0])
vol_info = heketi_volume_info(h_node, h_url, bhv, json=True)
bvols = vol_info['blockinfo']['blockvolume']
bvols_in_bhv.update(bvols)
self.assertEqual(bvols_pv, (bvols_in_bhv & bvols_pv))
# Expand BHV and verify bricks and size of BHV
if vol_expand:
vol_expand = False
heketi_volume_expand(h_node,
h_url,
bhv,
expand_size,
json=True)
vol_info = heketi_volume_info(h_node, h_url, bhv, json=True)
self.assertEqual(size + expand_size, vol_info['size'])
self.assertFalse(len(vol_info['bricks']) % 3)
self.assertLess(len(bricks), len(vol_info['bricks']))
# create more PVCs in expanded BHV
pvcs = self.create_and_wait_for_pvcs(pvc_size=(expand_size -
1),
pvc_amount=1)
def test_set_heketi_vol_size_and_brick_amount_limits(self):
# Get Heketi secret name
cmd_get_heketi_secret_name = (
"oc get dc -n %s %s -o jsonpath='{.spec.template.spec.volumes"
"[?(@.name==\"config\")].secret.secretName}'" %
(self.storage_project_name, self.heketi_dc_name))
heketi_secret_name = self.cmd_run(cmd_get_heketi_secret_name)
# Read Heketi secret data
self.node = self.ocp_master_node[0]
heketi_secret_data_str_base64 = oc_get_custom_resource(
self.node,
"secret",
":.data.'heketi\.json'", # noqa
name=heketi_secret_name)[0]
heketi_secret_data_str = self.cmd_run("echo %s | base64 -d" %
heketi_secret_data_str_base64)
heketi_secret_data = json.loads(heketi_secret_data_str)
# Update Heketi secret data
brick_min_size_gb, brick_max_size_gb = 2, 4
heketi_secret_data["glusterfs"].update({
"brick_min_size_gb": brick_min_size_gb,
"brick_max_size_gb": brick_max_size_gb,
"max_bricks_per_volume": 3,
})
heketi_secret_data_patched = json.dumps(heketi_secret_data)
heketi_secret_data_str_encoded = self.cmd_run(
"echo '%s' |base64" % heketi_secret_data_patched).replace(
'\n', '')
h_client, h_server = self.heketi_client_node, self.heketi_server_url
try:
# Patch Heketi secret
cmd_patch_heketi_secret = (
'oc patch secret -n %s %s -p '
'"{\\"data\\": {\\"heketi.json\\": \\"%s\\"}}"') % (
self.storage_project_name, heketi_secret_name, "%s")
self.cmd_run(cmd_patch_heketi_secret %
heketi_secret_data_str_encoded)
# Recreate the Heketi pod to make it reuse updated configuration
scale_dc_pod_amount_and_wait(self.node, self.heketi_dc_name, 0)
scale_dc_pod_amount_and_wait(self.node, self.heketi_dc_name, 1)
# Try to create too small and too big volumes
# It must fail because allowed range is not satisfied
for gb in (brick_min_size_gb - 1, brick_max_size_gb + 1):
try:
vol_1 = heketi_volume_create(h_client,
h_server,
size=gb,
json=True)
except AssertionError:
pass
else:
self.addCleanup(heketi_volume_delete, h_client, h_server,
vol_1['id'])
self.assertFalse(
vol_1,
"Volume '%s' got unexpectedly created. Heketi server "
"configuration haven't made required effect." %
(vol_1.get('id', 'failed_to_get_heketi_vol_id')))
# Create the smallest allowed volume
vol_2 = heketi_volume_create(h_client,
h_server,
size=brick_min_size_gb,
json=True)
self.addCleanup(heketi_volume_delete, h_client, h_server,
vol_2['id'])
# Try to expand volume, it must fail due to the brick amount limit
self.assertRaises(AssertionError, heketi_volume_expand, h_client,
h_server, vol_2['id'], 2)
# Create the largest allowed volume
vol_3 = heketi_volume_create(h_client,
h_server,
size=brick_max_size_gb,
json=True)
heketi_volume_delete(h_client, h_server, vol_3['id'])
finally:
# Revert the Heketi configuration back
self.cmd_run(cmd_patch_heketi_secret %
heketi_secret_data_str_base64)
scale_dc_pod_amount_and_wait(self.node, self.heketi_dc_name, 0)
scale_dc_pod_amount_and_wait(self.node, self.heketi_dc_name, 1)
# Create volume less than the old minimum limit
vol_4 = heketi_volume_create(h_client,
h_server,
size=(brick_min_size_gb - 1),
json=True)
self.addCleanup(heketi_volume_delete, h_client, h_server, vol_4['id'])
# Create volume bigger than the old maximum limit and expand it
vol_5 = heketi_volume_create(h_client,
h_server,
size=(brick_max_size_gb + 1),
json=True)
self.addCleanup(heketi_volume_delete, h_client, h_server, vol_5['id'])
heketi_volume_expand(h_client, h_server, vol_5['id'], 2)
def test_volume_expansion_rebalance_brick(self):
"""Validate volume expansion with brick and check rebalance"""
creation_info = heketi_ops.heketi_volume_create(
self.heketi_client_node, self.heketi_server_url, 10, json=True)
self.assertNotEqual(creation_info, False, "Volume creation failed")
volume_name = creation_info["name"]
volume_id = creation_info["id"]
free_space_after_creation = self.get_devices_summary_free_space()
volume_info_before_expansion = heketi_ops.heketi_volume_info(
self.heketi_client_node,
self.heketi_server_url,
volume_id, json=True)
self.assertNotEqual(volume_info_before_expansion, False,
"Volume info for %s failed" % volume_id)
heketi_vol_info_size_before_expansion = (
volume_info_before_expansion["size"])
self.get_brick_and_volume_status(volume_name)
num_of_bricks_before_expansion = self.get_num_of_bricks(volume_name)
expansion_info = heketi_ops.heketi_volume_expand(
self.heketi_client_node,
self.heketi_server_url,
volume_id, 5)
self.assertNotEqual(expansion_info, False,
"Volume expansion of %s failed" % volume_id)
free_space_after_expansion = self.get_devices_summary_free_space()
self.assertTrue(
free_space_after_creation > free_space_after_expansion,
"Free space not consumed after expansion of %s" % volume_id)
volume_info_after_expansion = heketi_ops.heketi_volume_info(
self.heketi_client_node,
self.heketi_server_url,
volume_id, json=True)
self.assertNotEqual(volume_info_after_expansion, False,
"Volume info failed for %s" % volume_id)
heketi_vol_info_size_after_expansion = (
volume_info_after_expansion["size"])
difference_size = (heketi_vol_info_size_after_expansion
- heketi_vol_info_size_before_expansion)
self.assertTrue(
difference_size > 0,
"Size not increased after expansion of %s" % volume_id)
self.get_brick_and_volume_status(volume_name)
num_of_bricks_after_expansion = self.get_num_of_bricks(volume_name)
num_of_bricks_added = (
num_of_bricks_after_expansion - num_of_bricks_before_expansion)
self.assertEqual(
num_of_bricks_added, 3,
"Number of bricks added is not 3 for %s" % volume_id)
self.get_rebalance_status(volume_name)
deletion_info = heketi_ops.heketi_volume_delete(
self.heketi_client_node, self.heketi_server_url,
volume_id, json=True)
self.assertNotEqual(deletion_info, False,
"Deletion of volume %s failed" % volume_id)
free_space_after_deletion = self.get_devices_summary_free_space()
self.assertTrue(
free_space_after_deletion > free_space_after_expansion,
"Free space is not reclaimed after volume deletion of %s"
% volume_id)
def test_volume_expansion_no_free_space(self):
"""Validate volume expansion when there is no free space"""
vol_size, expand_size, additional_devices_attached = None, 10, {}
h_node, h_server_url = self.heketi_client_node, self.heketi_server_url
# Get nodes info
heketi_node_id_list = heketi_ops.heketi_node_list(h_node, h_server_url)
if len(heketi_node_id_list) < 3:
self.skipTest("3 Heketi nodes are required.")
# Disable 4th and other nodes
for node_id in heketi_node_id_list[3:]:
heketi_ops.heketi_node_disable(h_node, h_server_url, node_id)
self.addCleanup(
heketi_ops.heketi_node_enable, h_node, h_server_url, node_id)
# Prepare first 3 nodes
smallest_size = None
err_msg = ''
for node_id in heketi_node_id_list[0:3]:
node_info = heketi_ops.heketi_node_info(
h_node, h_server_url, node_id, json=True)
# Disable second and other devices
devices = node_info["devices"]
self.assertTrue(
devices, "Node '%s' does not have devices." % node_id)
if devices[0]["state"].strip().lower() != "online":
self.skipTest("Test expects first device to be enabled.")
if (smallest_size is None
or devices[0]["storage"]["free"] < smallest_size):
smallest_size = devices[0]["storage"]["free"]
for device in node_info["devices"][1:]:
heketi_ops.heketi_device_disable(
h_node, h_server_url, device["id"])
self.addCleanup(
heketi_ops.heketi_device_enable,
h_node, h_server_url, device["id"])
# Gather info about additional devices
additional_device_name = None
for gluster_server in self.gluster_servers:
gluster_server_data = self.gluster_servers_info[gluster_server]
g_manage = gluster_server_data["manage"]
g_storage = gluster_server_data["storage"]
if not (g_manage in node_info["hostnames"]["manage"]
or g_storage in node_info["hostnames"]["storage"]):
continue
additional_device_name = ((
gluster_server_data.get("additional_devices") or [''])[0])
break
if not additional_device_name:
err_msg += ("No 'additional_devices' are configured for "
"'%s' node, which has following hostnames and "
"IP addresses: %s.\n" % (
node_id,
', '.join(
node_info["hostnames"]["manage"]
+ node_info["hostnames"]["storage"])))
continue
heketi_ops.heketi_device_add(
h_node, h_server_url, additional_device_name, node_id)
additional_devices_attached.update(
{node_id: additional_device_name})
# Schedule cleanup of the added devices
for node_id in additional_devices_attached.keys():
node_info = heketi_ops.heketi_node_info(
h_node, h_server_url, node_id, json=True)
for device in node_info["devices"]:
if device["name"] != additional_devices_attached[node_id]:
continue
self.addCleanup(self.detach_devices_attached, device["id"])
break
else:
self.fail("Could not find ID for added device on "
"'%s' node." % node_id)
if err_msg:
self.skipTest(err_msg)
# Temporary disable new devices
self.disable_devices(additional_devices_attached)
# Create volume and save info about it
vol_size = int(smallest_size / (1024**2)) - 1
creation_info = heketi_ops.heketi_volume_create(
h_node, h_server_url, vol_size, json=True)
volume_name, volume_id = creation_info["name"], creation_info["id"]
self.addCleanup(
heketi_ops.heketi_volume_delete,
h_node, h_server_url, volume_id, raise_on_error=False)
volume_info_before_expansion = heketi_ops.heketi_volume_info(
h_node, h_server_url, volume_id, json=True)
num_of_bricks_before_expansion = self.get_num_of_bricks(volume_name)
self.get_brick_and_volume_status(volume_name)
free_space_before_expansion = self.get_devices_summary_free_space()
# Try to expand volume with not enough device space
self.assertRaises(
AssertionError, heketi_ops.heketi_volume_expand,
h_node, h_server_url, volume_id, expand_size)
# Enable new devices to be able to expand our volume
self.enable_devices(additional_devices_attached)
# Expand volume and validate results
heketi_ops.heketi_volume_expand(
h_node, h_server_url, volume_id, expand_size, json=True)
free_space_after_expansion = self.get_devices_summary_free_space()
self.assertGreater(
free_space_before_expansion, free_space_after_expansion,
"Free space not consumed after expansion of %s" % volume_id)
num_of_bricks_after_expansion = self.get_num_of_bricks(volume_name)
self.get_brick_and_volume_status(volume_name)
volume_info_after_expansion = heketi_ops.heketi_volume_info(
h_node, h_server_url, volume_id, json=True)
self.assertGreater(
volume_info_after_expansion["size"],
volume_info_before_expansion["size"],
"Size of %s not increased" % volume_id)
self.assertGreater(
num_of_bricks_after_expansion, num_of_bricks_before_expansion)
self.assertEqual(
num_of_bricks_after_expansion % num_of_bricks_before_expansion, 0)
# Delete volume and validate release of the used space
heketi_ops.heketi_volume_delete(h_node, h_server_url, volume_id)
free_space_after_deletion = self.get_devices_summary_free_space()
self.assertGreater(
free_space_after_deletion, free_space_after_expansion,
"Free space not reclaimed after deletion of volume %s" % volume_id)
def test_expansion_of_block_hosting_volume_using_heketi(self):
"""Verify that after expanding block hosting volume we are able to
consume the expanded space"""
h_node = self.heketi_client_node
h_url = self.heketi_server_url
bvols_in_bhv = set([])
bvols_pv = set([])
BHVS = get_block_hosting_volume_list(h_node, h_url)
free_BHVS_count = 0
for vol in BHVS.keys():
info = heketi_volume_info(h_node, h_url, vol, json=True)
if info['blockinfo']['freesize'] > 0:
free_BHVS_count += 1
if free_BHVS_count > 1:
self.skipTest("Skip test case because there is more than one"
" Block Hosting Volume with free space")
# create block volume of 1gb
bvol_info = heketi_blockvolume_create(h_node, h_url, 1, json=True)
expand_size = 20
try:
self.verify_free_space(expand_size)
bhv = bvol_info['blockhostingvolume']
vol_info = heketi_volume_info(h_node, h_url, bhv, json=True)
bvols_in_bhv.update(vol_info['blockinfo']['blockvolume'])
finally:
# cleanup BHV if there is only one block volume inside it
if len(bvols_in_bhv) == 1:
self.addCleanup(
heketi_volume_delete, h_node, h_url, bhv, json=True)
self.addCleanup(
heketi_blockvolume_delete, h_node, h_url, bvol_info['id'])
size = vol_info['size']
free_size = vol_info['blockinfo']['freesize']
bvol_count = int(free_size / expand_size)
bricks = vol_info['bricks']
# create pvs to fill the BHV
pvcs = self.create_and_wait_for_pvcs(
pvc_size=(expand_size if bvol_count else free_size),
pvc_amount=(bvol_count or 1), timeout=300)
vol_expand = True
for i in range(2):
# get the vol ids from pvcs
for pvc in pvcs:
pv = get_pv_name_from_pvc(self.node, pvc)
custom = r':.metadata.annotations."gluster\.org\/volume-id"'
bvol_id = oc_get_custom_resource(self.node, 'pv', custom, pv)
bvols_pv.add(bvol_id[0])
vol_info = heketi_volume_info(h_node, h_url, bhv, json=True)
bvols = vol_info['blockinfo']['blockvolume']
bvols_in_bhv.update(bvols)
self.assertEqual(bvols_pv, (bvols_in_bhv & bvols_pv))
# Expand BHV and verify bricks and size of BHV
if vol_expand:
vol_expand = False
heketi_volume_expand(
h_node, h_url, bhv, expand_size, json=True)
vol_info = heketi_volume_info(h_node, h_url, bhv, json=True)
self.assertEqual(size + expand_size, vol_info['size'])
self.assertFalse(len(vol_info['bricks']) % 3)
self.assertLess(len(bricks), len(vol_info['bricks']))
# create more PVCs in expanded BHV
pvcs = self.create_and_wait_for_pvcs(
pvc_size=(expand_size - 1), pvc_amount=1)
