def test_sc_glusterfile_missing_parameter(self, parameter):
"""Validate glusterfile storage with missing parameters"""
node, sc = self.ocp_master_node[0], self.sc
secret_name = self.create_secret()
parameters = {'resturl': sc['resturl'], 'restuser': sc['restuser']}
if parameter == 'secretName':
parameters['secretName'] = secret_name
elif parameter == 'secretNamespace':
parameters['secretNamespace'] = sc['secretnamespace']
sc_name = oc_create_sc(node, **parameters)
self.addCleanup(oc_delete, node, 'sc', sc_name)
# Create PVC
pvc_name = oc_create_pvc(node, sc_name)
self.addCleanup(wait_for_resource_absence, node, 'pvc', pvc_name)
self.addCleanup(oc_delete, node, 'pvc', pvc_name)
# Wait for event with error
wait_for_events(node,
obj_name=pvc_name,
obj_type='PersistentVolumeClaim',
event_reason='ProvisioningFailed')
# Verify PVC did not get bound
with self.assertRaises(ExecutionError):
verify_pvc_status_is_bound(node, pvc_name, timeout=1)
def _pv_resize(self, exceed_free_space):
dir_path = "/mnt"
pvc_size_gb = 1
available_size_gb = self._available_disk_free_space()
# Create PVC
self.create_storage_class(allow_volume_expansion=True)
pvc_name = self.create_and_wait_for_pvc(pvc_size=pvc_size_gb)
# Create DC with POD and attached PVC to it
dc_name = oc_create_app_dc_with_io(
self.node, pvc_name, image=self.io_container_image_cirros)
self.addCleanup(oc_delete, self.node, 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, self.node, dc_name, 0)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
if exceed_free_space:
exceed_size = available_size_gb + 10
# Try to expand existing PVC exceeding free space
resize_pvc(self.node, pvc_name, exceed_size)
wait_for_events(self.node,
obj_name=pvc_name,
event_reason='VolumeResizeFailed')
# Check that app POD is up and runnig then try to write data
wait_for_pod_be_ready(self.node, pod_name)
cmd = ("dd if=/dev/urandom of=%s/autotest bs=100K count=1" %
dir_path)
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0,
"Failed to write data after failed attempt to expand PVC.")
else:
# Expand existing PVC using all the available free space
expand_size_gb = available_size_gb - pvc_size_gb
resize_pvc(self.node, pvc_name, expand_size_gb)
verify_pvc_size(self.node, pvc_name, expand_size_gb)
pv_name = get_pv_name_from_pvc(self.node, pvc_name)
verify_pv_size(self.node, pv_name, expand_size_gb)
wait_for_events(self.node,
obj_name=pvc_name,
event_reason='VolumeResizeSuccessful')
# Recreate app POD
oc_delete(self.node, 'pod', pod_name)
wait_for_resource_absence(self.node, 'pod', pod_name)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
# Write data on the expanded PVC
cmd = ("dd if=/dev/urandom of=%s/autotest "
"bs=1M count=1025" % dir_path)
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(ret, 0,
"Failed to write data on the expanded PVC")
def test_creation_of_block_vol_greater_than_the_default_size_of_BHV_neg(
self):
"""Verify that block volume creation fails when we create block
volume of size greater than the default size of BHV.
Verify that block volume creation succeed when we create BHV
of size greater than the default size of BHV.
"""
default_bhv_size = get_default_block_hosting_volume_size(
self.node, self.heketi_dc_name)
reserve_size = default_bhv_size * 0.02
reserve_size = int(math.ceil(reserve_size))
self.verify_free_space(default_bhv_size + reserve_size + 2)
with self.assertRaises(ExecutionError):
# create a block vol greater than default BHV size
bvol_info = heketi_blockvolume_create(
self.heketi_client_node, self.heketi_server_url,
(default_bhv_size + 1), json=True)
self.addCleanup(
heketi_blockvolume_delete, self.heketi_client_node,
self.heketi_server_url, bvol_info['id'])
sc_name = self.create_storage_class()
# create a block pvc greater than default BHV size
pvc_name = oc_create_pvc(
self.node, sc_name, pvc_size=(default_bhv_size + 1))
self.addCleanup(
wait_for_resource_absence, self.node, 'pvc', pvc_name)
self.addCleanup(
oc_delete, self.node, 'pvc', pvc_name, raise_on_absence=False)
wait_for_events(
self.node, pvc_name, obj_type='PersistentVolumeClaim',
event_type='Warning', event_reason='ProvisioningFailed')
# create block hosting volume greater than default BHV size
vol_info = heketi_volume_create(
self.heketi_client_node, self.heketi_server_url,
(default_bhv_size + reserve_size + 2), block=True,
json=True)
self.addCleanup(
heketi_volume_delete, self.heketi_client_node,
self.heketi_server_url, vol_info['id'])
# Cleanup PVC before block hosting volume to avoid failures
self.addCleanup(
wait_for_resource_absence, self.node, 'pvc', pvc_name)
self.addCleanup(
oc_delete, self.node, 'pvc', pvc_name, raise_on_absence=False)
verify_pvc_status_is_bound(self.node, pvc_name)
def test_creation_of_block_vol_greater_than_the_default_size_of_BHV_neg(
self):
"""Verify that block volume creation fails when we create block
volume of size greater than the default size of BHV.
Verify that block volume creation succeed when we create BHV
of size greater than the default size of BHV.
"""
default_bhv_size = get_default_block_hosting_volume_size(
self.node, self.heketi_dc_name)
reserve_size = default_bhv_size * 0.02
reserve_size = int(math.ceil(reserve_size))
self.verify_free_space(default_bhv_size + reserve_size + 2)
with self.assertRaises(AssertionError):
# create a block vol greater than default BHV size
bvol_info = heketi_blockvolume_create(
self.heketi_client_node, self.heketi_server_url,
(default_bhv_size + 1), json=True)
self.addCleanup(
heketi_blockvolume_delete, self.heketi_client_node,
self.heketi_server_url, bvol_info['id'])
sc_name = self.create_storage_class()
# create a block pvc greater than default BHV size
pvc_name = oc_create_pvc(
self.node, sc_name, pvc_size=(default_bhv_size + 1))
self.addCleanup(
wait_for_resource_absence, self.node, 'pvc', pvc_name)
self.addCleanup(
oc_delete, self.node, 'pvc', pvc_name, raise_on_absence=False)
wait_for_events(
self.node, pvc_name, obj_type='PersistentVolumeClaim',
event_type='Warning', event_reason='ProvisioningFailed')
# create block hosting volume greater than default BHV size
vol_info = heketi_volume_create(
self.heketi_client_node, self.heketi_server_url,
(default_bhv_size + reserve_size + 2), block=True,
json=True)
self.addCleanup(
heketi_volume_delete, self.heketi_client_node,
self.heketi_server_url, vol_info['id'])
# Cleanup PVC before block hosting volume to avoid failures
self.addCleanup(
wait_for_resource_absence, self.node, 'pvc', pvc_name)
self.addCleanup(
oc_delete, self.node, 'pvc', pvc_name, raise_on_absence=False)
verify_pvc_status_is_bound(self.node, pvc_name)
def test_creation_of_pvc_when_one_node_is_down(self):
"""Test PVC creation when one node is down than hacount"""
node_count = len(self.gluster_servers)
# Check at least 4 nodes available.
if node_count < 4:
self.skipTest("At least 4 nodes are required, found %s." %
node_count)
# Skip test if not able to connect to Cloud Provider
try:
find_vm_name_by_ip_or_hostname(self.node)
except (NotImplementedError, ConfigError) as e:
self.skipTest(e)
# Get gluster node on which heketi pod is not scheduled
heketi_node = oc_get_custom_resource(self.node,
'pod',
custom='.:status.hostIP',
selector='deploymentconfig=%s' %
self.heketi_dc_name)[0]
gluster_node = random.sample(
(set(self.gluster_servers) - set(heketi_node)), 1)[0]
gluster_hostname = self.gluster_servers_info[gluster_node]["manage"]
# Get VM name by VM hostname
vm_name = find_vm_name_by_ip_or_hostname(gluster_node)
# Power off one of the nodes
self.power_off_gluster_node_vm(vm_name, gluster_hostname)
# Create a PVC with SC of hacount equal to node count
sc_name = self.create_storage_class(hacount=node_count)
pvc_name = oc_create_pvc(self.node, sc_name=sc_name)
self.addCleanup(wait_for_resource_absence, self.node, 'pvc', pvc_name)
self.addCleanup(oc_delete, self.node, 'pvc', pvc_name)
events = wait_for_events(self.node,
obj_name=pvc_name,
obj_type='PersistentVolumeClaim',
event_type='Warning',
event_reason='ProvisioningFailed',
timeout=180)
error = 'insufficient block hosts online'
err_msg = ("Haven't found expected error message containing "
"following string: \n%s\nEvents: %s" % (error, events))
self.assertTrue(
list(filter((lambda e: error in e['message']), events)), err_msg)
# Create a PVC with SC of hacount less than node count
sc_name = self.create_storage_class(hacount=(node_count - 1))
self.create_and_wait_for_pvc(sc_name=sc_name)
def _pv_resize(self, exceed_free_space):
dir_path = "/mnt"
pvc_size_gb, min_free_space_gb = 1, 3
# Get available free space disabling redundant devices and nodes
heketi_url = self.heketi_server_url
node_id_list = heketi_ops.heketi_node_list(self.heketi_client_node,
heketi_url)
self.assertTrue(node_id_list)
nodes = {}
min_free_space = min_free_space_gb * 1024**2
for node_id in node_id_list:
node_info = heketi_ops.heketi_node_info(self.heketi_client_node,
heketi_url,
node_id,
json=True)
if (node_info['state'].lower() != 'online'
or not node_info['devices']):
continue
if len(nodes) > 2:
out = heketi_ops.heketi_node_disable(self.heketi_client_node,
heketi_url, node_id)
self.assertTrue(out)
self.addCleanup(heketi_ops.heketi_node_enable,
self.heketi_client_node, heketi_url, node_id)
for device in node_info['devices']:
if device['state'].lower() != 'online':
continue
free_space = device['storage']['free']
if (node_id in nodes.keys() or free_space < min_free_space):
out = heketi_ops.heketi_device_disable(
self.heketi_client_node, heketi_url, device['id'])
self.assertTrue(out)
self.addCleanup(heketi_ops.heketi_device_enable,
self.heketi_client_node, heketi_url,
device['id'])
continue
nodes[node_id] = free_space
if len(nodes) < 3:
raise self.skipTest("Could not find 3 online nodes with, "
"at least, 1 online device having free space "
"bigger than %dGb." % min_free_space_gb)
# Calculate maximum available size for PVC
available_size_gb = int(min(nodes.values()) / (1024**2))
# Create PVC
self.create_storage_class(allow_volume_expansion=True)
pvc_name = self.create_and_wait_for_pvc(pvc_size=pvc_size_gb)
# Create DC with POD and attached PVC to it
dc_name = oc_create_app_dc_with_io(self.node, pvc_name)
self.addCleanup(oc_delete, self.node, 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, self.node, dc_name, 0)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
if exceed_free_space:
# Try to expand existing PVC exceeding free space
resize_pvc(self.node, pvc_name, available_size_gb)
wait_for_events(self.node,
obj_name=pvc_name,
event_reason='VolumeResizeFailed')
# Check that app POD is up and runnig then try to write data
wait_for_pod_be_ready(self.node, pod_name)
cmd = ("dd if=/dev/urandom of=%s/autotest bs=100K count=1" %
dir_path)
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0,
"Failed to write data after failed attempt to expand PVC.")
else:
# Expand existing PVC using all the available free space
expand_size_gb = available_size_gb - pvc_size_gb
resize_pvc(self.node, pvc_name, expand_size_gb)
verify_pvc_size(self.node, pvc_name, expand_size_gb)
pv_name = get_pv_name_from_pvc(self.node, pvc_name)
verify_pv_size(self.node, pv_name, expand_size_gb)
wait_for_events(self.node,
obj_name=pvc_name,
event_reason='VolumeResizeSuccessful')
# Recreate app POD
oc_delete(self.node, 'pod', pod_name)
wait_for_resource_absence(self.node, 'pod', pod_name)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
# Write data on the expanded PVC
cmd = ("dd if=/dev/urandom of=%s/autotest "
"bs=1M count=1025" % dir_path)
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(ret, 0,
"Failed to write data on the expanded PVC")
def test_pv_resize_when_heketi_down(self):
"""Create a PVC and try to expand it when heketi is down, It should
fail. After heketi is up, expand PVC should work.
"""
self.create_storage_class(allow_volume_expansion=True)
pvc_name = self.create_and_wait_for_pvc()
dc_name, pod_name = self.create_dc_with_pvc(pvc_name)
pv_name = get_pv_name_from_pvc(self.node, pvc_name)
custom = (r':metadata.annotations.'
r'"gluster\.kubernetes\.io\/heketi-volume-id"')
vol_id = oc_get_custom_resource(self.node, 'pv', custom, pv_name)[0]
h_vol_info = heketi_ops.heketi_volume_info(self.heketi_client_node,
self.heketi_server_url,
vol_id,
json=True)
# Bring the heketi POD down
scale_dc_pod_amount_and_wait(self.node,
self.heketi_dc_name,
pod_amount=0)
self.addCleanup(scale_dc_pod_amount_and_wait,
self.node,
self.heketi_dc_name,
pod_amount=1)
cmd = 'dd if=/dev/urandom of=/mnt/%s bs=614400k count=1'
ret, out, err = oc_rsh(self.node, pod_name, cmd % 'file1')
self.assertFalse(ret, 'Not able to write file with err: %s' % err)
wait_for_pod_be_ready(self.node, pod_name, 10, 5)
resize_pvc(self.node, pvc_name, 2)
wait_for_events(self.node,
pvc_name,
obj_type='PersistentVolumeClaim',
event_type='Warning',
event_reason='VolumeResizeFailed')
# Verify volume was not expanded
vol_info = get_gluster_vol_info_by_pvc_name(self.node, pvc_name)
self.assertEqual(vol_info['gluster_vol_id'], h_vol_info['name'])
self.assertEqual(len(vol_info['bricks']['brick']),
len(h_vol_info['bricks']))
# Bring the heketi POD up
scale_dc_pod_amount_and_wait(self.node,
self.heketi_dc_name,
pod_amount=1)
# Verify volume expansion
verify_pvc_size(self.node, pvc_name, 2)
vol_info = get_gluster_vol_info_by_pvc_name(self.node, pvc_name)
self.assertFalse(len(vol_info['bricks']['brick']) % 3)
self.assertLess(len(h_vol_info['bricks']),
len(vol_info['bricks']['brick']))
# Wait for remount after expansion
for w in waiter.Waiter(timeout=30, interval=5):
ret, out, err = oc_rsh(self.node, pod_name,
"df -Ph /mnt | awk '{print $2}' | tail -1")
self.assertFalse(ret,
'Failed with err: %s and Output: %s' % (err, out))
if out.strip() == '2.0G':
break
# Write data making sure we have more space than it was
ret, out, err = oc_rsh(self.node, pod_name, cmd % 'file2')
self.assertFalse(ret, 'Not able to write file with err: %s' % err)
# Verify pod is running
wait_for_pod_be_ready(self.node, pod_name, 10, 5)
def create_sc_with_parameter(self, vol_type, success=False, parameter={}):
"""creates storage class, pvc and validates event
Args:
vol_type (str): storage type either gluster file or block
success (bool): if True check for successfull else failure
for pvc creation event
parameter (dict): dictionary with storage class parameters
"""
if vol_type == "glusterfile":
sc = self.storage_classes.get(
'storage_class1',
self.storage_classes.get('file_storage_class'))
# Create secret file for usage in storage class
self.secret_name = oc_create_secret(
self.ocp_master_node[0],
namespace=sc.get('secretnamespace', 'default'),
data_key=self.heketi_cli_key,
secret_type=sc.get('provisioner', 'kubernetes.io/glusterfs'))
self.addCleanup(oc_delete, self.ocp_master_node[0], 'secret',
self.secret_name)
sc_parameter = {
"secretnamespace": sc['secretnamespace'],
"secretname": self.secret_name,
"volumetype": "replicate:3"
}
elif vol_type == "glusterblock":
sc = self.storage_classes.get(
'storage_class2',
self.storage_classes.get('block_storage_class'))
# Create secret file for usage in storage class
self.secret_name = oc_create_secret(
self.ocp_master_node[0],
namespace=sc.get('restsecretnamespace', 'default'),
data_key=self.heketi_cli_key,
secret_type=sc.get('provisioner', 'gluster.org/glusterblock'))
self.addCleanup(oc_delete, self.ocp_master_node[0], 'secret',
self.secret_name)
sc_parameter = {
"provisioner": self.get_block_provisioner_for_sc(),
"restsecretnamespace": sc['restsecretnamespace'],
"restsecretname": self.secret_name,
"hacount": sc['hacount']
}
else:
err_msg = "invalid vol_type %s" % vol_type
g.log.error(err_msg)
raise AssertionError(err_msg)
sc_parameter['resturl'] = sc['resturl']
sc_parameter['restuser'] = sc['restuser']
sc_parameter.update(parameter)
# Create storage class
self.sc_name = oc_create_sc(self.ocp_master_node[0], **sc_parameter)
self.addCleanup(oc_delete, self.ocp_master_node[0], 'sc', self.sc_name)
# Create PVC
self.pvc_name = oc_create_pvc(self.ocp_client[0], self.sc_name)
self.addCleanup(wait_for_resource_absence, self.ocp_master_node[0],
'pvc', self.pvc_name)
self.addCleanup(oc_delete, self.ocp_master_node[0], 'pvc',
self.pvc_name)
# Wait for event with error
event_reason = 'ProvisioningFailed'
if success:
event_reason = 'ProvisioningSucceeded'
wait_for_events(self.ocp_master_node[0],
obj_name=self.pvc_name,
obj_type='PersistentVolumeClaim',
event_reason=event_reason)
def _pv_resize(self, exceed_free_space):
dir_path = "/mnt"
pvc_size_gb, min_free_space_gb = 1, 3
# Get available free space disabling redundant devices and nodes
heketi_url = self.heketi_server_url
node_id_list = heketi_ops.heketi_node_list(
self.heketi_client_node, heketi_url)
self.assertTrue(node_id_list)
nodes = {}
min_free_space = min_free_space_gb * 1024**2
for node_id in node_id_list:
node_info = heketi_ops.heketi_node_info(
self.heketi_client_node, heketi_url, node_id, json=True)
if (node_info['state'].lower() != 'online' or
not node_info['devices']):
continue
if len(nodes) > 2:
out = heketi_ops.heketi_node_disable(
self.heketi_client_node, heketi_url, node_id)
self.assertTrue(out)
self.addCleanup(
heketi_ops.heketi_node_enable,
self.heketi_client_node, heketi_url, node_id)
for device in node_info['devices']:
if device['state'].lower() != 'online':
continue
free_space = device['storage']['free']
if (node_id in nodes.keys() or free_space < min_free_space):
out = heketi_ops.heketi_device_disable(
self.heketi_client_node, heketi_url, device['id'])
self.assertTrue(out)
self.addCleanup(
heketi_ops.heketi_device_enable,
self.heketi_client_node, heketi_url, device['id'])
continue
nodes[node_id] = free_space
if len(nodes) < 3:
raise self.skipTest(
"Could not find 3 online nodes with, "
"at least, 1 online device having free space "
"bigger than %dGb." % min_free_space_gb)
# Calculate maximum available size for PVC
available_size_gb = int(min(nodes.values()) / (1024**2))
# Create PVC
self.create_storage_class(allow_volume_expansion=True)
pvc_name = self.create_and_wait_for_pvc(pvc_size=pvc_size_gb)
# Create DC with POD and attached PVC to it
dc_name = oc_create_app_dc_with_io(self.node, pvc_name)
self.addCleanup(oc_delete, self.node, 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, self.node, dc_name, 0)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
if exceed_free_space:
# Try to expand existing PVC exceeding free space
resize_pvc(self.node, pvc_name, available_size_gb)
wait_for_events(self.node, obj_name=pvc_name,
event_reason='VolumeResizeFailed')
# Check that app POD is up and runnig then try to write data
wait_for_pod_be_ready(self.node, pod_name)
cmd = (
"dd if=/dev/urandom of=%s/autotest bs=100K count=1" % dir_path)
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0,
"Failed to write data after failed attempt to expand PVC.")
else:
# Expand existing PVC using all the available free space
expand_size_gb = available_size_gb - pvc_size_gb
resize_pvc(self.node, pvc_name, expand_size_gb)
verify_pvc_size(self.node, pvc_name, expand_size_gb)
pv_name = get_pv_name_from_pvc(self.node, pvc_name)
verify_pv_size(self.node, pv_name, expand_size_gb)
wait_for_events(
self.node, obj_name=pvc_name,
event_reason='VolumeResizeSuccessful')
# Recreate app POD
oc_delete(self.node, 'pod', pod_name)
wait_for_resource_absence(self.node, 'pod', pod_name)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
# Write data on the expanded PVC
cmd = ("dd if=/dev/urandom of=%s/autotest "
"bs=1M count=1025" % dir_path)
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0, "Failed to write data on the expanded PVC")
def test_pv_resize_when_heketi_down(self):
"""Create a PVC and try to expand it when heketi is down, It should
fail. After heketi is up, expand PVC should work.
"""
self.create_storage_class(allow_volume_expansion=True)
pvc_name = self.create_and_wait_for_pvc()
dc_name, pod_name = self.create_dc_with_pvc(pvc_name)
pv_name = get_pv_name_from_pvc(self.node, pvc_name)
custom = (r':metadata.annotations.'
r'"gluster\.kubernetes\.io\/heketi-volume-id"')
vol_id = oc_get_custom_resource(self.node, 'pv', custom, pv_name)[0]
h_vol_info = heketi_ops.heketi_volume_info(
self.heketi_client_node, self.heketi_server_url, vol_id, json=True)
# Bring the heketi POD down
scale_dc_pod_amount_and_wait(
self.node, self.heketi_dc_name, pod_amount=0)
self.addCleanup(
scale_dc_pod_amount_and_wait, self.node,
self.heketi_dc_name, pod_amount=1)
cmd = 'dd if=/dev/urandom of=/mnt/%s bs=614400k count=1'
ret, out, err = oc_rsh(self.node, pod_name, cmd % 'file1')
self.assertFalse(ret, 'Not able to write file with err: %s' % err)
wait_for_pod_be_ready(self.node, pod_name, 10, 5)
resize_pvc(self.node, pvc_name, 2)
wait_for_events(
self.node, pvc_name, obj_type='PersistentVolumeClaim',
event_type='Warning', event_reason='VolumeResizeFailed')
# Verify volume was not expanded
vol_info = get_gluster_vol_info_by_pvc_name(self.node, pvc_name)
self.assertEqual(vol_info['gluster_vol_id'], h_vol_info['name'])
self.assertEqual(
len(vol_info['bricks']['brick']), len(h_vol_info['bricks']))
# Bring the heketi POD up
scale_dc_pod_amount_and_wait(
self.node, self.heketi_dc_name, pod_amount=1)
# Verify volume expansion
verify_pvc_size(self.node, pvc_name, 2)
vol_info = get_gluster_vol_info_by_pvc_name(self.node, pvc_name)
self.assertFalse(len(vol_info['bricks']['brick']) % 3)
self.assertLess(
len(h_vol_info['bricks']), len(vol_info['bricks']['brick']))
# Wait for remount after expansion
for w in waiter.Waiter(timeout=30, interval=5):
ret, out, err = oc_rsh(
self.node, pod_name,
"df -Ph /mnt | awk '{print $2}' | tail -1")
self.assertFalse(ret, 'Failed with err: %s and Output: %s' % (
err, out))
if out.strip() == '2.0G':
break
# Write data making sure we have more space than it was
ret, out, err = oc_rsh(self.node, pod_name, cmd % 'file2')
self.assertFalse(ret, 'Not able to write file with err: %s' % err)
# Verify pod is running
wait_for_pod_be_ready(self.node, pod_name, 10, 5)
def create_sc_with_parameter(self, vol_type, success=False, parameter={}):
"""creates storage class, pvc and validates event
Args:
vol_type (str): storage type either gluster file or block
success (bool): if True check for successfull else failure
for pvc creation event
parameter (dict): dictionary with storage class parameters
"""
if vol_type == "glusterfile":
sc = self.storage_classes.get(
'storage_class1',
self.storage_classes.get('file_storage_class'))
# Create secret file for usage in storage class
self.secret_name = oc_create_secret(
self.ocp_master_node[0],
namespace=sc.get('secretnamespace', 'default'),
data_key=self.heketi_cli_key,
secret_type=sc.get('provisioner', 'kubernetes.io/glusterfs'))
self.addCleanup(
oc_delete, self.ocp_master_node[0], 'secret', self.secret_name)
sc_parameter = {
"secretnamespace": sc['secretnamespace'],
"secretname": self.secret_name,
"volumetype": "replicate:3"
}
elif vol_type == "glusterblock":
sc = self.storage_classes.get(
'storage_class2',
self.storage_classes.get('block_storage_class'))
# Create secret file for usage in storage class
self.secret_name = oc_create_secret(
self.ocp_master_node[0],
namespace=sc.get('restsecretnamespace', 'default'),
data_key=self.heketi_cli_key,
secret_type=sc.get('provisioner', 'gluster.org/glusterblock'))
self.addCleanup(
oc_delete, self.ocp_master_node[0], 'secret', self.secret_name)
sc_parameter = {
"provisioner": "gluster.org/glusterblock",
"restsecretnamespace": sc['restsecretnamespace'],
"restsecretname": self.secret_name,
"hacount": sc['hacount']
}
else:
err_msg = "invalid vol_type %s" % vol_type
g.log.error(err_msg)
raise AssertionError(err_msg)
sc_parameter['resturl'] = sc['resturl']
sc_parameter['restuser'] = sc['restuser']
sc_parameter.update(parameter)
# Create storage class
self.sc_name = oc_create_sc(
self.ocp_master_node[0], **sc_parameter)
self.addCleanup(oc_delete, self.ocp_master_node[0], 'sc', self.sc_name)
# Create PVC
self.pvc_name = oc_create_pvc(self.ocp_client[0], self.sc_name)
self.addCleanup(
wait_for_resource_absence, self.ocp_master_node[0],
'pvc', self.pvc_name)
self.addCleanup(oc_delete, self.ocp_master_node[0],
'pvc', self.pvc_name)
# Wait for event with error
event_reason = 'ProvisioningFailed'
if success:
event_reason = 'ProvisioningSucceeded'
wait_for_events(self.ocp_master_node[0],
obj_name=self.pvc_name,
obj_type='PersistentVolumeClaim',
event_reason=event_reason)
def test_prometheus_pv_resize(self):
""" Validate prometheus metrics with pv resize"""
# Fetch the metrics and storing initial_metrics as dictionary
pvc_name, pod_name, initial_metrics = self._fetch_initial_metrics(
vol_name_prefix="for-pv-resize", volume_expansion=True)
# Write data on the pvc and confirm it is reflected in the prometheus
self._perform_io_and_fetch_metrics(
pod_name=pod_name, pvc_name=pvc_name,
filename="filename1", dirname="dirname1",
metric_data=initial_metrics, operation="create")
# Resize the pvc to 2GiB
openshift_ops.switch_oc_project(
self._master, self.storage_project_name)
pvc_size = 2
openshift_ops.resize_pvc(self._master, pvc_name, pvc_size)
openshift_ops.wait_for_events(self._master, obj_name=pvc_name,
event_reason='VolumeResizeSuccessful')
openshift_ops.verify_pvc_size(self._master, pvc_name, pvc_size)
pv_name = openshift_ops.get_pv_name_from_pvc(
self._master, pvc_name)
openshift_ops.verify_pv_size(self._master, pv_name, pvc_size)
heketi_volume_name = heketi_ops.heketi_volume_list_by_name_prefix(
self.heketi_client_node, self.heketi_server_url,
"for-pv-resize", json=True)[0][2]
self.assertIsNotNone(
heketi_volume_name, "Failed to fetch volume with prefix {}".
format("for-pv-resize"))
openshift_ops.oc_delete(self._master, 'pod', pod_name)
openshift_ops.wait_for_resource_absence(self._master, 'pod', pod_name)
pod_name = openshift_ops.get_pod_name_from_dc(
self._master, self.dc_name)
openshift_ops.wait_for_pod_be_ready(self._master, pod_name)
# Check whether the metrics are updated or not
for w in waiter.Waiter(120, 10):
resize_metrics = self._get_and_manipulate_metric_data(
self.metrics, pvc_name)
if bool(resize_metrics) and int(resize_metrics[
'kubelet_volume_stats_capacity_bytes']) > int(
initial_metrics['kubelet_volume_stats_capacity_bytes']):
break
if w.expired:
raise AssertionError("Failed to reflect PVC Size after resizing")
openshift_ops.switch_oc_project(
self._master, self.storage_project_name)
time.sleep(240)
# Lookup and trigger rebalance and wait for the its completion
for _ in range(100):
self.cmd_run("oc rsh {} ls /mnt/".format(pod_name))
self._rebalance_completion(heketi_volume_name)
# Write data on the resized pvc and compared with the resized_metrics
self._perform_io_and_fetch_metrics(
pod_name=pod_name, pvc_name=pvc_name,
filename="secondfilename", dirname="seconddirname",
metric_data=resize_metrics, operation="create")
