def validate_multipath_info(self, hacount):
"""validates multipath command on the pod node
Args:
hacount (int): hacount for which multipath to be checked
"""
# create pod using pvc created
dc_name = oc_create_app_dc_with_io(
self.ocp_master_node[0], self.pvc_name
)
pod_name = get_pod_name_from_dc(self.ocp_master_node[0], dc_name)
self.addCleanup(oc_delete, self.ocp_master_node[0], "dc", dc_name)
self.addCleanup(
scale_dc_pod_amount_and_wait, self.ocp_master_node[0], dc_name, 0
)
wait_for_pod_be_ready(
self.ocp_master_node[0], pod_name, timeout=120, wait_step=3
)
# validates multipath for pod created with hacount
self.assertTrue(
validate_multipath_pod(self.ocp_master_node[0], pod_name, hacount),
"multipath validation failed"
)
def dynamic_provisioning_glusterfile(self, create_vol_name_prefix):
# Create secret and storage class
self.create_storage_class(
create_vol_name_prefix=create_vol_name_prefix)
# Create PVC
pvc_name = self.create_and_wait_for_pvc()
# Create DC with POD and attached PVC to it.
dc_name = oc_create_app_dc_with_io(self.node, pvc_name)
self.addCleanup(oc_delete, self.node, 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, self.node, dc_name, 0)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
# Verify Heketi volume name for prefix presence if provided
if create_vol_name_prefix:
ret = verify_volume_name_prefix(self.node,
self.sc['volumenameprefix'],
self.sc['secretnamespace'],
pvc_name, self.sc['resturl'])
self.assertTrue(ret, "verify volnameprefix failed")
else:
# Get the volume name and volume id from PV
pv_name = get_pv_name_from_pvc(self.ocp_client[0], pvc_name)
custom = [
r':spec.glusterfs.path',
r':metadata.annotations.'
r'"gluster\.kubernetes\.io\/heketi-volume-id"'
]
pv_vol_name, vol_id = oc_get_custom_resource(
self.ocp_client[0], 'pv', custom, pv_name)
# check if the pv_volume_name is present in heketi
# Check if volume name is "vol_"+volumeid or not
heketi_vol_name = heketi_volume_info(
self.ocp_client[0], self.heketi_server_url, vol_id,
json=True)['name']
self.assertEqual(pv_vol_name, heketi_vol_name,
'Volume with vol_id = %s not found'
'in heketidb' % vol_id)
self.assertEqual(heketi_vol_name, 'vol_' + vol_id,
'Volume with vol_id = %s have a'
'custom perfix' % vol_id)
out = cmd_run_on_gluster_pod_or_node(self.ocp_master_node[0],
"gluster volume list")
self.assertIn(pv_vol_name, out,
"Volume with id %s does not exist" % vol_id)
# Make sure we are able to work with files on the mounted volume
filepath = "/mnt/file_for_testing_io.log"
for cmd in ("dd if=/dev/urandom of=%s bs=1K count=100",
"ls -lrt %s",
"rm -rf %s"):
cmd = cmd % filepath
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0,
"Failed to execute '%s' command on %s" % (cmd, self.node))
def test_pv_resize_try_shrink_pv_size(self):
"""Validate whether reducing the PV size is allowed"""
dir_path = "/mnt/"
node = self.ocp_master_node[0]
# Create PVC
pv_size = 5
self.create_storage_class(allow_volume_expansion=True)
pvc_name = self.create_and_wait_for_pvc(pvc_size=pv_size)
# Create DC with POD and attached PVC to it.
dc_name = oc_create_app_dc_with_io(node, pvc_name)
self.addCleanup(oc_delete, node, 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, node, dc_name, 0)
pod_name = get_pod_name_from_dc(node, dc_name)
wait_for_pod_be_ready(node, pod_name)
cmd = ("dd if=/dev/urandom of=%sfile " "bs=100K count=3000") % dir_path
ret, out, err = oc_rsh(node, pod_name, cmd)
self.assertEqual(ret, 0,
"failed to execute command %s on %s" % (cmd, node))
pvc_resize = 2
with self.assertRaises(ExecutionError):
resize_pvc(node, pvc_name, pvc_resize)
verify_pvc_size(node, pvc_name, pv_size)
pv_name = get_pv_name_from_pvc(node, pvc_name)
verify_pv_size(node, pv_name, pv_size)
cmd = ("dd if=/dev/urandom of=%sfile_new "
"bs=100K count=2000") % dir_path
ret, out, err = oc_rsh(node, pod_name, cmd)
self.assertEqual(ret, 0,
"failed to execute command %s on %s" % (cmd, node))
def test_pv_resize_try_shrink_pv_size(self):
"""Validate whether reducing the PV size is allowed"""
dir_path = "/mnt/"
node = self.ocp_master_node[0]
# Create PVC
pv_size = 5
self.create_storage_class(allow_volume_expansion=True)
pvc_name = self.create_and_wait_for_pvc(pvc_size=pv_size)
# Create DC with POD and attached PVC to it.
dc_name = oc_create_app_dc_with_io(node, pvc_name)
self.addCleanup(oc_delete, node, 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait,
node, dc_name, 0)
pod_name = get_pod_name_from_dc(node, dc_name)
wait_for_pod_be_ready(node, pod_name)
cmd = ("dd if=/dev/urandom of=%sfile "
"bs=100K count=3000") % dir_path
ret, out, err = oc_rsh(node, pod_name, cmd)
self.assertEqual(ret, 0, "failed to execute command %s on %s" % (
cmd, node))
pvc_resize = 2
with self.assertRaises(ExecutionError):
resize_pvc(node, pvc_name, pvc_resize)
verify_pvc_size(node, pvc_name, pv_size)
pv_name = get_pv_name_from_pvc(node, pvc_name)
verify_pv_size(node, pv_name, pv_size)
cmd = ("dd if=/dev/urandom of=%sfile_new "
"bs=100K count=2000") % dir_path
ret, out, err = oc_rsh(node, pod_name, cmd)
self.assertEqual(ret, 0, "failed to execute command %s on %s" % (
cmd, node))
def test_pv_resize_with_prefix_for_name_and_size(
self, create_vol_name_prefix=False, valid_size=True):
"""Validate PV resize with and without name prefix"""
dir_path = "/mnt/"
node = self.ocp_client[0]
# Create PVC
self.create_storage_class(
allow_volume_expansion=True,
create_vol_name_prefix=create_vol_name_prefix)
pvc_name = self.create_and_wait_for_pvc()
# Create DC with POD and attached PVC to it.
dc_name = oc_create_app_dc_with_io(node, pvc_name)
self.addCleanup(oc_delete, node, 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, node, dc_name, 0)
pod_name = get_pod_name_from_dc(node, dc_name)
wait_for_pod_be_ready(node, pod_name)
if create_vol_name_prefix:
ret = heketi_ops.verify_volume_name_prefix(
node, self.sc['volumenameprefix'], self.sc['secretnamespace'],
pvc_name, self.heketi_server_url)
self.assertTrue(ret, "verify volnameprefix failed")
cmd = ("dd if=/dev/urandom of=%sfile " "bs=100K count=1000") % dir_path
ret, out, err = oc_rsh(node, pod_name, cmd)
self.assertEqual(ret, 0,
"Failed to execute command %s on %s" % (cmd, node))
pv_name = get_pv_name_from_pvc(node, pvc_name)
# If resize size is invalid then size should not change
if valid_size:
cmd = ("dd if=/dev/urandom of=%sfile2 "
"bs=100K count=10000") % dir_path
with self.assertRaises(AssertionError):
ret, out, err = oc_rsh(node, pod_name, cmd)
msg = ("Command '%s' was expected to fail on '%s' node. "
"But it returned following: ret is '%s', err is '%s' "
"and out is '%s'" % (cmd, node, ret, err, out))
raise ExecutionError(msg)
pvc_size = 2
resize_pvc(node, pvc_name, pvc_size)
verify_pvc_size(node, pvc_name, pvc_size)
verify_pv_size(node, pv_name, pvc_size)
else:
invalid_pvc_size = 'ten'
with self.assertRaises(AssertionError):
resize_pvc(node, pvc_name, invalid_pvc_size)
verify_pvc_size(node, pvc_name, 1)
verify_pv_size(node, pv_name, 1)
oc_delete(node, 'pod', pod_name)
wait_for_resource_absence(node, 'pod', pod_name)
pod_name = get_pod_name_from_dc(node, dc_name)
wait_for_pod_be_ready(node, pod_name)
cmd = ("dd if=/dev/urandom of=%sfile_new "
"bs=50K count=10000") % dir_path
ret, out, err = oc_rsh(node, pod_name, cmd)
self.assertEqual(ret, 0,
"Failed to execute command %s on %s" % (cmd, node))
def _create_dcs_and_check_brick_placement(self, prefix, sc_name,
heketi_zone_checking,
zone_count):
app_pods, count, label = [], 5, "testlabel=autotest"
# Create multiple PVCs using storage class
pvc_names = self.create_and_wait_for_pvcs(pvc_name_prefix=prefix,
pvc_amount=count,
sc_name=sc_name)
# Create app dcs with I/O
for pvc_name in pvc_names:
app_dc = openshift_ops.oc_create_app_dc_with_io(
self.node,
pvc_name=pvc_name,
dc_name_prefix=prefix,
image=self.io_container_image_cirros)
self.addCleanup(openshift_ops.oc_delete, self.node, 'dc', app_dc)
# Get pod names and label them
pod_name = openshift_ops.get_pod_name_from_dc(self.node, app_dc)
openshift_ops.oc_label(self.node, 'pod', pod_name, label)
app_pods.append(pod_name)
# Wait for pods to be ready with the help of label selector
openshift_ops.wait_for_pods_be_ready(self.node, count, label)
# Validate brick placement in heketi zones
self._validate_brick_placement_in_correct_zone_or_with_expand_pvc(
heketi_zone_checking, pvc_name, zone_count)
return app_pods
def dynamic_provisioning_glusterfile(self, create_vol_name_prefix):
# Create secret and storage class
self.create_storage_class(
create_vol_name_prefix=create_vol_name_prefix)
# Create PVC
pvc_name = self.create_and_wait_for_pvc()
# Create DC with POD and attached PVC to it.
dc_name = oc_create_app_dc_with_io(self.node, pvc_name)
self.addCleanup(oc_delete, self.node, 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, self.node, dc_name, 0)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
# Verify Heketi volume name for prefix presence if provided
if create_vol_name_prefix:
ret = verify_volume_name_prefix(self.node,
self.sc['volumenameprefix'],
self.sc['secretnamespace'],
pvc_name, self.sc['resturl'])
self.assertTrue(ret, "verify volnameprefix failed")
else:
# Get the volume name and volume id from PV
pv_name = get_pv_name_from_pvc(self.ocp_client[0], pvc_name)
custom = [
r':spec.glusterfs.path',
r':metadata.annotations.'
r'"gluster\.kubernetes\.io\/heketi-volume-id"'
]
pv_vol_name, vol_id = oc_get_custom_resource(
self.ocp_client[0], 'pv', custom, pv_name)
# check if the pv_volume_name is present in heketi
# Check if volume name is "vol_"+volumeid or not
heketi_vol_name = heketi_volume_info(
self.ocp_client[0], self.heketi_server_url, vol_id,
json=True)['name']
self.assertEqual(pv_vol_name, heketi_vol_name,
'Volume with vol_id = %s not found'
'in heketidb' % vol_id)
self.assertEqual(heketi_vol_name, 'vol_' + vol_id,
'Volume with vol_id = %s have a'
'custom perfix' % vol_id)
out = cmd_run_on_gluster_pod_or_node(self.ocp_master_node[0],
"gluster volume list")
self.assertIn(pv_vol_name, out,
"Volume with id %s does not exist" % vol_id)
# Make sure we are able to work with files on the mounted volume
filepath = "/mnt/file_for_testing_io.log"
for cmd in ("dd if=/dev/urandom of=%s bs=1K count=100",
"ls -lrt %s",
"rm -rf %s"):
cmd = cmd % filepath
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0,
"Failed to execute '%s' command on %s" % (cmd, self.node))
def test_validate_pvc_in_multiple_app_pods(self):
"""Validate the use of a same claim in multiple app pods"""
replicas = 5
# Create PVC
sc_name = self.create_storage_class()
pvc_name = self.create_and_wait_for_pvc(sc_name=sc_name)
# Create DC with application PODs
dc_name = oc_create_app_dc_with_io(
self.node,
pvc_name,
replicas=replicas,
image=self.io_container_image_cirros)
self.addCleanup(oc_delete, self.node, 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, self.node, dc_name, 0)
# Wait for all the PODs to be ready
pod_names = get_pod_names_from_dc(self.node, dc_name)
self.assertEqual(replicas, len(pod_names))
for pod_name in pod_names:
wait_for_pod_be_ready(self.node, pod_name)
# Create files in each of the PODs
for pod_name in pod_names:
self.cmd_run("oc exec {0} -- touch /mnt/temp_{0}".format(pod_name))
# Check that all the created files are available at once
ls_out = self.cmd_run("oc exec %s -- ls /mnt" % pod_names[0]).split()
for pod_name in pod_names:
self.assertIn("temp_%s" % pod_name, ls_out)
def test_validate_pvc_in_multiple_app_pods(self):
"""Validate the use of a same claim in multiple app pods"""
replicas = 5
# Create PVC
sc_name = self.create_storage_class()
pvc_name = self.create_and_wait_for_pvc(sc_name=sc_name)
# Create DC with application PODs
dc_name = oc_create_app_dc_with_io(
self.node, pvc_name, replicas=replicas)
self.addCleanup(oc_delete, self.node, 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, self.node, dc_name, 0)
# Wait for all the PODs to be ready
pod_names = get_pod_names_from_dc(self.node, dc_name)
self.assertEqual(replicas, len(pod_names))
for pod_name in pod_names:
wait_for_pod_be_ready(self.node, pod_name)
# Create files in each of the PODs
for pod_name in pod_names:
self.cmd_run("oc exec {0} -- touch /mnt/temp_{0}".format(pod_name))
# Check that all the created files are available at once
ls_out = self.cmd_run("oc exec %s -- ls /mnt" % pod_names[0]).split()
for pod_name in pod_names:
self.assertIn("temp_%s" % pod_name, ls_out)
def _pv_resize(self, exceed_free_space):
dir_path = "/mnt"
pvc_size_gb = 1
available_size_gb = self._available_disk_free_space()
# Create PVC
self.create_storage_class(allow_volume_expansion=True)
pvc_name = self.create_and_wait_for_pvc(pvc_size=pvc_size_gb)
# Create DC with POD and attached PVC to it
dc_name = oc_create_app_dc_with_io(
self.node, pvc_name, image=self.io_container_image_cirros)
self.addCleanup(oc_delete, self.node, 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, self.node, dc_name, 0)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
if exceed_free_space:
exceed_size = available_size_gb + 10
# Try to expand existing PVC exceeding free space
resize_pvc(self.node, pvc_name, exceed_size)
wait_for_events(self.node,
obj_name=pvc_name,
event_reason='VolumeResizeFailed')
# Check that app POD is up and runnig then try to write data
wait_for_pod_be_ready(self.node, pod_name)
cmd = ("dd if=/dev/urandom of=%s/autotest bs=100K count=1" %
dir_path)
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0,
"Failed to write data after failed attempt to expand PVC.")
else:
# Expand existing PVC using all the available free space
expand_size_gb = available_size_gb - pvc_size_gb
resize_pvc(self.node, pvc_name, expand_size_gb)
verify_pvc_size(self.node, pvc_name, expand_size_gb)
pv_name = get_pv_name_from_pvc(self.node, pvc_name)
verify_pv_size(self.node, pv_name, expand_size_gb)
wait_for_events(self.node,
obj_name=pvc_name,
event_reason='VolumeResizeSuccessful')
# Recreate app POD
oc_delete(self.node, 'pod', pod_name)
wait_for_resource_absence(self.node, 'pod', pod_name)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
# Write data on the expanded PVC
cmd = ("dd if=/dev/urandom of=%s/autotest "
"bs=1M count=1025" % dir_path)
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(ret, 0,
"Failed to write data on the expanded PVC")
def create_dc_with_pvc(self, pvc_name, timeout=300, wait_step=10):
dc_name = oc_create_app_dc_with_io(self.ocp_client[0], pvc_name)
self.addCleanup(oc_delete, self.ocp_client[0], 'dc', dc_name)
self.addCleanup(
scale_dc_pod_amount_and_wait, self.ocp_client[0], dc_name, 0)
pod_name = get_pod_name_from_dc(self.ocp_client[0], dc_name)
wait_for_pod_be_ready(self.ocp_client[0], pod_name,
timeout=timeout, wait_step=wait_step)
return dc_name, pod_name
def test_dynamic_provisioning_glusterblock_reclaim_policy_retain(self):
"""Validate retain policy for gluster-block after PVC deletion"""
if get_openshift_version() < "3.9":
self.skipTest(
"'Reclaim' feature is not supported in OCP older than 3.9")
self.create_storage_class(reclaim_policy='Retain')
self.create_and_wait_for_pvc()
dc_name = oc_create_app_dc_with_io(self.node, self.pvc_name)
try:
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
finally:
scale_dc_pod_amount_and_wait(self.node, dc_name, pod_amount=0)
oc_delete(self.node, 'dc', dc_name)
# get the name of volume
pv_name = get_pv_name_from_pvc(self.node, self.pvc_name)
custom = [
r':.metadata.annotations."gluster\.org\/volume\-id"',
r':.spec.persistentVolumeReclaimPolicy'
]
vol_id, reclaim_policy = oc_get_custom_resource(
self.node, 'pv', custom, pv_name)
# checking the retainPolicy of pvc
self.assertEqual(reclaim_policy, 'Retain')
# delete the pvc
oc_delete(self.node, 'pvc', self.pvc_name)
# check if pv is also deleted or not
with self.assertRaises(ExecutionError):
wait_for_resource_absence(self.node,
'pvc',
self.pvc_name,
interval=3,
timeout=30)
# getting the blockvol list
blocklist = heketi_blockvolume_list(self.heketi_client_node,
self.heketi_server_url)
self.assertIn(vol_id, blocklist)
heketi_blockvolume_delete(self.heketi_client_node,
self.heketi_server_url, vol_id)
blocklist = heketi_blockvolume_list(self.heketi_client_node,
self.heketi_server_url)
self.assertNotIn(vol_id, blocklist)
oc_delete(self.node, 'pv', pv_name)
wait_for_resource_absence(self.node, 'pv', pv_name)
def create_dc_with_pvc(self, pvc_name, timeout=300, wait_step=10):
dc_name = oc_create_app_dc_with_io(self.ocp_client[0], pvc_name)
self.addCleanup(oc_delete, self.ocp_client[0], 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, self.ocp_client[0],
dc_name, 0)
pod_name = get_pod_name_from_dc(self.ocp_client[0], dc_name)
wait_for_pod_be_ready(self.ocp_client[0],
pod_name,
timeout=timeout,
wait_step=wait_step)
return dc_name, pod_name
def create_dcs_with_pvc(self,
pvc_names,
timeout=600,
wait_step=5,
dc_name_prefix='autotests-dc',
label=None,
skip_cleanup=False):
"""Create bunch of DCs with app PODs which use unique PVCs.
Args:
pvc_names (str/set/list/tuple): List/set/tuple of PVC names
to attach to app PODs as part of DCs.
timeout (int): timeout value, default value is 600 seconds.
wait_step( int): wait step, default value is 5 seconds.
dc_name_prefix(str): name prefix for deployement config.
lable (dict): keys and value for adding label into DC.
Returns: dictionary with following structure:
{
"pvc_name_1": ("dc_name_1", "pod_name_1"),
"pvc_name_2": ("dc_name_2", "pod_name_2"),
...
"pvc_name_n": ("dc_name_n", "pod_name_n"),
}
"""
pvc_names = (pvc_names if isinstance(pvc_names,
(list, set,
tuple)) else [pvc_names])
dc_and_pod_names, dc_names = {}, {}
for pvc_name in pvc_names:
dc_name = oc_create_app_dc_with_io(self.ocp_client[0],
pvc_name,
dc_name_prefix=dc_name_prefix,
label=label)
dc_names[pvc_name] = dc_name
if not skip_cleanup:
self.addCleanup(oc_delete, self.ocp_client[0], 'dc', dc_name)
if not skip_cleanup:
self.addCleanup(scale_dcs_pod_amount_and_wait,
self.ocp_client[0],
dc_names.values(),
0,
timeout=timeout,
wait_step=wait_step)
for pvc_name, dc_name in dc_names.items():
pod_name = get_pod_name_from_dc(self.ocp_client[0], dc_name)
dc_and_pod_names[pvc_name] = (dc_name, pod_name)
scale_dcs_pod_amount_and_wait(self.ocp_client[0],
dc_names.values(),
1,
timeout=timeout,
wait_step=wait_step)
return dc_and_pod_names
def test_pv_resize_with_prefix_for_name(self,
create_vol_name_prefix=False):
"""Validate PV resize with and without name prefix"""
dir_path = "/mnt/"
node = self.ocp_client[0]
# Create PVC
self.create_storage_class(
allow_volume_expansion=True,
create_vol_name_prefix=create_vol_name_prefix)
pvc_name = self.create_and_wait_for_pvc()
# Create DC with POD and attached PVC to it.
dc_name = oc_create_app_dc_with_io(node, pvc_name)
self.addCleanup(oc_delete, node, 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait,
node, dc_name, 0)
pod_name = get_pod_name_from_dc(node, dc_name)
wait_for_pod_be_ready(node, pod_name)
if create_vol_name_prefix:
ret = heketi_ops.verify_volume_name_prefix(
node, self.sc['volumenameprefix'],
self.sc['secretnamespace'],
pvc_name, self.heketi_server_url)
self.assertTrue(ret, "verify volnameprefix failed")
cmd = ("dd if=/dev/urandom of=%sfile "
"bs=100K count=1000") % dir_path
ret, out, err = oc_rsh(node, pod_name, cmd)
self.assertEqual(ret, 0, "failed to execute command %s on %s" % (
cmd, node))
cmd = ("dd if=/dev/urandom of=%sfile2 "
"bs=100K count=10000") % dir_path
with self.assertRaises(AssertionError):
ret, out, err = oc_rsh(node, pod_name, cmd)
msg = ("Command '%s' was expected to fail on '%s' node. "
"But it returned following: ret is '%s', err is '%s' "
"and out is '%s'" % (cmd, node, ret, err, out))
raise ExecutionError(msg)
pvc_size = 2
resize_pvc(node, pvc_name, pvc_size)
verify_pvc_size(node, pvc_name, pvc_size)
pv_name = get_pv_name_from_pvc(node, pvc_name)
verify_pv_size(node, pv_name, pvc_size)
oc_delete(node, 'pod', pod_name)
wait_for_resource_absence(node, 'pod', pod_name)
pod_name = get_pod_name_from_dc(node, dc_name)
wait_for_pod_be_ready(node, pod_name)
cmd = ("dd if=/dev/urandom of=%sfile_new "
"bs=50K count=10000") % dir_path
ret, out, err = oc_rsh(node, pod_name, cmd)
self.assertEqual(ret, 0, "failed to execute command %s on %s" % (
cmd, node))
def test_dynamic_provisioning_glusterfile_reclaim_policy_retain(self):
"""Validate retain policy for glusterfs after deletion of pvc"""
if get_openshift_version() < "3.9":
self.skipTest(
"'Reclaim' feature is not supported in OCP older than 3.9")
self.create_storage_class(reclaim_policy='Retain')
self.create_and_wait_for_pvc()
# get the name of the volume
pv_name = get_pv_name_from_pvc(self.node, self.pvc_name)
custom = [
r':.metadata.annotations.'
r'"gluster\.kubernetes\.io\/heketi\-volume\-id"',
r':.spec.persistentVolumeReclaimPolicy'
]
vol_id, reclaim_policy = oc_get_custom_resource(
self.node, 'pv', custom, pv_name)
self.assertEqual(reclaim_policy, 'Retain')
# Create DC with POD and attached PVC to it.
try:
dc_name = oc_create_app_dc_with_io(
self.node, self.pvc_name, image=self.io_container_image_cirros)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
finally:
scale_dc_pod_amount_and_wait(self.node, dc_name, 0)
oc_delete(self.node, 'dc', dc_name)
wait_for_resource_absence(self.node, 'pod', pod_name)
oc_delete(self.node, 'pvc', self.pvc_name)
with self.assertRaises(ExecutionError):
wait_for_resource_absence(self.node,
'pvc',
self.pvc_name,
interval=3,
timeout=30)
heketi_volume_delete(self.heketi_client_node, self.heketi_server_url,
vol_id)
vol_list = heketi_volume_list(self.heketi_client_node,
self.heketi_server_url)
self.assertNotIn(vol_id, vol_list)
oc_delete(self.node, 'pv', pv_name)
wait_for_resource_absence(self.node, 'pv', pv_name)
def test_dynamic_provisioning_glusterblock_reclaim_policy_retain(self):
"""Validate retain policy for gluster-block after PVC deletion"""
if get_openshift_version() < "3.9":
self.skipTest(
"'Reclaim' feature is not supported in OCP older than 3.9")
self.create_storage_class(reclaim_policy='Retain')
self.create_and_wait_for_pvc()
dc_name = oc_create_app_dc_with_io(self.node, self.pvc_name)
try:
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
finally:
scale_dc_pod_amount_and_wait(self.node, dc_name, pod_amount=0)
oc_delete(self.node, 'dc', dc_name)
# get the name of volume
pv_name = get_pv_name_from_pvc(self.node, self.pvc_name)
custom = [r':.metadata.annotations."gluster\.org\/volume\-id"',
r':.spec.persistentVolumeReclaimPolicy']
vol_id, reclaim_policy = oc_get_custom_resource(
self.node, 'pv', custom, pv_name)
# checking the retainPolicy of pvc
self.assertEqual(reclaim_policy, 'Retain')
# delete the pvc
oc_delete(self.node, 'pvc', self.pvc_name)
# check if pv is also deleted or not
with self.assertRaises(ExecutionError):
wait_for_resource_absence(
self.node, 'pvc', self.pvc_name, interval=3, timeout=30)
# getting the blockvol list
blocklist = heketi_blockvolume_list(self.heketi_client_node,
self.heketi_server_url)
self.assertIn(vol_id, blocklist)
heketi_blockvolume_delete(self.heketi_client_node,
self.heketi_server_url, vol_id)
blocklist = heketi_blockvolume_list(self.heketi_client_node,
self.heketi_server_url)
self.assertNotIn(vol_id, blocklist)
oc_delete(self.node, 'pv', pv_name)
wait_for_resource_absence(self.node, 'pv', pv_name)
def test_storage_class_mandatory_params_glusterfile(self):
"""Validate storage-class creation with mandatory parameters"""
# create secret
self.secret_name = oc_create_secret(
self.node,
namespace=self.sc.get('secretnamespace', 'default'),
data_key=self.heketi_cli_key,
secret_type=self.sc.get('provisioner', 'kubernetes.io/glusterfs'))
self.addCleanup(oc_delete, self.node, 'secret', self.secret_name)
# create storage class with mandatory parameters only
sc_name = oc_create_sc(self.node,
provisioner='kubernetes.io/glusterfs',
resturl=self.sc['resturl'],
restuser=self.sc['restuser'],
secretnamespace=self.sc['secretnamespace'],
secretname=self.secret_name)
self.addCleanup(oc_delete, self.node, 'sc', sc_name)
# Create PVC
pvc_name = self.create_and_wait_for_pvc(sc_name=sc_name)
# Create DC with POD and attached PVC to it.
dc_name = oc_create_app_dc_with_io(
self.node, pvc_name, image=self.io_container_image_cirros)
self.addCleanup(oc_delete, self.node, 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, self.node, dc_name, 0)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
# Make sure we are able to work with files on the mounted volume
filepath = "/mnt/file_for_testing_sc.log"
cmd = "dd if=/dev/urandom of=%s bs=1K count=100" % filepath
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0, "Failed to execute command %s on %s" % (cmd, self.node))
cmd = "ls -lrt %s" % filepath
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0, "Failed to execute command %s on %s" % (cmd, self.node))
cmd = "rm -rf %s" % filepath
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0, "Failed to execute command %s on %s" % (cmd, self.node))
def test_pv_resize_with_prefix_for_name(self,
create_vol_name_prefix=False):
"""Validate PV resize with and without name prefix"""
dir_path = "/mnt/"
node = self.ocp_client[0]
# Create PVC
self.create_storage_class(
allow_volume_expansion=True,
create_vol_name_prefix=create_vol_name_prefix)
pvc_name = self.create_and_wait_for_pvc()
# Create DC with POD and attached PVC to it.
dc_name = oc_create_app_dc_with_io(node, pvc_name)
self.addCleanup(oc_delete, node, 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, node, dc_name, 0)
pod_name = get_pod_name_from_dc(node, dc_name)
wait_for_pod_be_ready(node, pod_name)
if create_vol_name_prefix:
ret = heketi_ops.verify_volume_name_prefix(
node, self.sc['volumenameprefix'], self.sc['secretnamespace'],
pvc_name, self.heketi_server_url)
self.assertTrue(ret, "verify volnameprefix failed")
cmd = ("dd if=/dev/urandom of=%sfile " "bs=100K count=1000") % dir_path
ret, out, err = oc_rsh(node, pod_name, cmd)
self.assertEqual(ret, 0,
"failed to execute command %s on %s" % (cmd, node))
cmd = ("dd if=/dev/urandom of=%sfile2 "
"bs=100K count=10000") % dir_path
ret, out, err = oc_rsh(node, pod_name, cmd)
self.assertNotEqual(
ret, 0, " This IO did not fail as expected "
"command %s on %s" % (cmd, node))
pvc_size = 2
resize_pvc(node, pvc_name, pvc_size)
verify_pvc_size(node, pvc_name, pvc_size)
pv_name = get_pv_name_from_pvc(node, pvc_name)
verify_pv_size(node, pv_name, pvc_size)
oc_delete(node, 'pod', pod_name)
wait_for_resource_absence(node, 'pod', pod_name)
pod_name = get_pod_name_from_dc(node, dc_name)
wait_for_pod_be_ready(node, pod_name)
cmd = ("dd if=/dev/urandom of=%sfile_new "
"bs=50K count=10000") % dir_path
ret, out, err = oc_rsh(node, pod_name, cmd)
self.assertEqual(ret, 0,
"failed to execute command %s on %s" % (cmd, node))
def validate_multipath_info(self, hacount):
"""validates multipath command on the pod node
Args:
hacount (int): hacount for which multipath to be checked
"""
# create pod using pvc created
dc_name = oc_create_app_dc_with_io(
self.ocp_master_node[0],
self.pvc_name,
image=self.io_container_image_cirros)
pod_name = get_pod_name_from_dc(self.ocp_master_node[0], dc_name)
self.addCleanup(oc_delete, self.ocp_master_node[0], "dc", dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, self.ocp_master_node[0],
dc_name, 0)
wait_for_pod_be_ready(self.ocp_master_node[0],
pod_name,
timeout=120,
wait_step=3)
# Get pod info
pod_info = oc_get_pods(self.ocp_master_node[0],
selector='deploymentconfig=%s' % dc_name)
node = pod_info[pod_name]['node']
# Find iqn from volume info
pv_name = get_pv_name_from_pvc(self.ocp_master_node[0], self.pvc_name)
custom = [r':.metadata.annotations."gluster\.org\/volume\-id"']
vol_id = oc_get_custom_resource(self.ocp_master_node[0], 'pv', custom,
pv_name)[0]
vol_info = heketi_blockvolume_info(self.heketi_client_node,
self.heketi_server_url,
vol_id,
json=True)
iqn = vol_info['blockvolume']['iqn']
# Get the paths info from the node
devices = get_iscsi_block_devices_by_path(node, iqn).keys()
self.assertEqual(hacount, len(devices))
# Validate mpath
mpaths = set()
for device in devices:
mpaths.add(get_mpath_name_from_device_name(node, device))
self.assertEqual(1, len(mpaths))
validate_multipath_pod(self.ocp_master_node[0], pod_name, hacount,
list(mpaths)[0])
def test_storage_class_mandatory_params_glusterfile(self):
"""Validate storage-class creation with mandatory parameters"""
# create secret
self.secret_name = oc_create_secret(
self.node,
namespace=self.sc.get('secretnamespace', 'default'),
data_key=self.heketi_cli_key,
secret_type=self.sc.get('provisioner', 'kubernetes.io/glusterfs'))
self.addCleanup(
oc_delete, self.node, 'secret', self.secret_name)
# create storage class with mandatory parameters only
sc_name = oc_create_sc(
self.node, provisioner='kubernetes.io/glusterfs',
resturl=self.sc['resturl'], restuser=self.sc['restuser'],
secretnamespace=self.sc['secretnamespace'],
secretname=self.secret_name
)
self.addCleanup(oc_delete, self.node, 'sc', sc_name)
# Create PVC
pvc_name = self.create_and_wait_for_pvc(sc_name=sc_name)
# Create DC with POD and attached PVC to it.
dc_name = oc_create_app_dc_with_io(self.node, pvc_name)
self.addCleanup(oc_delete, self.node, 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, self.node, dc_name, 0)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
# Make sure we are able to work with files on the mounted volume
filepath = "/mnt/file_for_testing_sc.log"
cmd = "dd if=/dev/urandom of=%s bs=1K count=100" % filepath
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0, "Failed to execute command %s on %s" % (cmd, self.node))
cmd = "ls -lrt %s" % filepath
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0, "Failed to execute command %s on %s" % (cmd, self.node))
cmd = "rm -rf %s" % filepath
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0, "Failed to execute command %s on %s" % (cmd, self.node))
def test_dynamic_provisioning_glusterfile_reclaim_policy_retain(self):
"""Validate retain policy for glusterfs after deletion of pvc"""
if get_openshift_version() < "3.9":
self.skipTest(
"'Reclaim' feature is not supported in OCP older than 3.9")
self.create_storage_class(reclaim_policy='Retain')
self.create_and_wait_for_pvc()
# get the name of the volume
pv_name = get_pv_name_from_pvc(self.node, self.pvc_name)
custom = [r':.metadata.annotations.'
r'"gluster\.kubernetes\.io\/heketi\-volume\-id"',
r':.spec.persistentVolumeReclaimPolicy']
vol_id, reclaim_policy = oc_get_custom_resource(
self.node, 'pv', custom, pv_name)
self.assertEqual(reclaim_policy, 'Retain')
# Create DC with POD and attached PVC to it.
try:
dc_name = oc_create_app_dc_with_io(self.node, self.pvc_name)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
finally:
scale_dc_pod_amount_and_wait(self.node, dc_name, 0)
oc_delete(self.node, 'dc', dc_name)
wait_for_resource_absence(self.node, 'pod', pod_name)
oc_delete(self.node, 'pvc', self.pvc_name)
with self.assertRaises(ExecutionError):
wait_for_resource_absence(
self.node, 'pvc', self.pvc_name, interval=3, timeout=30)
heketi_volume_delete(self.heketi_client_node,
self.heketi_server_url, vol_id)
vol_list = heketi_volume_list(self.heketi_client_node,
self.heketi_server_url)
self.assertNotIn(vol_id, vol_list)
oc_delete(self.node, 'pv', pv_name)
wait_for_resource_absence(self.node, 'pv', pv_name)
def validate_multipath_info(self, hacount):
"""validates multipath command on the pod node
Args:
hacount (int): hacount for which multipath to be checked
"""
# create pod using pvc created
dc_name = oc_create_app_dc_with_io(
self.ocp_master_node[0], self.pvc_name
)
pod_name = get_pod_name_from_dc(self.ocp_master_node[0], dc_name)
self.addCleanup(oc_delete, self.ocp_master_node[0], "dc", dc_name)
self.addCleanup(
scale_dc_pod_amount_and_wait, self.ocp_master_node[0], dc_name, 0
)
wait_for_pod_be_ready(
self.ocp_master_node[0], pod_name, timeout=120, wait_step=3
)
# Get pod info
pod_info = oc_get_pods(
self.ocp_master_node[0], selector='deploymentconfig=%s' % dc_name)
node = pod_info[pod_name]['node']
# Find iqn from volume info
pv_name = get_pv_name_from_pvc(self.ocp_master_node[0], self.pvc_name)
custom = [r':.metadata.annotations."gluster\.org\/volume\-id"']
vol_id = oc_get_custom_resource(
self.ocp_master_node[0], 'pv', custom, pv_name)[0]
vol_info = heketi_blockvolume_info(
self.heketi_client_node, self.heketi_server_url, vol_id, json=True)
iqn = vol_info['blockvolume']['iqn']
# Get the paths info from the node
devices = get_iscsi_block_devices_by_path(node, iqn).keys()
self.assertEqual(hacount, len(devices))
# Validate mpath
mpaths = set()
for device in devices:
mpaths.add(get_mpath_name_from_device_name(node, device))
self.assertEqual(1, len(mpaths))
validate_multipath_pod(
self.ocp_master_node[0], pod_name, hacount, list(mpaths)[0])
def deploy_resouces(self):
"""Deploys required resources storage class, pvc and user app
with continous I/O runnig
Returns:
sc_name (str): deployed storage class name
pvc_name (str): deployed persistent volume claim name
dc_name (str): deployed deployment config name
secretname (str): created secret file name
"""
secretname = oc_create_secret(self.oc_node,
namespace=self.restsecretnamespace,
data_key=self.heketi_cli_key,
secret_type=self.provisioner)
self.addCleanup(oc_delete, self.oc_node, 'secret', secretname)
sc_name = oc_create_sc(self.oc_node,
sc_name_prefix=self.prefix,
provisioner=self.provisioner,
resturl=self.resturl,
restuser=self.restuser,
restsecretnamespace=self.restsecretnamespace,
restsecretname=secretname,
volumenameprefix=self.prefix)
self.addCleanup(oc_delete, self.oc_node, "sc", sc_name)
pvc_name = oc_create_pvc(self.oc_node,
sc_name,
pvc_name_prefix=self.prefix,
pvc_size=self.pvcsize)
self.addCleanup(wait_for_resource_absence,
self.oc_node,
"pvc",
pvc_name,
timeout=120,
interval=5)
self.addCleanup(oc_delete, self.oc_node, "pvc", pvc_name)
dc_name = oc_create_app_dc_with_io(self.oc_node,
pvc_name,
dc_name_prefix=self.prefix)
self.addCleanup(oc_delete, self.oc_node, "dc", dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, self.oc_node, dc_name, 0)
return sc_name, pvc_name, dc_name, secretname
def deploy_resouces(self):
"""Deploys required resources storage class, pvc and user app
with continous I/O runnig
Returns:
sc_name (str): deployed storage class name
pvc_name (str): deployed persistent volume claim name
dc_name (str): deployed deployment config name
secretname (str): created secret file name
"""
secretname = oc_create_secret(
self.oc_node, namespace=self.restsecretnamespace,
data_key=self.heketi_cli_key, secret_type=self.provisioner)
self.addCleanup(oc_delete, self.oc_node, 'secret', secretname)
sc_name = oc_create_sc(
self.oc_node,
sc_name_prefix=self.prefix, provisioner=self.provisioner,
resturl=self.resturl, restuser=self.restuser,
restsecretnamespace=self.restsecretnamespace,
restsecretname=secretname, volumenameprefix=self.prefix
)
self.addCleanup(oc_delete, self.oc_node, "sc", sc_name)
pvc_name = oc_create_pvc(
self.oc_node, sc_name,
pvc_name_prefix=self.prefix, pvc_size=self.pvcsize
)
self.addCleanup(
wait_for_resource_absence, self.oc_node, "pvc", pvc_name,
timeout=120, interval=5
)
self.addCleanup(oc_delete, self.oc_node, "pvc", pvc_name)
dc_name = oc_create_app_dc_with_io(
self.oc_node, pvc_name, dc_name_prefix=self.prefix
)
self.addCleanup(oc_delete, self.oc_node, "dc", dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, self.oc_node, dc_name, 0)
return sc_name, pvc_name, dc_name, secretname
def create_dcs_with_pvc(self, pvc_names, timeout=600, wait_step=5):
"""Create bunch of DCs with app PODs which use unique PVCs.
Args:
pvc_names (str/set/list/tuple): List/set/tuple of PVC names
to attach to app PODs as part of DCs.
timeout (int): timeout value, default value is 600 seconds.
wait_step( int): wait step, default value is 5 seconds.
Returns: dictionary with following structure:
{
"pvc_name_1": ("dc_name_1", "pod_name_1"),
"pvc_name_2": ("dc_name_2", "pod_name_2"),
...
"pvc_name_n": ("dc_name_n", "pod_name_n"),
}
"""
pvc_names = (
pvc_names
if isinstance(pvc_names, (list, set, tuple)) else [pvc_names])
dc_and_pod_names, dc_names = {}, {}
for pvc_name in pvc_names:
dc_name = oc_create_app_dc_with_io(self.ocp_client[0], pvc_name)
dc_names[pvc_name] = dc_name
self.addCleanup(oc_delete, self.ocp_client[0], 'dc', dc_name)
self.addCleanup(
scale_dcs_pod_amount_and_wait, self.ocp_client[0],
dc_names.values(), 0, timeout=timeout, wait_step=wait_step)
for pvc_name, dc_name in dc_names.items():
pod_name = get_pod_name_from_dc(self.ocp_client[0], dc_name)
dc_and_pod_names[pvc_name] = (dc_name, pod_name)
scale_dcs_pod_amount_and_wait(
self.ocp_client[0], dc_names.values(), 1,
timeout=timeout, wait_step=wait_step)
return dc_and_pod_names
def _pv_resize(self, exceed_free_space):
dir_path = "/mnt"
pvc_size_gb, min_free_space_gb = 1, 3
# Get available free space disabling redundant devices and nodes
heketi_url = self.heketi_server_url
node_id_list = heketi_ops.heketi_node_list(self.heketi_client_node,
heketi_url)
self.assertTrue(node_id_list)
nodes = {}
min_free_space = min_free_space_gb * 1024**2
for node_id in node_id_list:
node_info = heketi_ops.heketi_node_info(self.heketi_client_node,
heketi_url,
node_id,
json=True)
if (node_info['state'].lower() != 'online'
or not node_info['devices']):
continue
if len(nodes) > 2:
out = heketi_ops.heketi_node_disable(self.heketi_client_node,
heketi_url, node_id)
self.assertTrue(out)
self.addCleanup(heketi_ops.heketi_node_enable,
self.heketi_client_node, heketi_url, node_id)
for device in node_info['devices']:
if device['state'].lower() != 'online':
continue
free_space = device['storage']['free']
if (node_id in nodes.keys() or free_space < min_free_space):
out = heketi_ops.heketi_device_disable(
self.heketi_client_node, heketi_url, device['id'])
self.assertTrue(out)
self.addCleanup(heketi_ops.heketi_device_enable,
self.heketi_client_node, heketi_url,
device['id'])
continue
nodes[node_id] = free_space
if len(nodes) < 3:
raise self.skipTest("Could not find 3 online nodes with, "
"at least, 1 online device having free space "
"bigger than %dGb." % min_free_space_gb)
# Calculate maximum available size for PVC
available_size_gb = int(min(nodes.values()) / (1024**2))
# Create PVC
self.create_storage_class(allow_volume_expansion=True)
pvc_name = self.create_and_wait_for_pvc(pvc_size=pvc_size_gb)
# Create DC with POD and attached PVC to it
dc_name = oc_create_app_dc_with_io(self.node, pvc_name)
self.addCleanup(oc_delete, self.node, 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, self.node, dc_name, 0)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
if exceed_free_space:
# Try to expand existing PVC exceeding free space
resize_pvc(self.node, pvc_name, available_size_gb)
wait_for_events(self.node,
obj_name=pvc_name,
event_reason='VolumeResizeFailed')
# Check that app POD is up and runnig then try to write data
wait_for_pod_be_ready(self.node, pod_name)
cmd = ("dd if=/dev/urandom of=%s/autotest bs=100K count=1" %
dir_path)
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0,
"Failed to write data after failed attempt to expand PVC.")
else:
# Expand existing PVC using all the available free space
expand_size_gb = available_size_gb - pvc_size_gb
resize_pvc(self.node, pvc_name, expand_size_gb)
verify_pvc_size(self.node, pvc_name, expand_size_gb)
pv_name = get_pv_name_from_pvc(self.node, pvc_name)
verify_pv_size(self.node, pv_name, expand_size_gb)
wait_for_events(self.node,
obj_name=pvc_name,
event_reason='VolumeResizeSuccessful')
# Recreate app POD
oc_delete(self.node, 'pod', pod_name)
wait_for_resource_absence(self.node, 'pod', pod_name)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
# Write data on the expanded PVC
cmd = ("dd if=/dev/urandom of=%s/autotest "
"bs=1M count=1025" % dir_path)
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(ret, 0,
"Failed to write data on the expanded PVC")
def test_dynamic_provisioning_glusterblock_heketipod_failure(self):
"""Validate PVC with glusterblock creation when heketi pod is down"""
datafile_path = '/mnt/fake_file_for_%s' % self.id()
# Create DC with attached PVC
sc_name = self.create_storage_class()
app_1_pvc_name = self.create_and_wait_for_pvc(
pvc_name_prefix='autotest-block', sc_name=sc_name)
app_1_dc_name, app_1_pod_name = self.create_dc_with_pvc(app_1_pvc_name)
# Write test data
write_data_cmd = (
"dd if=/dev/urandom of=%s bs=1K count=100" % datafile_path)
ret, out, err = oc_rsh(self.node, app_1_pod_name, write_data_cmd)
self.assertEqual(
ret, 0,
"Failed to execute command %s on %s" % (write_data_cmd, self.node))
# Remove Heketi pod
heketi_down_cmd = "oc scale --replicas=0 dc/%s --namespace %s" % (
self.heketi_dc_name, self.storage_project_name)
heketi_up_cmd = "oc scale --replicas=1 dc/%s --namespace %s" % (
self.heketi_dc_name, self.storage_project_name)
self.addCleanup(self.cmd_run, heketi_up_cmd)
heketi_pod_name = get_pod_name_from_dc(
self.node, self.heketi_dc_name, timeout=10, wait_step=3)
self.cmd_run(heketi_down_cmd)
wait_for_resource_absence(self.node, 'pod', heketi_pod_name)
# Create second PVC
app_2_pvc_name = oc_create_pvc(
self.node, pvc_name_prefix='autotest-block2', sc_name=sc_name
)
self.addCleanup(
wait_for_resource_absence, self.node, 'pvc', app_2_pvc_name)
self.addCleanup(
oc_delete, self.node, 'pvc', app_2_pvc_name
)
# Create second app POD
app_2_dc_name = oc_create_app_dc_with_io(self.node, app_2_pvc_name)
self.addCleanup(oc_delete, self.node, 'dc', app_2_dc_name)
self.addCleanup(
scale_dc_pod_amount_and_wait, self.node, app_2_dc_name, 0)
app_2_pod_name = get_pod_name_from_dc(self.node, app_2_dc_name)
# Bring Heketi pod back
self.cmd_run(heketi_up_cmd)
# Wait for Heketi POD be up and running
new_heketi_pod_name = get_pod_name_from_dc(
self.node, self.heketi_dc_name, timeout=10, wait_step=2)
wait_for_pod_be_ready(
self.node, new_heketi_pod_name, wait_step=5, timeout=120)
# Wait for second PVC and app POD be ready
verify_pvc_status_is_bound(self.node, app_2_pvc_name)
wait_for_pod_be_ready(
self.node, app_2_pod_name, timeout=150, wait_step=3)
# Verify that we are able to write data
ret, out, err = oc_rsh(self.node, app_2_pod_name, write_data_cmd)
self.assertEqual(
ret, 0,
"Failed to execute command %s on %s" % (write_data_cmd, self.node))
def _pv_resize(self, exceed_free_space):
dir_path = "/mnt"
pvc_size_gb, min_free_space_gb = 1, 3
# Get available free space disabling redundant devices and nodes
heketi_url = self.heketi_server_url
node_id_list = heketi_ops.heketi_node_list(
self.heketi_client_node, heketi_url)
self.assertTrue(node_id_list)
nodes = {}
min_free_space = min_free_space_gb * 1024**2
for node_id in node_id_list:
node_info = heketi_ops.heketi_node_info(
self.heketi_client_node, heketi_url, node_id, json=True)
if (node_info['state'].lower() != 'online' or
not node_info['devices']):
continue
if len(nodes) > 2:
out = heketi_ops.heketi_node_disable(
self.heketi_client_node, heketi_url, node_id)
self.assertTrue(out)
self.addCleanup(
heketi_ops.heketi_node_enable,
self.heketi_client_node, heketi_url, node_id)
for device in node_info['devices']:
if device['state'].lower() != 'online':
continue
free_space = device['storage']['free']
if (node_id in nodes.keys() or free_space < min_free_space):
out = heketi_ops.heketi_device_disable(
self.heketi_client_node, heketi_url, device['id'])
self.assertTrue(out)
self.addCleanup(
heketi_ops.heketi_device_enable,
self.heketi_client_node, heketi_url, device['id'])
continue
nodes[node_id] = free_space
if len(nodes) < 3:
raise self.skipTest(
"Could not find 3 online nodes with, "
"at least, 1 online device having free space "
"bigger than %dGb." % min_free_space_gb)
# Calculate maximum available size for PVC
available_size_gb = int(min(nodes.values()) / (1024**2))
# Create PVC
self.create_storage_class(allow_volume_expansion=True)
pvc_name = self.create_and_wait_for_pvc(pvc_size=pvc_size_gb)
# Create DC with POD and attached PVC to it
dc_name = oc_create_app_dc_with_io(self.node, pvc_name)
self.addCleanup(oc_delete, self.node, 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, self.node, dc_name, 0)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
if exceed_free_space:
# Try to expand existing PVC exceeding free space
resize_pvc(self.node, pvc_name, available_size_gb)
wait_for_events(self.node, obj_name=pvc_name,
event_reason='VolumeResizeFailed')
# Check that app POD is up and runnig then try to write data
wait_for_pod_be_ready(self.node, pod_name)
cmd = (
"dd if=/dev/urandom of=%s/autotest bs=100K count=1" % dir_path)
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0,
"Failed to write data after failed attempt to expand PVC.")
else:
# Expand existing PVC using all the available free space
expand_size_gb = available_size_gb - pvc_size_gb
resize_pvc(self.node, pvc_name, expand_size_gb)
verify_pvc_size(self.node, pvc_name, expand_size_gb)
pv_name = get_pv_name_from_pvc(self.node, pvc_name)
verify_pv_size(self.node, pv_name, expand_size_gb)
wait_for_events(
self.node, obj_name=pvc_name,
event_reason='VolumeResizeSuccessful')
# Recreate app POD
oc_delete(self.node, 'pod', pod_name)
wait_for_resource_absence(self.node, 'pod', pod_name)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
# Write data on the expanded PVC
cmd = ("dd if=/dev/urandom of=%s/autotest "
"bs=1M count=1025" % dir_path)
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0, "Failed to write data on the expanded PVC")
def test_dynamic_provisioning_glusterblock_heketipod_failure(self):
"""Validate PVC with glusterblock creation when heketi pod is down"""
datafile_path = '/mnt/fake_file_for_%s' % self.id()
# Create DC with attached PVC
sc_name = self.create_storage_class()
app_1_pvc_name = self.create_and_wait_for_pvc(
pvc_name_prefix='autotest-block', sc_name=sc_name)
app_1_dc_name, app_1_pod_name = self.create_dc_with_pvc(app_1_pvc_name)
# Write test data
write_data_cmd = ("dd if=/dev/urandom of=%s bs=1K count=100" %
datafile_path)
ret, out, err = oc_rsh(self.node, app_1_pod_name, write_data_cmd)
self.assertEqual(
ret, 0,
"Failed to execute command %s on %s" % (write_data_cmd, self.node))
# Remove Heketi pod
heketi_down_cmd = "oc scale --replicas=0 dc/%s --namespace %s" % (
self.heketi_dc_name, self.storage_project_name)
heketi_up_cmd = "oc scale --replicas=1 dc/%s --namespace %s" % (
self.heketi_dc_name, self.storage_project_name)
self.addCleanup(self.cmd_run, heketi_up_cmd)
heketi_pod_name = get_pod_name_from_dc(self.node,
self.heketi_dc_name,
timeout=10,
wait_step=3)
self.cmd_run(heketi_down_cmd)
wait_for_resource_absence(self.node, 'pod', heketi_pod_name)
# Create second PVC
app_2_pvc_name = oc_create_pvc(self.node,
pvc_name_prefix='autotest-block2',
sc_name=sc_name)
self.addCleanup(wait_for_resource_absence, self.node, 'pvc',
app_2_pvc_name)
self.addCleanup(oc_delete, self.node, 'pvc', app_2_pvc_name)
# Create second app POD
app_2_dc_name = oc_create_app_dc_with_io(self.node, app_2_pvc_name)
self.addCleanup(oc_delete, self.node, 'dc', app_2_dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, self.node, app_2_dc_name,
0)
app_2_pod_name = get_pod_name_from_dc(self.node, app_2_dc_name)
# Bring Heketi pod back
self.cmd_run(heketi_up_cmd)
# Wait for Heketi POD be up and running
new_heketi_pod_name = get_pod_name_from_dc(self.node,
self.heketi_dc_name,
timeout=10,
wait_step=2)
wait_for_pod_be_ready(self.node,
new_heketi_pod_name,
wait_step=5,
timeout=120)
# Wait for second PVC and app POD be ready
verify_pvc_status_is_bound(self.node, app_2_pvc_name)
wait_for_pod_be_ready(self.node,
app_2_pod_name,
timeout=150,
wait_step=3)
# Verify that we are able to write data
ret, out, err = oc_rsh(self.node, app_2_pod_name, write_data_cmd)
self.assertEqual(
ret, 0,
"Failed to execute command %s on %s" % (write_data_cmd, self.node))
