def test_node_failure_pv_mounted(self):
"""Test node failure when PV is mounted with app pods running"""
filepath = "/mnt/file_for_testing_volume.log"
pvc_name = self.create_and_wait_for_pvc()
dc_and_pod_names = self.create_dcs_with_pvc(pvc_name)
dc_name, pod_name = dc_and_pod_names[pvc_name]
mount_point = "df -kh /mnt -P | tail -1 | awk '{{print $1}}'"
pod_cmd = "oc exec {} -- {}".format(pod_name, mount_point)
hostname = command.cmd_run(pod_cmd, hostname=self.node)
hostname = hostname.split(":")[0]
vm_name = find_vm_name_by_ip_or_hostname(hostname)
self.addCleanup(power_on_vm_by_name, vm_name)
power_off_vm_by_name(vm_name)
cmd = "dd if=/dev/urandom of={} bs=1K count=100".format(filepath)
ret, _, err = oc_rsh(self.node, pod_name, cmd)
self.assertFalse(
ret, "Failed to execute command {} on {} with error {}".format(
cmd, self.node, err))
oc_delete(self.node, 'pod', pod_name)
wait_for_resource_absence(self.node, 'pod', pod_name)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
ret, _, err = oc_rsh(self.node, pod_name, cmd)
self.assertFalse(
ret, "Failed to execute command {} on {} with error {}".format(
cmd, self.node, err))
def test_pv_resize_try_shrink_pv_size(self):
"""Validate whether reducing the PV size is allowed"""
dir_path = "/mnt/"
node = self.ocp_master_node[0]
# Create PVC
pv_size = 5
self.create_storage_class(allow_volume_expansion=True)
pvc_name = self.create_and_wait_for_pvc(pvc_size=pv_size)
# Create DC with POD and attached PVC to it.
dc_name = oc_create_app_dc_with_io(node, pvc_name)
self.addCleanup(oc_delete, node, 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait,
node, dc_name, 0)
pod_name = get_pod_name_from_dc(node, dc_name)
wait_for_pod_be_ready(node, pod_name)
cmd = ("dd if=/dev/urandom of=%sfile "
"bs=100K count=3000") % dir_path
ret, out, err = oc_rsh(node, pod_name, cmd)
self.assertEqual(ret, 0, "failed to execute command %s on %s" % (
cmd, node))
pvc_resize = 2
with self.assertRaises(ExecutionError):
resize_pvc(node, pvc_name, pvc_resize)
verify_pvc_size(node, pvc_name, pv_size)
pv_name = get_pv_name_from_pvc(node, pvc_name)
verify_pv_size(node, pv_name, pv_size)
cmd = ("dd if=/dev/urandom of=%sfile_new "
"bs=100K count=2000") % dir_path
ret, out, err = oc_rsh(node, pod_name, cmd)
self.assertEqual(ret, 0, "failed to execute command %s on %s" % (
cmd, node))
def test_pv_resize_try_shrink_pv_size(self):
"""Validate whether reducing the PV size is allowed"""
dir_path = "/mnt/"
node = self.ocp_master_node[0]
# Create PVC
pv_size = 5
self.create_storage_class(allow_volume_expansion=True)
pvc_name = self.create_and_wait_for_pvc(pvc_size=pv_size)
# Create DC with POD and attached PVC to it.
dc_name = oc_create_app_dc_with_io(node, pvc_name)
self.addCleanup(oc_delete, node, 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, node, dc_name, 0)
pod_name = get_pod_name_from_dc(node, dc_name)
wait_for_pod_be_ready(node, pod_name)
cmd = ("dd if=/dev/urandom of=%sfile " "bs=100K count=3000") % dir_path
ret, out, err = oc_rsh(node, pod_name, cmd)
self.assertEqual(ret, 0,
"failed to execute command %s on %s" % (cmd, node))
pvc_resize = 2
with self.assertRaises(ExecutionError):
resize_pvc(node, pvc_name, pvc_resize)
verify_pvc_size(node, pvc_name, pv_size)
pv_name = get_pv_name_from_pvc(node, pvc_name)
verify_pv_size(node, pv_name, pv_size)
cmd = ("dd if=/dev/urandom of=%sfile_new "
"bs=100K count=2000") % dir_path
ret, out, err = oc_rsh(node, pod_name, cmd)
self.assertEqual(ret, 0,
"failed to execute command %s on %s" % (cmd, node))
def test_logging_es_pod_pvc_all_freespace_utilization(self):
"""Validate logging by utilizing all the free space of block PVC bound
to elsaticsearch pod"""
# Fetch pod and validate iscsi and multipath
es_pod, _ = self._get_es_pod_and_verify_iscsi_sessions()
# Get the available free space
mount_point = '/elasticsearch/persistent'
cmd_free_space = (
"df -kh {} | awk '{{print $4}}' | tail -1".format(mount_point))
old_available_space = openshift_ops.oc_rsh(self._master, es_pod,
cmd_free_space)[1]
# Fill the all the available space
file_name = '{}/file'.format(mount_point)
cmd_fill_space = ("fallocate -l {} {}".format(old_available_space,
file_name))
with self.assertRaises(AssertionError):
openshift_ops.oc_rsh(self._master, es_pod, cmd_fill_space)
# Cleanup the filled space
cmd_remove_file = 'rm {}'.format(file_name)
self.addCleanup(openshift_ops.oc_rsh, self._master, es_pod,
cmd_remove_file)
def test_pv_resize_with_prefix_for_name_and_size(
self, create_vol_name_prefix=False, valid_size=True):
"""Validate PV resize with and without name prefix"""
dir_path = "/mnt/"
node = self.ocp_client[0]
# Create PVC
self.create_storage_class(
allow_volume_expansion=True,
create_vol_name_prefix=create_vol_name_prefix)
pvc_name = self.create_and_wait_for_pvc()
# Create DC with POD and attached PVC to it.
dc_name = oc_create_app_dc_with_io(node, pvc_name)
self.addCleanup(oc_delete, node, 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, node, dc_name, 0)
pod_name = get_pod_name_from_dc(node, dc_name)
wait_for_pod_be_ready(node, pod_name)
if create_vol_name_prefix:
ret = heketi_ops.verify_volume_name_prefix(
node, self.sc['volumenameprefix'], self.sc['secretnamespace'],
pvc_name, self.heketi_server_url)
self.assertTrue(ret, "verify volnameprefix failed")
cmd = ("dd if=/dev/urandom of=%sfile " "bs=100K count=1000") % dir_path
ret, out, err = oc_rsh(node, pod_name, cmd)
self.assertEqual(ret, 0,
"Failed to execute command %s on %s" % (cmd, node))
pv_name = get_pv_name_from_pvc(node, pvc_name)
# If resize size is invalid then size should not change
if valid_size:
cmd = ("dd if=/dev/urandom of=%sfile2 "
"bs=100K count=10000") % dir_path
with self.assertRaises(AssertionError):
ret, out, err = oc_rsh(node, pod_name, cmd)
msg = ("Command '%s' was expected to fail on '%s' node. "
"But it returned following: ret is '%s', err is '%s' "
"and out is '%s'" % (cmd, node, ret, err, out))
raise ExecutionError(msg)
pvc_size = 2
resize_pvc(node, pvc_name, pvc_size)
verify_pvc_size(node, pvc_name, pvc_size)
verify_pv_size(node, pv_name, pvc_size)
else:
invalid_pvc_size = 'ten'
with self.assertRaises(AssertionError):
resize_pvc(node, pvc_name, invalid_pvc_size)
verify_pvc_size(node, pvc_name, 1)
verify_pv_size(node, pv_name, 1)
oc_delete(node, 'pod', pod_name)
wait_for_resource_absence(node, 'pod', pod_name)
pod_name = get_pod_name_from_dc(node, dc_name)
wait_for_pod_be_ready(node, pod_name)
cmd = ("dd if=/dev/urandom of=%sfile_new "
"bs=50K count=10000") % dir_path
ret, out, err = oc_rsh(node, pod_name, cmd)
self.assertEqual(ret, 0,
"Failed to execute command %s on %s" % (cmd, node))
def test_metrics_cassandra_pod_pvc_all_freespace_utilization(self):
"""Validate metrics by utilizing all the free space of block PVC bound
to cassandra pod"""
# Validate iscsi and multipath
hawkular_cassandra, _, _, _, _ = (
self.verify_cassandra_pod_multipath_and_iscsi())
# Get the available free space
mount_point = '/cassandra_data'
cmd_free_space = (
"df -kh {} | awk '{{print $4}}' | tail -1".format(mount_point))
old_available_space = oc_rsh(self.master, hawkular_cassandra,
cmd_free_space)[1]
# Fill the all the available space
file_name = '{}/file'.format(mount_point)
cmd_fill_space = ("fallocate -l {} {}".format(old_available_space,
file_name))
with self.assertRaises(AssertionError):
oc_rsh(self.master, hawkular_cassandra, cmd_fill_space)
# Cleanup the filled space
cmd_remove_file = 'rm {}'.format(file_name)
self.addCleanup(oc_rsh, self.master, hawkular_cassandra,
cmd_remove_file)
def _pv_resize(self, exceed_free_space):
dir_path = "/mnt"
pvc_size_gb = 1
available_size_gb = self._available_disk_free_space()
# Create PVC
self.create_storage_class(allow_volume_expansion=True)
pvc_name = self.create_and_wait_for_pvc(pvc_size=pvc_size_gb)
# Create DC with POD and attached PVC to it
dc_name = oc_create_app_dc_with_io(
self.node, pvc_name, image=self.io_container_image_cirros)
self.addCleanup(oc_delete, self.node, 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, self.node, dc_name, 0)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
if exceed_free_space:
exceed_size = available_size_gb + 10
# Try to expand existing PVC exceeding free space
resize_pvc(self.node, pvc_name, exceed_size)
wait_for_events(self.node,
obj_name=pvc_name,
event_reason='VolumeResizeFailed')
# Check that app POD is up and runnig then try to write data
wait_for_pod_be_ready(self.node, pod_name)
cmd = ("dd if=/dev/urandom of=%s/autotest bs=100K count=1" %
dir_path)
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0,
"Failed to write data after failed attempt to expand PVC.")
else:
# Expand existing PVC using all the available free space
expand_size_gb = available_size_gb - pvc_size_gb
resize_pvc(self.node, pvc_name, expand_size_gb)
verify_pvc_size(self.node, pvc_name, expand_size_gb)
pv_name = get_pv_name_from_pvc(self.node, pvc_name)
verify_pv_size(self.node, pv_name, expand_size_gb)
wait_for_events(self.node,
obj_name=pvc_name,
event_reason='VolumeResizeSuccessful')
# Recreate app POD
oc_delete(self.node, 'pod', pod_name)
wait_for_resource_absence(self.node, 'pod', pod_name)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
# Write data on the expanded PVC
cmd = ("dd if=/dev/urandom of=%s/autotest "
"bs=1M count=1025" % dir_path)
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(ret, 0,
"Failed to write data on the expanded PVC")
def test_kill_bhv_fsd_while_es_pod_running(self):
"""Validate killing of bhv fsd won't effect es pod io's"""
# Fetch pod and PVC names and validate iscsi and multipath
es_pod, pvc_name = self._get_es_pod_and_verify_iscsi_sessions()
# Get the bhv name
gluster_node = list(self._registry_servers_info.keys())[0]
openshift_ops.switch_oc_project(self._master,
self._registry_project_name)
bhv_name = self.get_block_hosting_volume_by_pvc_name(
pvc_name,
heketi_server_url=self._registry_heketi_server_url,
gluster_node=gluster_node)
# Get one of the bricks pid of the bhv
gluster_volume_status = gluster_ops.get_gluster_vol_status(bhv_name)
pid = None
for g_node, g_node_data in gluster_volume_status.items():
if g_node != gluster_node:
continue
for process_name, process_data in g_node_data.items():
if not process_name.startswith("/var"):
continue
pid = process_data["pid"]
# When birck is down, pid of the brick is returned as -1.
# Which is unexepeted situation. So, add appropriate assertion.
self.assertNotEqual(
pid, "-1", "Got unexpected PID (-1) for '{}' gluster vol "
"on '{}' node.".format(bhv_name, gluster_node))
break
self.assertTrue(
pid, "Could not find 'pid' in Gluster vol data for '{}' "
"Gluster node. Data: {}".format(gluster_node,
gluster_volume_status))
break
# Kill gluster vol brick process using found pid
cmd_kill = "kill -9 {}".format(pid)
cmd_start_vol = "gluster v start {} force".format(bhv_name)
openshift_ops.cmd_run_on_gluster_pod_or_node(self._master, cmd_kill,
gluster_node)
self.addCleanup(openshift_ops.cmd_run_on_gluster_pod_or_node,
self._master, cmd_start_vol, gluster_node)
self.addCleanup(openshift_ops.switch_oc_project, self._master,
self._registry_project_name)
# Run I/O on ES pod
openshift_ops.switch_oc_project(self._master,
self._logging_project_name)
file_name = '/elasticsearch/persistent/file1'
cmd_run_io = 'dd if=/dev/urandom of={} bs=4k count=10000'.format(
file_name)
cmd_remove_file = 'rm {}'.format(file_name)
openshift_ops.oc_rsh(self._master, es_pod, cmd_run_io)
self.addCleanup(openshift_ops.oc_rsh, self._master, es_pod,
cmd_remove_file)
def test_pv_resize_with_prefix_for_name(self,
create_vol_name_prefix=False):
"""Validate PV resize with and without name prefix"""
dir_path = "/mnt/"
node = self.ocp_client[0]
# Create PVC
self.create_storage_class(
allow_volume_expansion=True,
create_vol_name_prefix=create_vol_name_prefix)
pvc_name = self.create_and_wait_for_pvc()
# Create DC with POD and attached PVC to it.
dc_name = oc_create_app_dc_with_io(node, pvc_name)
self.addCleanup(oc_delete, node, 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait,
node, dc_name, 0)
pod_name = get_pod_name_from_dc(node, dc_name)
wait_for_pod_be_ready(node, pod_name)
if create_vol_name_prefix:
ret = heketi_ops.verify_volume_name_prefix(
node, self.sc['volumenameprefix'],
self.sc['secretnamespace'],
pvc_name, self.heketi_server_url)
self.assertTrue(ret, "verify volnameprefix failed")
cmd = ("dd if=/dev/urandom of=%sfile "
"bs=100K count=1000") % dir_path
ret, out, err = oc_rsh(node, pod_name, cmd)
self.assertEqual(ret, 0, "failed to execute command %s on %s" % (
cmd, node))
cmd = ("dd if=/dev/urandom of=%sfile2 "
"bs=100K count=10000") % dir_path
with self.assertRaises(AssertionError):
ret, out, err = oc_rsh(node, pod_name, cmd)
msg = ("Command '%s' was expected to fail on '%s' node. "
"But it returned following: ret is '%s', err is '%s' "
"and out is '%s'" % (cmd, node, ret, err, out))
raise ExecutionError(msg)
pvc_size = 2
resize_pvc(node, pvc_name, pvc_size)
verify_pvc_size(node, pvc_name, pvc_size)
pv_name = get_pv_name_from_pvc(node, pvc_name)
verify_pv_size(node, pv_name, pvc_size)
oc_delete(node, 'pod', pod_name)
wait_for_resource_absence(node, 'pod', pod_name)
pod_name = get_pod_name_from_dc(node, dc_name)
wait_for_pod_be_ready(node, pod_name)
cmd = ("dd if=/dev/urandom of=%sfile_new "
"bs=50K count=10000") % dir_path
ret, out, err = oc_rsh(node, pod_name, cmd)
self.assertEqual(ret, 0, "failed to execute command %s on %s" % (
cmd, node))
def test_dynamic_provisioning_block_vol_with_custom_prefix(self):
"""Verify creation of block volume with custom prefix
"""
node = self.ocp_master_node[0]
prefix = "autotest-{}".format(utils.get_random_str())
# cmd to get available space
cmd_get_free_space = "df -h | grep '/mnt'| awk '{{print $4}}'"
# cmd to create a 100M file
cmd_run_io = 'dd if=/dev/zero of=/mnt/testfile bs=1024 count=102400'
# Create sc with prefix
sc_name = self.create_storage_class(sc_name_prefix=prefix,
create_vol_name_prefix=True,
vol_name_prefix=prefix)
# Create pvc and wait for it to be in bound state
pvc_name = self.create_and_wait_for_pvc(sc_name=sc_name, pvc_size=1)
# Verify blockvolume list with prefix
h_block_vol = heketi_blockvolume_list_by_name_prefix(
self.heketi_client_node, self.heketi_server_url, prefix)
self.assertIsNotNone(
h_block_vol,
"Failed to find blockvolume with prefix {}".format(prefix))
self.assertTrue(
h_block_vol[0][2].startswith(prefix),
"Failed to create blockvolume with the prefix {}".format(prefix))
# Create app pod
dc_name, pod_name = self.create_dc_with_pvc(pvc_name)
err_msg = ("Failed to get the free space for the mount point of the "
"app pod {} with error {}")
# Get free space of app pod before IO run
_, free_space_before, err = oc_rsh(node, pod_name, cmd_get_free_space)
self.assertTrue(free_space_before, err_msg.format(pod_name, err))
# Running IO on the app pod
ret, _, err = oc_rsh(node, pod_name, cmd_run_io)
self.assertFalse(
ret, "Failed to run the Io with the error msg {}".format(err))
# Get free space of app pod after IO run
_, free_space_after, err = oc_rsh(node, pod_name, cmd_get_free_space)
self.assertTrue(free_space_after, err_msg.format(pod_name, err))
self.assertGreaterEqual(
free_space_before, free_space_after,
"Expecting free space in app pod before {} should be greater than"
" {} as 100M file is created".format(free_space_before,
free_space_after))
def _check_fstab_and_df_entries(self, first_cmd, second_cmd):
# matches output of "df --out=target" and entries in fstab
# and vice-versa as per commands given in first_cmd and
# second_cmd
err_msg = "failed to execute command: %s with error: %s"
ret, out, err = oc_rsh(self.oc_node, self.gluster_pod_name, first_cmd)
self.assertEqual(ret, 0, err_msg % (first_cmd, err))
for mnt_path in (out.strip()).split("\n"):
ret, out, err = oc_rsh(self.oc_node, self.gluster_pod_name,
second_cmd % mnt_path)
self.assertEqual(ret, 0, err_msg % (second_cmd, err))
def _check_fstab_and_df_entries(self, first_cmd, second_cmd):
# matches output of "df --out=target" and entries in fstab
# and vice-versa as per commands given in first_cmd and
# second_cmd
err_msg = "failed to execute command: %s with error: %s"
ret, out, err = oc_rsh(self.oc_node, self.gluster_pod_name, first_cmd)
self.assertEqual(ret, 0, err_msg % (first_cmd, err))
for mnt_path in (out.strip()).split("\n"):
ret, out, err = oc_rsh(
self.oc_node, self.gluster_pod_name, second_cmd % mnt_path
)
self.assertEqual(ret, 0, err_msg % (second_cmd, err))
def test_storage_class_mandatory_params_glusterfile(self):
"""Validate storage-class creation with mandatory parameters"""
# create secret
self.secret_name = oc_create_secret(
self.node,
namespace=self.sc.get('secretnamespace', 'default'),
data_key=self.heketi_cli_key,
secret_type=self.sc.get('provisioner', 'kubernetes.io/glusterfs'))
self.addCleanup(oc_delete, self.node, 'secret', self.secret_name)
# create storage class with mandatory parameters only
sc_name = oc_create_sc(self.node,
provisioner='kubernetes.io/glusterfs',
resturl=self.sc['resturl'],
restuser=self.sc['restuser'],
secretnamespace=self.sc['secretnamespace'],
secretname=self.secret_name)
self.addCleanup(oc_delete, self.node, 'sc', sc_name)
# Create PVC
pvc_name = self.create_and_wait_for_pvc(sc_name=sc_name)
# Create DC with POD and attached PVC to it.
dc_name = oc_create_app_dc_with_io(
self.node, pvc_name, image=self.io_container_image_cirros)
self.addCleanup(oc_delete, self.node, 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, self.node, dc_name, 0)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
# Make sure we are able to work with files on the mounted volume
filepath = "/mnt/file_for_testing_sc.log"
cmd = "dd if=/dev/urandom of=%s bs=1K count=100" % filepath
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0, "Failed to execute command %s on %s" % (cmd, self.node))
cmd = "ls -lrt %s" % filepath
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0, "Failed to execute command %s on %s" % (cmd, self.node))
cmd = "rm -rf %s" % filepath
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0, "Failed to execute command %s on %s" % (cmd, self.node))
def test_pv_resize_with_prefix_for_name(self,
create_vol_name_prefix=False):
"""Validate PV resize with and without name prefix"""
dir_path = "/mnt/"
node = self.ocp_client[0]
# Create PVC
self.create_storage_class(
allow_volume_expansion=True,
create_vol_name_prefix=create_vol_name_prefix)
pvc_name = self.create_and_wait_for_pvc()
# Create DC with POD and attached PVC to it.
dc_name = oc_create_app_dc_with_io(node, pvc_name)
self.addCleanup(oc_delete, node, 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, node, dc_name, 0)
pod_name = get_pod_name_from_dc(node, dc_name)
wait_for_pod_be_ready(node, pod_name)
if create_vol_name_prefix:
ret = heketi_ops.verify_volume_name_prefix(
node, self.sc['volumenameprefix'], self.sc['secretnamespace'],
pvc_name, self.heketi_server_url)
self.assertTrue(ret, "verify volnameprefix failed")
cmd = ("dd if=/dev/urandom of=%sfile " "bs=100K count=1000") % dir_path
ret, out, err = oc_rsh(node, pod_name, cmd)
self.assertEqual(ret, 0,
"failed to execute command %s on %s" % (cmd, node))
cmd = ("dd if=/dev/urandom of=%sfile2 "
"bs=100K count=10000") % dir_path
ret, out, err = oc_rsh(node, pod_name, cmd)
self.assertNotEqual(
ret, 0, " This IO did not fail as expected "
"command %s on %s" % (cmd, node))
pvc_size = 2
resize_pvc(node, pvc_name, pvc_size)
verify_pvc_size(node, pvc_name, pvc_size)
pv_name = get_pv_name_from_pvc(node, pvc_name)
verify_pv_size(node, pv_name, pvc_size)
oc_delete(node, 'pod', pod_name)
wait_for_resource_absence(node, 'pod', pod_name)
pod_name = get_pod_name_from_dc(node, dc_name)
wait_for_pod_be_ready(node, pod_name)
cmd = ("dd if=/dev/urandom of=%sfile_new "
"bs=50K count=10000") % dir_path
ret, out, err = oc_rsh(node, pod_name, cmd)
self.assertEqual(ret, 0,
"failed to execute command %s on %s" % (cmd, node))
def test_storage_class_mandatory_params_glusterfile(self):
"""Validate storage-class creation with mandatory parameters"""
# create secret
self.secret_name = oc_create_secret(
self.node,
namespace=self.sc.get('secretnamespace', 'default'),
data_key=self.heketi_cli_key,
secret_type=self.sc.get('provisioner', 'kubernetes.io/glusterfs'))
self.addCleanup(
oc_delete, self.node, 'secret', self.secret_name)
# create storage class with mandatory parameters only
sc_name = oc_create_sc(
self.node, provisioner='kubernetes.io/glusterfs',
resturl=self.sc['resturl'], restuser=self.sc['restuser'],
secretnamespace=self.sc['secretnamespace'],
secretname=self.secret_name
)
self.addCleanup(oc_delete, self.node, 'sc', sc_name)
# Create PVC
pvc_name = self.create_and_wait_for_pvc(sc_name=sc_name)
# Create DC with POD and attached PVC to it.
dc_name = oc_create_app_dc_with_io(self.node, pvc_name)
self.addCleanup(oc_delete, self.node, 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, self.node, dc_name, 0)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
# Make sure we are able to work with files on the mounted volume
filepath = "/mnt/file_for_testing_sc.log"
cmd = "dd if=/dev/urandom of=%s bs=1K count=100" % filepath
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0, "Failed to execute command %s on %s" % (cmd, self.node))
cmd = "ls -lrt %s" % filepath
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0, "Failed to execute command %s on %s" % (cmd, self.node))
cmd = "rm -rf %s" % filepath
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0, "Failed to execute command %s on %s" % (cmd, self.node))
def test_lvm_script_and_wrapper_environments(self):
"""Validate lvm script present on glusterfs pods
lvm wrapper environment is present on heketi pod"""
# Check script /usr/sbin/exec-on-host is present in pod
if self.is_containerized_gluster():
cmd = "ls -lrt {}".format(ENV_VALUE)
ret, out, err = openshift_ops.oc_rsh(
self.oc_node, self.pod_name[0]['pod_name'], cmd)
self.assertFalse(
ret, "failed to execute command {} on pod {} with error:"
" {}".format(cmd, self.pod_name[0]['pod_name'], err))
self.assertIn(ENV_VALUE, out)
# Get a value associated with HEKETI_LVM_WRAPPER
custom = (r'":spec.containers[*].env[?(@.name==\"{}\")]'
r'.value"'.format(ENV_NAME))
env_var_value = openshift_ops.oc_get_custom_resource(
self.oc_node, "pod", custom, self.h_pod_name)
# Check value /usr/sbin/exec-on-host is present in converged mode
# and absent in independent mode deployment
err_msg = "Heketi LVM environment {} match failed".format(ENV_VALUE)
if self.is_containerized_gluster():
self.assertEqual(env_var_value[0], ENV_VALUE, err_msg)
else:
self.assertIsNotNone(env_var_value[0], err_msg)
def dynamic_provisioning_glusterfile(self, create_vol_name_prefix):
# Create secret and storage class
self.create_storage_class(
create_vol_name_prefix=create_vol_name_prefix)
# Create PVC
pvc_name = self.create_and_wait_for_pvc()
# Create DC with POD and attached PVC to it.
dc_name = oc_create_app_dc_with_io(self.node, pvc_name)
self.addCleanup(oc_delete, self.node, 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, self.node, dc_name, 0)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
# Verify Heketi volume name for prefix presence if provided
if create_vol_name_prefix:
ret = verify_volume_name_prefix(self.node,
self.sc['volumenameprefix'],
self.sc['secretnamespace'],
pvc_name, self.sc['resturl'])
self.assertTrue(ret, "verify volnameprefix failed")
else:
# Get the volume name and volume id from PV
pv_name = get_pv_name_from_pvc(self.ocp_client[0], pvc_name)
custom = [
r':spec.glusterfs.path',
r':metadata.annotations.'
r'"gluster\.kubernetes\.io\/heketi-volume-id"'
]
pv_vol_name, vol_id = oc_get_custom_resource(
self.ocp_client[0], 'pv', custom, pv_name)
# check if the pv_volume_name is present in heketi
# Check if volume name is "vol_"+volumeid or not
heketi_vol_name = heketi_volume_info(
self.ocp_client[0], self.heketi_server_url, vol_id,
json=True)['name']
self.assertEqual(pv_vol_name, heketi_vol_name,
'Volume with vol_id = %s not found'
'in heketidb' % vol_id)
self.assertEqual(heketi_vol_name, 'vol_' + vol_id,
'Volume with vol_id = %s have a'
'custom perfix' % vol_id)
out = cmd_run_on_gluster_pod_or_node(self.ocp_master_node[0],
"gluster volume list")
self.assertIn(pv_vol_name, out,
"Volume with id %s does not exist" % vol_id)
# Make sure we are able to work with files on the mounted volume
filepath = "/mnt/file_for_testing_io.log"
for cmd in ("dd if=/dev/urandom of=%s bs=1K count=100",
"ls -lrt %s",
"rm -rf %s"):
cmd = cmd % filepath
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0,
"Failed to execute '%s' command on %s" % (cmd, self.node))
def dynamic_provisioning_glusterfile(self, create_vol_name_prefix):
# Create secret and storage class
self.create_storage_class(
create_vol_name_prefix=create_vol_name_prefix)
# Create PVC
pvc_name = self.create_and_wait_for_pvc()
# Create DC with POD and attached PVC to it.
dc_name = oc_create_app_dc_with_io(self.node, pvc_name)
self.addCleanup(oc_delete, self.node, 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, self.node, dc_name, 0)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
# Verify Heketi volume name for prefix presence if provided
if create_vol_name_prefix:
ret = verify_volume_name_prefix(self.node,
self.sc['volumenameprefix'],
self.sc['secretnamespace'],
pvc_name, self.sc['resturl'])
self.assertTrue(ret, "verify volnameprefix failed")
else:
# Get the volume name and volume id from PV
pv_name = get_pv_name_from_pvc(self.ocp_client[0], pvc_name)
custom = [
r':spec.glusterfs.path',
r':metadata.annotations.'
r'"gluster\.kubernetes\.io\/heketi-volume-id"'
]
pv_vol_name, vol_id = oc_get_custom_resource(
self.ocp_client[0], 'pv', custom, pv_name)
# check if the pv_volume_name is present in heketi
# Check if volume name is "vol_"+volumeid or not
heketi_vol_name = heketi_volume_info(
self.ocp_client[0], self.heketi_server_url, vol_id,
json=True)['name']
self.assertEqual(pv_vol_name, heketi_vol_name,
'Volume with vol_id = %s not found'
'in heketidb' % vol_id)
self.assertEqual(heketi_vol_name, 'vol_' + vol_id,
'Volume with vol_id = %s have a'
'custom perfix' % vol_id)
out = cmd_run_on_gluster_pod_or_node(self.ocp_master_node[0],
"gluster volume list")
self.assertIn(pv_vol_name, out,
"Volume with id %s does not exist" % vol_id)
# Make sure we are able to work with files on the mounted volume
filepath = "/mnt/file_for_testing_io.log"
for cmd in ("dd if=/dev/urandom of=%s bs=1K count=100",
"ls -lrt %s",
"rm -rf %s"):
cmd = cmd % filepath
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0,
"Failed to execute '%s' command on %s" % (cmd, self.node))
def test_pvc_deletion_while_pod_is_running(self):
"""Validate PVC deletion while pod is running"""
if get_openshift_version() <= "3.9":
self.skipTest(
"PVC deletion while pod is running is not supported"
" in OCP older than 3.9")
# Create DC with POD and attached PVC to it
sc_name = self.create_storage_class()
pvc_name = self.create_and_wait_for_pvc(sc_name=sc_name)
dc_name, pod_name = self.create_dc_with_pvc(pvc_name)
# Delete PVC
oc_delete(self.node, 'pvc', self.pvc_name)
with self.assertRaises(ExecutionError):
wait_for_resource_absence(
self.node, 'pvc', self.pvc_name, interval=3, timeout=30)
# Make sure we are able to work with files on the mounted volume
# after deleting pvc.
filepath = "/mnt/file_for_testing_volume.log"
cmd = "dd if=/dev/urandom of=%s bs=1K count=100" % filepath
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0, "Failed to execute command %s on %s" % (cmd, self.node))
def _write_file(self, pod_name, filename, filesize, mnt_path):
# write file
cmd = ('dd if=/dev/zero of={}/{} bs={} count=1'.format(
mnt_path, filename, filesize))
ret, _, err = oc_rsh(self.node, pod_name, cmd)
self.assertFalse(ret, 'Failed to write file due to err {}'.format(err))
wait_for_pod_be_ready(self.node, pod_name)
def test_block_vol_offline_expand(self):
"""Test blockvol expansion while PVC is not in use"""
node = self.ocp_master_node[0]
pvc_name, dc_name, bvol_info = (
self._block_vol_expand_common_offline_vs_online(False))
# create and wait for job to be completed
jobname = oc_create_offline_block_volume_expand_job(node, pvc_name)
self.addCleanup(oc_delete, node, 'job', jobname)
for w in waiter.Waiter(300, 5):
if is_job_complete(node, jobname):
break
if w.expired:
raise AssertionError(
"block expand job {} is not completed".format(jobname))
# verify expand size
scale_dc_pod_amount_and_wait(node, dc_name[0], pod_amount=1)
pod_name = get_pod_name_from_dc(node, dc_name[0])
ret, size, _ = oc_rsh(
node, pod_name,
'df -kh /mnt | sed "/Filesystem/d" | awk \'{print $2}\' '
'| sed "s/G//"')
self.assertFalse(ret, "Failed to get size from client side")
self.assertEqual(
int(float(size)), bvol_info["size"], "new size is not "
"reflected at mount point after block volume expand")
def test_pvc_deletion_while_pod_is_running(self):
"""Validate PVC deletion while pod is running"""
if get_openshift_version() <= "3.9":
self.skipTest("PVC deletion while pod is running is not supported"
" in OCP older than 3.9")
# Create DC with POD and attached PVC to it
sc_name = self.create_storage_class()
pvc_name = self.create_and_wait_for_pvc(sc_name=sc_name)
dc_name, pod_name = self.create_dc_with_pvc(pvc_name)
# Delete PVC
oc_delete(self.node, 'pvc', self.pvc_name)
with self.assertRaises(ExecutionError):
wait_for_resource_absence(self.node,
'pvc',
self.pvc_name,
interval=3,
timeout=30)
# Make sure we are able to work with files on the mounted volume
# after deleting pvc.
filepath = "/mnt/file_for_testing_volume.log"
cmd = "dd if=/dev/urandom of=%s bs=1K count=100" % filepath
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0, "Failed to execute command %s on %s" % (cmd, self.node))
def _get_mount_size(self, pod_name, mnt_path):
cmd = ("df -h | grep {} | tail -1 | awk '{{print $4}}'".format(
mnt_path))
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertFalse(ret, 'Failed to get size due to err {} in {}'.format(
err, mnt_path))
return out
def test_lvm_script_executable_on_host(self):
"""Validate lvm script is executable on host instead
of container"""
# Skip the TC if independent mode deployment
if not self.is_containerized_gluster():
self.skipTest(
"Skipping this test as LVM script is not available in "
"independent mode deployment")
pod_name = self.pod_name[0]['pod_name']
gluster_pod_label = "glusterfs=storage-pod"
# Remove LVM banaries to validate /usr/sbin/exec-on-host script
# is execute LVM commands on host instead on pod
cmd = "rm /usr/sbin/lvm"
ret, _, err = openshift_ops.oc_rsh(self.oc_node, pod_name, cmd)
self.addCleanup(
openshift_ops.wait_for_pods_be_ready, self.oc_node,
len(self.gluster_servers), gluster_pod_label)
self.addCleanup(
openshift_ops.wait_for_resource_absence, self.oc_node, "pod",
pod_name)
self.addCleanup(
openshift_ops.oc_delete, self.oc_node, "pod", pod_name)
err_msg = (
"failed to execute command {} on pod {} with error: {}"
"".format(cmd, pod_name, err))
self.assertFalse(ret, err_msg)
# Validate LVM command is not executable in pod
cmd = "oc rsh {} lvs".format(pod_name)
stdout = command.cmd_run(cmd, self.oc_node, raise_on_error=False)
self.assertIn(
'exec: \\"lvs\\": executable file not found in $PATH', stdout)
# Run LVM command with /usr/sbin/exec-on-host
cmd = "{} lvs".format(ENV_VALUE)
ret, out, err = openshift_ops.oc_rsh(self.oc_node, pod_name, cmd)
err_msg = (
"failed to execute command {} on pod {} with error: {}"
"".format(cmd, pod_name, err))
self.assertFalse(ret, err_msg)
self.assertIn("VG", out)
def _create_volumes_with_io(self, pvc_cnt, timeout=120, wait_step=3):
pvc_names = self.create_and_wait_for_pvcs(
pvc_amount=pvc_cnt, sc_name=self.sc_name,
timeout=timeout, wait_step=wait_step
)
err_msg = "failed to execute command %s on pod %s with error: %s"
for pvc_name in pvc_names:
dc_name, pod_name = self.create_dc_with_pvc(pvc_name)
# Make sure we are able to work with files
# on the mounted volume
filepath = "/mnt/file_for_testing_io.log"
cmd = "dd if=/dev/urandom of=%s bs=1K count=100" % filepath
ret, out, err = oc_rsh(self.oc_node, pod_name, cmd)
self.assertEqual(ret, 0, err_msg % (cmd, pod_name, err))
cmd = "ls -lrt %s" % filepath
ret, out, err = oc_rsh(self.oc_node, pod_name, cmd)
self.assertEqual(ret, 0, err_msg % (cmd, pod_name, err))
return pvc_names
def _create_volumes_with_io(self, pvc_cnt, timeout=120, wait_step=3):
pvc_names = self.create_and_wait_for_pvcs(pvc_amount=pvc_cnt,
sc_name=self.sc_name,
timeout=timeout,
wait_step=wait_step)
err_msg = "failed to execute command %s on pod %s with error: %s"
for pvc_name in pvc_names:
dc_name, pod_name = self.create_dc_with_pvc(pvc_name)
# Make sure we are able to work with files
# on the mounted volume
filepath = "/mnt/file_for_testing_io.log"
cmd = "dd if=/dev/urandom of=%s bs=1K count=100" % filepath
ret, out, err = oc_rsh(self.oc_node, pod_name, cmd)
self.assertEqual(ret, 0, err_msg % (cmd, pod_name, err))
cmd = "ls -lrt %s" % filepath
ret, out, err = oc_rsh(self.oc_node, pod_name, cmd)
self.assertEqual(ret, 0, err_msg % (cmd, pod_name, err))
return pvc_names
def test_run_workload_with_logging(self):
"""Validate logs are being generated aifter running workload"""
# Get the size of used space of logs
es_pod = openshift_ops.get_pod_name_from_dc(
self._master, self._logging_es_dc)
mount_point = "/elasticsearch/persistent"
cmd_space_check = ('df -kh --output=used {} | sed "/Used/d" |'
'sed "s/G//"'.format(mount_point))
ret, initial_used_percent, err = openshift_ops.oc_rsh(
self._master, es_pod, cmd_space_check)
err_msg = "Failed to fetch the size of used space, error {}"
self.assertFalse(ret, err_msg.format(err))
# Create 20 pvcs and app pods with io
openshift_ops.switch_oc_project(
self._master, self.storage_project_name)
pvc_count, batch_count = 5, 4
for _ in range(batch_count):
pvcs = self.create_and_wait_for_pvcs(pvc_amount=pvc_count)
self.create_dcs_with_pvc(pvcs)
self.addCleanup(
openshift_ops.switch_oc_project,
self._master, self.storage_project_name)
# Get and verify the final used size of used space of logs
openshift_ops.switch_oc_project(
self._master, self._logging_project_name)
for w in waiter.Waiter(600, 30):
ret, final_used_percent, err = openshift_ops.oc_rsh(
self._master, es_pod, cmd_space_check)
self.assertFalse(ret, err_msg.format(err))
if int(initial_used_percent) < int(final_used_percent):
break
if w.expired:
raise AssertionError(
"Initial used space {} for logs is not less than final "
"used space {}".format(
initial_used_percent, final_used_percent))
def _fetch_metric_from_promtheus_pod(self, metric):
"""Fetch metric from prometheus pod using api call"""
prometheus_pods = list(openshift_ops.oc_get_pods(
self._master, selector=self._prometheus_resources_selector).keys())
fetch_metric_cmd = ("curl 'http://localhost:9090/api/v1/query"
"?query={}'".format(metric))
ret, metric_data, _ = openshift_ops.oc_rsh(
self._master, prometheus_pods[0], fetch_metric_cmd)
metric_result = json.loads(metric_data)["data"]["result"]
if (not metric_result) or ret:
raise exceptions.ExecutionError(
"Failed to fecth data for metric {}, output {}".format(
metric, metric_result))
return metric_result
def test_run_workload_with_metrics(self):
"""Validate if logs are being generated after running workload"""
# Get the size of used space of logs
cassandra_pod = get_pod_name_from_rc(
self.master, self.metrics_rc_hawkular_cassandra)
mount_point = "/cassandra_data"
cmd_space_check = ('df -k --output=used {} | sed "/Used/d" |'
'sed "s/G//"'.format(mount_point))
ret, initial_used_percent, err = oc_rsh(self.master, cassandra_pod,
cmd_space_check)
err_msg = "Failed to fetch the size of used space, error {}"
self.assertFalse(ret, err_msg.format(err))
# Create 20 PVCs and app pods with IO
switch_oc_project(self.master, self.storage_project_name)
pvc_count, batch_count = 5, 4
for _ in range(batch_count):
pvcs = self.create_and_wait_for_pvcs(pvc_amount=pvc_count)
self.create_dcs_with_pvc(pvcs)
self.addCleanup(switch_oc_project, self.master,
self.storage_project_name)
# Get and verify the final size of used space of logs
switch_oc_project(self.master, self.metrics_project_name)
for w in waiter.Waiter(600, 30):
ret, final_used_percent, err = oc_rsh(self.master, cassandra_pod,
cmd_space_check)
self.assertFalse(ret, err_msg.format(err))
if int(initial_used_percent) < int(final_used_percent):
break
if w.expired:
raise AssertionError(
"Initial used space {} for logs is not less than final "
"used space {}".format(initial_used_percent,
final_used_percent))
def test_prometheus_basic_validation(self):
""" Validate basic volume metrics using prometheus """
# Fetch the metrics and storing initial_metrics as dictionary
pvc_name, pod_name, initial_metrics = self._fetch_initial_metrics(
volume_expansion=False)
# Create 1000 files and fetch the metrics that the data is updated
self._perform_io_and_fetch_metrics(pod_name=pod_name,
pvc_name=pvc_name,
filename="filename1",
dirname="dirname1",
metric_data=initial_metrics,
operation="create")
# Write the IO half the size of the volume and validated from
# prometheus pod that the size change is reflected
size_to_write = int(
initial_metrics['kubelet_volume_stats_capacity_bytes']) // 2
openshift_ops.switch_oc_project(self._master,
self.storage_project_name)
cmd = ("dd if=/dev/urandom of=/mnt/large_file bs={} count=1024".format(
size_to_write // 1024))
ret, _, err = openshift_ops.oc_rsh(self._master, pod_name, cmd)
self.assertFalse(ret, 'Failed to write file due to err {}'.format(err))
# Fetching the metrics and validating the data change is reflected
for w in waiter.Waiter(120, 10):
half_io_metrics = self._get_and_manipulate_metric_data(
['kubelet_volume_stats_used_bytes'], pvc_name)
if bool(half_io_metrics) and (int(
half_io_metrics['kubelet_volume_stats_used_bytes']) >
size_to_write):
break
if w.expired:
raise AssertionError(
"After Data is written on the pvc, metrics like inodes used "
"and bytes used are not reflected in the prometheus")
# Delete the files from the volume and wait for the
# updated details reflected in prometheus
self._perform_io_and_fetch_metrics(pod_name=pod_name,
pvc_name=pvc_name,
filename="filename1",
dirname="dirname1",
metric_data=half_io_metrics,
operation="delete")
def test_block_vol_online_expand(self):
"""Test blockvol expansion while PVC is in use"""
node = self.ocp_master_node[0]
pvc_name, dc_name, bvol_info = (
self._block_vol_expand_common_offline_vs_online(True))
# get pod hostname
iqn, _, pod_hostname = self.verify_iscsi_sessions_and_multipath(
pvc_name, dc_name[0])
# Get the paths info from the node
device = list(
get_iscsi_block_devices_by_path(pod_hostname, iqn).keys())[0]
# Get mpath name
mpath = get_mpath_name_from_device_name(pod_hostname, device)
# rescan the devices on pod_hostname
cmd = "iscsiadm -m node -R -T {}".format(iqn)
self.cmd_run(cmd, pod_hostname)
# refresh multipath device size
cmd = "multipathd -k'resize map {}'".format(mpath)
self.cmd_run(cmd, pod_hostname)
# get mount point
cmd = "lsblk /dev/{} --output MOUNTPOINT --noheadings".format(device)
mount_point = self.cmd_run(cmd, pod_hostname)
cmd = "xfs_growfs {}".format(mount_point)
self.cmd_run(cmd, pod_hostname)
cmd = ("df -h {} | sed '/Filesystem/d' | awk '{{print $2}}' |"
" sed 's/G//'")
size = self.cmd_run(cmd.format(mount_point), pod_hostname)
self.assertEqual(
int(float(size)), bvol_info["size"], "new size is not "
"reflected at host mount point after block volume expand")
# verify expand size
pod_name = get_pod_name_from_dc(node, dc_name[0])
ret, size, _ = oc_rsh(node, pod_name, cmd.format("/mnt"))
self.assertFalse(ret, "Failed to get size from client side")
self.assertEqual(
int(float(size)), bvol_info["size"], "new size is not "
"reflected at mount point after block volume expand")
def get_io_time(self):
"""Gets last io time of io pod by listing log file directory
/mnt on pod
"""
ret, stdout, stderr = oc_rsh(self.oc_node, self.pod_name,
"ls -l /mnt/ | awk '{print $8}'")
if ret != 0:
err_msg = "failed to get io time for pod %s" % self.pod_name
g.log.error(err_msg)
raise AssertionError(err_msg)
get_time = None
try:
get_time = datetime.strptime(stdout.strip(), "%H:%M")
except Exception:
g.log.error("invalid time format ret %s, stout: %s, "
"stderr: %s" % (ret, stdout, stderr))
raise
return get_time
def get_io_time(self):
"""Gets last io time of io pod by listing log file directory
/mnt on pod
"""
ret, stdout, stderr = oc_rsh(
self.oc_node, self.pod_name, "ls -l /mnt/ | awk '{print $8}'"
)
if ret != 0:
err_msg = "failed to get io time for pod %s" % self.pod_name
g.log.error(err_msg)
raise AssertionError(err_msg)
get_time = None
try:
get_time = datetime.strptime(stdout.strip(), "%H:%M")
except Exception:
g.log.error("invalid time format ret %s, stout: %s, "
"stderr: %s" % (ret, stdout, stderr))
raise
return get_time
def dynamic_provisioning_glusterblock(
self, set_hacount, create_vol_name_prefix=False):
datafile_path = '/mnt/fake_file_for_%s' % self.id()
# Create DC with attached PVC
sc_name = self.create_storage_class(
set_hacount=set_hacount,
create_vol_name_prefix=create_vol_name_prefix)
pvc_name = self.create_and_wait_for_pvc(
pvc_name_prefix='autotest-block', sc_name=sc_name)
dc_name, pod_name = self.create_dc_with_pvc(pvc_name)
# Check that we can write data
for cmd in ("dd if=/dev/urandom of=%s bs=1K count=100",
"ls -lrt %s",
"rm -rf %s"):
cmd = cmd % datafile_path
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0,
"Failed to execute '%s' command on '%s'." % (cmd, self.node))
def dynamic_provisioning_glusterblock(self,
set_hacount,
create_vol_name_prefix=False):
datafile_path = '/mnt/fake_file_for_%s' % self.id()
# Create DC with attached PVC
sc_name = self.create_storage_class(
set_hacount=set_hacount,
create_vol_name_prefix=create_vol_name_prefix)
pvc_name = self.create_and_wait_for_pvc(
pvc_name_prefix='autotest-block', sc_name=sc_name)
dc_name, pod_name = self.create_dc_with_pvc(pvc_name)
# Check that we can write data
for cmd in ("dd if=/dev/urandom of=%s bs=1K count=100", "ls -lrt %s",
"rm -rf %s"):
cmd = cmd % datafile_path
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0,
"Failed to execute '%s' command on '%s'." % (cmd, self.node))
def test_pvc_resize_size_greater_than_available_space(self):
"""Re-size PVC to greater value than available volume size and then
expand volume to support maximum size.
"""
sc_name = self.create_storage_class(create_vol_name_prefix=True,
allow_volume_expansion=True)
pvc_name = self.create_and_wait_for_pvc(sc_name=sc_name, pvc_size=1)
dc_name, pod_name = self.create_dc_with_pvc(pvc_name)
# Verify the size of volume mounted
cmd_on_pod = "df -Ph /mnt | tail -1 | awk '{print $2}'"
_, stdout, _ = oc_rsh(self.node, pod_name, cmd_on_pod)
self.assertGreaterEqual(int(stdout.strip('.0M\n')), 1000,
"Size of %s not equal to 1G" % pvc_name)
self.assertLessEqual(int(stdout.strip('.0M\n')), 1024,
"Size of %s not equal to 1G" % pvc_name)
available_size_gb = self._available_disk_free_space()
with self.assertRaises(AssertionError):
resize_pvc(self.ocp_master_node[0], pvc_name,
available_size_gb + 1)
resize_pvc(self.ocp_master_node[0], pvc_name, available_size_gb)
verify_pvc_size(self.ocp_master_node[0], pvc_name, available_size_gb)
def test_pv_resize_when_heketi_down(self):
"""Create a PVC and try to expand it when heketi is down, It should
fail. After heketi is up, expand PVC should work.
"""
self.create_storage_class(allow_volume_expansion=True)
pvc_name = self.create_and_wait_for_pvc()
dc_name, pod_name = self.create_dc_with_pvc(pvc_name)
pv_name = get_pv_name_from_pvc(self.node, pvc_name)
custom = (r':metadata.annotations.'
r'"gluster\.kubernetes\.io\/heketi-volume-id"')
vol_id = oc_get_custom_resource(self.node, 'pv', custom, pv_name)[0]
h_vol_info = heketi_ops.heketi_volume_info(
self.heketi_client_node, self.heketi_server_url, vol_id, json=True)
# Bring the heketi POD down
scale_dc_pod_amount_and_wait(
self.node, self.heketi_dc_name, pod_amount=0)
self.addCleanup(
scale_dc_pod_amount_and_wait, self.node,
self.heketi_dc_name, pod_amount=1)
cmd = 'dd if=/dev/urandom of=/mnt/%s bs=614400k count=1'
ret, out, err = oc_rsh(self.node, pod_name, cmd % 'file1')
self.assertFalse(ret, 'Not able to write file with err: %s' % err)
wait_for_pod_be_ready(self.node, pod_name, 10, 5)
resize_pvc(self.node, pvc_name, 2)
wait_for_events(
self.node, pvc_name, obj_type='PersistentVolumeClaim',
event_type='Warning', event_reason='VolumeResizeFailed')
# Verify volume was not expanded
vol_info = get_gluster_vol_info_by_pvc_name(self.node, pvc_name)
self.assertEqual(vol_info['gluster_vol_id'], h_vol_info['name'])
self.assertEqual(
len(vol_info['bricks']['brick']), len(h_vol_info['bricks']))
# Bring the heketi POD up
scale_dc_pod_amount_and_wait(
self.node, self.heketi_dc_name, pod_amount=1)
# Verify volume expansion
verify_pvc_size(self.node, pvc_name, 2)
vol_info = get_gluster_vol_info_by_pvc_name(self.node, pvc_name)
self.assertFalse(len(vol_info['bricks']['brick']) % 3)
self.assertLess(
len(h_vol_info['bricks']), len(vol_info['bricks']['brick']))
# Wait for remount after expansion
for w in waiter.Waiter(timeout=30, interval=5):
ret, out, err = oc_rsh(
self.node, pod_name,
"df -Ph /mnt | awk '{print $2}' | tail -1")
self.assertFalse(ret, 'Failed with err: %s and Output: %s' % (
err, out))
if out.strip() == '2.0G':
break
# Write data making sure we have more space than it was
ret, out, err = oc_rsh(self.node, pod_name, cmd % 'file2')
self.assertFalse(ret, 'Not able to write file with err: %s' % err)
# Verify pod is running
wait_for_pod_be_ready(self.node, pod_name, 10, 5)
def test_dynamic_provisioning_glusterfile_heketipod_failure(self):
"""Validate dynamic provisioning for gluster file when heketi pod down
"""
mount_path = "/mnt"
datafile_path = '%s/fake_file_for_%s' % (mount_path, self.id())
# Create secret and storage class
sc_name = self.create_storage_class()
# Create PVC
app_1_pvc_name = self.create_and_wait_for_pvc(
pvc_name_prefix="autotest-file", sc_name=sc_name
)
# Create app POD with attached volume
app_1_pod_name = oc_create_tiny_pod_with_volume(
self.node, app_1_pvc_name, "test-pvc-mount-on-app-pod",
mount_path=mount_path)
self.addCleanup(
wait_for_resource_absence, self.node, 'pod', app_1_pod_name)
self.addCleanup(oc_delete, self.node, 'pod', app_1_pod_name)
# Wait for app POD be up and running
wait_for_pod_be_ready(
self.node, app_1_pod_name, timeout=60, wait_step=2)
# Write data to the app POD
write_data_cmd = (
"dd if=/dev/urandom of=%s bs=1K count=100" % datafile_path)
ret, out, err = oc_rsh(self.node, app_1_pod_name, write_data_cmd)
self.assertEqual(
ret, 0,
"Failed to execute command %s on %s" % (write_data_cmd, self.node))
# Remove Heketi pod
heketi_down_cmd = "oc scale --replicas=0 dc/%s --namespace %s" % (
self.heketi_dc_name, self.storage_project_name)
heketi_up_cmd = "oc scale --replicas=1 dc/%s --namespace %s" % (
self.heketi_dc_name, self.storage_project_name)
self.addCleanup(self.cmd_run, heketi_up_cmd)
heketi_pod_name = get_pod_name_from_dc(
self.node, self.heketi_dc_name, timeout=10, wait_step=3)
self.cmd_run(heketi_down_cmd)
wait_for_resource_absence(self.node, 'pod', heketi_pod_name)
app_2_pvc_name = oc_create_pvc(
self.node, pvc_name_prefix="autotest-file2", sc_name=sc_name
)
self.addCleanup(
wait_for_resource_absence, self.node, 'pvc', app_2_pvc_name)
self.addCleanup(
oc_delete, self.node, 'pvc', app_2_pvc_name, raise_on_absence=False
)
# Create second app POD
app_2_pod_name = oc_create_tiny_pod_with_volume(
self.node, app_2_pvc_name, "test-pvc-mount-on-app-pod",
mount_path=mount_path)
self.addCleanup(
wait_for_resource_absence, self.node, 'pod', app_2_pod_name)
self.addCleanup(oc_delete, self.node, 'pod', app_2_pod_name)
# Bring Heketi POD back
self.cmd_run(heketi_up_cmd)
# Wait for Heketi POD be up and running
new_heketi_pod_name = get_pod_name_from_dc(
self.node, self.heketi_dc_name, timeout=10, wait_step=2)
wait_for_pod_be_ready(
self.node, new_heketi_pod_name, wait_step=5, timeout=120)
# Wait for second PVC and app POD be ready
verify_pvc_status_is_bound(self.node, app_2_pvc_name)
wait_for_pod_be_ready(
self.node, app_2_pod_name, timeout=60, wait_step=2)
# Verify that we are able to write data
ret, out, err = oc_rsh(self.node, app_2_pod_name, write_data_cmd)
self.assertEqual(
ret, 0,
"Failed to execute command %s on %s" % (write_data_cmd, self.node))
def _pv_resize(self, exceed_free_space):
dir_path = "/mnt"
pvc_size_gb, min_free_space_gb = 1, 3
# Get available free space disabling redundant devices and nodes
heketi_url = self.heketi_server_url
node_id_list = heketi_ops.heketi_node_list(self.heketi_client_node,
heketi_url)
self.assertTrue(node_id_list)
nodes = {}
min_free_space = min_free_space_gb * 1024**2
for node_id in node_id_list:
node_info = heketi_ops.heketi_node_info(self.heketi_client_node,
heketi_url,
node_id,
json=True)
if (node_info['state'].lower() != 'online'
or not node_info['devices']):
continue
if len(nodes) > 2:
out = heketi_ops.heketi_node_disable(self.heketi_client_node,
heketi_url, node_id)
self.assertTrue(out)
self.addCleanup(heketi_ops.heketi_node_enable,
self.heketi_client_node, heketi_url, node_id)
for device in node_info['devices']:
if device['state'].lower() != 'online':
continue
free_space = device['storage']['free']
if (node_id in nodes.keys() or free_space < min_free_space):
out = heketi_ops.heketi_device_disable(
self.heketi_client_node, heketi_url, device['id'])
self.assertTrue(out)
self.addCleanup(heketi_ops.heketi_device_enable,
self.heketi_client_node, heketi_url,
device['id'])
continue
nodes[node_id] = free_space
if len(nodes) < 3:
raise self.skipTest("Could not find 3 online nodes with, "
"at least, 1 online device having free space "
"bigger than %dGb." % min_free_space_gb)
# Calculate maximum available size for PVC
available_size_gb = int(min(nodes.values()) / (1024**2))
# Create PVC
self.create_storage_class(allow_volume_expansion=True)
pvc_name = self.create_and_wait_for_pvc(pvc_size=pvc_size_gb)
# Create DC with POD and attached PVC to it
dc_name = oc_create_app_dc_with_io(self.node, pvc_name)
self.addCleanup(oc_delete, self.node, 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, self.node, dc_name, 0)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
if exceed_free_space:
# Try to expand existing PVC exceeding free space
resize_pvc(self.node, pvc_name, available_size_gb)
wait_for_events(self.node,
obj_name=pvc_name,
event_reason='VolumeResizeFailed')
# Check that app POD is up and runnig then try to write data
wait_for_pod_be_ready(self.node, pod_name)
cmd = ("dd if=/dev/urandom of=%s/autotest bs=100K count=1" %
dir_path)
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0,
"Failed to write data after failed attempt to expand PVC.")
else:
# Expand existing PVC using all the available free space
expand_size_gb = available_size_gb - pvc_size_gb
resize_pvc(self.node, pvc_name, expand_size_gb)
verify_pvc_size(self.node, pvc_name, expand_size_gb)
pv_name = get_pv_name_from_pvc(self.node, pvc_name)
verify_pv_size(self.node, pv_name, expand_size_gb)
wait_for_events(self.node,
obj_name=pvc_name,
event_reason='VolumeResizeSuccessful')
# Recreate app POD
oc_delete(self.node, 'pod', pod_name)
wait_for_resource_absence(self.node, 'pod', pod_name)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
# Write data on the expanded PVC
cmd = ("dd if=/dev/urandom of=%s/autotest "
"bs=1M count=1025" % dir_path)
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(ret, 0,
"Failed to write data on the expanded PVC")
def _pv_resize(self, exceed_free_space):
dir_path = "/mnt"
pvc_size_gb, min_free_space_gb = 1, 3
# Get available free space disabling redundant devices and nodes
heketi_url = self.heketi_server_url
node_id_list = heketi_ops.heketi_node_list(
self.heketi_client_node, heketi_url)
self.assertTrue(node_id_list)
nodes = {}
min_free_space = min_free_space_gb * 1024**2
for node_id in node_id_list:
node_info = heketi_ops.heketi_node_info(
self.heketi_client_node, heketi_url, node_id, json=True)
if (node_info['state'].lower() != 'online' or
not node_info['devices']):
continue
if len(nodes) > 2:
out = heketi_ops.heketi_node_disable(
self.heketi_client_node, heketi_url, node_id)
self.assertTrue(out)
self.addCleanup(
heketi_ops.heketi_node_enable,
self.heketi_client_node, heketi_url, node_id)
for device in node_info['devices']:
if device['state'].lower() != 'online':
continue
free_space = device['storage']['free']
if (node_id in nodes.keys() or free_space < min_free_space):
out = heketi_ops.heketi_device_disable(
self.heketi_client_node, heketi_url, device['id'])
self.assertTrue(out)
self.addCleanup(
heketi_ops.heketi_device_enable,
self.heketi_client_node, heketi_url, device['id'])
continue
nodes[node_id] = free_space
if len(nodes) < 3:
raise self.skipTest(
"Could not find 3 online nodes with, "
"at least, 1 online device having free space "
"bigger than %dGb." % min_free_space_gb)
# Calculate maximum available size for PVC
available_size_gb = int(min(nodes.values()) / (1024**2))
# Create PVC
self.create_storage_class(allow_volume_expansion=True)
pvc_name = self.create_and_wait_for_pvc(pvc_size=pvc_size_gb)
# Create DC with POD and attached PVC to it
dc_name = oc_create_app_dc_with_io(self.node, pvc_name)
self.addCleanup(oc_delete, self.node, 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, self.node, dc_name, 0)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
if exceed_free_space:
# Try to expand existing PVC exceeding free space
resize_pvc(self.node, pvc_name, available_size_gb)
wait_for_events(self.node, obj_name=pvc_name,
event_reason='VolumeResizeFailed')
# Check that app POD is up and runnig then try to write data
wait_for_pod_be_ready(self.node, pod_name)
cmd = (
"dd if=/dev/urandom of=%s/autotest bs=100K count=1" % dir_path)
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0,
"Failed to write data after failed attempt to expand PVC.")
else:
# Expand existing PVC using all the available free space
expand_size_gb = available_size_gb - pvc_size_gb
resize_pvc(self.node, pvc_name, expand_size_gb)
verify_pvc_size(self.node, pvc_name, expand_size_gb)
pv_name = get_pv_name_from_pvc(self.node, pvc_name)
verify_pv_size(self.node, pv_name, expand_size_gb)
wait_for_events(
self.node, obj_name=pvc_name,
event_reason='VolumeResizeSuccessful')
# Recreate app POD
oc_delete(self.node, 'pod', pod_name)
wait_for_resource_absence(self.node, 'pod', pod_name)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
# Write data on the expanded PVC
cmd = ("dd if=/dev/urandom of=%s/autotest "
"bs=1M count=1025" % dir_path)
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0, "Failed to write data on the expanded PVC")
def test_pv_resize_when_heketi_down(self):
"""Create a PVC and try to expand it when heketi is down, It should
fail. After heketi is up, expand PVC should work.
"""
self.create_storage_class(allow_volume_expansion=True)
pvc_name = self.create_and_wait_for_pvc()
dc_name, pod_name = self.create_dc_with_pvc(pvc_name)
pv_name = get_pv_name_from_pvc(self.node, pvc_name)
custom = (r':metadata.annotations.'
r'"gluster\.kubernetes\.io\/heketi-volume-id"')
vol_id = oc_get_custom_resource(self.node, 'pv', custom, pv_name)[0]
h_vol_info = heketi_ops.heketi_volume_info(self.heketi_client_node,
self.heketi_server_url,
vol_id,
json=True)
# Bring the heketi POD down
scale_dc_pod_amount_and_wait(self.node,
self.heketi_dc_name,
pod_amount=0)
self.addCleanup(scale_dc_pod_amount_and_wait,
self.node,
self.heketi_dc_name,
pod_amount=1)
cmd = 'dd if=/dev/urandom of=/mnt/%s bs=614400k count=1'
ret, out, err = oc_rsh(self.node, pod_name, cmd % 'file1')
self.assertFalse(ret, 'Not able to write file with err: %s' % err)
wait_for_pod_be_ready(self.node, pod_name, 10, 5)
resize_pvc(self.node, pvc_name, 2)
wait_for_events(self.node,
pvc_name,
obj_type='PersistentVolumeClaim',
event_type='Warning',
event_reason='VolumeResizeFailed')
# Verify volume was not expanded
vol_info = get_gluster_vol_info_by_pvc_name(self.node, pvc_name)
self.assertEqual(vol_info['gluster_vol_id'], h_vol_info['name'])
self.assertEqual(len(vol_info['bricks']['brick']),
len(h_vol_info['bricks']))
# Bring the heketi POD up
scale_dc_pod_amount_and_wait(self.node,
self.heketi_dc_name,
pod_amount=1)
# Verify volume expansion
verify_pvc_size(self.node, pvc_name, 2)
vol_info = get_gluster_vol_info_by_pvc_name(self.node, pvc_name)
self.assertFalse(len(vol_info['bricks']['brick']) % 3)
self.assertLess(len(h_vol_info['bricks']),
len(vol_info['bricks']['brick']))
# Wait for remount after expansion
for w in waiter.Waiter(timeout=30, interval=5):
ret, out, err = oc_rsh(self.node, pod_name,
"df -Ph /mnt | awk '{print $2}' | tail -1")
self.assertFalse(ret,
'Failed with err: %s and Output: %s' % (err, out))
if out.strip() == '2.0G':
break
# Write data making sure we have more space than it was
ret, out, err = oc_rsh(self.node, pod_name, cmd % 'file2')
self.assertFalse(ret, 'Not able to write file with err: %s' % err)
# Verify pod is running
wait_for_pod_be_ready(self.node, pod_name, 10, 5)
def test_dynamic_provisioning_glusterblock_heketipod_failure(self):
"""Validate PVC with glusterblock creation when heketi pod is down"""
datafile_path = '/mnt/fake_file_for_%s' % self.id()
# Create DC with attached PVC
sc_name = self.create_storage_class()
app_1_pvc_name = self.create_and_wait_for_pvc(
pvc_name_prefix='autotest-block', sc_name=sc_name)
app_1_dc_name, app_1_pod_name = self.create_dc_with_pvc(app_1_pvc_name)
# Write test data
write_data_cmd = (
"dd if=/dev/urandom of=%s bs=1K count=100" % datafile_path)
ret, out, err = oc_rsh(self.node, app_1_pod_name, write_data_cmd)
self.assertEqual(
ret, 0,
"Failed to execute command %s on %s" % (write_data_cmd, self.node))
# Remove Heketi pod
heketi_down_cmd = "oc scale --replicas=0 dc/%s --namespace %s" % (
self.heketi_dc_name, self.storage_project_name)
heketi_up_cmd = "oc scale --replicas=1 dc/%s --namespace %s" % (
self.heketi_dc_name, self.storage_project_name)
self.addCleanup(self.cmd_run, heketi_up_cmd)
heketi_pod_name = get_pod_name_from_dc(
self.node, self.heketi_dc_name, timeout=10, wait_step=3)
self.cmd_run(heketi_down_cmd)
wait_for_resource_absence(self.node, 'pod', heketi_pod_name)
# Create second PVC
app_2_pvc_name = oc_create_pvc(
self.node, pvc_name_prefix='autotest-block2', sc_name=sc_name
)
self.addCleanup(
wait_for_resource_absence, self.node, 'pvc', app_2_pvc_name)
self.addCleanup(
oc_delete, self.node, 'pvc', app_2_pvc_name
)
# Create second app POD
app_2_dc_name = oc_create_app_dc_with_io(self.node, app_2_pvc_name)
self.addCleanup(oc_delete, self.node, 'dc', app_2_dc_name)
self.addCleanup(
scale_dc_pod_amount_and_wait, self.node, app_2_dc_name, 0)
app_2_pod_name = get_pod_name_from_dc(self.node, app_2_dc_name)
# Bring Heketi pod back
self.cmd_run(heketi_up_cmd)
# Wait for Heketi POD be up and running
new_heketi_pod_name = get_pod_name_from_dc(
self.node, self.heketi_dc_name, timeout=10, wait_step=2)
wait_for_pod_be_ready(
self.node, new_heketi_pod_name, wait_step=5, timeout=120)
# Wait for second PVC and app POD be ready
verify_pvc_status_is_bound(self.node, app_2_pvc_name)
wait_for_pod_be_ready(
self.node, app_2_pod_name, timeout=150, wait_step=3)
# Verify that we are able to write data
ret, out, err = oc_rsh(self.node, app_2_pod_name, write_data_cmd)
self.assertEqual(
ret, 0,
"Failed to execute command %s on %s" % (write_data_cmd, self.node))
def test_dynamic_provisioning_glusterfile_heketipod_failure(self):
"""Validate dynamic provisioning for gluster file when heketi pod down
"""
mount_path = "/mnt"
datafile_path = '%s/fake_file_for_%s' % (mount_path, self.id())
# Create secret and storage class
sc_name = self.create_storage_class()
# Create PVC
app_1_pvc_name = self.create_and_wait_for_pvc(
pvc_name_prefix="autotest-file", sc_name=sc_name)
# Create app POD with attached volume
app_1_pod_name = oc_create_tiny_pod_with_volume(
self.node,
app_1_pvc_name,
"test-pvc-mount-on-app-pod",
mount_path=mount_path,
image=self.io_container_image_cirros)
self.addCleanup(wait_for_resource_absence, self.node, 'pod',
app_1_pod_name)
self.addCleanup(oc_delete, self.node, 'pod', app_1_pod_name)
# Wait for app POD be up and running
wait_for_pod_be_ready(self.node,
app_1_pod_name,
timeout=60,
wait_step=2)
# Write data to the app POD
write_data_cmd = ("dd if=/dev/urandom of=%s bs=1K count=100" %
datafile_path)
ret, out, err = oc_rsh(self.node, app_1_pod_name, write_data_cmd)
self.assertEqual(
ret, 0,
"Failed to execute command %s on %s" % (write_data_cmd, self.node))
# Remove Heketi pod
heketi_down_cmd = "oc scale --replicas=0 dc/%s --namespace %s" % (
self.heketi_dc_name, self.storage_project_name)
heketi_up_cmd = "oc scale --replicas=1 dc/%s --namespace %s" % (
self.heketi_dc_name, self.storage_project_name)
self.addCleanup(self.cmd_run, heketi_up_cmd)
heketi_pod_name = get_pod_name_from_dc(self.node,
self.heketi_dc_name,
timeout=10,
wait_step=3)
self.cmd_run(heketi_down_cmd)
wait_for_resource_absence(self.node, 'pod', heketi_pod_name)
app_2_pvc_name = oc_create_pvc(self.node,
pvc_name_prefix="autotest-file2",
sc_name=sc_name)
self.addCleanup(wait_for_resource_absence, self.node, 'pvc',
app_2_pvc_name)
self.addCleanup(oc_delete,
self.node,
'pvc',
app_2_pvc_name,
raise_on_absence=False)
# Create second app POD
app_2_pod_name = oc_create_tiny_pod_with_volume(
self.node,
app_2_pvc_name,
"test-pvc-mount-on-app-pod",
mount_path=mount_path,
image=self.io_container_image_cirros)
self.addCleanup(wait_for_resource_absence, self.node, 'pod',
app_2_pod_name)
self.addCleanup(oc_delete, self.node, 'pod', app_2_pod_name)
# Bring Heketi POD back
self.cmd_run(heketi_up_cmd)
# Wait for Heketi POD be up and running
new_heketi_pod_name = get_pod_name_from_dc(self.node,
self.heketi_dc_name,
timeout=10,
wait_step=2)
wait_for_pod_be_ready(self.node,
new_heketi_pod_name,
wait_step=5,
timeout=120)
# Wait for second PVC and app POD be ready
verify_pvc_status_is_bound(self.node, app_2_pvc_name)
wait_for_pod_be_ready(self.node,
app_2_pod_name,
timeout=60,
wait_step=2)
# Verify that we are able to write data
ret, out, err = oc_rsh(self.node, app_2_pod_name, write_data_cmd)
self.assertEqual(
ret, 0,
"Failed to execute command %s on %s" % (write_data_cmd, self.node))