def validate_multipath_info(self, hacount):
"""validates multipath command on the pod node
Args:
hacount (int): hacount for which multipath to be checked
"""
# create pod using pvc created
dc_name = oc_create_app_dc_with_io(
self.ocp_master_node[0], self.pvc_name
)
pod_name = get_pod_name_from_dc(self.ocp_master_node[0], dc_name)
self.addCleanup(oc_delete, self.ocp_master_node[0], "dc", dc_name)
self.addCleanup(
scale_dc_pod_amount_and_wait, self.ocp_master_node[0], dc_name, 0
)
wait_for_pod_be_ready(
self.ocp_master_node[0], pod_name, timeout=120, wait_step=3
)
# validates multipath for pod created with hacount
self.assertTrue(
validate_multipath_pod(self.ocp_master_node[0], pod_name, hacount),
"multipath validation failed"
)
def test_restart_gluster_block_provisioner_pod(self):
"""Restart gluster-block provisioner pod
"""
# create heketi block volume
vol_info = heketi_blockvolume_create(self.heketi_client_node,
self.heketi_server_url,
size=5,
json=True)
self.assertTrue(vol_info, "Failed to create heketi block"
"volume of size 5")
self.addCleanup(heketi_blockvolume_delete, self.heketi_client_node,
self.heketi_server_url, vol_info['id'])
# restart gluster-block-provisioner-pod
dc_name = "glusterblock-%s-provisioner-dc" % self.storage_project_name
pod_name = get_pod_name_from_dc(self.ocp_master_node, dc_name)
oc_delete(self.ocp_master_node, 'pod', pod_name)
wait_for_resource_absence(self.ocp_master_node, 'pod', pod_name)
# new gluster-pod name
pod_name = get_pod_name_from_dc(self.ocp_master_node, dc_name)
wait_for_pod_be_ready(self.ocp_master_node, pod_name)
# create new heketi block volume
vol_info = heketi_blockvolume_create(self.heketi_client_node,
self.heketi_server_url,
size=2,
json=True)
self.assertTrue(vol_info, "Failed to create heketi block"
"volume of size 2")
heketi_blockvolume_delete(self.heketi_client_node,
self.heketi_server_url, vol_info['id'])
def test_validate_pvc_in_multiple_app_pods(self):
"""Validate the use of a same claim in multiple app pods"""
replicas = 5
# Create PVC
sc_name = self.create_storage_class()
pvc_name = self.create_and_wait_for_pvc(sc_name=sc_name)
# Create DC with application PODs
dc_name = oc_create_app_dc_with_io(
self.node, pvc_name, replicas=replicas)
self.addCleanup(oc_delete, self.node, 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, self.node, dc_name, 0)
# Wait for all the PODs to be ready
pod_names = get_pod_names_from_dc(self.node, dc_name)
self.assertEqual(replicas, len(pod_names))
for pod_name in pod_names:
wait_for_pod_be_ready(self.node, pod_name)
# Create files in each of the PODs
for pod_name in pod_names:
self.cmd_run("oc exec {0} -- touch /mnt/temp_{0}".format(pod_name))
# Check that all the created files are available at once
ls_out = self.cmd_run("oc exec %s -- ls /mnt" % pod_names[0]).split()
for pod_name in pod_names:
self.assertIn("temp_%s" % pod_name, ls_out)
def test_restart_services_provision_volume_and_run_io(self, service):
"""Restart gluster service then validate volumes"""
block_hosting_vol = self.get_block_hosting_volume_by_pvc_name(
self.pvc_name)
g_nodes = get_gluster_vol_hosting_nodes(block_hosting_vol)
self.assertGreater(len(g_nodes), 2)
# restarts glusterfs service
restart_service_on_gluster_pod_or_node(self.oc_node, service,
g_nodes[0])
# wait for deployed user pod to be in Running state after restarting
# service
wait_for_pod_be_ready(self.oc_node,
self.pod_name,
timeout=60,
wait_step=5)
# checks if all glusterfs services are in running state
for g_node in g_nodes:
for service in (SERVICE_BLOCKD, SERVICE_TCMU, SERVICE_TARGET):
status = "exited" if service == SERVICE_TARGET else "running"
self.assertTrue(
wait_for_service_status_on_gluster_pod_or_node(
self.oc_node, service, status, g_node))
# validates pvc, pv, heketi block and gluster block count after
# service restarts
self.validate_volumes_and_blocks()
def setUp(self):
"""Deploys, Verifies and adds resources required for testcases
in cleanup method
"""
self.oc_node = self.ocp_master_node[0]
self.gluster_pod = get_ocp_gluster_pod_names(self.oc_node)[0]
self.gluster_pod_obj = podcmd.Pod(self.oc_node, self.gluster_pod)
# prefix used to create resources, generating using glusto_test_id
# which uses time and date of test case
self.prefix = "autotest-%s" % (self.glustotest_run_id.replace("_", ""))
_storage_class = self.storage_classes.get(
'storage_class2', self.storage_classes.get('block_storage_class'))
self.provisioner = _storage_class["provisioner"]
self.restsecretnamespace = _storage_class["restsecretnamespace"]
self.restuser = _storage_class["restuser"]
self.resturl = _storage_class["resturl"]
# using pvc size count as 1 by default
self.pvcsize = 1
# using pvc count as 10 by default
self.pvccount = 10
# create gluster block storage class, PVC and user app pod
self.sc_name, self.pvc_name, self.dc_name, self.secret_name = (
self.deploy_resouces())
# verify storage class
oc_get_yaml(self.oc_node, "sc", self.sc_name)
# verify pod creation, it's state and get the pod name
self.pod_name = get_pod_name_from_dc(self.oc_node,
self.dc_name,
timeout=180,
wait_step=3)
wait_for_pod_be_ready(self.oc_node,
self.pod_name,
timeout=180,
wait_step=3)
verify_pvc_status_is_bound(self.oc_node, self.pvc_name)
# create pvc's to test
self.pvc_list = []
for pvc in range(self.pvccount):
test_pvc_name = oc_create_pvc(self.oc_node,
self.sc_name,
pvc_name_prefix=self.prefix,
pvc_size=self.pvcsize)
self.pvc_list.append(test_pvc_name)
self.addCleanup(wait_for_resource_absence,
self.oc_node,
"pvc",
test_pvc_name,
timeout=600,
interval=10)
for pvc_name in self.pvc_list:
self.addCleanup(oc_delete, self.oc_node, "pvc", pvc_name)
def dynamic_provisioning_glusterfile(self, create_vol_name_prefix):
# Create secret and storage class
self.create_storage_class(
create_vol_name_prefix=create_vol_name_prefix)
# Create PVC
pvc_name = self.create_and_wait_for_pvc()
# Create DC with POD and attached PVC to it.
dc_name = oc_create_app_dc_with_io(self.node, pvc_name)
self.addCleanup(oc_delete, self.node, 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, self.node, dc_name, 0)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
# Verify Heketi volume name for prefix presence if provided
if create_vol_name_prefix:
ret = verify_volume_name_prefix(self.node,
self.sc['volumenameprefix'],
self.sc['secretnamespace'],
pvc_name, self.sc['resturl'])
self.assertTrue(ret, "verify volnameprefix failed")
# Make sure we are able to work with files on the mounted volume
filepath = "/mnt/file_for_testing_io.log"
for cmd in ("dd if=/dev/urandom of=%s bs=1K count=100",
"ls -lrt %s",
"rm -rf %s"):
cmd = cmd % filepath
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0,
"Failed to execute '%s' command on %s" % (cmd, self.node))
def test_pv_resize_try_shrink_pv_size(self):
"""Validate whether reducing the PV size is allowed"""
dir_path = "/mnt/"
node = self.ocp_master_node[0]
# Create PVC
pv_size = 5
self.create_storage_class(allow_volume_expansion=True)
pvc_name = self.create_and_wait_for_pvc(pvc_size=pv_size)
# Create DC with POD and attached PVC to it.
dc_name = oc_create_app_dc_with_io(node, pvc_name)
self.addCleanup(oc_delete, node, 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, node, dc_name, 0)
pod_name = get_pod_name_from_dc(node, dc_name)
wait_for_pod_be_ready(node, pod_name)
cmd = ("dd if=/dev/urandom of=%sfile " "bs=100K count=3000") % dir_path
ret, out, err = oc_rsh(node, pod_name, cmd)
self.assertEqual(ret, 0,
"failed to execute command %s on %s" % (cmd, node))
pvc_resize = 2
with self.assertRaises(ExecutionError):
resize_pvc(node, pvc_name, pvc_resize)
verify_pvc_size(node, pvc_name, pv_size)
pv_name = get_pv_name_from_pvc(node, pvc_name)
verify_pv_size(node, pv_name, pv_size)
cmd = ("dd if=/dev/urandom of=%sfile_new "
"bs=100K count=2000") % dir_path
ret, out, err = oc_rsh(node, pod_name, cmd)
self.assertEqual(ret, 0,
"failed to execute command %s on %s" % (cmd, node))
def test_dynamic_provisioning_glusterfile_glusterpod_failure(self):
"""Validate dynamic provisioning for gluster file when gluster pod down
"""
mount_path = "/mnt"
datafile_path = '%s/fake_file_for_%s' % (mount_path, self.id())
# Create secret and storage class
self.create_storage_class()
# Create PVC
pvc_name = self.create_and_wait_for_pvc()
# Create app POD with attached volume
pod_name = oc_create_tiny_pod_with_volume(
self.node, pvc_name, "test-pvc-mount-on-app-pod",
mount_path=mount_path)
self.addCleanup(
wait_for_resource_absence, self.node, 'pod', pod_name)
self.addCleanup(oc_delete, self.node, 'pod', pod_name)
# Wait for app POD be up and running
wait_for_pod_be_ready(
self.node, pod_name, timeout=60, wait_step=2)
# Run IO in background
io_cmd = "oc rsh %s dd if=/dev/urandom of=%s bs=1000K count=900" % (
pod_name, datafile_path)
async_io = g.run_async(self.node, io_cmd, "root")
# Pick up one of the hosts which stores PV brick (4+ nodes case)
gluster_pod_data = get_gluster_pod_names_by_pvc_name(
self.node, pvc_name)[0]
# Delete glusterfs POD from chosen host and wait for spawn of new one
oc_delete(self.node, 'pod', gluster_pod_data["pod_name"])
cmd = ("oc get pods -o wide | grep glusterfs | grep %s | "
"grep -v Terminating | awk '{print $1}'") % (
gluster_pod_data["host_name"])
for w in Waiter(600, 30):
out = self.cmd_run(cmd)
new_gluster_pod_name = out.strip().split("\n")[0].strip()
if not new_gluster_pod_name:
continue
else:
break
if w.expired:
error_msg = "exceeded timeout, new gluster pod not created"
g.log.error(error_msg)
raise ExecutionError(error_msg)
new_gluster_pod_name = out.strip().split("\n")[0].strip()
g.log.info("new gluster pod name is %s" % new_gluster_pod_name)
wait_for_pod_be_ready(self.node, new_gluster_pod_name)
# Check that async IO was not interrupted
ret, out, err = async_io.async_communicate()
self.assertEqual(ret, 0, "IO %s failed on %s" % (io_cmd, self.node))
def create_dc_with_pvc(self, pvc_name, timeout=300, wait_step=10):
dc_name = oc_create_app_dc_with_io(self.ocp_client[0], pvc_name)
self.addCleanup(oc_delete, self.ocp_client[0], 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, self.ocp_client[0],
dc_name, 0)
pod_name = get_pod_name_from_dc(self.ocp_client[0], dc_name)
wait_for_pod_be_ready(self.ocp_client[0],
pod_name,
timeout=timeout,
wait_step=wait_step)
return dc_name, pod_name
def test_restart_heketi_pod(self):
"""Validate restarting heketi pod"""
# create heketi volume
vol_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
size=1, json=True)
self.assertTrue(vol_info, "Failed to create heketi volume of size 1")
self.addCleanup(
heketi_volume_delete, self.heketi_client_node,
self.heketi_server_url, vol_info['id'], raise_on_error=False)
topo_info = heketi_topology_info(self.heketi_client_node,
self.heketi_server_url,
json=True)
# get heketi-pod name
heketi_pod_name = get_pod_name_from_dc(self.ocp_master_node[0],
self.heketi_dc_name)
# delete heketi-pod (it restarts the pod)
oc_delete(self.ocp_master_node[0], 'pod', heketi_pod_name)
wait_for_resource_absence(self.ocp_master_node[0],
'pod', heketi_pod_name)
# get new heketi-pod name
heketi_pod_name = get_pod_name_from_dc(self.ocp_master_node[0],
self.heketi_dc_name)
wait_for_pod_be_ready(self.ocp_master_node[0],
heketi_pod_name)
# check heketi server is running
self.assertTrue(
hello_heketi(self.heketi_client_node, self.heketi_server_url),
"Heketi server %s is not alive" % self.heketi_server_url
)
# compare the topology
new_topo_info = heketi_topology_info(self.heketi_client_node,
self.heketi_server_url,
json=True)
self.assertEqual(new_topo_info, topo_info, "topology info is not same,"
" difference - %s" % diff(topo_info, new_topo_info))
# create new volume
vol_info = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
size=2, json=True)
self.assertTrue(vol_info, "Failed to create heketi volume of size 20")
heketi_volume_delete(
self.heketi_client_node, self.heketi_server_url, vol_info['id'])
def test_storage_class_mandatory_params_glusterfile(self):
"""Validate storage-class creation with mandatory parameters"""
# create secret
self.secret_name = oc_create_secret(
self.node,
namespace=self.sc.get('secretnamespace', 'default'),
data_key=self.heketi_cli_key,
secret_type=self.sc.get('provisioner', 'kubernetes.io/glusterfs'))
self.addCleanup(
oc_delete, self.node, 'secret', self.secret_name)
# create storage class with mandatory parameters only
sc_name = oc_create_sc(
self.node, provisioner='kubernetes.io/glusterfs',
resturl=self.sc['resturl'], restuser=self.sc['restuser'],
secretnamespace=self.sc['secretnamespace'],
secretname=self.secret_name
)
self.addCleanup(oc_delete, self.node, 'sc', sc_name)
# Create PVC
pvc_name = self.create_and_wait_for_pvc(sc_name=sc_name)
# Create DC with POD and attached PVC to it.
dc_name = oc_create_app_dc_with_io(self.node, pvc_name)
self.addCleanup(oc_delete, self.node, 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, self.node, dc_name, 0)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
# Make sure we are able to work with files on the mounted volume
filepath = "/mnt/file_for_testing_sc.log"
cmd = "dd if=/dev/urandom of=%s bs=1K count=100" % filepath
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0, "Failed to execute command %s on %s" % (cmd, self.node))
cmd = "ls -lrt %s" % filepath
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0, "Failed to execute command %s on %s" % (cmd, self.node))
cmd = "rm -rf %s" % filepath
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0, "Failed to execute command %s on %s" % (cmd, self.node))
def test_pv_resize_with_prefix_for_name(self,
create_vol_name_prefix=False):
"""Validate PV resize with and without name prefix"""
dir_path = "/mnt/"
node = self.ocp_client[0]
# Create PVC
self.create_storage_class(
allow_volume_expansion=True,
create_vol_name_prefix=create_vol_name_prefix)
pvc_name = self.create_and_wait_for_pvc()
# Create DC with POD and attached PVC to it.
dc_name = oc_create_app_dc_with_io(node, pvc_name)
self.addCleanup(oc_delete, node, 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, node, dc_name, 0)
pod_name = get_pod_name_from_dc(node, dc_name)
wait_for_pod_be_ready(node, pod_name)
if create_vol_name_prefix:
ret = heketi_ops.verify_volume_name_prefix(
node, self.sc['volumenameprefix'], self.sc['secretnamespace'],
pvc_name, self.heketi_server_url)
self.assertTrue(ret, "verify volnameprefix failed")
cmd = ("dd if=/dev/urandom of=%sfile " "bs=100K count=1000") % dir_path
ret, out, err = oc_rsh(node, pod_name, cmd)
self.assertEqual(ret, 0,
"failed to execute command %s on %s" % (cmd, node))
cmd = ("dd if=/dev/urandom of=%sfile2 "
"bs=100K count=10000") % dir_path
ret, out, err = oc_rsh(node, pod_name, cmd)
self.assertNotEqual(
ret, 0, " This IO did not fail as expected "
"command %s on %s" % (cmd, node))
pvc_size = 2
resize_pvc(node, pvc_name, pvc_size)
verify_pvc_size(node, pvc_name, pvc_size)
pv_name = get_pv_name_from_pvc(node, pvc_name)
verify_pv_size(node, pv_name, pvc_size)
oc_delete(node, 'pod', pod_name)
wait_for_resource_absence(node, 'pod', pod_name)
pod_name = get_pod_name_from_dc(node, dc_name)
wait_for_pod_be_ready(node, pod_name)
cmd = ("dd if=/dev/urandom of=%sfile_new "
"bs=50K count=10000") % dir_path
ret, out, err = oc_rsh(node, pod_name, cmd)
self.assertEqual(ret, 0,
"failed to execute command %s on %s" % (cmd, node))
def test_dynamic_provisioning_glusterfile_reclaim_policy_retain(self):
"""Validate retain policy for glusterfs after deletion of pvc"""
self.create_storage_class(reclaim_policy='Retain')
self.create_and_wait_for_pvc()
# get the name of the volume
pv_name = get_pv_name_from_pvc(self.node, self.pvc_name)
custom = [
r':.metadata.annotations.'
r'"gluster\.kubernetes\.io\/heketi\-volume\-id"',
r':.spec.persistentVolumeReclaimPolicy'
]
vol_id, reclaim_policy = oc_get_custom_resource(
self.node, 'pv', custom, pv_name)
self.assertEqual(reclaim_policy, 'Retain')
# Create DC with POD and attached PVC to it.
try:
dc_name = oc_create_app_dc_with_io(self.node, self.pvc_name)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
finally:
scale_dc_pod_amount_and_wait(self.node, dc_name, 0)
oc_delete(self.node, 'dc', dc_name)
wait_for_resource_absence(self.node, 'pod', pod_name)
oc_delete(self.node, 'pvc', self.pvc_name)
with self.assertRaises(ExecutionError):
wait_for_resource_absence(self.node,
'pvc',
self.pvc_name,
interval=3,
timeout=30)
heketi_volume_delete(self.heketi_client_node, self.heketi_server_url,
vol_id)
vol_list = heketi_volume_list(self.heketi_client_node,
self.heketi_server_url)
self.assertNotIn(vol_id, vol_list)
oc_delete(self.node, 'pv', pv_name)
wait_for_resource_absence(self.node, 'pv', pv_name)
def test_heketi_metrics_heketipod_failure(self):
"""Validate heketi metrics after heketi pod failure"""
scale_dc_pod_amount_and_wait(self.ocp_master_node,
self.heketi_dc_name,
pod_amount=0)
self.addCleanup(scale_dc_pod_amount_and_wait,
self.ocp_master_node,
self.heketi_dc_name,
pod_amount=1)
# verify that metrics is not accessable when heketi pod is down
with self.assertRaises(exceptions.ExecutionError):
get_heketi_metrics(self.heketi_client_node,
self.heketi_server_url,
prometheus_format=True)
scale_dc_pod_amount_and_wait(self.ocp_master_node,
self.heketi_dc_name,
pod_amount=1)
pod_name = get_pod_name_from_dc(self.ocp_master_node,
self.heketi_dc_name,
self.heketi_dc_name)
wait_for_pod_be_ready(self.ocp_master_node, pod_name, wait_step=5)
for i in range(3):
vol = heketi_volume_create(self.heketi_client_node,
self.heketi_server_url,
1,
json=True)
self.assertTrue(vol)
self.addCleanup(heketi_volume_delete,
self.heketi_client_node,
self.heketi_server_url,
vol['id'],
raise_on_error=False)
vol_list = heketi_volume_list(self.heketi_client_node,
self.heketi_server_url)
self.assertIn(vol['id'], vol_list)
self.verify_heketi_metrics_with_topology_info()
def test_restart_services_provision_volume_and_run_io(self, service):
"""Restart gluster service then validate volumes"""
# restarts glusterfs service
restart_service_on_pod(self.oc_node, self.gluster_pod, service)
# wait for deployed user pod to be in Running state after restarting
# service
wait_for_pod_be_ready(self.oc_node,
self.pod_name,
timeout=60,
wait_step=5)
# checks if all glusterfs services are in running state
for service in (SERVICE_BLOCKD, SERVICE_TCMU, SERVICE_TARGET):
status = "exited" if service == SERVICE_TARGET else "running"
self.assertTrue(
check_service_status(self.oc_node, self.gluster_pod, service,
status),
"service %s is not in %s state" % (service, status))
# validates pvc, pv, heketi block and gluster block count after
# service restarts
self.validate_volumes_and_blocks()
def _wait_for_gluster_pod_to_be_ready(self):
for gluster_pod in self.gluster_pod_list:
for w in Waiter(timeout=600, interval=10):
try:
success = wait_for_pod_be_ready(self.oc_node,
gluster_pod,
timeout=1,
wait_step=1)
if success:
break
except ExecutionError as e:
g.log.info("exception %s while validating gluster "
"pod %s" % (e, gluster_pod))
if w.expired:
error_msg = ("exceeded timeout 600 sec, pod '%s' is "
"not in 'running' state" % gluster_pod)
g.log.error(error_msg)
raise ExecutionError(error_msg)
def test_dynamic_provisioning_glusterfile_heketipod_failure(self):
"""Validate dynamic provisioning for gluster file when heketi pod down
"""
mount_path = "/mnt"
datafile_path = '%s/fake_file_for_%s' % (mount_path, self.id())
# Create secret and storage class
sc_name = self.create_storage_class()
# Create PVC
app_1_pvc_name = self.create_and_wait_for_pvc(
pvc_name_prefix="autotest-file", sc_name=sc_name
)
# Create app POD with attached volume
app_1_pod_name = oc_create_tiny_pod_with_volume(
self.node, app_1_pvc_name, "test-pvc-mount-on-app-pod",
mount_path=mount_path)
self.addCleanup(
wait_for_resource_absence, self.node, 'pod', app_1_pod_name)
self.addCleanup(oc_delete, self.node, 'pod', app_1_pod_name)
# Wait for app POD be up and running
wait_for_pod_be_ready(
self.node, app_1_pod_name, timeout=60, wait_step=2)
# Write data to the app POD
write_data_cmd = (
"dd if=/dev/urandom of=%s bs=1K count=100" % datafile_path)
ret, out, err = oc_rsh(self.node, app_1_pod_name, write_data_cmd)
self.assertEqual(
ret, 0,
"Failed to execute command %s on %s" % (write_data_cmd, self.node))
# Remove Heketi pod
heketi_down_cmd = "oc scale --replicas=0 dc/%s --namespace %s" % (
self.heketi_dc_name, self.storage_project_name)
heketi_up_cmd = "oc scale --replicas=1 dc/%s --namespace %s" % (
self.heketi_dc_name, self.storage_project_name)
self.addCleanup(self.cmd_run, heketi_up_cmd)
heketi_pod_name = get_pod_name_from_dc(
self.node, self.heketi_dc_name, timeout=10, wait_step=3)
self.cmd_run(heketi_down_cmd)
wait_for_resource_absence(self.node, 'pod', heketi_pod_name)
app_2_pvc_name = oc_create_pvc(
self.node, pvc_name_prefix="autotest-file2", sc_name=sc_name
)
self.addCleanup(
wait_for_resource_absence, self.node, 'pvc', app_2_pvc_name)
self.addCleanup(
oc_delete, self.node, 'pvc', app_2_pvc_name, raise_on_absence=False
)
# Create second app POD
app_2_pod_name = oc_create_tiny_pod_with_volume(
self.node, app_2_pvc_name, "test-pvc-mount-on-app-pod",
mount_path=mount_path)
self.addCleanup(
wait_for_resource_absence, self.node, 'pod', app_2_pod_name)
self.addCleanup(oc_delete, self.node, 'pod', app_2_pod_name)
# Bring Heketi POD back
self.cmd_run(heketi_up_cmd)
# Wait for Heketi POD be up and running
new_heketi_pod_name = get_pod_name_from_dc(
self.node, self.heketi_dc_name, timeout=10, wait_step=2)
wait_for_pod_be_ready(
self.node, new_heketi_pod_name, wait_step=5, timeout=120)
# Wait for second PVC and app POD be ready
verify_pvc_status_is_bound(self.node, app_2_pvc_name)
wait_for_pod_be_ready(
self.node, app_2_pod_name, timeout=60, wait_step=2)
# Verify that we are able to write data
ret, out, err = oc_rsh(self.node, app_2_pod_name, write_data_cmd)
self.assertEqual(
ret, 0,
"Failed to execute command %s on %s" % (write_data_cmd, self.node))
def test_arbiter_pvc_mount_on_pod(self):
"""Validate new volume creation using app pod"""
# Create sc with gluster arbiter info
self.create_storage_class(is_arbiter_vol=True)
# Create PVC and wait for it to be in 'Bound' state
self.create_and_wait_for_pvc()
# Create POD with attached volume
mount_path = "/mnt"
pod_name = oc_create_tiny_pod_with_volume(
self.node,
self.pvc_name,
"test-arbiter-pvc-mount-on-app-pod",
mount_path=mount_path)
self.addCleanup(oc_delete, self.node, 'pod', pod_name)
# Wait for POD be up and running
wait_for_pod_be_ready(self.node, pod_name, timeout=60, wait_step=2)
# Get volume ID
vol_info = get_gluster_vol_info_by_pvc_name(self.node, self.pvc_name)
vol_id = vol_info["gluster_vol_id"]
# Verify that POD has volume mounted on it
cmd = "oc exec {0} -- df -PT {1} | grep {1}".format(
pod_name, mount_path)
out = self.cmd_run(cmd)
err_msg = ("Failed to get info about mounted '%s' volume. "
"Output is empty." % vol_id)
self.assertTrue(out, err_msg)
# Verify volume data on POD
# Filesystem Type Size Used Avail Cap Mounted on
# IP:vol_id fuse.glusterfs 1038336 33408 1004928 3% /mnt
data = [s for s in out.strip().split(' ') if s]
actual_vol_id = data[0].split(':')[-1]
self.assertEqual(
vol_id, actual_vol_id, "Volume ID does not match: expected is "
"'%s' and actual is '%s'." % (vol_id, actual_vol_id))
self.assertIn(
"gluster", data[1],
"Filesystem type is expected to be of 'glusterfs' type. "
"Actual value is '%s'." % data[1])
self.assertEqual(
mount_path, data[6],
"Unexpected mount path. Expected is '%s' and actual is '%s'." %
(mount_path, data[6]))
max_size = 1024**2
total_size = int(data[2])
self.assertLessEqual(
total_size, max_size,
"Volume has bigger size '%s' than expected - '%s'." %
(total_size, max_size))
min_available_size = int(max_size * 0.93)
available_size = int(data[4])
self.assertLessEqual(
min_available_size, available_size,
"Minimum available size (%s) not satisfied. Actual is '%s'." %
(min_available_size, available_size))
# Write data on mounted volume
write_data_cmd = ("dd if=/dev/zero of=%s/file$i bs=%s count=1; " %
(mount_path, available_size))
self.cmd_run(write_data_cmd)
def _pv_resize(self, exceed_free_space):
dir_path = "/mnt"
pvc_size_gb, min_free_space_gb = 1, 3
# Get available free space disabling redundant devices and nodes
heketi_url = self.heketi_server_url
node_id_list = heketi_ops.heketi_node_list(self.heketi_client_node,
heketi_url)
self.assertTrue(node_id_list)
nodes = {}
min_free_space = min_free_space_gb * 1024**2
for node_id in node_id_list:
node_info = heketi_ops.heketi_node_info(self.heketi_client_node,
heketi_url,
node_id,
json=True)
if (node_info['state'].lower() != 'online'
or not node_info['devices']):
continue
if len(nodes) > 2:
out = heketi_ops.heketi_node_disable(self.heketi_client_node,
heketi_url, node_id)
self.assertTrue(out)
self.addCleanup(heketi_ops.heketi_node_enable,
self.heketi_client_node, heketi_url, node_id)
for device in node_info['devices']:
if device['state'].lower() != 'online':
continue
free_space = device['storage']['free']
if (node_id in nodes.keys() or free_space < min_free_space):
out = heketi_ops.heketi_device_disable(
self.heketi_client_node, heketi_url, device['id'])
self.assertTrue(out)
self.addCleanup(heketi_ops.heketi_device_enable,
self.heketi_client_node, heketi_url,
device['id'])
continue
nodes[node_id] = free_space
if len(nodes) < 3:
raise self.skipTest("Could not find 3 online nodes with, "
"at least, 1 online device having free space "
"bigger than %dGb." % min_free_space_gb)
# Calculate maximum available size for PVC
available_size_gb = int(min(nodes.values()) / (1024**2))
# Create PVC
self.create_storage_class(allow_volume_expansion=True)
pvc_name = self.create_and_wait_for_pvc(pvc_size=pvc_size_gb)
# Create DC with POD and attached PVC to it
dc_name = oc_create_app_dc_with_io(self.node, pvc_name)
self.addCleanup(oc_delete, self.node, 'dc', dc_name)
self.addCleanup(scale_dc_pod_amount_and_wait, self.node, dc_name, 0)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
if exceed_free_space:
# Try to expand existing PVC exceeding free space
resize_pvc(self.node, pvc_name, available_size_gb)
wait_for_events(self.node,
obj_name=pvc_name,
event_reason='VolumeResizeFailed')
# Check that app POD is up and runnig then try to write data
wait_for_pod_be_ready(self.node, pod_name)
cmd = ("dd if=/dev/urandom of=%s/autotest bs=100K count=1" %
dir_path)
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(
ret, 0,
"Failed to write data after failed attempt to expand PVC.")
else:
# Expand existing PVC using all the available free space
expand_size_gb = available_size_gb - pvc_size_gb
resize_pvc(self.node, pvc_name, expand_size_gb)
verify_pvc_size(self.node, pvc_name, expand_size_gb)
pv_name = get_pv_name_from_pvc(self.node, pvc_name)
verify_pv_size(self.node, pv_name, expand_size_gb)
wait_for_events(self.node,
obj_name=pvc_name,
event_reason='VolumeResizeSuccessful')
# Recreate app POD
oc_delete(self.node, 'pod', pod_name)
wait_for_resource_absence(self.node, 'pod', pod_name)
pod_name = get_pod_name_from_dc(self.node, dc_name)
wait_for_pod_be_ready(self.node, pod_name)
# Write data on the expanded PVC
cmd = ("dd if=/dev/urandom of=%s/autotest "
"bs=1M count=1025" % dir_path)
ret, out, err = oc_rsh(self.node, pod_name, cmd)
self.assertEqual(ret, 0,
"Failed to write data on the expanded PVC")
