def test_prometheus_volume_metrics_on_pod_restart(self):
"""Validate volume metrics using prometheus before and after pod
restart"""
# Create PVC and wait for it to be in 'Bound' state
pvc_name = self.create_and_wait_for_pvc()
pod_name = openshift_ops.oc_create_tiny_pod_with_volume(
self._master, pvc_name, "autotest-volume",
image=self.io_container_image_cirros)
self.addCleanup(openshift_ops.oc_delete, self._master, 'pod', pod_name,
raise_on_absence=False)
# Wait for POD be up and running
openshift_ops.wait_for_pod_be_ready(
self._master, pod_name, timeout=60, wait_step=2)
# Write data on the volume and wait for 2 mins and sleep is must for
# prometheus to get the exact values of the metrics
self._run_io_on_the_pod(pod_name, 30)
time.sleep(120)
# Fetching the metrics and storing in initial_metrics as dictionary
initial_metrics = self._get_and_manipulate_metric_data(
self.metrics, pvc_name)
# Mark the current node unschedulable on which app pod is running
openshift_ops.switch_oc_project(
self._master, self.storage_project_name)
pod_info = openshift_ops.oc_get_pods(self._master, name=pod_name)
openshift_ops.oc_adm_manage_node(
self._master, '--schedulable=false',
nodes=[pod_info[pod_name]["node"]])
self.addCleanup(
openshift_ops.oc_adm_manage_node, self._master,
'--schedulable=true', nodes=[pod_info[pod_name]["node"]])
# Delete the existing pod and create a new pod
openshift_ops.oc_delete(self._master, 'pod', pod_name)
pod_name = openshift_ops.oc_create_tiny_pod_with_volume(
self._master, pvc_name, "autotest-volume")
self.addCleanup(openshift_ops.oc_delete, self._master, 'pod', pod_name)
# Wait for POD be up and running and prometheus to refresh the data
openshift_ops.wait_for_pod_be_ready(
self._master, pod_name, timeout=60, wait_step=2)
time.sleep(120)
# Fetching the metrics and storing in final_metrics as dictionary and
# validating with initial_metrics
final_metrics = self._get_and_manipulate_metric_data(
self.metrics, pvc_name)
self.assertEqual(dict(initial_metrics), dict(final_metrics),
"Metrics are different post pod restart")
def test_dynamic_provisioning_glusterfile_glusterpod_failure(self):
"""Create glusterblock PVC when gluster pod is down."""
# Check that we work with containerized Gluster
if not self.is_containerized_gluster():
self.skipTest("Only containerized Gluster clusters are supported.")
mount_path = "/mnt"
datafile_path = '%s/fake_file_for_%s' % (mount_path, self.id())
# Create secret and storage class
self.create_storage_class()
# Create PVC
pvc_name = self.create_and_wait_for_pvc()
# Create app POD with attached volume
pod_name = oc_create_tiny_pod_with_volume(
self.node, pvc_name, "test-pvc-mount-on-app-pod",
mount_path=mount_path)
self.addCleanup(
wait_for_resource_absence, self.node, 'pod', pod_name)
self.addCleanup(oc_delete, self.node, 'pod', pod_name)
# Wait for app POD be up and running
wait_for_pod_be_ready(
self.node, pod_name, timeout=60, wait_step=2)
# Run IO in background
io_cmd = "oc rsh %s dd if=/dev/urandom of=%s bs=1000K count=900" % (
pod_name, datafile_path)
async_io = g.run_async(self.node, io_cmd, "root")
# Pick up one of the hosts which stores PV brick (4+ nodes case)
gluster_pod_data = get_gluster_pod_names_by_pvc_name(
self.node, pvc_name)[0]
# Delete glusterfs POD from chosen host and wait for spawn of new one
oc_delete(self.node, 'pod', gluster_pod_data["pod_name"])
cmd = ("oc get pods -o wide | grep glusterfs | grep %s | "
"grep -v Terminating | awk '{print $1}'") % (
gluster_pod_data["host_name"])
for w in Waiter(600, 15):
out = self.cmd_run(cmd)
new_gluster_pod_name = out.strip().split("\n")[0].strip()
if not new_gluster_pod_name:
continue
else:
break
if w.expired:
error_msg = "exceeded timeout, new gluster pod not created"
g.log.error(error_msg)
raise ExecutionError(error_msg)
new_gluster_pod_name = out.strip().split("\n")[0].strip()
g.log.info("new gluster pod name is %s" % new_gluster_pod_name)
wait_for_pod_be_ready(self.node, new_gluster_pod_name)
# Check that async IO was not interrupted
ret, out, err = async_io.async_communicate()
self.assertEqual(ret, 0, "IO %s failed on %s" % (io_cmd, self.node))
def test_dynamic_provisioning_glusterfile_glusterpod_failure(self):
"""Create glusterblock PVC when gluster pod is down."""
# Check that we work with containerized Gluster
if not self.is_containerized_gluster():
self.skipTest("Only containerized Gluster clusters are supported.")
mount_path = "/mnt"
datafile_path = '%s/fake_file_for_%s' % (mount_path, self.id())
# Create secret and storage class
self.create_storage_class()
# Create PVC
pvc_name = self.create_and_wait_for_pvc()
# Create app POD with attached volume
pod_name = oc_create_tiny_pod_with_volume(self.node,
pvc_name,
"test-pvc-mount-on-app-pod",
mount_path=mount_path)
self.addCleanup(wait_for_resource_absence, self.node, 'pod', pod_name)
self.addCleanup(oc_delete, self.node, 'pod', pod_name)
# Wait for app POD be up and running
wait_for_pod_be_ready(self.node, pod_name, timeout=60, wait_step=2)
# Run IO in background
io_cmd = "oc rsh %s dd if=/dev/urandom of=%s bs=1000K count=900" % (
pod_name, datafile_path)
async_io = g.run_async(self.node, io_cmd, "root")
# Pick up one of the hosts which stores PV brick (4+ nodes case)
gluster_pod_data = get_gluster_pod_names_by_pvc_name(
self.node, pvc_name)[0]
# Delete glusterfs POD from chosen host and wait for spawn of new one
oc_delete(self.node, 'pod', gluster_pod_data["pod_name"])
cmd = ("oc get pods -o wide | grep glusterfs | grep %s | "
"grep -v Terminating | awk '{print $1}'") % (
gluster_pod_data["host_name"])
for w in Waiter(600, 15):
out = self.cmd_run(cmd)
new_gluster_pod_name = out.strip().split("\n")[0].strip()
if not new_gluster_pod_name:
continue
else:
break
if w.expired:
error_msg = "exceeded timeout, new gluster pod not created"
g.log.error(error_msg)
raise ExecutionError(error_msg)
new_gluster_pod_name = out.strip().split("\n")[0].strip()
g.log.info("new gluster pod name is %s" % new_gluster_pod_name)
wait_for_pod_be_ready(self.node, new_gluster_pod_name)
# Check that async IO was not interrupted
ret, out, err = async_io.async_communicate()
self.assertEqual(ret, 0, "IO %s failed on %s" % (io_cmd, self.node))
def test_arbiter_pvc_mount_on_pod(self):
"""Validate new volume creation using app pod"""
# Create sc with gluster arbiter info
self.create_storage_class(is_arbiter_vol=True)
# Create PVC and wait for it to be in 'Bound' state
self.create_and_wait_for_pvc()
# Create POD with attached volume
mount_path = "/mnt"
pod_name = openshift_ops.oc_create_tiny_pod_with_volume(
self.node,
self.pvc_name,
"test-arbiter-pvc-mount-on-app-pod",
mount_path=mount_path)
self.addCleanup(openshift_ops.oc_delete, self.node, 'pod', pod_name)
# Wait for POD be up and running
openshift_ops.wait_for_pod_be_ready(self.node,
pod_name,
timeout=60,
wait_step=2)
# Get volume ID
vol_info = openshift_ops.get_gluster_vol_info_by_pvc_name(
self.node, self.pvc_name)
vol_id = vol_info["gluster_vol_id"]
# Verify that POD has volume mounted on it
cmd = "oc exec {0} -- df -PT {1} | grep {1}".format(
pod_name, mount_path)
out = self.cmd_run(cmd)
err_msg = ("Failed to get info about mounted '%s' volume. "
"Output is empty." % vol_id)
self.assertTrue(out, err_msg)
# Verify volume data on POD
# Filesystem Type Size Used Avail Cap Mounted on
# IP:vol_id fuse.glusterfs 1038336 33408 1004928 3% /mnt
data = [s for s in out.strip().split(' ') if s]
actual_vol_id = data[0].split(':')[-1]
self.assertEqual(
vol_id, actual_vol_id, "Volume ID does not match: expected is "
"'%s' and actual is '%s'." % (vol_id, actual_vol_id))
self.assertIn(
"gluster", data[1],
"Filesystem type is expected to be of 'glusterfs' type. "
"Actual value is '%s'." % data[1])
self.assertEqual(
mount_path, data[6],
"Unexpected mount path. Expected is '%s' and actual is '%s'." %
(mount_path, data[6]))
max_size = 1024**2
total_size = int(data[2])
self.assertLessEqual(
total_size, max_size,
"Volume has bigger size '%s' than expected - '%s'." %
(total_size, max_size))
min_available_size = int(max_size * 0.93)
available_size = int(data[4])
self.assertLessEqual(
min_available_size, available_size,
"Minimum available size (%s) not satisfied. Actual is '%s'." %
(min_available_size, available_size))
# Write data on mounted volume
write_data_cmd = ("dd if=/dev/zero of=%s/file$i bs=%s count=1; " %
(mount_path, available_size))
self.cmd_run(write_data_cmd)
def test_dynamic_provisioning_glusterfile_gluster_pod_or_node_failure(
self):
"""Create glusterblock PVC when gluster pod or node is down."""
mount_path = "/mnt"
datafile_path = '%s/fake_file_for_%s' % (mount_path, self.id())
# Create secret and storage class
self.create_storage_class()
# Create PVC
pvc_name = self.create_and_wait_for_pvc()
# Create app POD with attached volume
pod_name = oc_create_tiny_pod_with_volume(
self.node,
pvc_name,
"test-pvc-mount-on-app-pod",
mount_path=mount_path,
image=self.io_container_image_cirros)
self.addCleanup(wait_for_resource_absence, self.node, 'pod', pod_name)
self.addCleanup(oc_delete, self.node, 'pod', pod_name)
# Wait for app POD be up and running
wait_for_pod_be_ready(self.node, pod_name, timeout=60, wait_step=2)
# Run IO in background
io_cmd = "oc rsh %s dd if=/dev/urandom of=%s bs=1000K count=900" % (
pod_name, datafile_path)
async_io = g.run_async(self.node, io_cmd, "root")
# Check for containerized Gluster
if self.is_containerized_gluster():
# Pick up one of the hosts which stores PV brick (4+ nodes case)
gluster_pod_data = get_gluster_pod_names_by_pvc_name(
self.node, pvc_name)[0]
# Delete glusterfs POD from chosen host and wait for
# spawn of new one
oc_delete(self.node, 'pod', gluster_pod_data["pod_name"])
cmd = ("oc get pods -o wide | grep glusterfs | grep %s | "
"grep -v Terminating | awk '{print $1}'") % (
gluster_pod_data["pod_hostname"])
for w in Waiter(600, 15):
new_gluster_pod_name = self.cmd_run(cmd)
if new_gluster_pod_name:
break
if w.expired:
error_msg = "exceeded timeout, new gluster pod not created"
g.log.error(error_msg)
raise AssertionError(error_msg)
g.log.info("new gluster pod name is %s" % new_gluster_pod_name)
wait_for_pod_be_ready(self.node, new_gluster_pod_name)
else:
pvc_hosting_node_ip = get_gluster_host_ips_by_pvc_name(
self.node, pvc_name)[0]
heketi_nodes = heketi_node_list(self.heketi_client_node,
self.heketi_server_url)
node_ip_for_reboot = None
for heketi_node in heketi_nodes:
heketi_node_ip = heketi_node_info(
self.heketi_client_node,
self.heketi_server_url,
heketi_node,
json=True)["hostnames"]["storage"][0]
if heketi_node_ip == pvc_hosting_node_ip:
node_ip_for_reboot = heketi_node_ip
break
if not node_ip_for_reboot:
raise AssertionError(
"Gluster node IP %s not matched with heketi node %s" %
(pvc_hosting_node_ip, heketi_node_ip))
node_reboot_by_command(node_ip_for_reboot)
# Check that async IO was not interrupted
ret, out, err = async_io.async_communicate()
self.assertEqual(ret, 0, "IO %s failed on %s" % (io_cmd, self.node))
def test_dynamic_provisioning_glusterfile_heketipod_failure(self):
"""Validate dynamic provisioning for gluster file when heketi pod down
"""
mount_path = "/mnt"
datafile_path = '%s/fake_file_for_%s' % (mount_path, self.id())
# Create secret and storage class
sc_name = self.create_storage_class()
# Create PVC
app_1_pvc_name = self.create_and_wait_for_pvc(
pvc_name_prefix="autotest-file", sc_name=sc_name)
# Create app POD with attached volume
app_1_pod_name = oc_create_tiny_pod_with_volume(
self.node,
app_1_pvc_name,
"test-pvc-mount-on-app-pod",
mount_path=mount_path,
image=self.io_container_image_cirros)
self.addCleanup(wait_for_resource_absence, self.node, 'pod',
app_1_pod_name)
self.addCleanup(oc_delete, self.node, 'pod', app_1_pod_name)
# Wait for app POD be up and running
wait_for_pod_be_ready(self.node,
app_1_pod_name,
timeout=60,
wait_step=2)
# Write data to the app POD
write_data_cmd = ("dd if=/dev/urandom of=%s bs=1K count=100" %
datafile_path)
ret, out, err = oc_rsh(self.node, app_1_pod_name, write_data_cmd)
self.assertEqual(
ret, 0,
"Failed to execute command %s on %s" % (write_data_cmd, self.node))
# Remove Heketi pod
heketi_down_cmd = "oc scale --replicas=0 dc/%s --namespace %s" % (
self.heketi_dc_name, self.storage_project_name)
heketi_up_cmd = "oc scale --replicas=1 dc/%s --namespace %s" % (
self.heketi_dc_name, self.storage_project_name)
self.addCleanup(self.cmd_run, heketi_up_cmd)
heketi_pod_name = get_pod_name_from_dc(self.node,
self.heketi_dc_name,
timeout=10,
wait_step=3)
self.cmd_run(heketi_down_cmd)
wait_for_resource_absence(self.node, 'pod', heketi_pod_name)
app_2_pvc_name = oc_create_pvc(self.node,
pvc_name_prefix="autotest-file2",
sc_name=sc_name)
self.addCleanup(wait_for_resource_absence, self.node, 'pvc',
app_2_pvc_name)
self.addCleanup(oc_delete,
self.node,
'pvc',
app_2_pvc_name,
raise_on_absence=False)
# Create second app POD
app_2_pod_name = oc_create_tiny_pod_with_volume(
self.node,
app_2_pvc_name,
"test-pvc-mount-on-app-pod",
mount_path=mount_path,
image=self.io_container_image_cirros)
self.addCleanup(wait_for_resource_absence, self.node, 'pod',
app_2_pod_name)
self.addCleanup(oc_delete, self.node, 'pod', app_2_pod_name)
# Bring Heketi POD back
self.cmd_run(heketi_up_cmd)
# Wait for Heketi POD be up and running
new_heketi_pod_name = get_pod_name_from_dc(self.node,
self.heketi_dc_name,
timeout=10,
wait_step=2)
wait_for_pod_be_ready(self.node,
new_heketi_pod_name,
wait_step=5,
timeout=120)
# Wait for second PVC and app POD be ready
verify_pvc_status_is_bound(self.node, app_2_pvc_name)
wait_for_pod_be_ready(self.node,
app_2_pod_name,
timeout=60,
wait_step=2)
# Verify that we are able to write data
ret, out, err = oc_rsh(self.node, app_2_pod_name, write_data_cmd)
self.assertEqual(
ret, 0,
"Failed to execute command %s on %s" % (write_data_cmd, self.node))
def test_dynamic_provisioning_glusterfile_heketipod_failure(self):
"""Validate dynamic provisioning for gluster file when heketi pod down
"""
mount_path = "/mnt"
datafile_path = '%s/fake_file_for_%s' % (mount_path, self.id())
# Create secret and storage class
sc_name = self.create_storage_class()
# Create PVC
app_1_pvc_name = self.create_and_wait_for_pvc(
pvc_name_prefix="autotest-file", sc_name=sc_name
)
# Create app POD with attached volume
app_1_pod_name = oc_create_tiny_pod_with_volume(
self.node, app_1_pvc_name, "test-pvc-mount-on-app-pod",
mount_path=mount_path)
self.addCleanup(
wait_for_resource_absence, self.node, 'pod', app_1_pod_name)
self.addCleanup(oc_delete, self.node, 'pod', app_1_pod_name)
# Wait for app POD be up and running
wait_for_pod_be_ready(
self.node, app_1_pod_name, timeout=60, wait_step=2)
# Write data to the app POD
write_data_cmd = (
"dd if=/dev/urandom of=%s bs=1K count=100" % datafile_path)
ret, out, err = oc_rsh(self.node, app_1_pod_name, write_data_cmd)
self.assertEqual(
ret, 0,
"Failed to execute command %s on %s" % (write_data_cmd, self.node))
# Remove Heketi pod
heketi_down_cmd = "oc scale --replicas=0 dc/%s --namespace %s" % (
self.heketi_dc_name, self.storage_project_name)
heketi_up_cmd = "oc scale --replicas=1 dc/%s --namespace %s" % (
self.heketi_dc_name, self.storage_project_name)
self.addCleanup(self.cmd_run, heketi_up_cmd)
heketi_pod_name = get_pod_name_from_dc(
self.node, self.heketi_dc_name, timeout=10, wait_step=3)
self.cmd_run(heketi_down_cmd)
wait_for_resource_absence(self.node, 'pod', heketi_pod_name)
app_2_pvc_name = oc_create_pvc(
self.node, pvc_name_prefix="autotest-file2", sc_name=sc_name
)
self.addCleanup(
wait_for_resource_absence, self.node, 'pvc', app_2_pvc_name)
self.addCleanup(
oc_delete, self.node, 'pvc', app_2_pvc_name, raise_on_absence=False
)
# Create second app POD
app_2_pod_name = oc_create_tiny_pod_with_volume(
self.node, app_2_pvc_name, "test-pvc-mount-on-app-pod",
mount_path=mount_path)
self.addCleanup(
wait_for_resource_absence, self.node, 'pod', app_2_pod_name)
self.addCleanup(oc_delete, self.node, 'pod', app_2_pod_name)
# Bring Heketi POD back
self.cmd_run(heketi_up_cmd)
# Wait for Heketi POD be up and running
new_heketi_pod_name = get_pod_name_from_dc(
self.node, self.heketi_dc_name, timeout=10, wait_step=2)
wait_for_pod_be_ready(
self.node, new_heketi_pod_name, wait_step=5, timeout=120)
# Wait for second PVC and app POD be ready
verify_pvc_status_is_bound(self.node, app_2_pvc_name)
wait_for_pod_be_ready(
self.node, app_2_pod_name, timeout=60, wait_step=2)
# Verify that we are able to write data
ret, out, err = oc_rsh(self.node, app_2_pod_name, write_data_cmd)
self.assertEqual(
ret, 0,
"Failed to execute command %s on %s" % (write_data_cmd, self.node))
def test_prometheus_volume_metrics_on_node_reboot(self):
"""Validate volume metrics using prometheus before and after node
reboot"""
# Pod name for the entire test
prefix = "autotest-{}".format(utils.get_random_str())
# Create I/O pod with PVC
pvc_name = self.create_and_wait_for_pvc()
pod_name = openshift_ops.oc_create_tiny_pod_with_volume(
self._master, pvc_name, prefix,
image=self.io_container_image_cirros)
self.addCleanup(openshift_ops.oc_delete, self._master, 'pod', pod_name,
raise_on_absence=False)
openshift_ops.wait_for_pod_be_ready(
self._master, pod_name, timeout=60, wait_step=5)
# Write data on the volume and wait for 2 mins and sleep is must for
# prometheus to get the exact values of the metrics
ret, _, err = openshift_ops.oc_rsh(
self._master, pod_name, "touch /mnt/file{1..1000}")
self.assertEqual(
ret, 0, "Failed to create files in the app pod "
"with {}".format(err))
time.sleep(120)
# Fetch the metrics and store in initial_metrics as dictionary
initial_metrics = self._get_and_manipulate_metric_data(
self.metrics, pvc_name)
openshift_ops.switch_oc_project(
self._master, self.storage_project_name)
# Get the hostname to reboot where the pod is running
pod_info = openshift_ops.oc_get_pods(self._master, name=pod_name)
node_for_reboot = pod_info[pod_name]['node']
# Get the vm name by the hostname
vm_name = node_ops.find_vm_name_by_ip_or_hostname(node_for_reboot)
# power off and on the vm, based on the vm type(either gluster or not)
if node_for_reboot in self.gluster_servers:
self.power_off_gluster_node_vm(vm_name, node_for_reboot)
self.power_on_gluster_node_vm(vm_name, node_for_reboot)
else:
self.power_off_vm(vm_name)
self.power_on_vm(vm_name)
openshift_ops.wait_for_ocp_node_be_ready(
self._master, node_for_reboot)
# Create the new pod and validate the prometheus metrics
pod_name = openshift_ops.oc_create_tiny_pod_with_volume(
self._master, pvc_name, prefix)
self.addCleanup(openshift_ops.oc_delete, self._master, 'pod', pod_name)
# Wait for POD be up and running and prometheus to refresh the data
openshift_ops.wait_for_pod_be_ready(
self._master, pod_name, timeout=60, wait_step=5)
time.sleep(120)
# Fetching the metrics and storing in final_metrics as dictionary and
# validating with initial_metrics
final_metrics = self._get_and_manipulate_metric_data(
self.metrics, pvc_name)
self.assertEqual(dict(initial_metrics), dict(final_metrics),
"Metrics are different post node reboot")
def test_arbiter_pvc_mount_on_pod(self):
"""Validate new volume creation using app pod"""
# Create sc with gluster arbiter info
self.create_storage_class(is_arbiter_vol=True)
# Create PVC and wait for it to be in 'Bound' state
self.create_and_wait_for_pvc()
# Create POD with attached volume
mount_path = "/mnt"
pod_name = oc_create_tiny_pod_with_volume(
self.node, self.pvc_name, "test-arbiter-pvc-mount-on-app-pod",
mount_path=mount_path)
self.addCleanup(oc_delete, self.node, 'pod', pod_name)
# Wait for POD be up and running
wait_for_pod_be_ready(self.node, pod_name, timeout=60, wait_step=2)
# Get volume ID
vol_info = get_gluster_vol_info_by_pvc_name(self.node, self.pvc_name)
vol_id = vol_info["gluster_vol_id"]
# Verify that POD has volume mounted on it
cmd = "oc exec {0} -- df -PT {1} | grep {1}".format(
pod_name, mount_path)
out = self.cmd_run(cmd)
err_msg = ("Failed to get info about mounted '%s' volume. "
"Output is empty." % vol_id)
self.assertTrue(out, err_msg)
# Verify volume data on POD
# Filesystem Type Size Used Avail Cap Mounted on
# IP:vol_id fuse.glusterfs 1038336 33408 1004928 3% /mnt
data = [s for s in out.strip().split(' ') if s]
actual_vol_id = data[0].split(':')[-1]
self.assertEqual(
vol_id, actual_vol_id,
"Volume ID does not match: expected is "
"'%s' and actual is '%s'." % (vol_id, actual_vol_id))
self.assertIn(
"gluster", data[1],
"Filesystem type is expected to be of 'glusterfs' type. "
"Actual value is '%s'." % data[1])
self.assertEqual(
mount_path, data[6],
"Unexpected mount path. Expected is '%s' and actual is '%s'." % (
mount_path, data[6]))
max_size = 1024 ** 2
total_size = int(data[2])
self.assertLessEqual(
total_size, max_size,
"Volume has bigger size '%s' than expected - '%s'." % (
total_size, max_size))
min_available_size = int(max_size * 0.93)
available_size = int(data[4])
self.assertLessEqual(
min_available_size, available_size,
"Minimum available size (%s) not satisfied. Actual is '%s'." % (
min_available_size, available_size))
# Write data on mounted volume
write_data_cmd = (
"dd if=/dev/zero of=%s/file$i bs=%s count=1; " % (
mount_path, available_size))
self.cmd_run(write_data_cmd)