def test_prometheus_volume_metrics_on_pod_restart(self): """Validate volume metrics using prometheus before and after pod restart""" # Create PVC and wait for it to be in 'Bound' state pvc_name = self.create_and_wait_for_pvc() pod_name = openshift_ops.oc_create_tiny_pod_with_volume( self._master, pvc_name, "autotest-volume", image=self.io_container_image_cirros) self.addCleanup(openshift_ops.oc_delete, self._master, 'pod', pod_name, raise_on_absence=False) # Wait for POD be up and running openshift_ops.wait_for_pod_be_ready( self._master, pod_name, timeout=60, wait_step=2) # Write data on the volume and wait for 2 mins and sleep is must for # prometheus to get the exact values of the metrics self._run_io_on_the_pod(pod_name, 30) time.sleep(120) # Fetching the metrics and storing in initial_metrics as dictionary initial_metrics = self._get_and_manipulate_metric_data( self.metrics, pvc_name) # Mark the current node unschedulable on which app pod is running openshift_ops.switch_oc_project( self._master, self.storage_project_name) pod_info = openshift_ops.oc_get_pods(self._master, name=pod_name) openshift_ops.oc_adm_manage_node( self._master, '--schedulable=false', nodes=[pod_info[pod_name]["node"]]) self.addCleanup( openshift_ops.oc_adm_manage_node, self._master, '--schedulable=true', nodes=[pod_info[pod_name]["node"]]) # Delete the existing pod and create a new pod openshift_ops.oc_delete(self._master, 'pod', pod_name) pod_name = openshift_ops.oc_create_tiny_pod_with_volume( self._master, pvc_name, "autotest-volume") self.addCleanup(openshift_ops.oc_delete, self._master, 'pod', pod_name) # Wait for POD be up and running and prometheus to refresh the data openshift_ops.wait_for_pod_be_ready( self._master, pod_name, timeout=60, wait_step=2) time.sleep(120) # Fetching the metrics and storing in final_metrics as dictionary and # validating with initial_metrics final_metrics = self._get_and_manipulate_metric_data( self.metrics, pvc_name) self.assertEqual(dict(initial_metrics), dict(final_metrics), "Metrics are different post pod restart")
def test_dynamic_provisioning_glusterfile_glusterpod_failure(self): """Create glusterblock PVC when gluster pod is down.""" # Check that we work with containerized Gluster if not self.is_containerized_gluster(): self.skipTest("Only containerized Gluster clusters are supported.") mount_path = "/mnt" datafile_path = '%s/fake_file_for_%s' % (mount_path, self.id()) # Create secret and storage class self.create_storage_class() # Create PVC pvc_name = self.create_and_wait_for_pvc() # Create app POD with attached volume pod_name = oc_create_tiny_pod_with_volume( self.node, pvc_name, "test-pvc-mount-on-app-pod", mount_path=mount_path) self.addCleanup( wait_for_resource_absence, self.node, 'pod', pod_name) self.addCleanup(oc_delete, self.node, 'pod', pod_name) # Wait for app POD be up and running wait_for_pod_be_ready( self.node, pod_name, timeout=60, wait_step=2) # Run IO in background io_cmd = "oc rsh %s dd if=/dev/urandom of=%s bs=1000K count=900" % ( pod_name, datafile_path) async_io = g.run_async(self.node, io_cmd, "root") # Pick up one of the hosts which stores PV brick (4+ nodes case) gluster_pod_data = get_gluster_pod_names_by_pvc_name( self.node, pvc_name)[0] # Delete glusterfs POD from chosen host and wait for spawn of new one oc_delete(self.node, 'pod', gluster_pod_data["pod_name"]) cmd = ("oc get pods -o wide | grep glusterfs | grep %s | " "grep -v Terminating | awk '{print $1}'") % ( gluster_pod_data["host_name"]) for w in Waiter(600, 15): out = self.cmd_run(cmd) new_gluster_pod_name = out.strip().split("\n")[0].strip() if not new_gluster_pod_name: continue else: break if w.expired: error_msg = "exceeded timeout, new gluster pod not created" g.log.error(error_msg) raise ExecutionError(error_msg) new_gluster_pod_name = out.strip().split("\n")[0].strip() g.log.info("new gluster pod name is %s" % new_gluster_pod_name) wait_for_pod_be_ready(self.node, new_gluster_pod_name) # Check that async IO was not interrupted ret, out, err = async_io.async_communicate() self.assertEqual(ret, 0, "IO %s failed on %s" % (io_cmd, self.node))
def test_dynamic_provisioning_glusterfile_glusterpod_failure(self): """Create glusterblock PVC when gluster pod is down.""" # Check that we work with containerized Gluster if not self.is_containerized_gluster(): self.skipTest("Only containerized Gluster clusters are supported.") mount_path = "/mnt" datafile_path = '%s/fake_file_for_%s' % (mount_path, self.id()) # Create secret and storage class self.create_storage_class() # Create PVC pvc_name = self.create_and_wait_for_pvc() # Create app POD with attached volume pod_name = oc_create_tiny_pod_with_volume(self.node, pvc_name, "test-pvc-mount-on-app-pod", mount_path=mount_path) self.addCleanup(wait_for_resource_absence, self.node, 'pod', pod_name) self.addCleanup(oc_delete, self.node, 'pod', pod_name) # Wait for app POD be up and running wait_for_pod_be_ready(self.node, pod_name, timeout=60, wait_step=2) # Run IO in background io_cmd = "oc rsh %s dd if=/dev/urandom of=%s bs=1000K count=900" % ( pod_name, datafile_path) async_io = g.run_async(self.node, io_cmd, "root") # Pick up one of the hosts which stores PV brick (4+ nodes case) gluster_pod_data = get_gluster_pod_names_by_pvc_name( self.node, pvc_name)[0] # Delete glusterfs POD from chosen host and wait for spawn of new one oc_delete(self.node, 'pod', gluster_pod_data["pod_name"]) cmd = ("oc get pods -o wide | grep glusterfs | grep %s | " "grep -v Terminating | awk '{print $1}'") % ( gluster_pod_data["host_name"]) for w in Waiter(600, 15): out = self.cmd_run(cmd) new_gluster_pod_name = out.strip().split("\n")[0].strip() if not new_gluster_pod_name: continue else: break if w.expired: error_msg = "exceeded timeout, new gluster pod not created" g.log.error(error_msg) raise ExecutionError(error_msg) new_gluster_pod_name = out.strip().split("\n")[0].strip() g.log.info("new gluster pod name is %s" % new_gluster_pod_name) wait_for_pod_be_ready(self.node, new_gluster_pod_name) # Check that async IO was not interrupted ret, out, err = async_io.async_communicate() self.assertEqual(ret, 0, "IO %s failed on %s" % (io_cmd, self.node))
def test_arbiter_pvc_mount_on_pod(self): """Validate new volume creation using app pod""" # Create sc with gluster arbiter info self.create_storage_class(is_arbiter_vol=True) # Create PVC and wait for it to be in 'Bound' state self.create_and_wait_for_pvc() # Create POD with attached volume mount_path = "/mnt" pod_name = openshift_ops.oc_create_tiny_pod_with_volume( self.node, self.pvc_name, "test-arbiter-pvc-mount-on-app-pod", mount_path=mount_path) self.addCleanup(openshift_ops.oc_delete, self.node, 'pod', pod_name) # Wait for POD be up and running openshift_ops.wait_for_pod_be_ready(self.node, pod_name, timeout=60, wait_step=2) # Get volume ID vol_info = openshift_ops.get_gluster_vol_info_by_pvc_name( self.node, self.pvc_name) vol_id = vol_info["gluster_vol_id"] # Verify that POD has volume mounted on it cmd = "oc exec {0} -- df -PT {1} | grep {1}".format( pod_name, mount_path) out = self.cmd_run(cmd) err_msg = ("Failed to get info about mounted '%s' volume. " "Output is empty." % vol_id) self.assertTrue(out, err_msg) # Verify volume data on POD # Filesystem Type Size Used Avail Cap Mounted on # IP:vol_id fuse.glusterfs 1038336 33408 1004928 3% /mnt data = [s for s in out.strip().split(' ') if s] actual_vol_id = data[0].split(':')[-1] self.assertEqual( vol_id, actual_vol_id, "Volume ID does not match: expected is " "'%s' and actual is '%s'." % (vol_id, actual_vol_id)) self.assertIn( "gluster", data[1], "Filesystem type is expected to be of 'glusterfs' type. " "Actual value is '%s'." % data[1]) self.assertEqual( mount_path, data[6], "Unexpected mount path. Expected is '%s' and actual is '%s'." % (mount_path, data[6])) max_size = 1024**2 total_size = int(data[2]) self.assertLessEqual( total_size, max_size, "Volume has bigger size '%s' than expected - '%s'." % (total_size, max_size)) min_available_size = int(max_size * 0.93) available_size = int(data[4]) self.assertLessEqual( min_available_size, available_size, "Minimum available size (%s) not satisfied. Actual is '%s'." % (min_available_size, available_size)) # Write data on mounted volume write_data_cmd = ("dd if=/dev/zero of=%s/file$i bs=%s count=1; " % (mount_path, available_size)) self.cmd_run(write_data_cmd)
def test_dynamic_provisioning_glusterfile_gluster_pod_or_node_failure( self): """Create glusterblock PVC when gluster pod or node is down.""" mount_path = "/mnt" datafile_path = '%s/fake_file_for_%s' % (mount_path, self.id()) # Create secret and storage class self.create_storage_class() # Create PVC pvc_name = self.create_and_wait_for_pvc() # Create app POD with attached volume pod_name = oc_create_tiny_pod_with_volume( self.node, pvc_name, "test-pvc-mount-on-app-pod", mount_path=mount_path, image=self.io_container_image_cirros) self.addCleanup(wait_for_resource_absence, self.node, 'pod', pod_name) self.addCleanup(oc_delete, self.node, 'pod', pod_name) # Wait for app POD be up and running wait_for_pod_be_ready(self.node, pod_name, timeout=60, wait_step=2) # Run IO in background io_cmd = "oc rsh %s dd if=/dev/urandom of=%s bs=1000K count=900" % ( pod_name, datafile_path) async_io = g.run_async(self.node, io_cmd, "root") # Check for containerized Gluster if self.is_containerized_gluster(): # Pick up one of the hosts which stores PV brick (4+ nodes case) gluster_pod_data = get_gluster_pod_names_by_pvc_name( self.node, pvc_name)[0] # Delete glusterfs POD from chosen host and wait for # spawn of new one oc_delete(self.node, 'pod', gluster_pod_data["pod_name"]) cmd = ("oc get pods -o wide | grep glusterfs | grep %s | " "grep -v Terminating | awk '{print $1}'") % ( gluster_pod_data["pod_hostname"]) for w in Waiter(600, 15): new_gluster_pod_name = self.cmd_run(cmd) if new_gluster_pod_name: break if w.expired: error_msg = "exceeded timeout, new gluster pod not created" g.log.error(error_msg) raise AssertionError(error_msg) g.log.info("new gluster pod name is %s" % new_gluster_pod_name) wait_for_pod_be_ready(self.node, new_gluster_pod_name) else: pvc_hosting_node_ip = get_gluster_host_ips_by_pvc_name( self.node, pvc_name)[0] heketi_nodes = heketi_node_list(self.heketi_client_node, self.heketi_server_url) node_ip_for_reboot = None for heketi_node in heketi_nodes: heketi_node_ip = heketi_node_info( self.heketi_client_node, self.heketi_server_url, heketi_node, json=True)["hostnames"]["storage"][0] if heketi_node_ip == pvc_hosting_node_ip: node_ip_for_reboot = heketi_node_ip break if not node_ip_for_reboot: raise AssertionError( "Gluster node IP %s not matched with heketi node %s" % (pvc_hosting_node_ip, heketi_node_ip)) node_reboot_by_command(node_ip_for_reboot) # Check that async IO was not interrupted ret, out, err = async_io.async_communicate() self.assertEqual(ret, 0, "IO %s failed on %s" % (io_cmd, self.node))
def test_dynamic_provisioning_glusterfile_heketipod_failure(self): """Validate dynamic provisioning for gluster file when heketi pod down """ mount_path = "/mnt" datafile_path = '%s/fake_file_for_%s' % (mount_path, self.id()) # Create secret and storage class sc_name = self.create_storage_class() # Create PVC app_1_pvc_name = self.create_and_wait_for_pvc( pvc_name_prefix="autotest-file", sc_name=sc_name) # Create app POD with attached volume app_1_pod_name = oc_create_tiny_pod_with_volume( self.node, app_1_pvc_name, "test-pvc-mount-on-app-pod", mount_path=mount_path, image=self.io_container_image_cirros) self.addCleanup(wait_for_resource_absence, self.node, 'pod', app_1_pod_name) self.addCleanup(oc_delete, self.node, 'pod', app_1_pod_name) # Wait for app POD be up and running wait_for_pod_be_ready(self.node, app_1_pod_name, timeout=60, wait_step=2) # Write data to the app POD write_data_cmd = ("dd if=/dev/urandom of=%s bs=1K count=100" % datafile_path) ret, out, err = oc_rsh(self.node, app_1_pod_name, write_data_cmd) self.assertEqual( ret, 0, "Failed to execute command %s on %s" % (write_data_cmd, self.node)) # Remove Heketi pod heketi_down_cmd = "oc scale --replicas=0 dc/%s --namespace %s" % ( self.heketi_dc_name, self.storage_project_name) heketi_up_cmd = "oc scale --replicas=1 dc/%s --namespace %s" % ( self.heketi_dc_name, self.storage_project_name) self.addCleanup(self.cmd_run, heketi_up_cmd) heketi_pod_name = get_pod_name_from_dc(self.node, self.heketi_dc_name, timeout=10, wait_step=3) self.cmd_run(heketi_down_cmd) wait_for_resource_absence(self.node, 'pod', heketi_pod_name) app_2_pvc_name = oc_create_pvc(self.node, pvc_name_prefix="autotest-file2", sc_name=sc_name) self.addCleanup(wait_for_resource_absence, self.node, 'pvc', app_2_pvc_name) self.addCleanup(oc_delete, self.node, 'pvc', app_2_pvc_name, raise_on_absence=False) # Create second app POD app_2_pod_name = oc_create_tiny_pod_with_volume( self.node, app_2_pvc_name, "test-pvc-mount-on-app-pod", mount_path=mount_path, image=self.io_container_image_cirros) self.addCleanup(wait_for_resource_absence, self.node, 'pod', app_2_pod_name) self.addCleanup(oc_delete, self.node, 'pod', app_2_pod_name) # Bring Heketi POD back self.cmd_run(heketi_up_cmd) # Wait for Heketi POD be up and running new_heketi_pod_name = get_pod_name_from_dc(self.node, self.heketi_dc_name, timeout=10, wait_step=2) wait_for_pod_be_ready(self.node, new_heketi_pod_name, wait_step=5, timeout=120) # Wait for second PVC and app POD be ready verify_pvc_status_is_bound(self.node, app_2_pvc_name) wait_for_pod_be_ready(self.node, app_2_pod_name, timeout=60, wait_step=2) # Verify that we are able to write data ret, out, err = oc_rsh(self.node, app_2_pod_name, write_data_cmd) self.assertEqual( ret, 0, "Failed to execute command %s on %s" % (write_data_cmd, self.node))
def test_dynamic_provisioning_glusterfile_heketipod_failure(self): """Validate dynamic provisioning for gluster file when heketi pod down """ mount_path = "/mnt" datafile_path = '%s/fake_file_for_%s' % (mount_path, self.id()) # Create secret and storage class sc_name = self.create_storage_class() # Create PVC app_1_pvc_name = self.create_and_wait_for_pvc( pvc_name_prefix="autotest-file", sc_name=sc_name ) # Create app POD with attached volume app_1_pod_name = oc_create_tiny_pod_with_volume( self.node, app_1_pvc_name, "test-pvc-mount-on-app-pod", mount_path=mount_path) self.addCleanup( wait_for_resource_absence, self.node, 'pod', app_1_pod_name) self.addCleanup(oc_delete, self.node, 'pod', app_1_pod_name) # Wait for app POD be up and running wait_for_pod_be_ready( self.node, app_1_pod_name, timeout=60, wait_step=2) # Write data to the app POD write_data_cmd = ( "dd if=/dev/urandom of=%s bs=1K count=100" % datafile_path) ret, out, err = oc_rsh(self.node, app_1_pod_name, write_data_cmd) self.assertEqual( ret, 0, "Failed to execute command %s on %s" % (write_data_cmd, self.node)) # Remove Heketi pod heketi_down_cmd = "oc scale --replicas=0 dc/%s --namespace %s" % ( self.heketi_dc_name, self.storage_project_name) heketi_up_cmd = "oc scale --replicas=1 dc/%s --namespace %s" % ( self.heketi_dc_name, self.storage_project_name) self.addCleanup(self.cmd_run, heketi_up_cmd) heketi_pod_name = get_pod_name_from_dc( self.node, self.heketi_dc_name, timeout=10, wait_step=3) self.cmd_run(heketi_down_cmd) wait_for_resource_absence(self.node, 'pod', heketi_pod_name) app_2_pvc_name = oc_create_pvc( self.node, pvc_name_prefix="autotest-file2", sc_name=sc_name ) self.addCleanup( wait_for_resource_absence, self.node, 'pvc', app_2_pvc_name) self.addCleanup( oc_delete, self.node, 'pvc', app_2_pvc_name, raise_on_absence=False ) # Create second app POD app_2_pod_name = oc_create_tiny_pod_with_volume( self.node, app_2_pvc_name, "test-pvc-mount-on-app-pod", mount_path=mount_path) self.addCleanup( wait_for_resource_absence, self.node, 'pod', app_2_pod_name) self.addCleanup(oc_delete, self.node, 'pod', app_2_pod_name) # Bring Heketi POD back self.cmd_run(heketi_up_cmd) # Wait for Heketi POD be up and running new_heketi_pod_name = get_pod_name_from_dc( self.node, self.heketi_dc_name, timeout=10, wait_step=2) wait_for_pod_be_ready( self.node, new_heketi_pod_name, wait_step=5, timeout=120) # Wait for second PVC and app POD be ready verify_pvc_status_is_bound(self.node, app_2_pvc_name) wait_for_pod_be_ready( self.node, app_2_pod_name, timeout=60, wait_step=2) # Verify that we are able to write data ret, out, err = oc_rsh(self.node, app_2_pod_name, write_data_cmd) self.assertEqual( ret, 0, "Failed to execute command %s on %s" % (write_data_cmd, self.node))
def test_prometheus_volume_metrics_on_node_reboot(self): """Validate volume metrics using prometheus before and after node reboot""" # Pod name for the entire test prefix = "autotest-{}".format(utils.get_random_str()) # Create I/O pod with PVC pvc_name = self.create_and_wait_for_pvc() pod_name = openshift_ops.oc_create_tiny_pod_with_volume( self._master, pvc_name, prefix, image=self.io_container_image_cirros) self.addCleanup(openshift_ops.oc_delete, self._master, 'pod', pod_name, raise_on_absence=False) openshift_ops.wait_for_pod_be_ready( self._master, pod_name, timeout=60, wait_step=5) # Write data on the volume and wait for 2 mins and sleep is must for # prometheus to get the exact values of the metrics ret, _, err = openshift_ops.oc_rsh( self._master, pod_name, "touch /mnt/file{1..1000}") self.assertEqual( ret, 0, "Failed to create files in the app pod " "with {}".format(err)) time.sleep(120) # Fetch the metrics and store in initial_metrics as dictionary initial_metrics = self._get_and_manipulate_metric_data( self.metrics, pvc_name) openshift_ops.switch_oc_project( self._master, self.storage_project_name) # Get the hostname to reboot where the pod is running pod_info = openshift_ops.oc_get_pods(self._master, name=pod_name) node_for_reboot = pod_info[pod_name]['node'] # Get the vm name by the hostname vm_name = node_ops.find_vm_name_by_ip_or_hostname(node_for_reboot) # power off and on the vm, based on the vm type(either gluster or not) if node_for_reboot in self.gluster_servers: self.power_off_gluster_node_vm(vm_name, node_for_reboot) self.power_on_gluster_node_vm(vm_name, node_for_reboot) else: self.power_off_vm(vm_name) self.power_on_vm(vm_name) openshift_ops.wait_for_ocp_node_be_ready( self._master, node_for_reboot) # Create the new pod and validate the prometheus metrics pod_name = openshift_ops.oc_create_tiny_pod_with_volume( self._master, pvc_name, prefix) self.addCleanup(openshift_ops.oc_delete, self._master, 'pod', pod_name) # Wait for POD be up and running and prometheus to refresh the data openshift_ops.wait_for_pod_be_ready( self._master, pod_name, timeout=60, wait_step=5) time.sleep(120) # Fetching the metrics and storing in final_metrics as dictionary and # validating with initial_metrics final_metrics = self._get_and_manipulate_metric_data( self.metrics, pvc_name) self.assertEqual(dict(initial_metrics), dict(final_metrics), "Metrics are different post node reboot")
def test_arbiter_pvc_mount_on_pod(self): """Validate new volume creation using app pod""" # Create sc with gluster arbiter info self.create_storage_class(is_arbiter_vol=True) # Create PVC and wait for it to be in 'Bound' state self.create_and_wait_for_pvc() # Create POD with attached volume mount_path = "/mnt" pod_name = oc_create_tiny_pod_with_volume( self.node, self.pvc_name, "test-arbiter-pvc-mount-on-app-pod", mount_path=mount_path) self.addCleanup(oc_delete, self.node, 'pod', pod_name) # Wait for POD be up and running wait_for_pod_be_ready(self.node, pod_name, timeout=60, wait_step=2) # Get volume ID vol_info = get_gluster_vol_info_by_pvc_name(self.node, self.pvc_name) vol_id = vol_info["gluster_vol_id"] # Verify that POD has volume mounted on it cmd = "oc exec {0} -- df -PT {1} | grep {1}".format( pod_name, mount_path) out = self.cmd_run(cmd) err_msg = ("Failed to get info about mounted '%s' volume. " "Output is empty." % vol_id) self.assertTrue(out, err_msg) # Verify volume data on POD # Filesystem Type Size Used Avail Cap Mounted on # IP:vol_id fuse.glusterfs 1038336 33408 1004928 3% /mnt data = [s for s in out.strip().split(' ') if s] actual_vol_id = data[0].split(':')[-1] self.assertEqual( vol_id, actual_vol_id, "Volume ID does not match: expected is " "'%s' and actual is '%s'." % (vol_id, actual_vol_id)) self.assertIn( "gluster", data[1], "Filesystem type is expected to be of 'glusterfs' type. " "Actual value is '%s'." % data[1]) self.assertEqual( mount_path, data[6], "Unexpected mount path. Expected is '%s' and actual is '%s'." % ( mount_path, data[6])) max_size = 1024 ** 2 total_size = int(data[2]) self.assertLessEqual( total_size, max_size, "Volume has bigger size '%s' than expected - '%s'." % ( total_size, max_size)) min_available_size = int(max_size * 0.93) available_size = int(data[4]) self.assertLessEqual( min_available_size, available_size, "Minimum available size (%s) not satisfied. Actual is '%s'." % ( min_available_size, available_size)) # Write data on mounted volume write_data_cmd = ( "dd if=/dev/zero of=%s/file$i bs=%s count=1; " % ( mount_path, available_size)) self.cmd_run(write_data_cmd)