def test_promethoues_pods_and_pvcs(self): """Validate prometheus pods and PVC""" # Wait for PVCs to be bound pod_names, pvc_names = self._get_pod_names_and_pvc_names() openshift_ops.wait_for_pvcs_be_bound(self._master, pvc_names) # Validate that there should be no or zero pods in non-running state field_selector, pod_count = "status.phase!=Running", 0 openshift_ops.wait_for_pods_be_ready( self._master, pod_count, field_selector=field_selector) # Validate iscsi and multipath for (pvc_name, pod_name) in zip(pvc_names, pod_names): self.verify_iscsi_sessions_and_multipath( pvc_name, pod_name[0], rtype='pod', heketi_server_url=self._registry_heketi_server_url, is_registry_gluster=True) # Try to fetch metric from prometheus pod self._fetch_metric_from_promtheus_pod(metric='kube_node_info')
def create_and_wait_for_pvcs( self, pvc_size=1, pvc_name_prefix="autotests-pvc", pvc_amount=1, sc_name=None, timeout=600, wait_step=10, skip_waiting=False, skip_cleanup=False): """Create multiple PVC's not waiting for it Args: pvc_size (int): size of PVC, default value is 1 pvc_name_prefix (str): volume prefix for each PVC, default value is 'autotests-pvc' pvc_amount (int): number of PVC's, default value is 1 sc_name (str): storage class to create PVC, default value is None, which will cause automatic creation of sc. timeout (int): timeout time for waiting for PVC's to get bound wait_step (int): waiting time between each try of PVC status check skip_waiting (bool): boolean value which defines whether we need to wait for PVC creation or not. Returns: List: list of PVC names """ node = self.ocp_client[0] # Create storage class if not specified if not sc_name: if getattr(self, "sc_name", ""): sc_name = self.sc_name else: sc_name = self.create_storage_class(skip_cleanup=skip_cleanup) # Create PVCs pvc_names = [] for i in range(pvc_amount): pvc_name = oc_create_pvc( node, sc_name, pvc_name_prefix=pvc_name_prefix, pvc_size=pvc_size) pvc_names.append(pvc_name) if not skip_cleanup: self.addCleanup( wait_for_resources_absence, node, 'pvc', pvc_names) # Wait for PVCs to be in bound state try: if not skip_waiting: wait_for_pvcs_be_bound(node, pvc_names, timeout, wait_step) finally: if skip_cleanup: return pvc_names if get_openshift_version() < "3.9": reclaim_policy = "Delete" else: reclaim_policy = oc_get_custom_resource( node, 'sc', ':.reclaimPolicy', sc_name)[0] for pvc_name in pvc_names: if reclaim_policy == 'Retain': pv_name = get_pv_name_from_pvc(node, pvc_name) if not pv_name and skip_waiting: continue self.addCleanup(oc_delete, node, 'pv', pv_name, raise_on_absence=False) custom = (r':.metadata.annotations."gluster\.kubernetes' r'\.io\/heketi\-volume\-id"') vol_id = oc_get_custom_resource( node, 'pv', custom, pv_name)[0] if self.sc.get('provisioner') == "kubernetes.io/glusterfs": self.addCleanup(heketi_volume_delete, self.heketi_client_node, self.heketi_server_url, vol_id, raise_on_error=False) else: self.addCleanup(heketi_blockvolume_delete, self.heketi_client_node, self.heketi_server_url, vol_id, raise_on_error=False) self.addCleanup(oc_delete, node, 'pvc', pvc_name, raise_on_absence=False) return pvc_names
def test_targetcli_failure_during_block_pvc_creation(self): h_node, h_server = self.heketi_client_node, self.heketi_server_url # Disable redundant nodes and leave just 3 nodes online h_node_id_list = heketi_node_list(h_node, h_server) self.assertGreater(len(h_node_id_list), 2) for node_id in h_node_id_list[3:]: heketi_node_disable(h_node, h_server, node_id) self.addCleanup(heketi_node_enable, h_node, h_server, node_id) # Gather info about the Gluster node we are going to use for killing # targetcli processes. chosen_g_node_id = h_node_id_list[0] chosen_g_node_info = heketi_node_info(h_node, h_server, chosen_g_node_id, json=True) chosen_g_node_ip = chosen_g_node_info['hostnames']['storage'][0] chosen_g_node_hostname = chosen_g_node_info['hostnames']['manage'][0] chosen_g_node_ip_and_hostname = set( (chosen_g_node_ip, chosen_g_node_hostname)) g_pods = oc_get_custom_resource( self.node, 'pod', [ ':.metadata.name', ':.status.hostIP', ':.status.podIP', ':.spec.nodeName' ], selector='glusterfs-node=pod') if g_pods and g_pods[0]: for g_pod in g_pods: if chosen_g_node_ip_and_hostname.intersection(set(g_pod[1:])): host_to_run_cmds = self.node g_pod_prefix, g_pod = 'oc exec %s -- ' % g_pod[0], g_pod[0] break else: err_msg = ( 'Failed to find Gluster pod filtering it by following IPs ' 'and hostnames: %s\nFound following Gluster pods: %s') % ( chosen_g_node_ip_and_hostname, g_pods) g.log.error(err_msg) raise AssertionError(err_msg) else: host_to_run_cmds, g_pod_prefix, g_pod = chosen_g_node_ip, '', '' # Schedule deletion of targetcli process file_for_bkp, pvc_number = "~/.targetcli/prefs.bin", 10 self.cmd_run("%scp %s %s_backup" % (g_pod_prefix, file_for_bkp, file_for_bkp), hostname=host_to_run_cmds) self.addCleanup(self.cmd_run, "%srm -f %s_backup" % (g_pod_prefix, file_for_bkp), hostname=host_to_run_cmds) kill_targetcli_services_cmd = ( "while true; do " " %spkill targetcli || echo 'failed to kill targetcli process'; " "done" % g_pod_prefix) loop_for_killing_targetcli_process = g.run_async( host_to_run_cmds, kill_targetcli_services_cmd, "root") try: # Create bunch of PVCs sc_name, pvc_names = self.create_storage_class(), [] for i in range(pvc_number): pvc_names.append(oc_create_pvc(self.node, sc_name, pvc_size=1)) self.addCleanup(wait_for_resources_absence, self.node, 'pvc', pvc_names) self.addCleanup(oc_delete, self.node, 'pvc', ' '.join(pvc_names)) # Check that we get expected number of provisioning errors timeout, wait_step, succeeded_pvcs, failed_pvcs = 120, 1, [], [] _waiter, err_msg = Waiter(timeout=timeout, interval=wait_step), "" for pvc_name in pvc_names: _waiter._attempt = 0 for w in _waiter: events = get_events(self.node, pvc_name, obj_type="PersistentVolumeClaim") for event in events: if event['reason'] == 'ProvisioningSucceeded': succeeded_pvcs.append(pvc_name) break elif event['reason'] == 'ProvisioningFailed': failed_pvcs.append(pvc_name) break else: continue break if w.expired: err_msg = ( "Failed to get neither 'ProvisioningSucceeded' nor " "'ProvisioningFailed' statuses for all the PVCs in " "time. Timeout was %ss, interval was %ss." % (timeout, wait_step)) g.log.error(err_msg) raise AssertionError(err_msg) self.assertGreater(len(failed_pvcs), len(succeeded_pvcs)) finally: # Restore targetcli workability loop_for_killing_targetcli_process._proc.terminate() # Revert breakage back which can be caused by BZ-1769426 check_bkp_file_size_cmd = ("%sls -lah %s | awk '{print $5}'" % (g_pod_prefix, file_for_bkp)) bkp_file_size = self.cmd_run(check_bkp_file_size_cmd, hostname=host_to_run_cmds).strip() if bkp_file_size == "0": self.cmd_run("%smv %s_backup %s" % (g_pod_prefix, file_for_bkp, file_for_bkp), hostname=host_to_run_cmds) breakage_err_msg = ( "File located at '%s' was corrupted (zero size) on the " "%s. Looks like BZ-1769426 took effect. \n" "Don't worry, it has been restored after test failure." % (file_for_bkp, "'%s' Gluster pod" % g_pod if g_pod else "'%s' Gluster node" % chosen_g_node_ip)) g.log.error(breakage_err_msg) if err_msg: breakage_err_msg = "%s\n%s" % (err_msg, breakage_err_msg) raise AssertionError(breakage_err_msg) # Wait for all the PVCs to be in bound state wait_for_pvcs_be_bound(self.node, pvc_names, timeout=300, wait_step=5)
def test_dev_path_file_volume_delete(self): """Validate device path name changes the deletion of already existing file volumes """ pvc_size, pvc_amount = 2, 5 vol_details, pvc_names = [], [] # Create PVC's sc_name = self.create_storage_class() for i in range(0, pvc_amount): pvc_name = openshift_ops.oc_create_pvc(self.node, sc_name, pvc_size=pvc_size) pvc_names.append(pvc_name) self.addCleanup(openshift_ops.wait_for_resource_absence, self.node, 'pvc', pvc_name) self.addCleanup(openshift_ops.oc_delete, self.node, 'pvc', pvc_name, raise_on_absence=False) # Wait for PVC's to be bound openshift_ops.wait_for_pvcs_be_bound(self.node, pvc_names) # Get Volumes name and validate volumes count for pvc_name in pvc_names: pv_name = openshift_ops.get_pv_name_from_pvc(self.node, pvc_name) volume_name = openshift_ops.get_vol_names_from_pv( self.node, pv_name) vol_details.append(volume_name) # Verify file volumes count self.validate_file_volumes_count(self.h_node, self.h_server, self.node_ip) # Collect pvs info and detach disks and get pvs info pvs_info_before = openshift_storage_libs.get_pvs_info( self.node, self.node_ip, self.devices_list, raise_on_error=False) self.detach_and_attach_vmdk(self.vm_name, self.node_hostname, self.devices_list) pvs_info_after = openshift_storage_libs.get_pvs_info( self.node, self.node_ip, self.devices_list, raise_on_error=False) # Compare pvs info before and after for (path, uuid, vg_name), (_path, _uuid, _vg_name) in zip(pvs_info_before[:-1], pvs_info_after[1:]): self.assertEqual( uuid, _uuid, "pv_uuid check failed. Expected:{}," "Actual: {}".format(uuid, _uuid)) self.assertEqual( vg_name, _vg_name, "vg_name check failed. Expected:" "{}, Actual:{}".format(vg_name, _vg_name)) # Delete created PVC's for pvc_name in pvc_names: openshift_ops.oc_delete(self.node, 'pvc', pvc_name) # Wait for resource absence and get volume list openshift_ops.wait_for_resources_absence(self.node, 'pvc', pvc_names) vol_list = volume_ops.get_volume_list(self.node_ip) self.assertIsNotNone(vol_list, "Failed to get volumes list") # Validate volumes created are not present for vol in vol_details: self.assertNotIn(vol, vol_list, "Failed to delete volume {}".format(vol))