def _get_newly_deployed_gluster_pod(self, g_pod_list_before): # Fetch pod after delete g_pod_list_after = [ pod["pod_name"] for pod in openshift_ops.get_ocp_gluster_pod_details(self._master)] # Fetch the new gluster pod g_new_pod = list(set(g_pod_list_after) - set(g_pod_list_before)) self.assertTrue(g_new_pod, "No new gluster pod deployed after delete") return g_new_pod
def run(target, command, user=None, log_level=None, orig_run=g.run): """Function that runs a command on a host or in a pod via a host. Wraps glusto's run function. Args: target (str|Pod): If target is str object and it equals to 'auto_get_gluster_endpoint', then Gluster endpoint gets autocalculated to be any of Gluster PODs or nodes depending on the deployment type of a Gluster cluster. If it is str object with other value, then it is considered to be an endpoint for command. If 'target' is of the 'Pod' type, then command will run on the specified POD. command (str|list): Command to run. user (str|None): user to be passed on to glusto's run method log_level (str|None): log level to be passed on to glusto's run method orig_run (function): The default implementation of the run method. Will be used when target is not a pod. Returns: A tuple of the command's return code, stdout, and stderr. """ # NOTE: orig_run captures the glusto run method at function # definition time in order to capture the method before # any additional monkeypatching by other code ocp_client_node = list(g.config['ocp_servers']['client'].keys())[0] with mock.patch.object(g, 'run', new=orig_run): gluster_pods = openshift_ops.get_ocp_gluster_pod_details( ocp_client_node) if target == 'auto_get_gluster_endpoint': if gluster_pods: target = Pod(ocp_client_node, gluster_pods[0]["pod_name"]) else: target = list(g.config.get("gluster_servers", {}).keys())[0] elif not isinstance(target, Pod) and gluster_pods: for g_pod in gluster_pods: if target in (g_pod['pod_host_ip'], g_pod['pod_hostname']): target = Pod(ocp_client_node, g_pod['pod_name']) break if isinstance(target, Pod): prefix = ['oc', 'rsh', target.podname] if isinstance(command, six.string_types): cmd = ' '.join(prefix + [command]) else: cmd = prefix + command # unpack the tuple to make sure our return value exactly matches # our docstring return g.run(target.node, cmd, user=user, log_level=log_level) else: return orig_run(target, command, user=user, log_level=log_level)
def test_verify_metrics_data_during_gluster_pod_respin(self): # Add check for CRS version switch_oc_project(self.master, self.registry_project_name) if not self.is_containerized_gluster(): self.skipTest("Skipping this test case as CRS version check " "can not be implemented") # Verify multipath and iscsi for cassandra pod switch_oc_project(self.master, self.metrics_project_name) hawkular_cassandra, pvc_name, iqn, _, node = ( self.verify_cassandra_pod_multipath_and_iscsi()) # Get the ip of active path device_and_ip = get_iscsi_block_devices_by_path(node, iqn) mpath = get_mpath_name_from_device_name(node, list(device_and_ip.keys())[0]) active_passive_dict = get_active_and_enabled_devices_from_mpath( node, mpath) node_ip = device_and_ip[active_passive_dict['active'][0]] # Get the name of gluster pod from the ip switch_oc_project(self.master, self.registry_project_name) gluster_pods = get_ocp_gluster_pod_details(self.master) pod_name = list( filter(lambda pod: (pod["pod_host_ip"] == node_ip), gluster_pods))[0]["pod_name"] err_msg = "Failed to get the gluster pod name {} with active path" self.assertTrue(pod_name, err_msg.format(pod_name)) # Delete the pod oc_delete(self.master, 'pod', pod_name) wait_for_resource_absence(self.master, 'pod', pod_name) # Wait for new pod to come up pod_count = len(self.registry_servers_info.keys()) selector = "glusterfs-node=pod" wait_for_pods_be_ready(self.master, pod_count, selector) # Validate cassandra pod state, multipath and issci switch_oc_project(self.master, self.metrics_project_name) wait_for_pod_be_ready(self.master, hawkular_cassandra, timeout=2) self.verify_iscsi_sessions_and_multipath( pvc_name, self.metrics_rc_hawkular_cassandra, rtype='rc', heketi_server_url=self.registry_heketi_server_url, is_registry_gluster=True)
def test_resping_gluster_pod(self): """Validate gluster pod restart with no disruption to elasticsearch pod """ restart_custom = ":status.containerStatuses[0].restartCount" # Fetch pod and validate iscsi and multipath es_pod, _ = self._get_es_pod_and_verify_iscsi_sessions() # Fetch the restart count for the es pod restart_count_before = openshift_ops.oc_get_custom_resource( self._master, "pod", restart_custom, es_pod)[0] # Switch to gluster project openshift_ops.switch_oc_project(self._master, self._registry_project_name) # Fetch the gluster pod list before g_pod_list_before = [ pod["pod_name"] for pod in openshift_ops.get_ocp_gluster_pod_details(self._master) ] # Respin a gluster pod openshift_ops.oc_delete(self._master, "pod", g_pod_list_before[0]) self.addCleanup(self._guster_pod_delete_cleanup, g_pod_list_before) # Wait for pod to get absent openshift_ops.wait_for_resource_absence(self._master, "pod", g_pod_list_before[0]) # Fetch gluster pod after delete g_new_pod = self._get_newly_deployed_gluster_pod(g_pod_list_before) openshift_ops.wait_for_pod_be_ready(self._master, g_new_pod[0]) # Switch to logging project openshift_ops.switch_oc_project(self._master, self._logging_project_name) # Fetch the restart count for the es pod restart_count_after = openshift_ops.oc_get_custom_resource( self._master, "pod", restart_custom, es_pod)[0] self.assertEqual( restart_count_before, restart_count_after, "Failed disruption to es pod found expecting restart count before" " {} and after {} for es pod to be equal after gluster pod" " respin".format(restart_count_before, restart_count_after))
def setUp(self): super(TestNodeRestart, self).setUp() self.oc_node = self.ocp_master_node[0] self.gluster_pod_list = [ pod["pod_name"] for pod in get_ocp_gluster_pod_details(self.oc_node)] if not self.gluster_pod_list: self.skipTest("Standalone Gluster is not supported by this test.") self.gluster_pod_name = self.gluster_pod_list[0] if get_openshift_storage_version() < "3.11.1": self.skipTest("Skipping this test case due to bug BZ-1635736") self.sc_name = self.create_storage_class() self.pvc_names = self._create_volumes_with_io(3)
def setUp(self): super(TestHeketiLvmWrapper, self).setUp() self.oc_node = self.ocp_master_node[0] self.pod_name = openshift_ops.get_ocp_gluster_pod_details(self.oc_node) self.h_pod_name = openshift_ops.get_pod_name_from_dc( self.oc_node, self.heketi_dc_name) self.volume_size = 2 ocp_version = openshift_version.get_openshift_version() if ocp_version < "3.11.170": self.skipTest("Heketi LVM Wrapper functionality does not " "support on OCP {}".format(ocp_version.v_str)) h_version = heketi_version.get_heketi_version(self.heketi_client_node) if h_version < '9.0.0-9': self.skipTest("heketi-client package {} does not support Heketi " "LVM Wrapper functionality".format(h_version.v_str))
def _get_gluster_cmd(target, command): if isinstance(command, six.string_types): command = [command] ocp_client_node = list(g.config['ocp_servers']['client'].keys())[0] gluster_pods = get_ocp_gluster_pod_details(ocp_client_node) if target == 'auto_get_gluster_endpoint': if gluster_pods: target = podcmd.Pod(ocp_client_node, gluster_pods[0]["pod_name"]) else: target = list(g.config.get("gluster_servers", {}).keys())[0] elif not isinstance(target, podcmd.Pod) and gluster_pods: for g_pod in gluster_pods: if target in (g_pod['pod_host_ip'], g_pod['pod_hostname']): target = podcmd.Pod(ocp_client_node, g_pod['pod_name']) break if isinstance(target, podcmd.Pod): return target.node, ' '.join(['oc', 'rsh', target.podname] + command) return target, ' '.join(command)
def reboot_gluster_node_and_wait_for_services(self): gluster_node_ip = ( g.config["gluster_servers"][self.gluster_servers[0]]["storage"]) gluster_pod = list( filter(lambda pod: (pod["pod_host_ip"] == gluster_node_ip), get_ocp_gluster_pod_details(self.oc_node))) if not gluster_pod: raise ExecutionError("Gluster pod Host IP '%s' not matched." % gluster_node_ip) gluster_pod = gluster_pod[0]["pod_name"] self.addCleanup(wait_for_pod_be_ready, self.oc_node, gluster_pod) node_reboot_by_command(gluster_node_ip, timeout=600, wait_step=10) # wait for the gluster pod to be in 'Running' state wait_for_pod_be_ready(self.oc_node, gluster_pod) # glusterd and gluster-blockd service should be up and running services = (("glusterd", "running"), ("gluster-blockd", "running"), ("tcmu-runner", "running"), ("gluster-block-target", "exited")) for service, state in services: check_service_status_on_pod(self.oc_node, gluster_pod, service, "active", state)
def test_restart_prometheus_glusterfs_pod(self): """Validate restarting glusterfs pod""" # Add check for CRS version openshift_ops.switch_oc_project( self._master, self._registry_project_name) if not self.is_containerized_gluster(): self.skipTest( "Skipping this test case as CRS version check " "can not be implemented") # Get one of the prometheus pod name and respective pvc name openshift_ops.switch_oc_project( self._master, self._prometheus_project_name) prometheus_pods = openshift_ops.oc_get_pods( self._master, selector=self._prometheus_resources_selector) if not prometheus_pods: self.skipTest( prometheus_pods, "Skipping test as prometheus" " pod is not present") prometheus_pod = list(prometheus_pods.keys())[0] pvc_name = openshift_ops.oc_get_custom_resource( self._master, "pod", ":.spec.volumes[*].persistentVolumeClaim.claimName", prometheus_pod)[0] self.assertTrue( pvc_name, "Failed to get pvc name from {} pod".format(prometheus_pod)) iqn, _, node = self.verify_iscsi_sessions_and_multipath( pvc_name, prometheus_pod, rtype='pod', heketi_server_url=self._registry_heketi_server_url, is_registry_gluster=True) # Get the ip of active path devices = openshift_storage_libs.get_iscsi_block_devices_by_path( node, iqn) mpath = openshift_storage_libs.get_mpath_name_from_device_name( node, list(devices.keys())[0]) mpath_dev = ( openshift_storage_libs.get_active_and_enabled_devices_from_mpath( node, mpath)) node_ip = devices[mpath_dev['active'][0]] # Get the name of gluster pod from the ip openshift_ops.switch_oc_project( self._master, self._registry_project_name) gluster_pods = openshift_ops.get_ocp_gluster_pod_details( self._master) active_pod_name = list( filter(lambda pod: (pod["pod_host_ip"] == node_ip), gluster_pods) )[0]["pod_name"] err_msg = "Failed to get the gluster pod name {} with active path" self.assertTrue(active_pod_name, err_msg.format(active_pod_name)) g_pods = [pod['pod_name'] for pod in gluster_pods] g_pods.remove(active_pod_name) pod_list = [active_pod_name, g_pods[0]] for pod_name in pod_list: # Delete the glusterfs pods openshift_ops.switch_oc_project( self._master, self._prometheus_project_name) self._fetch_metric_from_promtheus_pod( metric='heketi_device_brick_count') openshift_ops.switch_oc_project( self._master, self._registry_project_name) g_pod_list_before = [ pod["pod_name"] for pod in openshift_ops.get_ocp_gluster_pod_details( self._master)] openshift_ops.oc_delete(self._master, 'pod', pod_name) self.addCleanup( self._guster_pod_delete, g_pod_list_before) # Wait for gluster pod to be absent openshift_ops.wait_for_resource_absence( self._master, 'pod', pod_name) # Try to fetch metric from prometheus pod openshift_ops.switch_oc_project( self._master, self._prometheus_project_name) self._fetch_metric_from_promtheus_pod( metric='heketi_device_brick_count') # Wait for new pod to come up openshift_ops.switch_oc_project( self._master, self._registry_project_name) self.assertTrue(self._get_newly_deployed_gluster_pod( g_pod_list_before), "Failed to get new pod") self._wait_for_gluster_pod_be_ready(g_pod_list_before) # Validate iscsi and multipath openshift_ops.switch_oc_project( self._master, self._prometheus_project_name) self.verify_iscsi_sessions_and_multipath( pvc_name, prometheus_pod, rtype='pod', heketi_server_url=self._registry_heketi_server_url, is_registry_gluster=True) # Try to fetch metric from prometheus pod self._fetch_metric_from_promtheus_pod( metric='heketi_device_brick_count')
def test_heketi_node_add_with_valid_cluster(self): """Test heketi node add operation with valid cluster id""" if (openshift_storage_version.get_openshift_storage_version() < "3.11.4"): self.skipTest( "This test case is not supported for < OCS 3.11.4 builds due " "to bug BZ-1732831") h_client, h_server = self.heketi_client_node, self.heketi_server_url ocp_node = self.ocp_master_node[0] # Get heketi endpoints before adding node h_volume_ids = heketi_ops.heketi_volume_list( h_client, h_server, json=True) h_endpoints_before_new_node = heketi_ops.heketi_volume_endpoint_patch( h_client, h_server, h_volume_ids["volumes"][0]) cluster_info = heketi_ops.heketi_cluster_list( h_client, h_server, json=True) storage_hostname, storage_ip = self.add_heketi_node_to_cluster( cluster_info["clusters"][0]) # Get heketi nodes and validate for newly added node h_node_ids = heketi_ops.heketi_node_list(h_client, h_server, json=True) for h_node_id in h_node_ids: node_hostname = heketi_ops.heketi_node_info( h_client, h_server, h_node_id, json=True) if node_hostname["hostnames"]["manage"][0] == storage_hostname: break node_hostname = None err_msg = ("Newly added heketi node %s not found in heketi node " "list %s" % (storage_hostname, h_node_ids)) self.assertTrue(node_hostname, err_msg) # Check gluster peer status for newly added node if self.is_containerized_gluster(): gluster_pods = openshift_ops.get_ocp_gluster_pod_details(ocp_node) gluster_pod = [ gluster_pod["pod_name"] for gluster_pod in gluster_pods if gluster_pod["pod_hostname"] == storage_hostname][0] gluster_peer_status = peer_ops.get_peer_status( podcmd.Pod(ocp_node, gluster_pod)) else: gluster_peer_status = peer_ops.get_peer_status( storage_hostname) self.assertEqual( len(gluster_peer_status), len(self.gluster_servers)) err_msg = "Expected peer status is 1 and actual is %s" for peer in gluster_peer_status: peer_status = int(peer["connected"]) self.assertEqual(peer_status, 1, err_msg % peer_status) # Get heketi endpoints after adding node h_endpoints_after_new_node = heketi_ops.heketi_volume_endpoint_patch( h_client, h_server, h_volume_ids["volumes"][0]) # Get openshift openshift endpoints and patch with heketi endpoints heketi_db_endpoint = openshift_ops.oc_get_custom_resource( ocp_node, "dc", name=self.heketi_dc_name, custom=".:spec.template.spec.volumes[*].glusterfs.endpoints")[0] openshift_ops.oc_patch( ocp_node, "ep", heketi_db_endpoint, h_endpoints_after_new_node) self.addCleanup( openshift_ops.oc_patch, ocp_node, "ep", heketi_db_endpoint, h_endpoints_before_new_node) ep_addresses = openshift_ops.oc_get_custom_resource( ocp_node, "ep", name=heketi_db_endpoint, custom=".:subsets[*].addresses[*].ip")[0].split(",") err_msg = "Hostname %s not present in endpoints %s" % ( storage_ip, ep_addresses) self.assertIn(storage_ip, ep_addresses, err_msg)