def test_validate_logging_pods_and_pvc(self): """Validate logging pods and PVC""" # Wait for kibana pod to be ready kibana_pod = openshift_ops.get_pod_name_from_dc( self._master, self._logging_kibana_dc) openshift_ops.wait_for_pod_be_ready(self._master, kibana_pod) # Wait for fluentd pods to be ready fluentd_custom = [ ":.status.desiredNumberScheduled", ":.spec.template.metadata.labels" ] count_and_selector = openshift_ops.oc_get_custom_resource( self._master, "ds", fluentd_custom, self._logging_fluentd_ds) selector = count_and_selector[1][4:].replace(":", "=") openshift_ops.wait_for_pods_be_ready(self._master, int(count_and_selector[0]), selector) # Wait for PVC to be bound and elasticsearch pod to be ready es_pod = openshift_ops.get_pod_name_from_dc(self._master, self._logging_es_dc) pvc_custom = ":.spec.volumes[*].persistentVolumeClaim.claimName" pvc_name = openshift_ops.oc_get_custom_resource( self._master, "pod", pvc_custom, es_pod)[0] openshift_ops.verify_pvc_status_is_bound(self._master, pvc_name) openshift_ops.wait_for_pod_be_ready(self._master, es_pod) # Validate iscsi and multipath self.verify_iscsi_sessions_and_multipath( pvc_name, self._logging_es_dc, heketi_server_url=self._registry_heketi_server_url, is_registry_gluster=True)
def configure_node_to_run_gluster_pod(self, storage_hostname): ports = ("24010", "3260", "111", "2222", "24007", "24008", "49152:49664") iptables_rule_pattern = ( "-p tcp -m state --state NEW -m %s --%s %s -j ACCEPT") iptables_rule_chain = "OS_FIREWALL_ALLOW" iptables_rules = [] for port in ports: if ":" in port: iptables_rules.append(iptables_rule_pattern % ("multiport", "dports", port)) else: iptables_rules.append(iptables_rule_pattern % ("tcp", "dport", port)) node_add_iptables_rules(storage_hostname, iptables_rule_chain, iptables_rules) self.addCleanup(node_delete_iptables_rules, storage_hostname, iptables_rule_chain, iptables_rules) gluster_host_label = "glusterfs=storage-host" gluster_pod_label = "glusterfs=storage-pod" oc_label(self.ocp_client[0], "node", storage_hostname, gluster_host_label) self.addCleanup(wait_for_pods_be_ready, self.ocp_client[0], len(self.gluster_servers), selector=gluster_pod_label) self.addCleanup(oc_label, self.ocp_client[0], "node", storage_hostname, "glusterfs-") wait_for_pods_be_ready(self.ocp_client[0], len(self.gluster_servers) + 1, selector=gluster_pod_label)
def _create_dcs_and_check_brick_placement(self, prefix, sc_name, heketi_zone_checking, zone_count): app_pods, count, label = [], 5, "testlabel=autotest" # Create multiple PVCs using storage class pvc_names = self.create_and_wait_for_pvcs(pvc_name_prefix=prefix, pvc_amount=count, sc_name=sc_name) # Create app dcs with I/O for pvc_name in pvc_names: app_dc = openshift_ops.oc_create_app_dc_with_io( self.node, pvc_name=pvc_name, dc_name_prefix=prefix, image=self.io_container_image_cirros) self.addCleanup(openshift_ops.oc_delete, self.node, 'dc', app_dc) # Get pod names and label them pod_name = openshift_ops.get_pod_name_from_dc(self.node, app_dc) openshift_ops.oc_label(self.node, 'pod', pod_name, label) app_pods.append(pod_name) # Wait for pods to be ready with the help of label selector openshift_ops.wait_for_pods_be_ready(self.node, count, label) # Validate brick placement in heketi zones self._validate_brick_placement_in_correct_zone_or_with_expand_pvc( heketi_zone_checking, pvc_name, zone_count) return app_pods
def test_respin_prometheus_pod(self, motive="delete"): """Validate respin of prometheus pod""" if motive == 'drain': # Get the number of infra nodes infra_node_count_cmd = ( 'oc get nodes ' '--no-headers -l node-role.kubernetes.io/infra=true|wc -l') infra_node_count = command.cmd_run( infra_node_count_cmd, self._master) # Skip test case if number infra nodes are less than #2 if int(infra_node_count) < 2: self.skipTest('Available number of infra nodes "{}", it should' ' be more than 1'.format(infra_node_count)) # Get PVC names and pod names pod_names, pvc_names = self._get_pod_names_and_pvc_names() # Validate iscsi and multipath for (pvc_name, pod_name) in zip(pvc_names, pod_names): _, _, node = self.verify_iscsi_sessions_and_multipath( pvc_name, pod_name[0], rtype='pod', heketi_server_url=self._registry_heketi_server_url, is_registry_gluster=True) # Delete the prometheus pods if motive == 'delete': for pod_name in pod_names: openshift_ops.oc_delete(self._master, 'pod', pod_name[0]) # Drain the node elif motive == 'drain': drain_cmd = ('oc adm drain {} --force=true --ignore-daemonsets ' '--delete-local-data'.format(node)) command.cmd_run(drain_cmd, hostname=self._master) # Cleanup to make node schedulable cmd_schedule = ( 'oc adm manage-node {} --schedulable=true'.format(node)) self.addCleanup( command.cmd_run, cmd_schedule, hostname=self._master) # Validate that there should be no or zero pods in non-running state field_selector, pod_count = "status.phase!=Running", 0 openshift_ops.wait_for_pods_be_ready( self._master, pod_count, field_selector=field_selector) # Validate iscsi and multipath for (pvc_name, pod_name) in zip(pvc_names, pod_names): self.verify_iscsi_sessions_and_multipath( pvc_name, pod_name[0], rtype='pod', heketi_server_url=self._registry_heketi_server_url, is_registry_gluster=True) # Try to fetch metric from prometheus pod self._fetch_metric_from_promtheus_pod(metric='kube_node_info')
def _wait_for_gluster_pod_be_ready(self, g_pod_list_before): """Wait for the gluster pods to be in ready state""" openshift_ops.switch_oc_project( self._master, self._registry_project_name) # Check if the gluster pods are in ready state try: pod_count = len(self._registry_servers_info.keys()) openshift_ops.wait_for_pods_be_ready( self._master, pod_count, "glusterfs-node=pod", timeout=120, wait_step=6) except exceptions.ExecutionError: self._guster_pod_delete(g_pod_list_before)
def test_docker_service_restart(self): """Validate docker service should not fail after restart""" # Skip the TC if independent mode deployment if not self.is_containerized_gluster(): self.skipTest( "Skipping this test case as LVM script is not available in " "independent mode deployment") # Skip the TC if docker storage driver other than devicemapper pod_host_ip = self.pod_name[0]["pod_host_ip"] cmd = "docker info -f '{{json .Driver}}'" device_driver = command.cmd_run(cmd, pod_host_ip) if device_driver != '"devicemapper"': self.skipTest( "Skipping this test case as docker storage driver is not " "set to devicemapper") # Validate LVM environment is present custom = (r'":spec.containers[*].env[?(@.name==\"{}\")]' r'.value"'.format(ENV_NAME)) env_var_value = openshift_ops.oc_get_custom_resource( self.oc_node, "pod", custom, self.h_pod_name)[0] err_msg = "Heketi {} environment should has {}".format( ENV_NAME, ENV_VALUE) self.assertEqual(env_var_value, ENV_VALUE, err_msg) # Check docker status is active command.cmd_run(DOCKER_SERVICE.format("is-active"), pod_host_ip) # Restart the docker service self.addCleanup(self._check_docker_status_is_active, pod_host_ip) command.cmd_run(DOCKER_SERVICE.format("restart"), pod_host_ip) # Wait for docker service to become active self._wait_for_docker_service_status(pod_host_ip, "active", "running") # Wait for glusterfs pods to be ready openshift_ops.wait_for_pods_be_ready(self.oc_node, len(self.gluster_servers), "glusterfs=storage-pod") # Check the docker pool is available after docker restart cmd = "ls -lrt /dev/docker-vg/docker-pool" command.cmd_run(cmd, pod_host_ip) # Create PVC after docker restart self.create_and_wait_for_pvcs()
def test_verify_metrics_data_during_gluster_pod_respin(self): # Add check for CRS version switch_oc_project(self.master, self.registry_project_name) if not self.is_containerized_gluster(): self.skipTest("Skipping this test case as CRS version check " "can not be implemented") # Verify multipath and iscsi for cassandra pod switch_oc_project(self.master, self.metrics_project_name) hawkular_cassandra, pvc_name, iqn, _, node = ( self.verify_cassandra_pod_multipath_and_iscsi()) # Get the ip of active path device_and_ip = get_iscsi_block_devices_by_path(node, iqn) mpath = get_mpath_name_from_device_name(node, list(device_and_ip.keys())[0]) active_passive_dict = get_active_and_enabled_devices_from_mpath( node, mpath) node_ip = device_and_ip[active_passive_dict['active'][0]] # Get the name of gluster pod from the ip switch_oc_project(self.master, self.registry_project_name) gluster_pods = get_ocp_gluster_pod_details(self.master) pod_name = list( filter(lambda pod: (pod["pod_host_ip"] == node_ip), gluster_pods))[0]["pod_name"] err_msg = "Failed to get the gluster pod name {} with active path" self.assertTrue(pod_name, err_msg.format(pod_name)) # Delete the pod oc_delete(self.master, 'pod', pod_name) wait_for_resource_absence(self.master, 'pod', pod_name) # Wait for new pod to come up pod_count = len(self.registry_servers_info.keys()) selector = "glusterfs-node=pod" wait_for_pods_be_ready(self.master, pod_count, selector) # Validate cassandra pod state, multipath and issci switch_oc_project(self.master, self.metrics_project_name) wait_for_pod_be_ready(self.master, hawkular_cassandra, timeout=2) self.verify_iscsi_sessions_and_multipath( pvc_name, self.metrics_rc_hawkular_cassandra, rtype='rc', heketi_server_url=self.registry_heketi_server_url, is_registry_gluster=True)
def test_promethoues_pods_and_pvcs(self): """Validate prometheus pods and PVC""" # Wait for PVCs to be bound pod_names, pvc_names = self._get_pod_names_and_pvc_names() openshift_ops.wait_for_pvcs_be_bound(self._master, pvc_names) # Validate that there should be no or zero pods in non-running state field_selector, pod_count = "status.phase!=Running", 0 openshift_ops.wait_for_pods_be_ready( self._master, pod_count, field_selector=field_selector) # Validate iscsi and multipath for (pvc_name, pod_name) in zip(pvc_names, pod_names): self.verify_iscsi_sessions_and_multipath( pvc_name, pod_name[0], rtype='pod', heketi_server_url=self._registry_heketi_server_url, is_registry_gluster=True) # Try to fetch metric from prometheus pod self._fetch_metric_from_promtheus_pod(metric='kube_node_info')