def pod_health_check(): """ Check health of all pods and get logs of failed pods """ api = kube_api() namespace_list = settings.getValue('airship_namespace_list') result = { 'category': 'platform', 'case_name': 'pod_health_check', 'criteria': 'pass', 'details': [] } for namespace in namespace_list: pod_list = api.list_namespaced_pod(namespace) for pod in pod_list.items: pod_stats = pod_status(pod) if pod_stats['criteria'] == 'fail': pod_stats['logs'] = get_logs(pod) result['criteria'] = 'fail' result['details'].append(pod_stats) store_result(result) return result
def helmv2_disabled_check(): """ Checks for helm v2 support """ result = { 'category': 'platform', 'case_name': 'helmv2_disabled_check', 'criteria': 'pass', 'details': [] } kube = kube_api() logger = logging.getLogger(__name__) res = False pod_details = kube.list_pod_for_all_namespaces() pods = pod_details.items version_support = settings.getValue( 'pdf_file')['vim_functional']['legacy_helm_support'] if 'YES' in version_support: for pod in pods: if 'tiller' in pod.metadata.name: res = True result['details'].append(pod) if res is False: result['criteria'] = 'fail' store_result(logger, result) return result
def collectd_check(): """ Checks for collectd pods present and their state of being """ api_instance = kube_api() pod_details = api_instance.list_pod_for_all_namespaces() pods = pod_details.items result = { 'category': 'observability', 'case_name': 'collectd_check', 'criteria': 'pass', 'details': [] } logger = logging.getLogger(__name__) status = [] flag = False for pod in pods: if 'collectd' in pod.metadata.name: stats = health_checker(pod, api_instance, logger, result) status.append(stats) flag = True if flag is False: result['criteria'] = 'fail' result['details'].append(status) store_result(logger, result) return result
def kubevirt_check(): """ Checks for existence kubevirt namespace and checks health of the pods within """ k8s_api = kube_api() namespaces = k8s_api.list_namespace() ns_names = [] for nspace in namespaces.items: ns_names.append(nspace.metadata.name) result = { 'category': 'platform', 'case_name': 'kubevirt_check', 'criteria': 'pass', 'details': [] } logger = logging.getLogger(__name__) if 'kubevirt' in ns_names: result['criteria'] = 'pass' result['details'].append(ns_names) pod_list = k8s_api.list_namespaced_pod('kubevirt') for pod in pod_list.items: pod_stats = pod_status(logger, pod) if pod_stats['criteria'] == 'fail': pod_stats['logs'] = get_logs(k8s_api, pod) result['criteria'] = 'fail' result['details'].append(pod_stats) else: result['criteria'] = 'fail' store_result(logger, result) return result
def host_network_check(): """ Checks if the pods can share the network with their host """ kube = kube_api() logger = logging.getLogger(__name__) pod_manifest = { 'apiVersion': 'v1', 'kind': 'Pod', 'metadata': { 'name': 'security-host-network-demo', }, 'spec': { 'hostNetwork': True, 'containers': [{ 'image': 'k8s.gcr.io/pause', 'name': 'security-host-network-demo', 'command': ["/bin/sh", "-c", "sleep 60m"], }], 'restartPolicy': 'Always' } } result = { 'category': 'platform', 'case_name': 'host_network_check', 'criteria': 'pass', 'details': [] } status = [] try: pod_nw = kube.create_namespaced_pod(body=pod_manifest, namespace='default') time.sleep(6) kube.delete_namespaced_pod(name=pod_nw.metadata.name, namespace='default') result['criteria'] = 'fail' except KeyError as error: status.append(error) except RuntimeError as error: status.append(error) except Exception as error: kube.delete_namespaced_pod(name=pod_nw.metadata.name, namespace='default') result['criteria'] = 'fail' status.append(error) result['details'].append(status) store_result(logger, result) return result
def cpu_manager_policy_check(): """ Checks cpu manager settings """ api = kube_api() logger = logging.getLogger(__name__) node_list = api.list_node() nodes = [] for node in node_list.items: nodes.append(node.metadata.name) result = { 'category': 'compute', 'case_name': 'cpu_manager_policy_check', 'criteria': 'pass', 'details': [] } for node in nodes: configz = api.connect_get_node_proxy_with_path(node, "configz") configz = ast.literal_eval(configz) res = {'node': node, 'criteria': 'pass', 'config': []} status = [] flag = True cpu_manager = settings.getValue( 'pdf_file')['vim_functional']['cpu_manager_policy'] if cpu_manager['type'] == configz['kubeletconfig']['cpuManagerPolicy']: if cpu_manager['type'] == 'static': if cpu_manager['reconcile_period'] == configz['kubeletconfig'][ 'cpuManagerReconcilePeriod']: if cpu_manager['full_pcpus'] == configz['kubeletconfig'][ 'full-pcpus-only']: flag = flag and True else: flag = flag and False else: flag = flag and True else: flag = flag and False if flag is False: res['criteria'] = 'fail' status.append(cpu_manager) res['config'] = status result['details'].append(res) if flag is False: result['criteria'] = 'fail' store_result(logger, result) return result
def node_exporter_check(): """ Checks existence & health of node exporter pods """ kube = kube_api() namespaces = kube.list_namespace() ns_names = [] for nspace in namespaces.items: ns_names.append(nspace.metadata.name) result = {'category': 'observability', 'case_name': 'node_exporter_check', 'criteria': 'pass', 'details': [] } status = [] flag = False logger = logging.getLogger(__name__) if 'monitoring' in ns_names: pod_list = kube.list_namespaced_pod('monitoring', watch=False) pods = pod_list.items for pod in pods: if 'node-exporter' in pod.metadata.name: pod_stats = pod_status(logger, pod) if pod_stats['criteria'] == 'fail': pod_stats['logs'] = get_logs(kube, pod) result['criteria'] = 'fail' status.append(pod.metadata.name) status.append(pod_stats) flag = True else: for nspace in namespaces.items: pod_list = kube.list_namespaced_pod(nspace.metadata.name, watch=False) pods = pod_list.items for pod in pods: if 'node-exporter' in pod.metadata.name: pod_stats = pod_status(logger, pod) if pod_stats['criteria'] == 'fail': pod_stats['logs'] = get_logs(kube, pod) result['criteria'] = 'fail' status.append(pod.metadata.name) status.append(pod_stats) flag = True if flag is False: result['criteria'] = 'fail' result['details'].append(status) store_result(logger, result) return result
def cni_plugin_check(): """ Checks for CNI plugins and validate against PDF """ apps_instance = client.AppsV1Api() api_instance = kube_api() result = { 'category': 'network', 'case_name': 'cni_plugin_check', 'criteria': 'pass', 'details': [] } logger = logging.getLogger(__name__) create_daemonset(apps_instance) pod_details = api_instance.list_namespaced_pod('default', watch=False) pods = pod_details.items daemon_pods = [] status = [] cmd = ['ls', '/opt/cni/bin'] cni_plugins = settings.getValue( 'pdf_file')['vim_functional']['cnis_supported'] for pod in pods: if 'plugin-check-test-set' in pod.metadata.name: try: list_of_cni_from_dir = kube_exec(pod, cmd) for plugin in cni_plugins: if plugin not in list_of_cni_from_dir: result['criteria'] = 'fail' status.append(list_of_cni_from_dir) daemon_pods.append(pod.metadata.name) except ConnectionError as error: status.append(error) except RuntimeError as error: status.append(error) except Exception as error: result['criteria'] = 'fail' status.append(error) apps_instance.delete_namespaced_daemon_set('plugin-check-test-set', 'default') result['details'].append(daemon_pods) result['details'].append(status) store_result(logger, result) return result
def pod_health_check(): """ Check health of all pods and get logs of failed pods """ logger = logging.getLogger(__name__) api = kube_api() namespace_list = settings.getValue('airship_namespace_list') result = checks.pod_health_check(logger, api, namespace_list) store_result(logger, result) return result
def multi_interface_cni_check(): """ Checks if multi interface cni is enabled """ apps_instance = client.AppsV1Api() api_instance = kube_api() logger = logging.getLogger(__name__) result = { 'category': 'network', 'case_name': 'multi_interface_cni_check', 'criteria': 'pass', 'details': [] } create_daemonset(apps_instance) pod_details = api_instance.list_namespaced_pod('default', watch=False) pods = pod_details.items status = [] cmd = ['ls', '/etc/cni/net.d'] for pod in pods: if 'plugin-check-test-set' in pod.metadata.name: try: list_of_plugin_conf = kube_exec(pod, cmd) list_of_plugin_conf = list_of_plugin_conf.split("\n") cmd3 = ['cat', "/etc/cni/net.d/" + list_of_plugin_conf[0]] multi_interface_conf = kube_exec(pod, cmd3) if 'multus' not in multi_interface_conf: result['criteria'] = 'fail' status.append(list_of_plugin_conf) status.append(multi_interface_conf) except ConnectionError as error: status.append(error) except RuntimeError as error: status.append(error) except Exception as error: result['criteria'] = 'fail' status.append(error) apps_instance.delete_namespaced_daemon_set('plugin-check-test-set', 'default') result['details'].append(status) store_result(logger, result) return result
def monitoring_agent_check(): """ Checks existence & health of prometheus pods """ api_instance = kube_api() namespaces = api_instance.list_namespace() ns_names = [] for nspace in namespaces.items: ns_names.append(nspace.metadata.name) result = { 'category': 'observability', 'case_name': 'prometheus_check', 'criteria': 'pass', 'details': [] } status = [] flag = False logger = logging.getLogger(__name__) if 'monitoring' in ns_names: pod_details = api_instance.list_namespaced_pod('monitoring', watch=False) pods = pod_details.items for pod in pods: if 'prometheus' in pod.metadata.name: stats = health_checker(pod, api_instance, logger, result) status.append(stats) flag = True else: for name in ns_names: pod_details = api_instance.list_namespaced_pod(name, watch=False) pods = pod_details.items for pod in pods: if 'prometheus' in pod.metadata.name: stats = health_checker(pod, api_instance, logger, result) status.append(stats) flag = True if flag is False: result['criteria'] = 'fail' result['details'].append(status) store_result(logger, result) return result
def liveness_probe_check(): """ Checks whether the liveness probe is configured for all overcloud components deployed as pods on undercloud Kubernetes. """ logger = logging.getLogger(__name__) api = kube_api() namespace_list = settings.getValue('airship_namespace_list') result = { 'category': 'platform', 'case_name': 'liveness_probe_check', 'criteria': 'pass', 'details': [] } for namespace in namespace_list: pod_list = api.list_namespaced_pod(namespace) for pod in pod_list.items: pod_stats = { 'criteria': 'pass', 'name': pod.metadata.name, 'namespace': pod.metadata.namespace, 'node': pod.spec.node_name, 'containers': [] } for container in pod.spec.containers: container_stats = { 'name': container.name, 'liveness_probe': None } if hasattr(container, 'liveness_probe' ) and container.liveness_probe is not None: container_stats[ 'liveness_probe'] = container.liveness_probe else: result['criteria'] = 'fail' pod_stats['criteria'] = 'fail' pod_stats['containers'].append(container_stats) result['details'].append(pod_stats) store_result(logger, result) return result
def get_logs(pod): """ Collects logs of all containers in ``pod`` """ api = kube_api() logs = [] if pod.status.container_statuses is not None: for container in pod.status.container_statuses: con = {'container': container.name} if container.state.waiting is not None and \ container.state.waiting.reason == 'PodInitializing': log = 'Not found, status: waiting, reason: PodInitializing' else: log = api.read_namespaced_pod_log( name=pod.metadata.name, namespace=pod.metadata.namespace, container=container.name) con['log'] = rfile(log) logs.append(con) return logs
def privilege_check(): """ Checks if privileged pods are possible to created """ kube = kube_api() logger = logging.getLogger(__name__) pod_manifest = { 'apiVersion': 'v1', 'kind': 'Pod', 'metadata': { 'name': 'security-privileges-demo', }, 'spec': { 'containers': [{ 'image': 'alpine:3.2', 'name': 'security-privileges-demo', 'command': ["/bin/sh", "-c", "sleep 60m"], 'securityContext': { 'privileged': True } }] } } result = { 'category': 'platform', 'case_name': 'privilege_check', 'criteria': 'pass', 'details': [] } status = [] try: pod_priv = kube.create_namespaced_pod(body=pod_manifest, namespace='default') time.sleep(5) cmd = ['ps', 'aux'] response = kube_exec(pod_priv, cmd) if "root" in response: result['criteria'] = 'fail' status.append(response) kube.delete_namespaced_pod(name=pod_priv.metadata.name, namespace='default') except KeyError as error: status.append(error) except RuntimeError as error: status.append(error) except Exception as error: kube.delete_namespaced_pod(name=pod_priv.metadata.name, namespace='default') result['criteria'] = 'fail' status.append(error) result['details'].append(status) store_result(logger, result) return result
def host_path_vol_check(): """ Checks if pods can be mounted to a host directory """ kube = kube_api() logger = logging.getLogger(__name__) pod_manifest = { 'apiVersion': 'v1', 'kind': 'Pod', 'metadata': { 'name': 'security-host-path-volume-demo', }, 'spec': { 'hostNetwork': True, 'containers': [{ 'image': 'k8s.gcr.io/pause', 'name': 'security-host-path-volume-demo', 'command': ["/bin/sh", "-c", "sleep 60m"], }], 'volumes': [{ 'name': 'test-vol', 'hostpath': { 'path': 'home', 'type': 'Directory' } }] } } result = { 'category': 'platform', 'case_name': 'host_path_dir_vol_check', 'criteria': 'pass', 'details': [] } status = [] try: pod_vol = kube.create_namespaced_pod(body=pod_manifest, namespace='default') time.sleep(5) kube.delete_namespaced_pod(name=pod_vol.metadata.name, namespace='default') result['criteria'] = 'fail' except KeyError as error: status.append(error) except RuntimeError as error: status.append(error) except Exception as error: kube.delete_namespaced_pod(name=pod_vol.metadata.name, namespace='default') result['criteria'] = 'fail' status.append(error) result['details'].append(status) store_result(logger, result) return result
def capability_check(): """ Checks if creation of pods with particular capabilties is possible """ kube = kube_api() logger = logging.getLogger(__name__) pod_manifest = { 'apiVersion': 'v1', 'kind': 'Pod', 'metadata': { 'name': 'security-capability-demo', }, 'spec': { 'containers': [{ 'image': 'alpine:3.2', 'name': 'security-capability-demo', 'command': ["/bin/sh", "-c", "sleep 60m"], 'securityContext': { 'capabilities': { 'drop': ["ALL"], 'add': ['NET_ADMIN', 'NET_RAW'] } } }] } } result = { 'category': 'platform', 'case_name': 'capability_check', 'criteria': 'pass', 'details': [] } status = [] try: pod_cap = kube.create_namespaced_pod(body=pod_manifest, namespace='default') time.sleep(6) cmd = ['cat', '/proc/1/status'] response = kube_exec(pod_cap, cmd) if "0000000000003000" in response: result['criteria'] = 'fail' status.append(pod_cap) kube.delete_namespaced_pod(name=pod_cap.metadata.name, namespace='default') except KeyError as error: status.append(error) except RuntimeError as error: status.append(error) except Exception as error: kube.delete_namespaced_pod(name=pod_cap.metadata.name, namespace='default') result['criteria'] = 'fail' status.append(error) result['details'].append(status) store_result(logger, result) return result