Exemple #1
0
def pod_health_check():
    """
    Check health of all pods and get logs of failed pods
    """
    api = kube_api()
    namespace_list = settings.getValue('airship_namespace_list')

    result = {
        'category': 'platform',
        'case_name': 'pod_health_check',
        'criteria': 'pass',
        'details': []
    }

    for namespace in namespace_list:
        pod_list = api.list_namespaced_pod(namespace)
        for pod in pod_list.items:
            pod_stats = pod_status(pod)
            if pod_stats['criteria'] == 'fail':
                pod_stats['logs'] = get_logs(pod)
                result['criteria'] = 'fail'
            result['details'].append(pod_stats)

    store_result(result)
    return result
Exemple #2
0
def helmv2_disabled_check():
    """
    Checks for helm v2 support
    """
    result = {
        'category': 'platform',
        'case_name': 'helmv2_disabled_check',
        'criteria': 'pass',
        'details': []
    }
    kube = kube_api()
    logger = logging.getLogger(__name__)
    res = False
    pod_details = kube.list_pod_for_all_namespaces()
    pods = pod_details.items
    version_support = settings.getValue(
        'pdf_file')['vim_functional']['legacy_helm_support']
    if 'YES' in version_support:
        for pod in pods:
            if 'tiller' in pod.metadata.name:
                res = True
                result['details'].append(pod)
    if res is False:
        result['criteria'] = 'fail'
    store_result(logger, result)
    return result
def collectd_check():
    """
    Checks for collectd pods present and their state of being
    """
    api_instance = kube_api()
    pod_details = api_instance.list_pod_for_all_namespaces()
    pods = pod_details.items

    result = {
        'category': 'observability',
        'case_name': 'collectd_check',
        'criteria': 'pass',
        'details': []
    }

    logger = logging.getLogger(__name__)

    status = []

    flag = False
    for pod in pods:
        if 'collectd' in pod.metadata.name:
            stats = health_checker(pod, api_instance, logger, result)
            status.append(stats)
            flag = True

    if flag is False:
        result['criteria'] = 'fail'

    result['details'].append(status)
    store_result(logger, result)
    return result
def kubevirt_check():
    """
    Checks for existence kubevirt namespace and checks health of the pods within

    """
    k8s_api = kube_api()
    namespaces = k8s_api.list_namespace()
    ns_names = []
    for nspace in namespaces.items:
        ns_names.append(nspace.metadata.name)

    result = {
        'category': 'platform',
        'case_name': 'kubevirt_check',
        'criteria': 'pass',
        'details': []
    }

    logger = logging.getLogger(__name__)

    if 'kubevirt' in ns_names:
        result['criteria'] = 'pass'
        result['details'].append(ns_names)
        pod_list = k8s_api.list_namespaced_pod('kubevirt')
        for pod in pod_list.items:
            pod_stats = pod_status(logger, pod)
            if pod_stats['criteria'] == 'fail':
                pod_stats['logs'] = get_logs(k8s_api, pod)
                result['criteria'] = 'fail'
            result['details'].append(pod_stats)
    else:
        result['criteria'] = 'fail'

    store_result(logger, result)
    return result
Exemple #5
0
def host_network_check():
    """
    Checks if the pods can share the network with their host
    """
    kube = kube_api()
    logger = logging.getLogger(__name__)

    pod_manifest = {
        'apiVersion': 'v1',
        'kind': 'Pod',
        'metadata': {
            'name': 'security-host-network-demo',
        },
        'spec': {
            'hostNetwork':
            True,
            'containers': [{
                'image': 'k8s.gcr.io/pause',
                'name': 'security-host-network-demo',
                'command': ["/bin/sh", "-c", "sleep 60m"],
            }],
            'restartPolicy':
            'Always'
        }
    }
    result = {
        'category': 'platform',
        'case_name': 'host_network_check',
        'criteria': 'pass',
        'details': []
    }

    status = []

    try:
        pod_nw = kube.create_namespaced_pod(body=pod_manifest,
                                            namespace='default')
        time.sleep(6)

        kube.delete_namespaced_pod(name=pod_nw.metadata.name,
                                   namespace='default')
        result['criteria'] = 'fail'

    except KeyError as error:
        status.append(error)

    except RuntimeError as error:
        status.append(error)

    except Exception as error:
        kube.delete_namespaced_pod(name=pod_nw.metadata.name,
                                   namespace='default')
        result['criteria'] = 'fail'
        status.append(error)

    result['details'].append(status)

    store_result(logger, result)
    return result
Exemple #6
0
def cpu_manager_policy_check():
    """
    Checks cpu manager settings
    """
    api = kube_api()
    logger = logging.getLogger(__name__)
    node_list = api.list_node()
    nodes = []

    for node in node_list.items:
        nodes.append(node.metadata.name)

    result = {
        'category': 'compute',
        'case_name': 'cpu_manager_policy_check',
        'criteria': 'pass',
        'details': []
    }

    for node in nodes:
        configz = api.connect_get_node_proxy_with_path(node, "configz")
        configz = ast.literal_eval(configz)
        res = {'node': node, 'criteria': 'pass', 'config': []}

        status = []

        flag = True

        cpu_manager = settings.getValue(
            'pdf_file')['vim_functional']['cpu_manager_policy']

        if cpu_manager['type'] == configz['kubeletconfig']['cpuManagerPolicy']:
            if cpu_manager['type'] == 'static':
                if cpu_manager['reconcile_period'] == configz['kubeletconfig'][
                        'cpuManagerReconcilePeriod']:
                    if cpu_manager['full_pcpus'] == configz['kubeletconfig'][
                            'full-pcpus-only']:
                        flag = flag and True
                else:
                    flag = flag and False
            else:
                flag = flag and True
        else:
            flag = flag and False

        if flag is False:
            res['criteria'] = 'fail'

        status.append(cpu_manager)
        res['config'] = status
        result['details'].append(res)

    if flag is False:
        result['criteria'] = 'fail'

    store_result(logger, result)
    return result
def node_exporter_check():
    """
    Checks existence & health of node exporter pods
    """
    kube = kube_api()
    namespaces = kube.list_namespace()
    ns_names = []
    for nspace in namespaces.items:
        ns_names.append(nspace.metadata.name)

    result = {'category':  'observability',
              'case_name': 'node_exporter_check',
              'criteria':  'pass',
              'details': []
             }

    status = []

    flag = False

    logger = logging.getLogger(__name__)

    if 'monitoring' in ns_names:
        pod_list = kube.list_namespaced_pod('monitoring', watch=False)
        pods = pod_list.items
        for pod in pods:
            if 'node-exporter' in pod.metadata.name:
                pod_stats = pod_status(logger, pod)
                if pod_stats['criteria'] == 'fail':
                    pod_stats['logs'] = get_logs(kube, pod)
                    result['criteria'] = 'fail'
                status.append(pod.metadata.name)
                status.append(pod_stats)
                flag = True
    else:
        for nspace in namespaces.items:
            pod_list = kube.list_namespaced_pod(nspace.metadata.name, watch=False)
            pods = pod_list.items
            for pod in pods:
                if 'node-exporter' in pod.metadata.name:
                    pod_stats = pod_status(logger, pod)
                    if pod_stats['criteria'] == 'fail':
                        pod_stats['logs'] = get_logs(kube, pod)
                        result['criteria'] = 'fail'
                    status.append(pod.metadata.name)
                    status.append(pod_stats)
                    flag = True

    if flag is False:
        result['criteria'] = 'fail'

    result['details'].append(status)

    store_result(logger, result)
    return result
Exemple #8
0
def cni_plugin_check():
    """
    Checks for CNI plugins and validate against PDF
    """
    apps_instance = client.AppsV1Api()
    api_instance = kube_api()

    result = {
        'category': 'network',
        'case_name': 'cni_plugin_check',
        'criteria': 'pass',
        'details': []
    }

    logger = logging.getLogger(__name__)
    create_daemonset(apps_instance)
    pod_details = api_instance.list_namespaced_pod('default', watch=False)
    pods = pod_details.items
    daemon_pods = []
    status = []
    cmd = ['ls', '/opt/cni/bin']
    cni_plugins = settings.getValue(
        'pdf_file')['vim_functional']['cnis_supported']

    for pod in pods:
        if 'plugin-check-test-set' in pod.metadata.name:
            try:
                list_of_cni_from_dir = kube_exec(pod, cmd)

                for plugin in cni_plugins:
                    if plugin not in list_of_cni_from_dir:
                        result['criteria'] = 'fail'

                status.append(list_of_cni_from_dir)
                daemon_pods.append(pod.metadata.name)

            except ConnectionError as error:
                status.append(error)

            except RuntimeError as error:
                status.append(error)

            except Exception as error:
                result['criteria'] = 'fail'
                status.append(error)

    apps_instance.delete_namespaced_daemon_set('plugin-check-test-set',
                                               'default')

    result['details'].append(daemon_pods)
    result['details'].append(status)
    store_result(logger, result)
    return result
Exemple #9
0
def pod_health_check():
    """
    Check health of all pods and get logs of failed pods
    """
    logger = logging.getLogger(__name__)
    api = kube_api()
    namespace_list = settings.getValue('airship_namespace_list')

    result = checks.pod_health_check(logger, api, namespace_list)

    store_result(logger, result)
    return result
Exemple #10
0
def multi_interface_cni_check():
    """
    Checks if multi interface cni is enabled
    """
    apps_instance = client.AppsV1Api()
    api_instance = kube_api()
    logger = logging.getLogger(__name__)

    result = {
        'category': 'network',
        'case_name': 'multi_interface_cni_check',
        'criteria': 'pass',
        'details': []
    }

    create_daemonset(apps_instance)
    pod_details = api_instance.list_namespaced_pod('default', watch=False)
    pods = pod_details.items
    status = []
    cmd = ['ls', '/etc/cni/net.d']

    for pod in pods:
        if 'plugin-check-test-set' in pod.metadata.name:
            try:
                list_of_plugin_conf = kube_exec(pod, cmd)
                list_of_plugin_conf = list_of_plugin_conf.split("\n")

                cmd3 = ['cat', "/etc/cni/net.d/" + list_of_plugin_conf[0]]
                multi_interface_conf = kube_exec(pod, cmd3)

                if 'multus' not in multi_interface_conf:
                    result['criteria'] = 'fail'

                status.append(list_of_plugin_conf)
                status.append(multi_interface_conf)

            except ConnectionError as error:
                status.append(error)

            except RuntimeError as error:
                status.append(error)

            except Exception as error:
                result['criteria'] = 'fail'
                status.append(error)

    apps_instance.delete_namespaced_daemon_set('plugin-check-test-set',
                                               'default')
    result['details'].append(status)
    store_result(logger, result)
    return result
def monitoring_agent_check():
    """
    Checks existence & health of prometheus pods
    """
    api_instance = kube_api()
    namespaces = api_instance.list_namespace()
    ns_names = []

    for nspace in namespaces.items:
        ns_names.append(nspace.metadata.name)

    result = {
        'category': 'observability',
        'case_name': 'prometheus_check',
        'criteria': 'pass',
        'details': []
    }

    status = []
    flag = False
    logger = logging.getLogger(__name__)
    if 'monitoring' in ns_names:
        pod_details = api_instance.list_namespaced_pod('monitoring',
                                                       watch=False)
        pods = pod_details.items
        for pod in pods:
            if 'prometheus' in pod.metadata.name:
                stats = health_checker(pod, api_instance, logger, result)
                status.append(stats)
                flag = True
    else:
        for name in ns_names:
            pod_details = api_instance.list_namespaced_pod(name, watch=False)
            pods = pod_details.items
            for pod in pods:
                if 'prometheus' in pod.metadata.name:
                    stats = health_checker(pod, api_instance, logger, result)
                    status.append(stats)
                    flag = True

    if flag is False:
        result['criteria'] = 'fail'

    result['details'].append(status)
    store_result(logger, result)
    return result
Exemple #12
0
def liveness_probe_check():
    """
    Checks whether the liveness probe is configured for all overcloud
    components deployed as pods on undercloud Kubernetes.
    """
    logger = logging.getLogger(__name__)
    api = kube_api()
    namespace_list = settings.getValue('airship_namespace_list')

    result = {
        'category': 'platform',
        'case_name': 'liveness_probe_check',
        'criteria': 'pass',
        'details': []
    }

    for namespace in namespace_list:
        pod_list = api.list_namespaced_pod(namespace)
        for pod in pod_list.items:
            pod_stats = {
                'criteria': 'pass',
                'name': pod.metadata.name,
                'namespace': pod.metadata.namespace,
                'node': pod.spec.node_name,
                'containers': []
            }

            for container in pod.spec.containers:
                container_stats = {
                    'name': container.name,
                    'liveness_probe': None
                }
                if hasattr(container, 'liveness_probe'
                           ) and container.liveness_probe is not None:
                    container_stats[
                        'liveness_probe'] = container.liveness_probe
                else:
                    result['criteria'] = 'fail'
                    pod_stats['criteria'] = 'fail'
                pod_stats['containers'].append(container_stats)
            result['details'].append(pod_stats)

    store_result(logger, result)
    return result
Exemple #13
0
def get_logs(pod):
    """
    Collects logs of all containers in ``pod``
    """
    api = kube_api()
    logs = []
    if pod.status.container_statuses is not None:
        for container in pod.status.container_statuses:
            con = {'container': container.name}
            if container.state.waiting is not None and \
               container.state.waiting.reason == 'PodInitializing':
                log = 'Not found, status: waiting, reason: PodInitializing'
            else:
                log = api.read_namespaced_pod_log(
                    name=pod.metadata.name,
                    namespace=pod.metadata.namespace,
                    container=container.name)
            con['log'] = rfile(log)
            logs.append(con)
    return logs
Exemple #14
0
def privilege_check():
    """
    Checks if privileged pods are possible to created
    """
    kube = kube_api()
    logger = logging.getLogger(__name__)

    pod_manifest = {
        'apiVersion': 'v1',
        'kind': 'Pod',
        'metadata': {
            'name': 'security-privileges-demo',
        },
        'spec': {
            'containers': [{
                'image': 'alpine:3.2',
                'name': 'security-privileges-demo',
                'command': ["/bin/sh", "-c", "sleep 60m"],
                'securityContext': {
                    'privileged': True
                }
            }]
        }
    }
    result = {
        'category': 'platform',
        'case_name': 'privilege_check',
        'criteria': 'pass',
        'details': []
    }

    status = []

    try:
        pod_priv = kube.create_namespaced_pod(body=pod_manifest,
                                              namespace='default')
        time.sleep(5)
        cmd = ['ps', 'aux']

        response = kube_exec(pod_priv, cmd)

        if "root" in response:
            result['criteria'] = 'fail'
            status.append(response)

        kube.delete_namespaced_pod(name=pod_priv.metadata.name,
                                   namespace='default')

    except KeyError as error:
        status.append(error)

    except RuntimeError as error:
        status.append(error)

    except Exception as error:
        kube.delete_namespaced_pod(name=pod_priv.metadata.name,
                                   namespace='default')
        result['criteria'] = 'fail'
        status.append(error)

    result['details'].append(status)

    store_result(logger, result)
    return result
Exemple #15
0
def host_path_vol_check():
    """
    Checks if pods can be mounted to a host directory
    """
    kube = kube_api()
    logger = logging.getLogger(__name__)

    pod_manifest = {
        'apiVersion': 'v1',
        'kind': 'Pod',
        'metadata': {
            'name': 'security-host-path-volume-demo',
        },
        'spec': {
            'hostNetwork':
            True,
            'containers': [{
                'image': 'k8s.gcr.io/pause',
                'name': 'security-host-path-volume-demo',
                'command': ["/bin/sh", "-c", "sleep 60m"],
            }],
            'volumes': [{
                'name': 'test-vol',
                'hostpath': {
                    'path': 'home',
                    'type': 'Directory'
                }
            }]
        }
    }
    result = {
        'category': 'platform',
        'case_name': 'host_path_dir_vol_check',
        'criteria': 'pass',
        'details': []
    }

    status = []

    try:
        pod_vol = kube.create_namespaced_pod(body=pod_manifest,
                                             namespace='default')

        time.sleep(5)

        kube.delete_namespaced_pod(name=pod_vol.metadata.name,
                                   namespace='default')
        result['criteria'] = 'fail'

    except KeyError as error:
        status.append(error)

    except RuntimeError as error:
        status.append(error)

    except Exception as error:
        kube.delete_namespaced_pod(name=pod_vol.metadata.name,
                                   namespace='default')
        result['criteria'] = 'fail'
        status.append(error)

    result['details'].append(status)

    store_result(logger, result)
    return result
Exemple #16
0
def capability_check():
    """
    Checks if creation of pods with particular capabilties is possible
    """
    kube = kube_api()
    logger = logging.getLogger(__name__)
    pod_manifest = {
        'apiVersion': 'v1',
        'kind': 'Pod',
        'metadata': {
            'name': 'security-capability-demo',
        },
        'spec': {
            'containers': [{
                'image': 'alpine:3.2',
                'name': 'security-capability-demo',
                'command': ["/bin/sh", "-c", "sleep 60m"],
                'securityContext': {
                    'capabilities': {
                        'drop': ["ALL"],
                        'add': ['NET_ADMIN', 'NET_RAW']
                    }
                }
            }]
        }
    }
    result = {
        'category': 'platform',
        'case_name': 'capability_check',
        'criteria': 'pass',
        'details': []
    }
    status = []
    try:
        pod_cap = kube.create_namespaced_pod(body=pod_manifest,
                                             namespace='default')
        time.sleep(6)
        cmd = ['cat', '/proc/1/status']

        response = kube_exec(pod_cap, cmd)
        if "0000000000003000" in response:
            result['criteria'] = 'fail'
            status.append(pod_cap)
        kube.delete_namespaced_pod(name=pod_cap.metadata.name,
                                   namespace='default')

    except KeyError as error:
        status.append(error)

    except RuntimeError as error:
        status.append(error)

    except Exception as error:
        kube.delete_namespaced_pod(name=pod_cap.metadata.name,
                                   namespace='default')
        result['criteria'] = 'fail'
        status.append(error)

    result['details'].append(status)
    store_result(logger, result)
    return result