def pod_health_check():
"""
Check health of all pods and get logs of failed pods
"""
api = kube_api()
namespace_list = settings.getValue('airship_namespace_list')
result = {
'category': 'platform',
'case_name': 'pod_health_check',
'criteria': 'pass',
'details': []
}
for namespace in namespace_list:
pod_list = api.list_namespaced_pod(namespace)
for pod in pod_list.items:
pod_stats = pod_status(pod)
if pod_stats['criteria'] == 'fail':
pod_stats['logs'] = get_logs(pod)
result['criteria'] = 'fail'
result['details'].append(pod_stats)
store_result(result)
return result
def helmv2_disabled_check():
"""
Checks for helm v2 support
"""
result = {
'category': 'platform',
'case_name': 'helmv2_disabled_check',
'criteria': 'pass',
'details': []
}
kube = kube_api()
logger = logging.getLogger(__name__)
res = False
pod_details = kube.list_pod_for_all_namespaces()
pods = pod_details.items
version_support = settings.getValue(
'pdf_file')['vim_functional']['legacy_helm_support']
if 'YES' in version_support:
for pod in pods:
if 'tiller' in pod.metadata.name:
res = True
result['details'].append(pod)
if res is False:
result['criteria'] = 'fail'
store_result(logger, result)
return result
def collectd_check():
"""
Checks for collectd pods present and their state of being
"""
api_instance = kube_api()
pod_details = api_instance.list_pod_for_all_namespaces()
pods = pod_details.items
result = {
'category': 'observability',
'case_name': 'collectd_check',
'criteria': 'pass',
'details': []
}
logger = logging.getLogger(__name__)
status = []
flag = False
for pod in pods:
if 'collectd' in pod.metadata.name:
stats = health_checker(pod, api_instance, logger, result)
status.append(stats)
flag = True
if flag is False:
result['criteria'] = 'fail'
result['details'].append(status)
store_result(logger, result)
return result
def kubevirt_check():
"""
Checks for existence kubevirt namespace and checks health of the pods within
"""
k8s_api = kube_api()
namespaces = k8s_api.list_namespace()
ns_names = []
for nspace in namespaces.items:
ns_names.append(nspace.metadata.name)
result = {
'category': 'platform',
'case_name': 'kubevirt_check',
'criteria': 'pass',
'details': []
}
logger = logging.getLogger(__name__)
if 'kubevirt' in ns_names:
result['criteria'] = 'pass'
result['details'].append(ns_names)
pod_list = k8s_api.list_namespaced_pod('kubevirt')
for pod in pod_list.items:
pod_stats = pod_status(logger, pod)
if pod_stats['criteria'] == 'fail':
pod_stats['logs'] = get_logs(k8s_api, pod)
result['criteria'] = 'fail'
result['details'].append(pod_stats)
else:
result['criteria'] = 'fail'
store_result(logger, result)
return result
def host_network_check():
"""
Checks if the pods can share the network with their host
"""
kube = kube_api()
logger = logging.getLogger(__name__)
pod_manifest = {
'apiVersion': 'v1',
'kind': 'Pod',
'metadata': {
'name': 'security-host-network-demo',
},
'spec': {
'hostNetwork':
True,
'containers': [{
'image': 'k8s.gcr.io/pause',
'name': 'security-host-network-demo',
'command': ["/bin/sh", "-c", "sleep 60m"],
}],
'restartPolicy':
'Always'
}
}
result = {
'category': 'platform',
'case_name': 'host_network_check',
'criteria': 'pass',
'details': []
}
status = []
try:
pod_nw = kube.create_namespaced_pod(body=pod_manifest,
namespace='default')
time.sleep(6)
kube.delete_namespaced_pod(name=pod_nw.metadata.name,
namespace='default')
result['criteria'] = 'fail'
except KeyError as error:
status.append(error)
except RuntimeError as error:
status.append(error)
except Exception as error:
kube.delete_namespaced_pod(name=pod_nw.metadata.name,
namespace='default')
result['criteria'] = 'fail'
status.append(error)
result['details'].append(status)
store_result(logger, result)
return result
def cpu_manager_policy_check():
"""
Checks cpu manager settings
"""
api = kube_api()
logger = logging.getLogger(__name__)
node_list = api.list_node()
nodes = []
for node in node_list.items:
nodes.append(node.metadata.name)
result = {
'category': 'compute',
'case_name': 'cpu_manager_policy_check',
'criteria': 'pass',
'details': []
}
for node in nodes:
configz = api.connect_get_node_proxy_with_path(node, "configz")
configz = ast.literal_eval(configz)
res = {'node': node, 'criteria': 'pass', 'config': []}
status = []
flag = True
cpu_manager = settings.getValue(
'pdf_file')['vim_functional']['cpu_manager_policy']
if cpu_manager['type'] == configz['kubeletconfig']['cpuManagerPolicy']:
if cpu_manager['type'] == 'static':
if cpu_manager['reconcile_period'] == configz['kubeletconfig'][
'cpuManagerReconcilePeriod']:
if cpu_manager['full_pcpus'] == configz['kubeletconfig'][
'full-pcpus-only']:
flag = flag and True
else:
flag = flag and False
else:
flag = flag and True
else:
flag = flag and False
if flag is False:
res['criteria'] = 'fail'
status.append(cpu_manager)
res['config'] = status
result['details'].append(res)
if flag is False:
result['criteria'] = 'fail'
store_result(logger, result)
return result
def node_exporter_check():
"""
Checks existence & health of node exporter pods
"""
kube = kube_api()
namespaces = kube.list_namespace()
ns_names = []
for nspace in namespaces.items:
ns_names.append(nspace.metadata.name)
result = {'category': 'observability',
'case_name': 'node_exporter_check',
'criteria': 'pass',
'details': []
}
status = []
flag = False
logger = logging.getLogger(__name__)
if 'monitoring' in ns_names:
pod_list = kube.list_namespaced_pod('monitoring', watch=False)
pods = pod_list.items
for pod in pods:
if 'node-exporter' in pod.metadata.name:
pod_stats = pod_status(logger, pod)
if pod_stats['criteria'] == 'fail':
pod_stats['logs'] = get_logs(kube, pod)
result['criteria'] = 'fail'
status.append(pod.metadata.name)
status.append(pod_stats)
flag = True
else:
for nspace in namespaces.items:
pod_list = kube.list_namespaced_pod(nspace.metadata.name, watch=False)
pods = pod_list.items
for pod in pods:
if 'node-exporter' in pod.metadata.name:
pod_stats = pod_status(logger, pod)
if pod_stats['criteria'] == 'fail':
pod_stats['logs'] = get_logs(kube, pod)
result['criteria'] = 'fail'
status.append(pod.metadata.name)
status.append(pod_stats)
flag = True
if flag is False:
result['criteria'] = 'fail'
result['details'].append(status)
store_result(logger, result)
return result
def cni_plugin_check():
"""
Checks for CNI plugins and validate against PDF
"""
apps_instance = client.AppsV1Api()
api_instance = kube_api()
result = {
'category': 'network',
'case_name': 'cni_plugin_check',
'criteria': 'pass',
'details': []
}
logger = logging.getLogger(__name__)
create_daemonset(apps_instance)
pod_details = api_instance.list_namespaced_pod('default', watch=False)
pods = pod_details.items
daemon_pods = []
status = []
cmd = ['ls', '/opt/cni/bin']
cni_plugins = settings.getValue(
'pdf_file')['vim_functional']['cnis_supported']
for pod in pods:
if 'plugin-check-test-set' in pod.metadata.name:
try:
list_of_cni_from_dir = kube_exec(pod, cmd)
for plugin in cni_plugins:
if plugin not in list_of_cni_from_dir:
result['criteria'] = 'fail'
status.append(list_of_cni_from_dir)
daemon_pods.append(pod.metadata.name)
except ConnectionError as error:
status.append(error)
except RuntimeError as error:
status.append(error)
except Exception as error:
result['criteria'] = 'fail'
status.append(error)
apps_instance.delete_namespaced_daemon_set('plugin-check-test-set',
'default')
result['details'].append(daemon_pods)
result['details'].append(status)
store_result(logger, result)
return result
def pod_health_check():
"""
Check health of all pods and get logs of failed pods
"""
logger = logging.getLogger(__name__)
api = kube_api()
namespace_list = settings.getValue('airship_namespace_list')
result = checks.pod_health_check(logger, api, namespace_list)
store_result(logger, result)
return result
def multi_interface_cni_check():
"""
Checks if multi interface cni is enabled
"""
apps_instance = client.AppsV1Api()
api_instance = kube_api()
logger = logging.getLogger(__name__)
result = {
'category': 'network',
'case_name': 'multi_interface_cni_check',
'criteria': 'pass',
'details': []
}
create_daemonset(apps_instance)
pod_details = api_instance.list_namespaced_pod('default', watch=False)
pods = pod_details.items
status = []
cmd = ['ls', '/etc/cni/net.d']
for pod in pods:
if 'plugin-check-test-set' in pod.metadata.name:
try:
list_of_plugin_conf = kube_exec(pod, cmd)
list_of_plugin_conf = list_of_plugin_conf.split("\n")
cmd3 = ['cat', "/etc/cni/net.d/" + list_of_plugin_conf[0]]
multi_interface_conf = kube_exec(pod, cmd3)
if 'multus' not in multi_interface_conf:
result['criteria'] = 'fail'
status.append(list_of_plugin_conf)
status.append(multi_interface_conf)
except ConnectionError as error:
status.append(error)
except RuntimeError as error:
status.append(error)
except Exception as error:
result['criteria'] = 'fail'
status.append(error)
apps_instance.delete_namespaced_daemon_set('plugin-check-test-set',
'default')
result['details'].append(status)
store_result(logger, result)
return result
def monitoring_agent_check():
"""
Checks existence & health of prometheus pods
"""
api_instance = kube_api()
namespaces = api_instance.list_namespace()
ns_names = []
for nspace in namespaces.items:
ns_names.append(nspace.metadata.name)
result = {
'category': 'observability',
'case_name': 'prometheus_check',
'criteria': 'pass',
'details': []
}
status = []
flag = False
logger = logging.getLogger(__name__)
if 'monitoring' in ns_names:
pod_details = api_instance.list_namespaced_pod('monitoring',
watch=False)
pods = pod_details.items
for pod in pods:
if 'prometheus' in pod.metadata.name:
stats = health_checker(pod, api_instance, logger, result)
status.append(stats)
flag = True
else:
for name in ns_names:
pod_details = api_instance.list_namespaced_pod(name, watch=False)
pods = pod_details.items
for pod in pods:
if 'prometheus' in pod.metadata.name:
stats = health_checker(pod, api_instance, logger, result)
status.append(stats)
flag = True
if flag is False:
result['criteria'] = 'fail'
result['details'].append(status)
store_result(logger, result)
return result
def liveness_probe_check():
"""
Checks whether the liveness probe is configured for all overcloud
components deployed as pods on undercloud Kubernetes.
"""
logger = logging.getLogger(__name__)
api = kube_api()
namespace_list = settings.getValue('airship_namespace_list')
result = {
'category': 'platform',
'case_name': 'liveness_probe_check',
'criteria': 'pass',
'details': []
}
for namespace in namespace_list:
pod_list = api.list_namespaced_pod(namespace)
for pod in pod_list.items:
pod_stats = {
'criteria': 'pass',
'name': pod.metadata.name,
'namespace': pod.metadata.namespace,
'node': pod.spec.node_name,
'containers': []
}
for container in pod.spec.containers:
container_stats = {
'name': container.name,
'liveness_probe': None
}
if hasattr(container, 'liveness_probe'
) and container.liveness_probe is not None:
container_stats[
'liveness_probe'] = container.liveness_probe
else:
result['criteria'] = 'fail'
pod_stats['criteria'] = 'fail'
pod_stats['containers'].append(container_stats)
result['details'].append(pod_stats)
store_result(logger, result)
return result
def get_logs(pod):
"""
Collects logs of all containers in ``pod``
"""
api = kube_api()
logs = []
if pod.status.container_statuses is not None:
for container in pod.status.container_statuses:
con = {'container': container.name}
if container.state.waiting is not None and \
container.state.waiting.reason == 'PodInitializing':
log = 'Not found, status: waiting, reason: PodInitializing'
else:
log = api.read_namespaced_pod_log(
name=pod.metadata.name,
namespace=pod.metadata.namespace,
container=container.name)
con['log'] = rfile(log)
logs.append(con)
return logs
def privilege_check():
"""
Checks if privileged pods are possible to created
"""
kube = kube_api()
logger = logging.getLogger(__name__)
pod_manifest = {
'apiVersion': 'v1',
'kind': 'Pod',
'metadata': {
'name': 'security-privileges-demo',
},
'spec': {
'containers': [{
'image': 'alpine:3.2',
'name': 'security-privileges-demo',
'command': ["/bin/sh", "-c", "sleep 60m"],
'securityContext': {
'privileged': True
}
}]
}
}
result = {
'category': 'platform',
'case_name': 'privilege_check',
'criteria': 'pass',
'details': []
}
status = []
try:
pod_priv = kube.create_namespaced_pod(body=pod_manifest,
namespace='default')
time.sleep(5)
cmd = ['ps', 'aux']
response = kube_exec(pod_priv, cmd)
if "root" in response:
result['criteria'] = 'fail'
status.append(response)
kube.delete_namespaced_pod(name=pod_priv.metadata.name,
namespace='default')
except KeyError as error:
status.append(error)
except RuntimeError as error:
status.append(error)
except Exception as error:
kube.delete_namespaced_pod(name=pod_priv.metadata.name,
namespace='default')
result['criteria'] = 'fail'
status.append(error)
result['details'].append(status)
store_result(logger, result)
return result
def host_path_vol_check():
"""
Checks if pods can be mounted to a host directory
"""
kube = kube_api()
logger = logging.getLogger(__name__)
pod_manifest = {
'apiVersion': 'v1',
'kind': 'Pod',
'metadata': {
'name': 'security-host-path-volume-demo',
},
'spec': {
'hostNetwork':
True,
'containers': [{
'image': 'k8s.gcr.io/pause',
'name': 'security-host-path-volume-demo',
'command': ["/bin/sh", "-c", "sleep 60m"],
}],
'volumes': [{
'name': 'test-vol',
'hostpath': {
'path': 'home',
'type': 'Directory'
}
}]
}
}
result = {
'category': 'platform',
'case_name': 'host_path_dir_vol_check',
'criteria': 'pass',
'details': []
}
status = []
try:
pod_vol = kube.create_namespaced_pod(body=pod_manifest,
namespace='default')
time.sleep(5)
kube.delete_namespaced_pod(name=pod_vol.metadata.name,
namespace='default')
result['criteria'] = 'fail'
except KeyError as error:
status.append(error)
except RuntimeError as error:
status.append(error)
except Exception as error:
kube.delete_namespaced_pod(name=pod_vol.metadata.name,
namespace='default')
result['criteria'] = 'fail'
status.append(error)
result['details'].append(status)
store_result(logger, result)
return result
def capability_check():
"""
Checks if creation of pods with particular capabilties is possible
"""
kube = kube_api()
logger = logging.getLogger(__name__)
pod_manifest = {
'apiVersion': 'v1',
'kind': 'Pod',
'metadata': {
'name': 'security-capability-demo',
},
'spec': {
'containers': [{
'image': 'alpine:3.2',
'name': 'security-capability-demo',
'command': ["/bin/sh", "-c", "sleep 60m"],
'securityContext': {
'capabilities': {
'drop': ["ALL"],
'add': ['NET_ADMIN', 'NET_RAW']
}
}
}]
}
}
result = {
'category': 'platform',
'case_name': 'capability_check',
'criteria': 'pass',
'details': []
}
status = []
try:
pod_cap = kube.create_namespaced_pod(body=pod_manifest,
namespace='default')
time.sleep(6)
cmd = ['cat', '/proc/1/status']
response = kube_exec(pod_cap, cmd)
if "0000000000003000" in response:
result['criteria'] = 'fail'
status.append(pod_cap)
kube.delete_namespaced_pod(name=pod_cap.metadata.name,
namespace='default')
except KeyError as error:
status.append(error)
except RuntimeError as error:
status.append(error)
except Exception as error:
kube.delete_namespaced_pod(name=pod_cap.metadata.name,
namespace='default')
result['criteria'] = 'fail'
status.append(error)
result['details'].append(status)
store_result(logger, result)
return result
