def prometheus_alert_manager_check():
"""
Check health of Alert Manager
"""
logger = logging.getLogger(__name__)
service = "alerts-engine"
namespace = "osh-infra"
health = "fail" #default
res = kube_curl("-sL", "-m", "3", f'{service}.{namespace}:9093/-/healthy')
if 'Prometheus is Healthy' in res:
health = "pass"
readiness = "fail" #default
res = kube_curl("-sL", "-m", "3", f'{service}.{namespace}:9093/-/ready')
if 'Prometheus is Ready' in res:
readiness = "pass"
if health == "pass" and readiness == "pass":
state = "pass"
else:
state = "fail"
result = {'category': 'platform',
'case_name': 'prometheus_alert_manager_check',
'criteria': state,
'details': {'health': health, 'readiness': readiness}
}
store_result(logger, result)
return result
def prometheus_check():
"""
Check health of Prometheus
"""
logger = logging.getLogger(__name__)
username = "******"
password = "******"
service = "prom-metrics"
namespace = "osh-infra"
health = "fail" #default
res = kube_curl("-sL", "-m", "3", "-u", f'{username}:{password}', f'{service}.{namespace}/-/healthy')
if 'Prometheus is Healthy' in res:
health = "pass"
readiness = "fail" #default
res = kube_curl("-sL", "-m", "3", "-u", f'{username}:{password}', f'{service}.{namespace}/-/ready')
if 'Prometheus is Ready' in res:
readiness = "pass"
if health == "pass" and readiness == "pass":
state = "pass"
else:
state = "fail"
result = {'category': 'platform',
'case_name': 'prometheus_check',
'criteria': state,
'details': {'health': health, 'readiness': readiness}
}
store_result(logger, result)
return result
def grafana_check():
"""
Check health of Grafana
"""
logger = logging.getLogger(__name__)
username = "******"
password = "******"
service = "grafana-dashboard"
namespace = "osh-infra"
state = "fail" #default
res = kube_curl("-sL", "-m", "3", "-w", "%{http_code}",\
"-o", "/dev/null", "-u", \
f'{username}:{password}', \
f'{service}.{namespace}:3000/api/health')
if res == '200':
state = "pass"
result = {'category': 'platform',
'case_name': 'grafana_check',
'criteria': state
}
store_result(logger, result)
return result
def nagios_check():
"""
Check health of Nagios
"""
logger = logging.getLogger(__name__)
username = "******"
password = "******"
service = "nagios-metrics"
namespace = "osh-infra"
state = "fail" #default
res = kube_curl("-sL", "-m", "3", "-w", "%{http_code}",\
"-o", "/dev/null", "-u", \
f'{username}:{password}', \
f'{service}.{namespace}')
if res == '200':
state = "pass"
result = {'category': 'platform',
'case_name': 'nagios_check',
'criteria': state
}
store_result(logger, result)
return result
def kibana_check():
"""
Check health of Kibana
"""
logger = logging.getLogger(__name__)
username = "******"
password = "******"
service = "kibana-dash"
namespace = "osh-infra"
state = "fail" #default
res = kube_curl("-sL", "-m", "3", "-u", f'{username}:{password}', f'{service}.{namespace}/api/status')
if res == '':
res = 'kibana not reachable'
else:
res = ast.literal_eval(res)
if res['status']['overall']['state'] == 'green':
state = "pass"
result = {'category': 'platform',
'case_name': 'kibana_check',
'criteria': state,
'details': rfile(str(res))
}
store_result(logger, result)
return result
def elasticsearch_check():
"""
Check health of Elasticsearch cluster
"""
logger = logging.getLogger(__name__)
username = "******"
password = "******"
service = "elasticsearch"
namespace = "osh-infra"
state = "fail" #default
res = kube_curl("-sL", "-m", "3", "-u", f'{username}:{password}', f'{service}.{namespace}/_cluster/health')
if res == '':
res = 'Elasticsearch not reachable'
else:
res = ast.literal_eval(res)
if res['status'] == 'green':
state = "pass"
result = {'category': 'platform',
'case_name': 'elasticsearch_check',
'criteria': state,
'details': res
}
store_result(logger, result)
return result
def get_report(self):
"""
Return final report as dict
"""
self._report = super(KuberefValidator, self).get_report()
store_result(self._logger, self._report)
return self._report
def ceph_health_check():
"""
Check health of Ceph
"""
logger = logging.getLogger(__name__)
pod = get_pod_with_labels('application=ceph,component=mon')
cmd = ['ceph', 'health', '-f', 'json']
response = kube_exec(pod, cmd)
response = ast.literal_eval(response)
result = {
'category': 'storage',
'case_name': 'ceph_health_check',
'details': []
}
if response['status'] == 'HEALTH_OK':
result['criteria'] = 'pass'
result['details'] = 'HEALTH_OK'
else:
result['criteria'] = 'fail'
result['details'] = response
store_result(logger, result)
return result
def physical_network_check():
"""
physical_network_check
"""
logger = logging.getLogger(__name__)
ml2_config = neutron_ml2_config()
physical_networks = settings.getValue('pdf_file')['physical_networks']
type_drivers = ml2_config.get('ml2', 'type_drivers').split(',')
flat_networks = ml2_config.get('ml2_type_flat', 'flat_networks').split(',')
vlan_networks = []
network_vlan_ranges = ml2_config.get('ml2_type_vlan',
'network_vlan_ranges').split(',')
for network in network_vlan_ranges:
vlan_networks.append(network.split(':')[0])
result = {
'category': 'network',
'case_name': 'physical_network_check',
'criteria': 'pass',
'details': []
}
for physnet in physical_networks:
res = {
'network_name': physnet['name'],
'type': physnet['type'],
'criteria': 'fail'
}
if physnet['type'] in type_drivers:
if physnet['type'] == 'flat':
if physnet['name'] in flat_networks or '*' in flat_networks:
res['criteria'] = 'pass'
else:
res['details'] = 'physical network name not found'
if physnet['type'] == 'vlan':
if physnet['name'] in vlan_networks:
res['criteria'] = 'pass'
else:
res['details'] = 'physical network name not found'
else:
res['details'] = 'physical network type not found'
result['details'].append(res)
if res['criteria'] == 'fail':
result['criteria'] = 'fail'
store_result(logger, result)
return result
def pod_health_check():
"""
Check health of all pods and get logs of failed pods
"""
logger = logging.getLogger(__name__)
api = kube_api()
namespace_list = settings.getValue('airship_namespace_list')
result = checks.pod_health_check(logger, api, namespace_list)
store_result(logger, result)
return result
def fluentd_exporter_check():
"""
Check health of Fluentd Exporter
"""
logger = logging.getLogger(__name__)
service = "fluentd-exporter"
namespace = "osh-infra"
state = "fail" #default
res = kube_curl("-sL", "-m", "3", "-w", "%{http_code}", "-o", "/dev/null", f'{service}.{namespace}:9309/metrics')
if res == '200':
state = "pass"
result = {'category': 'platform',
'case_name': 'fluentd_exporter_check',
'criteria': state
}
store_result(logger, result)
return result