def prometheus_alert_manager_check(): """ Check health of Alert Manager """ logger = logging.getLogger(__name__) service = "alerts-engine" namespace = "osh-infra" health = "fail" #default res = kube_curl("-sL", "-m", "3", f'{service}.{namespace}:9093/-/healthy') if 'Prometheus is Healthy' in res: health = "pass" readiness = "fail" #default res = kube_curl("-sL", "-m", "3", f'{service}.{namespace}:9093/-/ready') if 'Prometheus is Ready' in res: readiness = "pass" if health == "pass" and readiness == "pass": state = "pass" else: state = "fail" result = {'category': 'platform', 'case_name': 'prometheus_alert_manager_check', 'criteria': state, 'details': {'health': health, 'readiness': readiness} } store_result(logger, result) return result
def prometheus_check(): """ Check health of Prometheus """ logger = logging.getLogger(__name__) username = "******" password = "******" service = "prom-metrics" namespace = "osh-infra" health = "fail" #default res = kube_curl("-sL", "-m", "3", "-u", f'{username}:{password}', f'{service}.{namespace}/-/healthy') if 'Prometheus is Healthy' in res: health = "pass" readiness = "fail" #default res = kube_curl("-sL", "-m", "3", "-u", f'{username}:{password}', f'{service}.{namespace}/-/ready') if 'Prometheus is Ready' in res: readiness = "pass" if health == "pass" and readiness == "pass": state = "pass" else: state = "fail" result = {'category': 'platform', 'case_name': 'prometheus_check', 'criteria': state, 'details': {'health': health, 'readiness': readiness} } store_result(logger, result) return result
def grafana_check(): """ Check health of Grafana """ logger = logging.getLogger(__name__) username = "******" password = "******" service = "grafana-dashboard" namespace = "osh-infra" state = "fail" #default res = kube_curl("-sL", "-m", "3", "-w", "%{http_code}",\ "-o", "/dev/null", "-u", \ f'{username}:{password}', \ f'{service}.{namespace}:3000/api/health') if res == '200': state = "pass" result = {'category': 'platform', 'case_name': 'grafana_check', 'criteria': state } store_result(logger, result) return result
def nagios_check(): """ Check health of Nagios """ logger = logging.getLogger(__name__) username = "******" password = "******" service = "nagios-metrics" namespace = "osh-infra" state = "fail" #default res = kube_curl("-sL", "-m", "3", "-w", "%{http_code}",\ "-o", "/dev/null", "-u", \ f'{username}:{password}', \ f'{service}.{namespace}') if res == '200': state = "pass" result = {'category': 'platform', 'case_name': 'nagios_check', 'criteria': state } store_result(logger, result) return result
def kibana_check(): """ Check health of Kibana """ logger = logging.getLogger(__name__) username = "******" password = "******" service = "kibana-dash" namespace = "osh-infra" state = "fail" #default res = kube_curl("-sL", "-m", "3", "-u", f'{username}:{password}', f'{service}.{namespace}/api/status') if res == '': res = 'kibana not reachable' else: res = ast.literal_eval(res) if res['status']['overall']['state'] == 'green': state = "pass" result = {'category': 'platform', 'case_name': 'kibana_check', 'criteria': state, 'details': rfile(str(res)) } store_result(logger, result) return result
def elasticsearch_check(): """ Check health of Elasticsearch cluster """ logger = logging.getLogger(__name__) username = "******" password = "******" service = "elasticsearch" namespace = "osh-infra" state = "fail" #default res = kube_curl("-sL", "-m", "3", "-u", f'{username}:{password}', f'{service}.{namespace}/_cluster/health') if res == '': res = 'Elasticsearch not reachable' else: res = ast.literal_eval(res) if res['status'] == 'green': state = "pass" result = {'category': 'platform', 'case_name': 'elasticsearch_check', 'criteria': state, 'details': res } store_result(logger, result) return result
def get_report(self): """ Return final report as dict """ self._report = super(KuberefValidator, self).get_report() store_result(self._logger, self._report) return self._report
def ceph_health_check(): """ Check health of Ceph """ logger = logging.getLogger(__name__) pod = get_pod_with_labels('application=ceph,component=mon') cmd = ['ceph', 'health', '-f', 'json'] response = kube_exec(pod, cmd) response = ast.literal_eval(response) result = { 'category': 'storage', 'case_name': 'ceph_health_check', 'details': [] } if response['status'] == 'HEALTH_OK': result['criteria'] = 'pass' result['details'] = 'HEALTH_OK' else: result['criteria'] = 'fail' result['details'] = response store_result(logger, result) return result
def physical_network_check(): """ physical_network_check """ logger = logging.getLogger(__name__) ml2_config = neutron_ml2_config() physical_networks = settings.getValue('pdf_file')['physical_networks'] type_drivers = ml2_config.get('ml2', 'type_drivers').split(',') flat_networks = ml2_config.get('ml2_type_flat', 'flat_networks').split(',') vlan_networks = [] network_vlan_ranges = ml2_config.get('ml2_type_vlan', 'network_vlan_ranges').split(',') for network in network_vlan_ranges: vlan_networks.append(network.split(':')[0]) result = { 'category': 'network', 'case_name': 'physical_network_check', 'criteria': 'pass', 'details': [] } for physnet in physical_networks: res = { 'network_name': physnet['name'], 'type': physnet['type'], 'criteria': 'fail' } if physnet['type'] in type_drivers: if physnet['type'] == 'flat': if physnet['name'] in flat_networks or '*' in flat_networks: res['criteria'] = 'pass' else: res['details'] = 'physical network name not found' if physnet['type'] == 'vlan': if physnet['name'] in vlan_networks: res['criteria'] = 'pass' else: res['details'] = 'physical network name not found' else: res['details'] = 'physical network type not found' result['details'].append(res) if res['criteria'] == 'fail': result['criteria'] = 'fail' store_result(logger, result) return result
def pod_health_check(): """ Check health of all pods and get logs of failed pods """ logger = logging.getLogger(__name__) api = kube_api() namespace_list = settings.getValue('airship_namespace_list') result = checks.pod_health_check(logger, api, namespace_list) store_result(logger, result) return result
def fluentd_exporter_check(): """ Check health of Fluentd Exporter """ logger = logging.getLogger(__name__) service = "fluentd-exporter" namespace = "osh-infra" state = "fail" #default res = kube_curl("-sL", "-m", "3", "-w", "%{http_code}", "-o", "/dev/null", f'{service}.{namespace}:9309/metrics') if res == '200': state = "pass" result = {'category': 'platform', 'case_name': 'fluentd_exporter_check', 'criteria': state } store_result(logger, result) return result