def check(self, instance):
self.kubelet_conn_info = get_connection_info()
endpoint = self.kubelet_conn_info.get('url')
if endpoint is None:
raise CheckException(
"Unable to find metrics_endpoint in config "
"file or detect the kubelet URL automatically.")
self.metrics_url = instance.get('metrics_endpoint') or urljoin(
endpoint, CADVISOR_METRICS_PATH)
self.kube_health_url = urljoin(endpoint, KUBELET_HEALTH_PATH)
self.node_spec_url = urljoin(endpoint, NODE_SPEC_PATH)
self.pod_list_url = urljoin(endpoint, POD_LIST_PATH)
# By default we send the buckets.
send_buckets = instance.get('send_histograms_buckets', True)
if send_buckets is not None and str(send_buckets).lower() == 'false':
send_buckets = False
else:
send_buckets = True
try:
self.pod_list = self.retrieve_pod_list()
except Exception:
self.pod_list = None
instance_tags = instance.get('tags', [])
self._perform_kubelet_check(instance_tags)
self._report_node_metrics(instance_tags)
self._report_pods_running(self.pod_list, instance_tags)
self._report_container_spec_metrics(self.pod_list, instance_tags)
self.process(self.metrics_url,
send_histograms_buckets=send_buckets,
instance=instance)
def check(self, _):
kubelet_conn_info = get_connection_info()
endpoint = kubelet_conn_info.get('url')
if endpoint is None:
raise CheckException(
"Unable to detect the kubelet URL automatically: " +
kubelet_conn_info.get('err', ''))
self.pod_list_url = endpoint.strip("/") + POD_LIST_PATH
self.kubelet_credentials = KubeletCredentials(kubelet_conn_info)
if self.fargate_mode:
pod_list = self.retrieve_pod_list()
for pod in pod_list.get('items', []):
pod_id = pod.get('metadata', {}).get('uid')
tagger_tags = tagger.tag('kubernetes_pod_uid://%s' % pod_id,
tagger.ORCHESTRATOR) or []
tagger_tags.extend(self.tags)
tags = set(tagger_tags)
# Submit the heartbeat metric for fargate virtual nodes.
self.gauge(self.NAMESPACE + '.pods.running', 1, tags)
pod_annotations = pod.get('metadata', {}).get('annotations')
if CAPACITY_ANNOTATION_KEY not in pod_annotations:
continue
cpu_val, mem_val = extract_resource_values(
pod_annotations.get(CAPACITY_ANNOTATION_KEY))
if cpu_val == 0 or mem_val == 0:
continue
self.gauge(self.NAMESPACE + '.cpu.capacity', cpu_val, tags)
self.gauge(self.NAMESPACE + '.memory.capacity', mem_val, tags)
def check(self, instance):
self.kubelet_conn_info = get_connection_info()
endpoint = self.kubelet_conn_info.get('url')
if endpoint is None:
raise CheckException(
"Unable to find metrics_endpoint in config "
"file or detect the kubelet URL automatically.")
self.metrics_url = instance.get(
'metrics_endpoint', urljoin(endpoint, CADVISOR_METRICS_PATH))
self.kube_health_url = urljoin(endpoint, KUBELET_HEALTH_PATH)
self.node_spec_url = urljoin(endpoint, NODE_SPEC_PATH)
self.pod_list_url = urljoin(endpoint, POD_LIST_PATH)
# Legacy cadvisor support
try:
self.cadvisor_legacy_url = self.detect_cadvisor(
endpoint, self.cadvisor_legacy_port)
except Exception as e:
self.log.debug(
'cAdvisor not found, running in prometheus mode: %s' % str(e))
# By default we send the buckets.
send_buckets = instance.get('send_histograms_buckets', True)
if send_buckets is not None and str(send_buckets).lower() == 'false':
send_buckets = False
else:
send_buckets = True
try:
self.pod_list = self.retrieve_pod_list()
if self.pod_list.get("items") is None:
# Sanitize input: if no pod are running, 'items' is a NoneObject
self.pod_list['items'] = []
except Exception:
self.pod_list = None
self.container_filter = ContainerFilter(self.pod_list)
self.instance_tags = instance.get('tags', [])
self._perform_kubelet_check(self.instance_tags)
self._report_node_metrics(self.instance_tags)
self._report_pods_running(self.pod_list, self.instance_tags)
self._report_container_spec_metrics(self.pod_list, self.instance_tags)
if self.cadvisor_legacy_url: # Legacy cAdvisor
self.process_cadvisor(instance, self.cadvisor_legacy_url,
self.pod_list, self.container_filter)
elif self.metrics_url: # Prometheus
self.process(self.metrics_url,
send_histograms_buckets=send_buckets,
instance=instance)
# Free up memory
self.pod_list = None
self.container_filter = None
def check(self, instance):
creds = get_connection_info()
if creds.get("url"):
self.warning("Found kubelet at " + creds.get("url"))
else:
self.warning("Kubelet not found")
if creds.get("verify_tls") == "false":
self.warning("no tls verification")
if creds.get("ca_cert"):
self.warning("ca_cert:" + creds.get("ca_cert"))
if creds.get("client_crt"):
self.warning("client_crt:" + creds.get("client_crt"))
if creds.get("client_key"):
self.warning("client_key:" + creds.get("client_key"))
if creds.get("token"):
self.warning("token:" + creds.get("token"))
def __init__(self, name, init_config, instances):
super(EksFargateCheck, self).__init__(name, init_config, instances)
self.NAMESPACE = 'eks.fargate'
virtual_node = os.getenv(KUBELET_NODE_ENV_VAR, "")
self.fargate_mode = 'fargate' in virtual_node
kubelet_conn_info = get_connection_info()
endpoint = kubelet_conn_info.get('url')
if endpoint is None:
raise CheckException(
"Unable to detect the kubelet URL automatically: " +
kubelet_conn_info.get('err', ''))
self.pod_list_url = endpoint.strip("/") + POD_LIST_PATH
self.kubelet_credentials = KubeletCredentials(kubelet_conn_info)
if self.fargate_mode:
self.tags = []
self.tags.append('virtual_node:' + virtual_node)
self.tags.extend(instances[0].get('tags', []))
def check(self, instance):
kubelet_conn_info = get_connection_info()
endpoint = kubelet_conn_info.get('url')
if endpoint is None:
raise CheckException(
"Unable to detect the kubelet URL automatically.")
self.kube_health_url = urljoin(endpoint, KUBELET_HEALTH_PATH)
self.node_spec_url = urljoin(endpoint, NODE_SPEC_PATH)
self.pod_list_url = urljoin(endpoint, POD_LIST_PATH)
self.instance_tags = instance.get('tags', [])
self.kubelet_credentials = KubeletCredentials(kubelet_conn_info)
# Test the kubelet health ASAP
self._perform_kubelet_check(self.instance_tags)
if 'cadvisor_metrics_endpoint' in instance:
self.cadvisor_scraper_config['prometheus_url'] = instance.get(
'cadvisor_metrics_endpoint',
urljoin(endpoint, CADVISOR_METRICS_PATH))
else:
self.cadvisor_scraper_config['prometheus_url'] = instance.get(
'metrics_endpoint', urljoin(endpoint, CADVISOR_METRICS_PATH))
if 'metrics_endpoint' in instance:
self.log.warning(
'metrics_endpoint is deprecated, please specify cadvisor_metrics_endpoint instead.'
)
self.kubelet_scraper_config['prometheus_url'] = instance.get(
'kubelet_metrics_endpoint', urljoin(endpoint,
KUBELET_METRICS_PATH))
# Kubelet credentials handling
self.kubelet_credentials.configure_scraper(
self.cadvisor_scraper_config)
self.kubelet_credentials.configure_scraper(self.kubelet_scraper_config)
# Legacy cadvisor support
try:
self.cadvisor_legacy_url = self.detect_cadvisor(
endpoint, self.cadvisor_legacy_port)
except Exception as e:
self.log.debug(
'cAdvisor not found, running in prometheus mode: %s' % str(e))
self.pod_list = self.retrieve_pod_list()
self.pod_list_utils = PodListUtils(self.pod_list)
self._report_node_metrics(self.instance_tags)
self._report_pods_running(self.pod_list, self.instance_tags)
self._report_container_spec_metrics(self.pod_list, self.instance_tags)
self._report_container_state_metrics(self.pod_list, self.instance_tags)
if self.cadvisor_legacy_url: # Legacy cAdvisor
self.log.debug('processing legacy cadvisor metrics')
self.process_cadvisor(instance, self.cadvisor_legacy_url,
self.pod_list, self.pod_list_utils)
elif self.cadvisor_scraper_config['prometheus_url']: # Prometheus
self.log.debug('processing cadvisor metrics')
self.process(self.cadvisor_scraper_config,
metric_transformers=self.CADVISOR_METRIC_TRANSFORMERS)
if self.kubelet_scraper_config['prometheus_url']: # Prometheus
self.log.debug('processing kubelet metrics')
self.process(self.kubelet_scraper_config)
# Free up memory
self.pod_list = None
self.pod_list_utils = None
def check(self, instance):
# Kubelet credential defaults are determined dynamically during every
# check run so we must make sure that configuration is always reset
self.reset_http_config()
kubelet_conn_info = get_connection_info()
endpoint = kubelet_conn_info.get('url')
if endpoint is None:
raise CheckException("Unable to detect the kubelet URL automatically: " + kubelet_conn_info.get('err', ''))
self.kube_health_url = urljoin(endpoint, KUBELET_HEALTH_PATH)
self.node_spec_url = urljoin(endpoint, NODE_SPEC_PATH)
self.pod_list_url = urljoin(endpoint, POD_LIST_PATH)
self.stats_url = urljoin(endpoint, STATS_PATH)
self.instance_tags = instance.get('tags', [])
self.kubelet_credentials = KubeletCredentials(kubelet_conn_info)
# Test the kubelet health ASAP
self._perform_kubelet_check(self.instance_tags)
if 'cadvisor_metrics_endpoint' in instance:
self.cadvisor_scraper_config['prometheus_url'] = instance.get(
'cadvisor_metrics_endpoint', urljoin(endpoint, CADVISOR_METRICS_PATH)
)
else:
self.cadvisor_scraper_config['prometheus_url'] = instance.get(
'metrics_endpoint', urljoin(endpoint, CADVISOR_METRICS_PATH)
)
if 'metrics_endpoint' in instance:
self.log.warning('metrics_endpoint is deprecated, please specify cadvisor_metrics_endpoint instead.')
self.kubelet_scraper_config['prometheus_url'] = instance.get(
'kubelet_metrics_endpoint', urljoin(endpoint, KUBELET_METRICS_PATH)
)
# Kubelet credentials handling
self.kubelet_credentials.configure_scraper(self.cadvisor_scraper_config)
self.kubelet_credentials.configure_scraper(self.kubelet_scraper_config)
# Legacy cadvisor support
try:
self.cadvisor_legacy_url = self.detect_cadvisor(endpoint, self.cadvisor_legacy_port)
except Exception as e:
self.log.debug('cAdvisor not found, running in prometheus mode: %s', e)
self.pod_list = self.retrieve_pod_list()
self.pod_list_utils = PodListUtils(self.pod_list)
self.pod_tags_by_pvc = self._create_pod_tags_by_pvc(self.pod_list)
self._report_node_metrics(self.instance_tags)
self._report_pods_running(self.pod_list, self.instance_tags)
self._report_container_spec_metrics(self.pod_list, self.instance_tags)
self._report_container_state_metrics(self.pod_list, self.instance_tags)
self.stats = self._retrieve_stats()
self.process_stats_summary(
self.pod_list_utils, self.stats, self.instance_tags, self.use_stats_summary_as_source
)
if self.cadvisor_legacy_url: # Legacy cAdvisor
self.log.debug('processing legacy cadvisor metrics')
self.process_cadvisor(instance, self.cadvisor_legacy_url, self.pod_list, self.pod_list_utils)
elif self.cadvisor_scraper_config['prometheus_url']: # Prometheus
self.log.debug('processing cadvisor metrics')
self.process(self.cadvisor_scraper_config, metric_transformers=self.transformers)
if self.kubelet_scraper_config['prometheus_url']: # Prometheus
self.log.debug('processing kubelet metrics')
self.process(self.kubelet_scraper_config, metric_transformers=self.transformers)
# Free up memory
self.pod_list = None
self.pod_list_utils = None
def check(self, instance):
kubelet_conn_info = get_connection_info()
endpoint = kubelet_conn_info.get('url')
if endpoint is None:
raise CheckException(
"Unable to detect the kubelet URL automatically.")
if 'cadvisor_metrics_endpoint' in instance:
self.cadvisor_metrics_url = \
instance.get('cadvisor_metrics_endpoint', urljoin(endpoint, CADVISOR_METRICS_PATH))
else:
self.cadvisor_metrics_url = instance.get(
'metrics_endpoint', urljoin(endpoint, CADVISOR_METRICS_PATH))
if 'metrics_endpoint' in instance:
self.log.warning(
'metrics_endpoint is deprecated, please specify cadvisor_metrics_endpoint instead.'
)
self.kubelet_metrics_url = instance.get(
'kubelet_metrics_endpoint', urljoin(endpoint,
KUBELET_METRICS_PATH))
self.kube_health_url = urljoin(endpoint, KUBELET_HEALTH_PATH)
self.node_spec_url = urljoin(endpoint, NODE_SPEC_PATH)
self.pod_list_url = urljoin(endpoint, POD_LIST_PATH)
# Kubelet credentials handling
self.kubelet_credentials = KubeletCredentials(kubelet_conn_info)
self.kubelet_credentials.configure_scraper(self.cadvisor_scraper,
self.cadvisor_metrics_url)
self.kubelet_credentials.configure_scraper(self.kubelet_scraper,
self.kubelet_metrics_url)
# Legacy cadvisor support
try:
self.cadvisor_legacy_url = self.detect_cadvisor(
endpoint, self.cadvisor_legacy_port)
except Exception as e:
self.log.debug(
'cAdvisor not found, running in prometheus mode: %s' % str(e))
# By default we send the buckets.
send_buckets = instance.get('send_histograms_buckets', True)
if send_buckets is not None and str(send_buckets).lower() == 'false':
send_buckets = False
else:
send_buckets = True
self.pod_list = self.retrieve_pod_list()
self.container_filter = ContainerFilter(self.pod_list)
self.instance_tags = instance.get('tags', [])
self._perform_kubelet_check(self.instance_tags)
self._report_node_metrics(self.instance_tags)
self._report_pods_running(self.pod_list, self.instance_tags)
self._report_container_spec_metrics(self.pod_list, self.instance_tags)
if self.cadvisor_legacy_url: # Legacy cAdvisor
self.log.debug('processing legacy cadvisor metrics')
self.process_cadvisor(instance, self.cadvisor_legacy_url,
self.pod_list, self.container_filter)
elif self.cadvisor_metrics_url: # Prometheus
self.log.debug('processing cadvisor metrics')
self.cadvisor_scraper.process(
self.cadvisor_metrics_url,
send_histograms_buckets=send_buckets,
instance=instance,
pod_list=self.pod_list,
container_filter=self.container_filter)
if self.kubelet_metrics_url: # Prometheus
self.log.debug('processing kubelet metrics')
self.kubelet_scraper.process(self.kubelet_metrics_url,
send_histograms_buckets=send_buckets,
instance=instance,
ignore_unmapped=True)
# Free up memory
self.pod_list = None
self.container_filter = None
def check(self, instance):
creds = get_connection_info()
if creds.get("url"):
self.warning("Found kubelet at " + creds.get("url"))
else:
self.warning("Kubelet not found")
