def check(self, _):
kubelet_conn_info = get_connection_info()
endpoint = kubelet_conn_info.get('url')
if endpoint is None:
raise CheckException(
"Unable to detect the kubelet URL automatically: " +
kubelet_conn_info.get('err', ''))
self.pod_list_url = endpoint.strip("/") + POD_LIST_PATH
self.kubelet_credentials = KubeletCredentials(kubelet_conn_info)
if self.fargate_mode:
pod_list = self.retrieve_pod_list()
for pod in pod_list.get('items', []):
pod_id = pod.get('metadata', {}).get('uid')
tagger_tags = tagger.tag('kubernetes_pod_uid://%s' % pod_id,
tagger.ORCHESTRATOR) or []
tagger_tags.extend(self.tags)
tags = set(tagger_tags)
# Submit the heartbeat metric for fargate virtual nodes.
self.gauge(self.NAMESPACE + '.pods.running', 1, tags)
pod_annotations = pod.get('metadata', {}).get('annotations')
if CAPACITY_ANNOTATION_KEY not in pod_annotations:
continue
cpu_val, mem_val = extract_resource_values(
pod_annotations.get(CAPACITY_ANNOTATION_KEY))
if cpu_val == 0 or mem_val == 0:
continue
self.gauge(self.NAMESPACE + '.cpu.capacity', cpu_val, tags)
self.gauge(self.NAMESPACE + '.memory.capacity', mem_val, tags)
def test_perform_kubelet_check(monkeypatch):
check = KubeletCheck('kubelet', {}, [{}])
check.kube_health_url = "http://127.0.0.1:10255/healthz"
check.kubelet_credentials = KubeletCredentials({})
monkeypatch.setattr(check, 'service_check', mock.Mock())
instance_tags = ["one:1"]
get = MockResponse()
with mock.patch("requests.get", side_effect=get):
check._perform_kubelet_check(instance_tags)
get.assert_has_calls([
mock.call(
'http://127.0.0.1:10255/healthz',
auth=None,
cert=None,
headers=None,
params={'verbose': True},
proxies=None,
stream=False,
timeout=(10.0, 10.0),
verify=None,
)
])
calls = [mock.call('kubernetes.kubelet.check', 0, tags=instance_tags)]
check.service_check.assert_has_calls(calls)
def test_credentials_token_noverify():
expected_headers = {'Authorization': 'Bearer mytoken'}
creds = KubeletCredentials({
"verify_tls": "false",
"ca_cert": "ca_cert",
"client_crt": "ignore_me",
"token": "mytoken"
})
assert creds.verify() is False
assert creds.cert_pair() is None
assert creds.headers("https://dummy") == expected_headers
# Make sure we don't leak the token over http
assert creds.headers("http://dummy") is None
instance = {'prometheus_url': 'https://dummy', 'namespace': 'foo'}
scraper = OpenMetricsBaseCheck('prometheus', {}, [instance])
scraper_config = scraper.create_scraper_configuration(instance)
creds.configure_scraper(scraper_config)
assert scraper_config['ssl_ca_cert'] is False
assert scraper_config['ssl_cert'] is None
assert scraper_config['ssl_private_key'] is None
assert scraper_config['extra_headers'] == expected_headers
# Make sure we don't leak the token over http
scraper_config['prometheus_url'] = "http://dummy"
creds.configure_scraper(scraper_config)
assert scraper_config['ssl_ca_cert'] is False
assert scraper_config['ssl_cert'] is None
assert scraper_config['ssl_private_key'] is None
assert scraper_config['extra_headers'] == {}
def test_silent_tls_warning(caplog, monkeypatch, aggregator):
check = KubeletCheck('kubelet', {}, [{}])
check.kube_health_url = "https://example.com/"
check.kubelet_credentials = KubeletCredentials({'verify_tls': 'false'})
with caplog.at_level(logging.DEBUG):
check._perform_kubelet_check([])
expected_message = 'An unverified HTTPS request is being made to https://example.com/'
for _, _, message in caplog.record_tuples:
assert message != expected_message
def test_report_node_metrics_kubernetes1_18(monkeypatch, aggregator):
check = KubeletCheck('kubelet', {}, [{}])
check.kubelet_credentials = KubeletCredentials({'verify_tls': 'false'})
check.node_spec_url = "http://localhost:10255/spec"
get = mock.MagicMock(status_code=404,
iter_lines=lambda **kwargs: "Error Code")
get.raise_for_status.side_effect = requests.HTTPError('error')
with mock.patch('requests.get', return_value=get):
check._report_node_metrics(['foo:bar'])
aggregator.assert_all_metrics_covered()
def test_credentials_empty():
creds = KubeletCredentials({})
assert creds.verify() is None
assert creds.cert_pair() is None
assert creds.headers("https://dummy") is None
instance = {'prometheus_url': 'https://dummy', 'namespace': 'foo'}
scraper = OpenMetricsBaseCheck('prometheus', {}, [instance])
scraper_config = scraper.create_scraper_configuration(instance)
creds.configure_scraper(scraper_config)
assert scraper_config['ssl_ca_cert'] is None
assert scraper_config['ssl_cert'] is None
assert scraper_config['ssl_private_key'] is None
assert scraper_config['extra_headers'] == {}
def test_credentials_certificates():
creds = KubeletCredentials({
"verify_tls": "true",
"ca_cert": "ca_cert",
"client_crt": "crt",
"client_key": "key",
"token": "ignore_me"
})
assert creds.verify() == "ca_cert"
assert creds.cert_pair() == ("crt", "key")
assert creds.headers("https://dummy") is None
instance = {'prometheus_url': 'https://dummy', 'namespace': 'foo'}
scraper = OpenMetricsBaseCheck('prometheus', {}, [instance])
scraper_config = scraper.create_scraper_configuration(instance)
creds.configure_scraper(scraper_config)
assert scraper_config['ssl_ca_cert'] == "ca_cert"
assert scraper_config['ssl_cert'] == "crt"
assert scraper_config['ssl_private_key'] == "key"
assert scraper_config['extra_headers'] == {}
def check(self, instance):
# Kubelet credential defaults are determined dynamically during every
# check run so we must make sure that configuration is always reset
self.reset_http_config()
kubelet_conn_info = get_connection_info()
endpoint = kubelet_conn_info.get('url')
if endpoint is None:
raise CheckException(
"Unable to detect the kubelet URL automatically: " +
kubelet_conn_info.get('err', ''))
self.kube_health_url = urljoin(endpoint, KUBELET_HEALTH_PATH)
self.node_spec_url = urljoin(endpoint, NODE_SPEC_PATH)
self.pod_list_url = urljoin(endpoint, POD_LIST_PATH)
self.stats_url = urljoin(endpoint, STATS_PATH)
self.instance_tags = instance.get('tags', [])
self.kubelet_credentials = KubeletCredentials(kubelet_conn_info)
# Test the kubelet health ASAP
self._perform_kubelet_check(self.instance_tags)
if 'cadvisor_metrics_endpoint' in instance:
self.cadvisor_scraper_config['prometheus_url'] = instance.get(
'cadvisor_metrics_endpoint',
urljoin(endpoint, CADVISOR_METRICS_PATH))
else:
self.cadvisor_scraper_config['prometheus_url'] = instance.get(
'metrics_endpoint', urljoin(endpoint, CADVISOR_METRICS_PATH))
if 'metrics_endpoint' in instance:
self.log.warning(
'metrics_endpoint is deprecated, please specify cadvisor_metrics_endpoint instead.'
)
self.kubelet_scraper_config['prometheus_url'] = instance.get(
'kubelet_metrics_endpoint', urljoin(endpoint,
KUBELET_METRICS_PATH))
# Kubelet credentials handling
self.kubelet_credentials.configure_scraper(
self.cadvisor_scraper_config)
self.kubelet_credentials.configure_scraper(self.kubelet_scraper_config)
# Legacy cadvisor support
try:
self.cadvisor_legacy_url = self.detect_cadvisor(
endpoint, self.cadvisor_legacy_port)
except Exception as e:
self.log.debug(
'cAdvisor not found, running in prometheus mode: %s', e)
self.pod_list = self.retrieve_pod_list()
self.pod_list_utils = PodListUtils(self.pod_list)
self.pod_tags_by_pvc = self._create_pod_tags_by_pvc(self.pod_list)
self._report_node_metrics(self.instance_tags)
self._report_pods_running(self.pod_list, self.instance_tags)
self._report_container_spec_metrics(self.pod_list, self.instance_tags)
self._report_container_state_metrics(self.pod_list, self.instance_tags)
self.stats = self._retrieve_stats()
self.process_stats_summary(self.pod_list_utils, self.stats,
self.instance_tags,
self.use_stats_summary_as_source)
if self.cadvisor_legacy_url: # Legacy cAdvisor
self.log.debug('processing legacy cadvisor metrics')
self.process_cadvisor(instance, self.cadvisor_legacy_url,
self.pod_list, self.pod_list_utils)
elif self.cadvisor_scraper_config['prometheus_url']: # Prometheus
self.log.debug('processing cadvisor metrics')
self.process(self.cadvisor_scraper_config,
metric_transformers=self.transformers)
if self.kubelet_scraper_config['prometheus_url']: # Prometheus
self.log.debug('processing kubelet metrics')
self.process(self.kubelet_scraper_config,
metric_transformers=self.transformers)
# Free up memory
self.pod_list = None
self.pod_list_utils = None