class Kubernetes(AgentCheck):
""" Collect metrics and events from kubelet """
pod_names_by_container = {}
def __init__(self, name, init_config, agentConfig, instances=None):
if instances is not None and len(instances) > 1:
raise Exception('Kubernetes check only supports one configured instance.')
AgentCheck.__init__(self, name, init_config, agentConfig, instances)
inst = instances[0] if instances is not None else None
self.kubeutil = KubeUtil(instance=inst)
if not self.kubeutil.host:
raise Exception('Unable to retrieve Docker hostname and host parameter is not set')
def _perform_kubelet_checks(self, url):
service_check_base = NAMESPACE + '.kubelet.check'
is_ok = True
try:
r = requests.get(url)
for line in r.iter_lines():
# avoid noise; this check is expected to fail since we override the container hostname
if line.find('hostname') != -1:
continue
matches = re.match('\[(.)\]([^\s]+) (.*)?', line)
if not matches or len(matches.groups()) < 2:
continue
service_check_name = service_check_base + '.' + matches.group(2)
status = matches.group(1)
if status == '+':
self.service_check(service_check_name, AgentCheck.OK)
else:
self.service_check(service_check_name, AgentCheck.CRITICAL)
is_ok = False
except Exception as e:
self.log.warning('kubelet check %s failed: %s' % (url, str(e)))
self.service_check(service_check_base, AgentCheck.CRITICAL,
message='Kubelet check %s failed: %s' % (url, str(e)))
else:
if is_ok:
self.service_check(service_check_base, AgentCheck.OK)
else:
self.service_check(service_check_base, AgentCheck.CRITICAL)
def check(self, instance):
self.max_depth = instance.get('max_depth', DEFAULT_MAX_DEPTH)
enabled_gauges = instance.get('enabled_gauges', DEFAULT_ENABLED_GAUGES)
self.enabled_gauges = ["{0}.{1}".format(NAMESPACE, x) for x in enabled_gauges]
enabled_rates = instance.get('enabled_rates', DEFAULT_ENABLED_RATES)
self.enabled_rates = ["{0}.{1}".format(NAMESPACE, x) for x in enabled_rates]
self.publish_aliases = _is_affirmative(instance.get('publish_aliases', DEFAULT_PUBLISH_ALIASES))
self.use_histogram = _is_affirmative(instance.get('use_histogram', DEFAULT_USE_HISTOGRAM))
self.publish_rate = FUNC_MAP[RATE][self.use_histogram]
self.publish_gauge = FUNC_MAP[GAUGE][self.use_histogram]
pods_list = self.kubeutil.retrieve_pods_list()
# kubelet health checks
self._perform_kubelet_checks(self.kubeutil.kube_health_url)
# kubelet metrics
self._update_metrics(instance, pods_list)
# kubelet events
if _is_affirmative(instance.get('collect_events', DEFAULT_COLLECT_EVENTS)):
self._process_events(instance, pods_list)
def _publish_raw_metrics(self, metric, dat, tags, depth=0):
if depth >= self.max_depth:
self.log.warning('Reached max depth on metric=%s' % metric)
return
if isinstance(dat, numbers.Number):
if self.enabled_rates and any([fnmatch(metric, pat) for pat in self.enabled_rates]):
self.publish_rate(self, metric, float(dat), tags)
elif self.enabled_gauges and any([fnmatch(metric, pat) for pat in self.enabled_gauges]):
self.publish_gauge(self, metric, float(dat), tags)
elif isinstance(dat, dict):
for k, v in dat.iteritems():
self._publish_raw_metrics(metric + '.%s' % k.lower(), v, tags, depth + 1)
elif isinstance(dat, list):
self._publish_raw_metrics(metric, dat[-1], tags, depth + 1)
@staticmethod
def _shorten_name(name):
# shorten docker image id
return re.sub('([0-9a-fA-F]{64,})', lambda x: x.group(1)[0:12], name)
def _get_post_1_2_tags(self, cont_labels, subcontainer, kube_labels):
tags = []
pod_name = cont_labels[KubeUtil.POD_NAME_LABEL]
pod_namespace = cont_labels[KubeUtil.NAMESPACE_LABEL]
tags.append(u"pod_name:{0}/{1}".format(pod_namespace, pod_name))
tags.append(u"kube_namespace:{0}".format(pod_namespace))
kube_labels_key = "{0}/{1}".format(pod_namespace, pod_name)
pod_labels = kube_labels.get(kube_labels_key)
if pod_labels:
tags += list(pod_labels)
if "-" in pod_name:
replication_controller = "-".join(pod_name.split("-")[:-1])
tags.append("kube_replication_controller:%s" % replication_controller)
if self.publish_aliases and subcontainer.get("aliases"):
for alias in subcontainer['aliases'][1:]:
# we don't add the first alias as it will be the container_name
tags.append('container_alias:%s' % (self._shorten_name(alias)))
return tags
def _get_pre_1_2_tags(self, cont_labels, subcontainer, kube_labels):
tags = []
pod_name = cont_labels[KubeUtil.POD_NAME_LABEL]
tags.append(u"pod_name:{0}".format(pod_name))
pod_labels = kube_labels.get(pod_name)
if pod_labels:
tags.extend(list(pod_labels))
if "-" in pod_name:
replication_controller = "-".join(pod_name.split("-")[:-1])
if "/" in replication_controller:
namespace, replication_controller = replication_controller.split("/", 1)
tags.append(u"kube_namespace:%s" % namespace)
tags.append(u"kube_replication_controller:%s" % replication_controller)
if self.publish_aliases and subcontainer.get("aliases"):
for alias in subcontainer['aliases'][1:]:
# we don't add the first alias as it will be the container_name
tags.append(u"container_alias:%s" % (self._shorten_name(alias)))
return tags
def _update_container_metrics(self, instance, subcontainer, kube_labels):
tags = list(instance.get('tags', [])) # add support for custom tags
if len(subcontainer.get('aliases', [])) >= 1:
# The first alias seems to always match the docker container name
container_name = subcontainer['aliases'][0]
else:
# We default to the container id
container_name = subcontainer['name']
tags.append('container_name:%s' % container_name)
try:
cont_labels = subcontainer['spec']['labels']
except KeyError:
self.log.debug("Subcontainer, doesn't have any labels")
cont_labels = {}
# Collect pod names, namespaces, rc...
if KubeUtil.NAMESPACE_LABEL in cont_labels and KubeUtil.POD_NAME_LABEL in cont_labels:
# Kubernetes >= 1.2
tags += self._get_post_1_2_tags(cont_labels, subcontainer, kube_labels)
elif KubeUtil.POD_NAME_LABEL in cont_labels:
# Kubernetes <= 1.1
tags += self._get_pre_1_2_tags(cont_labels, subcontainer, kube_labels)
else:
# Those are containers that are not part of a pod.
# They are top aggregate views and don't have the previous metadata.
tags.append("pod_name:no_pod")
stats = subcontainer['stats'][-1] # take the latest
self._publish_raw_metrics(NAMESPACE, stats, tags)
if subcontainer.get("spec", {}).get("has_filesystem"):
fs = stats['filesystem'][-1]
fs_utilization = float(fs['usage'])/float(fs['capacity'])
self.publish_gauge(self, NAMESPACE + '.filesystem.usage_pct', fs_utilization, tags)
if subcontainer.get("spec", {}).get("has_network"):
net = stats['network']
self.publish_rate(self, NAMESPACE + '.network_errors',
sum(float(net[x]) for x in NET_ERRORS),
tags)
return tags
def _update_metrics(self, instance, pods_list):
metrics = self.kubeutil.retrieve_metrics()
excluded_labels = instance.get('excluded_labels')
kube_labels = self.kubeutil.extract_kube_labels(pods_list, excluded_keys=excluded_labels)
if not metrics:
raise Exception('No metrics retrieved cmd=%s' % self.metrics_cmd)
# container metrics from Cadvisor
container_tags = {}
for subcontainer in metrics:
c_id = subcontainer.get('id')
try:
tags = self._update_container_metrics(instance, subcontainer, kube_labels)
if c_id:
container_tags[c_id] = tags
# also store tags for aliases
for alias in subcontainer.get('aliases', []):
container_tags[alias] = tags
except Exception, e:
self.log.error("Unable to collect metrics for container: {0} ({1}".format(c_id, e))
# container metrics from kubernetes API: limits and requests
for pod in pods_list['items']:
try:
containers = pod['spec']['containers']
name2id = {}
for cs in pod['status'].get('containerStatuses', []):
c_id = cs.get('containerID', '').split('//')[-1]
name = cs.get('name')
if name:
name2id[name] = c_id
except KeyError:
self.log.debug("Pod %s does not have containers specs, skipping...", pod['metadata'].get('name'))
continue
for container in containers:
c_name = container.get('name')
_tags = container_tags.get(name2id.get(c_name), [])
prog = re.compile(r'[-+]?\d+[\.]?\d*')
# limits
try:
for limit, value_str in container['resources']['limits'].iteritems():
values = [float(s) for s in prog.findall(value_str)]
if len(values) != 1:
self.log.warning("Error parsing limits value string: %s", value_str)
continue
self.publish_gauge(self, '{}.{}.limits'.format(NAMESPACE, limit), values[0], _tags)
except (KeyError, AttributeError) as e:
self.log.debug("Unable to retrieve container limits for %s: %s", c_name, e)
self.log.debug("Container object for {}: {}".format(c_name, container))
# requests
try:
for request, value_str in container['resources']['requests'].iteritems():
values = [float(s) for s in prog.findall(value_str)]
if len(values) != 1:
self.log.warning("Error parsing requests value string: %s", value_str)
continue
self.publish_gauge(self, '{}.{}.requests'.format(NAMESPACE, request), values[0], _tags)
except (KeyError, AttributeError) as e:
self.log.error("Unable to retrieve container requests for %s: %s", c_name, e)
self.log.debug("Container object for {}: {}".format(c_name, container))
self._update_pods_metrics(instance, pods_list)
class TestKubeutil(unittest.TestCase):
def setUp(self):
self.kubeutil = KubeUtil()
@mock.patch('utils.kubeutil.KubeUtil.retrieve_pods_list',
side_effect=['foo'])
@mock.patch('utils.kubeutil.KubeUtil.extract_kube_labels')
def test_get_kube_labels(self, extract_kube_labels, retrieve_pods_list):
self.kubeutil.get_kube_labels(excluded_keys='bar')
retrieve_pods_list.assert_called_once()
extract_kube_labels.assert_called_once_with('foo', excluded_keys='bar')
def test_extract_kube_labels(self):
"""
Test with both 1.1 and 1.2 version payloads
"""
res = self.kubeutil.extract_kube_labels({}, ['foo'])
self.assertEqual(len(res), 0)
pods = json.loads(
Fixtures.read_file("pods_list_1.1.json", string_escape=False))
res = self.kubeutil.extract_kube_labels(pods, ['foo'])
labels = set(inn for out in res.values() for inn in out)
self.assertEqual(len(labels), 8)
res = self.kubeutil.extract_kube_labels(pods, ['k8s-app'])
labels = set(inn for out in res.values() for inn in out)
self.assertEqual(len(labels), 6)
pods = json.loads(
Fixtures.read_file("pods_list_1.2.json", string_escape=False))
res = self.kubeutil.extract_kube_labels(pods, ['foo'])
labels = set(inn for out in res.values() for inn in out)
self.assertEqual(len(labels), 3)
res = self.kubeutil.extract_kube_labels(pods, ['k8s-app'])
labels = set(inn for out in res.values() for inn in out)
self.assertEqual(len(labels), 3)
def test_extract_meta(self):
"""
Test with both 1.1 and 1.2 version payloads
"""
res = self.kubeutil.extract_meta({}, 'foo')
self.assertEqual(len(res), 0)
pods = json.loads(
Fixtures.read_file("pods_list_1.1.json", string_escape=False))
res = self.kubeutil.extract_meta(pods, 'foo')
self.assertEqual(len(res), 0)
res = self.kubeutil.extract_meta(pods, 'uid')
self.assertEqual(len(res), 6)
pods = json.loads(
Fixtures.read_file("pods_list_1.2.json", string_escape=False))
res = self.kubeutil.extract_meta(pods, 'foo')
self.assertEqual(len(res), 0)
res = self.kubeutil.extract_meta(pods, 'uid')
self.assertEqual(len(res), 4)
@mock.patch('utils.kubeutil.retrieve_json')
def test_retrieve_pods_list(self, retrieve_json):
self.kubeutil.retrieve_pods_list()
retrieve_json.assert_called_once_with(self.kubeutil.pods_list_url)
@mock.patch('utils.kubeutil.retrieve_json')
def test_retrieve_metrics(self, retrieve_json):
self.kubeutil.retrieve_metrics()
retrieve_json.assert_called_once_with(self.kubeutil.metrics_url)
def test_filter_pods_list(self):
"""
Test with both 1.1 and 1.2 version payloads
"""
res = self.kubeutil.filter_pods_list({}, 'foo')
self.assertEqual(len(res.get('items')), 0)
pods = json.loads(
Fixtures.read_file("pods_list_1.1.json", string_escape=False))
res = self.kubeutil.filter_pods_list(pods, '10.240.0.9')
self.assertEqual(len(res.get('items')), 5)
pods = json.loads(
Fixtures.read_file("pods_list_1.1.json", string_escape=False))
res = self.kubeutil.filter_pods_list(pods, 'foo')
self.assertEqual(len(res.get('items')), 0)
pods = json.loads(
Fixtures.read_file("pods_list_1.2.json", string_escape=False))
res = self.kubeutil.filter_pods_list(pods, '10.240.0.5')
self.assertEqual(len(res.get('items')), 1)
pods = json.loads(
Fixtures.read_file("pods_list_1.2.json", string_escape=False))
res = self.kubeutil.filter_pods_list(pods, 'foo')
self.assertEqual(len(res.get('items')), 0)
@mock.patch('utils.kubeutil.requests')
def test_retrieve_json_auth(self, r):
self.kubeutil.retrieve_json_auth('url', 'foo_tok')
r.get.assert_called_once_with(
'url',
verify=False,
timeout=10,
headers={'Authorization': 'Bearer foo_tok'})
self.kubeutil.CA_CRT_PATH = __file__
self.kubeutil.retrieve_json_auth('url', 'foo_tok')
r.get.assert_called_with('url',
verify=__file__,
timeout=10,
headers={'Authorization': 'Bearer foo_tok'})
def test_get_node_info(self):
with mock.patch('utils.kubeutil.KubeUtil._fetch_host_data') as f:
self.kubeutil.get_node_info()
f.assert_called_once()
f.reset_mock()
self.kubeutil._node_ip = 'foo'
self.kubeutil._node_name = 'bar'
ip, name = self.kubeutil.get_node_info()
self.assertEqual(ip, 'foo')
self.assertEqual(name, 'bar')
f.assert_not_called()
def test__fetch_host_data(self):
"""
Test with both 1.1 and 1.2 version payloads
"""
with mock.patch(
'utils.kubeutil.KubeUtil.retrieve_pods_list') as mock_pods:
self.kubeutil.host_name = 'dd-agent-1rxlh'
mock_pods.return_value = json.loads(
Fixtures.read_file("pods_list_1.2.json", string_escape=False))
self.kubeutil._fetch_host_data()
self.assertEqual(self.kubeutil._node_ip, '10.240.0.9')
self.assertEqual(self.kubeutil._node_name,
'kubernetes-massi-minion-k23m')
self.kubeutil.host_name = 'heapster-v11-l8sh1'
mock_pods.return_value = json.loads(
Fixtures.read_file("pods_list_1.1.json", string_escape=False))
self.kubeutil._fetch_host_data()
self.assertEqual(self.kubeutil._node_ip, '10.240.0.9')
self.assertEqual(self.kubeutil._node_name,
'gke-cluster-1-8046fdfa-node-ld35')
def test_get_auth_token(self):
KubeUtil.AUTH_TOKEN_PATH = '/foo/bar'
self.assertIsNone(KubeUtil.get_auth_token())
KubeUtil.AUTH_TOKEN_PATH = Fixtures.file(
'events.json') # any file could do the trick
self.assertIsNotNone(KubeUtil.get_auth_token())
def test_is_k8s(self):
os.unsetenv('KUBERNETES_PORT')
self.assertFalse(Platform.is_k8s())
os.environ['KUBERNETES_PORT'] = '999'
self.assertTrue(Platform.is_k8s())
def test_extract_event_tags(self):
events = json.loads(
Fixtures.read_file("events.json", string_escape=False))['items']
for ev in events:
tags = KubeUtil().extract_event_tags(ev)
# there should be 4 tags except for some events where source.host is missing
self.assertTrue(len(tags) >= 3)
tag_names = [tag.split(':')[0] for tag in tags]
self.assertIn('reason', tag_names)
self.assertIn('namespace', tag_names)
self.assertIn('object_type', tag_names)
if len(tags) == 4:
self.assertIn('node_name', tag_names)
class Kubernetes(AgentCheck):
""" Collect metrics and events from kubelet """
pod_names_by_container = {}
def __init__(self, name, init_config, agentConfig, instances=None):
if instances is not None and len(instances) > 1:
raise Exception(
'Kubernetes check only supports one configured instance.')
AgentCheck.__init__(self, name, init_config, agentConfig, instances)
self.kubeutil = KubeUtil()
if not self.kubeutil.host:
raise Exception(
'Unable to get default router and host parameter is not set')
def _perform_kubelet_checks(self, url):
service_check_base = NAMESPACE + '.kubelet.check'
is_ok = True
try:
r = requests.get(url)
for line in r.iter_lines():
# avoid noise; this check is expected to fail since we override the container hostname
if line.find('hostname') != -1:
continue
matches = re.match('\[(.)\]([^\s]+) (.*)?', line)
if not matches or len(matches.groups()) < 2:
continue
service_check_name = service_check_base + '.' + matches.group(
2)
status = matches.group(1)
if status == '+':
self.service_check(service_check_name, AgentCheck.OK)
else:
self.service_check(service_check_name, AgentCheck.CRITICAL)
is_ok = False
except Exception as e:
self.log.warning('kubelet check %s failed: %s' % (url, str(e)))
self.service_check(service_check_base,
AgentCheck.CRITICAL,
message='Kubelet check %s failed: %s' %
(url, str(e)))
else:
if is_ok:
self.service_check(service_check_base, AgentCheck.OK)
else:
self.service_check(service_check_base, AgentCheck.CRITICAL)
def check(self, instance):
self.max_depth = instance.get('max_depth', DEFAULT_MAX_DEPTH)
enabled_gauges = instance.get('enabled_gauges', DEFAULT_ENABLED_GAUGES)
self.enabled_gauges = [
"{0}.{1}".format(NAMESPACE, x) for x in enabled_gauges
]
enabled_rates = instance.get('enabled_rates', DEFAULT_ENABLED_RATES)
self.enabled_rates = [
"{0}.{1}".format(NAMESPACE, x) for x in enabled_rates
]
self.publish_aliases = _is_affirmative(
instance.get('publish_aliases', DEFAULT_PUBLISH_ALIASES))
self.use_histogram = _is_affirmative(
instance.get('use_histogram', DEFAULT_USE_HISTOGRAM))
self.publish_rate = FUNC_MAP[RATE][self.use_histogram]
self.publish_gauge = FUNC_MAP[GAUGE][self.use_histogram]
# kubelet health checks
self._perform_kubelet_checks(self.kubeutil.kube_health_url)
# kubelet metrics
self._update_metrics(instance)
def _publish_raw_metrics(self, metric, dat, tags, depth=0):
if depth >= self.max_depth:
self.log.warning('Reached max depth on metric=%s' % metric)
return
if isinstance(dat, numbers.Number):
if self.enabled_rates and any(
[fnmatch(metric, pat) for pat in self.enabled_rates]):
self.publish_rate(self, metric, float(dat), tags)
elif self.enabled_gauges and any(
[fnmatch(metric, pat) for pat in self.enabled_gauges]):
self.publish_gauge(self, metric, float(dat), tags)
elif isinstance(dat, dict):
for k, v in dat.iteritems():
self._publish_raw_metrics(metric + '.%s' % k.lower(), v, tags,
depth + 1)
elif isinstance(dat, list):
self._publish_raw_metrics(metric, dat[-1], tags, depth + 1)
@staticmethod
def _shorten_name(name):
# shorten docker image id
return re.sub('([0-9a-fA-F]{64,})', lambda x: x.group(1)[0:12], name)
def _get_post_1_2_tags(self, cont_labels, subcontainer, kube_labels):
tags = []
pod_name = cont_labels[KubeUtil.POD_NAME_LABEL]
pod_namespace = cont_labels[KubeUtil.NAMESPACE_LABEL]
tags.append(u"pod_name:{0}/{1}".format(pod_namespace, pod_name))
tags.append(u"kube_namespace:{0}".format(pod_namespace))
kube_labels_key = "{0}/{1}".format(pod_namespace, pod_name)
pod_labels = kube_labels.get(kube_labels_key)
if pod_labels:
tags += list(pod_labels)
if "-" in pod_name:
replication_controller = "-".join(pod_name.split("-")[:-1])
tags.append("kube_replication_controller:%s" %
replication_controller)
if self.publish_aliases and subcontainer.get("aliases"):
for alias in subcontainer['aliases'][1:]:
# we don't add the first alias as it will be the container_name
tags.append('container_alias:%s' % (self._shorten_name(alias)))
return tags
def _get_pre_1_2_tags(self, cont_labels, subcontainer, kube_labels):
tags = []
pod_name = cont_labels[KubeUtil.POD_NAME_LABEL]
tags.append(u"pod_name:{0}".format(pod_name))
pod_labels = kube_labels.get(pod_name)
if pod_labels:
tags.extend(list(pod_labels))
if "-" in pod_name:
replication_controller = "-".join(pod_name.split("-")[:-1])
if "/" in replication_controller:
namespace, replication_controller = replication_controller.split(
"/", 1)
tags.append(u"kube_namespace:%s" % namespace)
tags.append(u"kube_replication_controller:%s" %
replication_controller)
if self.publish_aliases and subcontainer.get("aliases"):
for alias in subcontainer['aliases'][1:]:
# we don't add the first alias as it will be the container_name
tags.append(u"container_alias:%s" %
(self._shorten_name(alias)))
return tags
def _update_container_metrics(self, instance, subcontainer, kube_labels):
tags = list(instance.get('tags', [])) # add support for custom tags
if len(subcontainer.get('aliases', [])) >= 1:
# The first alias seems to always match the docker container name
container_name = subcontainer['aliases'][0]
else:
# We default to the container id
container_name = subcontainer['name']
tags.append('container_name:%s' % container_name)
try:
cont_labels = subcontainer['spec']['labels']
except KeyError:
self.log.debug("Subcontainer, doesn't have any labels")
cont_labels = {}
# Collect pod names, namespaces, rc...
if KubeUtil.NAMESPACE_LABEL in cont_labels and KubeUtil.POD_NAME_LABEL in cont_labels:
# Kubernetes >= 1.2
tags += self._get_post_1_2_tags(cont_labels, subcontainer,
kube_labels)
elif KubeUtil.POD_NAME_LABEL in cont_labels:
# Kubernetes <= 1.1
tags += self._get_pre_1_2_tags(cont_labels, subcontainer,
kube_labels)
else:
# Those are containers that are not part of a pod.
# They are top aggregate views and don't have the previous metadata.
tags.append("pod_name:no_pod")
stats = subcontainer['stats'][-1] # take the latest
self._publish_raw_metrics(NAMESPACE, stats, tags)
if subcontainer.get("spec", {}).get("has_filesystem"):
fs = stats['filesystem'][-1]
fs_utilization = float(fs['usage']) / float(fs['capacity'])
self.publish_gauge(self, NAMESPACE + '.filesystem.usage_pct',
fs_utilization, tags)
if subcontainer.get("spec", {}).get("has_network"):
net = stats['network']
self.publish_rate(self, NAMESPACE + '.network_errors',
sum(float(net[x]) for x in NET_ERRORS), tags)
def _retrieve_metrics(self, url):
return retrieve_json(url)
def _update_metrics(self, instance):
pods_list = self.kubeutil.retrieve_pods_list()
metrics = self._retrieve_metrics(self.kubeutil.metrics_url)
excluded_labels = instance.get('excluded_labels')
kube_labels = self.kubeutil.extract_kube_labels(
pods_list, excluded_keys=excluded_labels)
if not metrics:
raise Exception('No metrics retrieved cmd=%s' % self.metrics_cmd)
for subcontainer in metrics:
try:
self._update_container_metrics(instance, subcontainer,
kube_labels)
except Exception as e:
self.log.error(
"Unable to collect metrics for container: {0} ({1}".format(
subcontainer.get('name'), e))
self._update_pods_metrics(instance, pods_list)
def _update_pods_metrics(self, instance, pods):
supported_kinds = [
"DaemonSet",
"Deployment",
"Job",
"ReplicationController",
"ReplicaSet",
]
controllers_map = defaultdict(int)
for pod in pods['items']:
try:
created_by = json.loads(
pod['metadata']['annotations']['kubernetes.io/created-by'])
kind = created_by['reference']['kind']
if kind in supported_kinds:
controllers_map[created_by['reference']['name']] += 1
except KeyError:
continue
tags = instance.get('tags', [])
for ctrl, pod_count in controllers_map.iteritems():
_tags = tags[:] # copy base tags
_tags.append('kube_replication_controller:{0}'.format(ctrl))
self.publish_gauge(self, NAMESPACE + '.pods.running', pod_count,
_tags)
class TestKubeutil(unittest.TestCase):
def setUp(self):
self.kubeutil = KubeUtil()
@mock.patch("utils.kubeutil.KubeUtil.retrieve_pods_list", side_effect=["foo"])
@mock.patch("utils.kubeutil.KubeUtil.extract_kube_labels")
def test_get_kube_labels(self, extract_kube_labels, retrieve_pods_list):
self.kubeutil.get_kube_labels(excluded_keys="bar")
retrieve_pods_list.assert_called_once()
extract_kube_labels.assert_called_once_with("foo", excluded_keys="bar")
def test_extract_kube_labels(self):
"""
Test with both 1.1 and 1.2 version payloads
"""
res = self.kubeutil.extract_kube_labels({}, ["foo"])
self.assertEqual(len(res), 0)
pods = json.loads(Fixtures.read_file("pods_list_1.1.json", string_escape=False))
res = self.kubeutil.extract_kube_labels(pods, ["foo"])
labels = set(inn for out in res.values() for inn in out)
self.assertEqual(len(labels), 8)
res = self.kubeutil.extract_kube_labels(pods, ["k8s-app"])
labels = set(inn for out in res.values() for inn in out)
self.assertEqual(len(labels), 6)
pods = json.loads(Fixtures.read_file("pods_list_1.2.json", string_escape=False))
res = self.kubeutil.extract_kube_labels(pods, ["foo"])
labels = set(inn for out in res.values() for inn in out)
self.assertEqual(len(labels), 3)
res = self.kubeutil.extract_kube_labels(pods, ["k8s-app"])
labels = set(inn for out in res.values() for inn in out)
self.assertEqual(len(labels), 3)
def test_extract_meta(self):
"""
Test with both 1.1 and 1.2 version payloads
"""
res = self.kubeutil.extract_meta({}, "foo")
self.assertEqual(len(res), 0)
pods = json.loads(Fixtures.read_file("pods_list_1.1.json", string_escape=False))
res = self.kubeutil.extract_meta(pods, "foo")
self.assertEqual(len(res), 0)
res = self.kubeutil.extract_meta(pods, "uid")
self.assertEqual(len(res), 6)
pods = json.loads(Fixtures.read_file("pods_list_1.2.json", string_escape=False))
res = self.kubeutil.extract_meta(pods, "foo")
self.assertEqual(len(res), 0)
res = self.kubeutil.extract_meta(pods, "uid")
self.assertEqual(len(res), 4)
@mock.patch("utils.kubeutil.retrieve_json")
def test_retrieve_pods_list(self, retrieve_json):
self.kubeutil.retrieve_pods_list()
retrieve_json.assert_called_once_with(self.kubeutil.pods_list_url)
@mock.patch("utils.kubeutil.retrieve_json")
def test_retrieve_metrics(self, retrieve_json):
self.kubeutil.retrieve_metrics()
retrieve_json.assert_called_once_with(self.kubeutil.metrics_url)
def test_filter_pods_list(self):
"""
Test with both 1.1 and 1.2 version payloads
"""
res = self.kubeutil.filter_pods_list({}, "foo")
self.assertEqual(len(res.get("items")), 0)
pods = json.loads(Fixtures.read_file("pods_list_1.1.json", string_escape=False))
res = self.kubeutil.filter_pods_list(pods, "10.240.0.9")
self.assertEqual(len(res.get("items")), 5)
pods = json.loads(Fixtures.read_file("pods_list_1.1.json", string_escape=False))
res = self.kubeutil.filter_pods_list(pods, "foo")
self.assertEqual(len(res.get("items")), 0)
pods = json.loads(Fixtures.read_file("pods_list_1.2.json", string_escape=False))
res = self.kubeutil.filter_pods_list(pods, "10.240.0.5")
self.assertEqual(len(res.get("items")), 1)
pods = json.loads(Fixtures.read_file("pods_list_1.2.json", string_escape=False))
res = self.kubeutil.filter_pods_list(pods, "foo")
self.assertEqual(len(res.get("items")), 0)
@mock.patch("utils.kubeutil.requests")
def test_retrieve_json_auth(self, r):
self.kubeutil.retrieve_json_auth("url", "foo_tok")
r.get.assert_called_once_with("url", verify=False, timeout=10, headers={"Authorization": "Bearer foo_tok"})
self.kubeutil.CA_CRT_PATH = __file__
self.kubeutil.retrieve_json_auth("url", "foo_tok")
r.get.assert_called_with("url", verify=__file__, timeout=10, headers={"Authorization": "Bearer foo_tok"})
def test_get_node_info(self):
with mock.patch("utils.kubeutil.KubeUtil._fetch_host_data") as f:
self.kubeutil.get_node_info()
f.assert_called_once()
f.reset_mock()
self.kubeutil._node_ip = "foo"
self.kubeutil._node_name = "bar"
ip, name = self.kubeutil.get_node_info()
self.assertEqual(ip, "foo")
self.assertEqual(name, "bar")
f.assert_not_called()
def test__fetch_host_data(self):
"""
Test with both 1.1 and 1.2 version payloads
"""
with mock.patch("utils.kubeutil.KubeUtil.retrieve_pods_list") as mock_pods:
self.kubeutil.host_name = "dd-agent-1rxlh"
mock_pods.return_value = json.loads(Fixtures.read_file("pods_list_1.2.json", string_escape=False))
self.kubeutil._fetch_host_data()
self.assertEqual(self.kubeutil._node_ip, "10.240.0.9")
self.assertEqual(self.kubeutil._node_name, "kubernetes-massi-minion-k23m")
self.kubeutil.host_name = "heapster-v11-l8sh1"
mock_pods.return_value = json.loads(Fixtures.read_file("pods_list_1.1.json", string_escape=False))
self.kubeutil._fetch_host_data()
self.assertEqual(self.kubeutil._node_ip, "10.240.0.9")
self.assertEqual(self.kubeutil._node_name, "gke-cluster-1-8046fdfa-node-ld35")
def test_get_auth_token(self):
KubeUtil.AUTH_TOKEN_PATH = "/foo/bar"
self.assertIsNone(KubeUtil.get_auth_token())
KubeUtil.AUTH_TOKEN_PATH = Fixtures.file("events.json") # any file could do the trick
self.assertIsNotNone(KubeUtil.get_auth_token())
def test_is_k8s(self):
os.unsetenv("KUBERNETES_PORT")
self.assertFalse(is_k8s())
os.environ["KUBERNETES_PORT"] = "999"
self.assertTrue(is_k8s())
class TestKubeutil(unittest.TestCase):
def setUp(self):
self.kubeutil = KubeUtil()
@mock.patch('utils.kubeutil.KubeUtil.retrieve_pods_list', side_effect=['foo'])
@mock.patch('utils.kubeutil.KubeUtil.extract_kube_labels')
def test_get_kube_labels(self, extract_kube_labels, retrieve_pods_list):
self.kubeutil.get_kube_labels(excluded_keys='bar')
retrieve_pods_list.assert_called_once()
extract_kube_labels.assert_called_once_with('foo', excluded_keys='bar')
def test_extract_kube_labels(self):
"""
Test with both 1.1 and 1.2 version payloads
"""
res = self.kubeutil.extract_kube_labels({}, ['foo'])
self.assertEqual(len(res), 0)
pods = json.loads(Fixtures.read_file("pods_list_1.1.json", string_escape=False))
res = self.kubeutil.extract_kube_labels(pods, ['foo'])
labels = set(inn for out in res.values() for inn in out)
self.assertEqual(len(labels), 8)
res = self.kubeutil.extract_kube_labels(pods, ['k8s-app'])
labels = set(inn for out in res.values() for inn in out)
self.assertEqual(len(labels), 6)
pods = json.loads(Fixtures.read_file("pods_list_1.2.json", string_escape=False))
res = self.kubeutil.extract_kube_labels(pods, ['foo'])
labels = set(inn for out in res.values() for inn in out)
self.assertEqual(len(labels), 3)
res = self.kubeutil.extract_kube_labels(pods, ['k8s-app'])
labels = set(inn for out in res.values() for inn in out)
self.assertEqual(len(labels), 3)
def test_extract_meta(self):
"""
Test with both 1.1 and 1.2 version payloads
"""
res = self.kubeutil.extract_meta({}, 'foo')
self.assertEqual(len(res), 0)
pods = json.loads(Fixtures.read_file("pods_list_1.1.json", string_escape=False))
res = self.kubeutil.extract_meta(pods, 'foo')
self.assertEqual(len(res), 0)
res = self.kubeutil.extract_meta(pods, 'uid')
self.assertEqual(len(res), 6)
pods = json.loads(Fixtures.read_file("pods_list_1.2.json", string_escape=False))
res = self.kubeutil.extract_meta(pods, 'foo')
self.assertEqual(len(res), 0)
res = self.kubeutil.extract_meta(pods, 'uid')
self.assertEqual(len(res), 4)
@mock.patch('utils.kubeutil.retrieve_json')
def test_retrieve_pods_list(self, retrieve_json):
self.kubeutil.retrieve_pods_list()
retrieve_json.assert_called_once_with(self.kubeutil.pods_list_url)
@mock.patch('utils.kubeutil.retrieve_json')
def test_retrieve_metrics(self, retrieve_json):
self.kubeutil.retrieve_metrics()
retrieve_json.assert_called_once_with(self.kubeutil.metrics_url)
def test_filter_pods_list(self):
"""
Test with both 1.1 and 1.2 version payloads
"""
res = self.kubeutil.filter_pods_list({}, 'foo')
self.assertEqual(len(res.get('items')), 0)
pods = json.loads(Fixtures.read_file("pods_list_1.1.json", string_escape=False))
res = self.kubeutil.filter_pods_list(pods, '10.240.0.9')
self.assertEqual(len(res.get('items')), 5)
pods = json.loads(Fixtures.read_file("pods_list_1.1.json", string_escape=False))
res = self.kubeutil.filter_pods_list(pods, 'foo')
self.assertEqual(len(res.get('items')), 0)
pods = json.loads(Fixtures.read_file("pods_list_1.2.json", string_escape=False))
res = self.kubeutil.filter_pods_list(pods, '10.240.0.5')
self.assertEqual(len(res.get('items')), 1)
pods = json.loads(Fixtures.read_file("pods_list_1.2.json", string_escape=False))
res = self.kubeutil.filter_pods_list(pods, 'foo')
self.assertEqual(len(res.get('items')), 0)
@mock.patch('utils.kubeutil.requests')
def test_retrieve_json_auth(self, r):
self.kubeutil.retrieve_json_auth('url', 'foo_tok')
r.get.assert_called_once_with('url', verify=False, timeout=10, headers={'Authorization': 'Bearer foo_tok'})
self.kubeutil.CA_CRT_PATH = __file__
self.kubeutil.retrieve_json_auth('url', 'foo_tok')
r.get.assert_called_with('url', verify=__file__, timeout=10, headers={'Authorization': 'Bearer foo_tok'})
def test_get_node_info(self):
with mock.patch('utils.kubeutil.KubeUtil._fetch_host_data') as f:
self.kubeutil.get_node_info()
f.assert_called_once()
f.reset_mock()
self.kubeutil._node_ip = 'foo'
self.kubeutil._node_name = 'bar'
ip, name = self.kubeutil.get_node_info()
self.assertEqual(ip, 'foo')
self.assertEqual(name, 'bar')
f.assert_not_called()
def test__fetch_host_data(self):
"""
Test with both 1.1 and 1.2 version payloads
"""
with mock.patch('utils.kubeutil.KubeUtil.retrieve_pods_list') as mock_pods:
self.kubeutil.host_name = 'dd-agent-1rxlh'
mock_pods.return_value = json.loads(Fixtures.read_file("pods_list_1.2.json", string_escape=False))
self.kubeutil._fetch_host_data()
self.assertEqual(self.kubeutil._node_ip, '10.240.0.9')
self.assertEqual(self.kubeutil._node_name, 'kubernetes-massi-minion-k23m')
self.kubeutil.host_name = 'heapster-v11-l8sh1'
mock_pods.return_value = json.loads(Fixtures.read_file("pods_list_1.1.json", string_escape=False))
self.kubeutil._fetch_host_data()
self.assertEqual(self.kubeutil._node_ip, '10.240.0.9')
self.assertEqual(self.kubeutil._node_name, 'gke-cluster-1-8046fdfa-node-ld35')
def test_get_auth_token(self):
KubeUtil.AUTH_TOKEN_PATH = '/foo/bar'
self.assertIsNone(KubeUtil.get_auth_token())
KubeUtil.AUTH_TOKEN_PATH = Fixtures.file('events.json') # any file could do the trick
self.assertIsNotNone(KubeUtil.get_auth_token())
def test_is_k8s(self):
os.unsetenv('KUBERNETES_PORT')
self.assertFalse(Platform.is_k8s())
os.environ['KUBERNETES_PORT'] = '999'
self.assertTrue(Platform.is_k8s())
class Kubernetes(AgentCheck):
""" Collect metrics and events from kubelet """
pod_names_by_container = {}
def __init__(self, name, init_config, agentConfig, instances=None):
if instances is not None and len(instances) > 1:
raise Exception('Kubernetes check only supports one configured instance.')
AgentCheck.__init__(self, name, init_config, agentConfig, instances)
self.kubeutil = KubeUtil()
if not self.kubeutil.host:
raise Exception('Unable to get default router and host parameter is not set')
def _perform_kubelet_checks(self, url):
service_check_base = NAMESPACE + '.kubelet.check'
is_ok = True
try:
r = requests.get(url)
for line in r.iter_lines():
# avoid noise; this check is expected to fail since we override the container hostname
if line.find('hostname') != -1:
continue
matches = re.match('\[(.)\]([^\s]+) (.*)?', line)
if not matches or len(matches.groups()) < 2:
continue
service_check_name = service_check_base + '.' + matches.group(2)
status = matches.group(1)
if status == '+':
self.service_check(service_check_name, AgentCheck.OK)
else:
self.service_check(service_check_name, AgentCheck.CRITICAL)
is_ok = False
except Exception as e:
self.log.warning('kubelet check failed: %s' % str(e))
self.service_check(service_check_base, AgentCheck.CRITICAL,
message='Kubelet check failed: %s' % str(e))
else:
if is_ok:
self.service_check(service_check_base, AgentCheck.OK)
else:
self.service_check(service_check_base, AgentCheck.CRITICAL)
def check(self, instance):
self.max_depth = instance.get('max_depth', DEFAULT_MAX_DEPTH)
enabled_gauges = instance.get('enabled_gauges', DEFAULT_ENABLED_GAUGES)
self.enabled_gauges = ["{0}.{1}".format(NAMESPACE, x) for x in enabled_gauges]
enabled_rates = instance.get('enabled_rates', DEFAULT_ENABLED_RATES)
self.enabled_rates = ["{0}.{1}".format(NAMESPACE, x) for x in enabled_rates]
self.publish_aliases = _is_affirmative(instance.get('publish_aliases', DEFAULT_PUBLISH_ALIASES))
self.use_histogram = _is_affirmative(instance.get('use_histogram', DEFAULT_USE_HISTOGRAM))
self.publish_rate = FUNC_MAP[RATE][self.use_histogram]
self.publish_gauge = FUNC_MAP[GAUGE][self.use_histogram]
# kubelet health checks
self._perform_kubelet_checks(self.kubeutil.kube_health_url)
# kubelet metrics
self._update_metrics(instance)
def _publish_raw_metrics(self, metric, dat, tags, depth=0):
if depth >= self.max_depth:
self.log.warning('Reached max depth on metric=%s' % metric)
return
if isinstance(dat, numbers.Number):
if self.enabled_rates and any([fnmatch(metric, pat) for pat in self.enabled_rates]):
self.publish_rate(self, metric, float(dat), tags)
elif self.enabled_gauges and any([fnmatch(metric, pat) for pat in self.enabled_gauges]):
self.publish_gauge(self, metric, float(dat), tags)
elif isinstance(dat, dict):
for k, v in dat.iteritems():
self._publish_raw_metrics(metric + '.%s' % k.lower(), v, tags, depth + 1)
elif isinstance(dat, list):
self._publish_raw_metrics(metric, dat[-1], tags, depth + 1)
@staticmethod
def _shorten_name(name):
# shorten docker image id
return re.sub('([0-9a-fA-F]{64,})', lambda x: x.group(1)[0:12], name)
def _get_post_1_2_tags(self, cont_labels, subcontainer, kube_labels):
tags = []
pod_name = cont_labels[KubeUtil.POD_NAME_LABEL]
pod_namespace = cont_labels[KubeUtil.NAMESPACE_LABEL]
tags.append(u"pod_name:{0}/{1}".format(pod_namespace, pod_name))
tags.append(u"kube_namespace:{0}".format(pod_namespace))
kube_labels_key = "{0}/{1}".format(pod_namespace, pod_name)
pod_labels = kube_labels.get(kube_labels_key)
if pod_labels:
tags += list(pod_labels)
if "-" in pod_name:
replication_controller = "-".join(pod_name.split("-")[:-1])
tags.append("kube_replication_controller:%s" % replication_controller)
if self.publish_aliases and subcontainer.get("aliases"):
for alias in subcontainer['aliases'][1:]:
# we don't add the first alias as it will be the container_name
tags.append('container_alias:%s' % (self._shorten_name(alias)))
return tags
def _get_pre_1_2_tags(self, cont_labels, subcontainer, kube_labels):
tags = []
pod_name = cont_labels[KubeUtil.POD_NAME_LABEL]
tags.append(u"pod_name:{0}".format(pod_name))
pod_labels = kube_labels.get(pod_name)
if pod_labels:
tags.extend(list(pod_labels))
if "-" in pod_name:
replication_controller = "-".join(pod_name.split("-")[:-1])
if "/" in replication_controller:
namespace, replication_controller = replication_controller.split("/", 1)
tags.append(u"kube_namespace:%s" % namespace)
tags.append(u"kube_replication_controller:%s" % replication_controller)
if self.publish_aliases and subcontainer.get("aliases"):
for alias in subcontainer['aliases'][1:]:
# we don't add the first alias as it will be the container_name
tags.append(u"container_alias:%s" % (self._shorten_name(alias)))
return tags
def _update_container_metrics(self, instance, subcontainer, kube_labels):
tags = list(instance.get('tags', [])) # add support for custom tags
if len(subcontainer.get('aliases', [])) >= 1:
# The first alias seems to always match the docker container name
container_name = subcontainer['aliases'][0]
else:
# We default to the container id
container_name = subcontainer['name']
tags.append('container_name:%s' % container_name)
try:
cont_labels = subcontainer['spec']['labels']
except KeyError:
self.log.debug("Subcontainer, doesn't have any labels")
cont_labels = {}
# Collect pod names, namespaces, rc...
if KubeUtil.NAMESPACE_LABEL in cont_labels and KubeUtil.POD_NAME_LABEL in cont_labels:
# Kubernetes >= 1.2
tags += self._get_post_1_2_tags(cont_labels, subcontainer, kube_labels)
elif KubeUtil.POD_NAME_LABEL in cont_labels:
# Kubernetes <= 1.1
tags += self._get_pre_1_2_tags(cont_labels, subcontainer, kube_labels)
else:
# Those are containers that are not part of a pod.
# They are top aggregate views and don't have the previous metadata.
tags.append("pod_name:no_pod")
stats = subcontainer['stats'][-1] # take the latest
self._publish_raw_metrics(NAMESPACE, stats, tags)
if subcontainer.get("spec", {}).get("has_filesystem"):
fs = stats['filesystem'][-1]
fs_utilization = float(fs['usage'])/float(fs['capacity'])
self.publish_gauge(self, NAMESPACE + '.filesystem.usage_pct', fs_utilization, tags)
if subcontainer.get("spec", {}).get("has_network"):
net = stats['network']
self.publish_rate(self, NAMESPACE + '.network_errors',
sum(float(net[x]) for x in NET_ERRORS),
tags)
def _retrieve_metrics(self, url):
return retrieve_json(url)
def _update_metrics(self, instance):
pods_list = self.kubeutil.retrieve_pods_list()
metrics = self._retrieve_metrics(self.kubeutil.metrics_url)
excluded_labels = instance.get('excluded_labels')
kube_labels = self.kubeutil.extract_kube_labels(pods_list, excluded_keys=excluded_labels)
if not metrics:
raise Exception('No metrics retrieved cmd=%s' % self.metrics_cmd)
for subcontainer in metrics:
try:
self._update_container_metrics(instance, subcontainer, kube_labels)
except Exception as e:
self.log.error("Unable to collect metrics for container: {0} ({1}".format(
subcontainer.get('name'), e))
self._update_pods_metrics(instance, pods_list)
def _update_pods_metrics(self, instance, pods):
supported_kinds = [
"DaemonSet",
"Deployment",
"Job",
"ReplicationController",
"ReplicaSet",
]
controllers_map = defaultdict(int)
for pod in pods['items']:
try:
created_by = json.loads(pod['metadata']['annotations']['kubernetes.io/created-by'])
kind = created_by['reference']['kind']
if kind in supported_kinds:
controllers_map[created_by['reference']['name']] += 1
except KeyError:
continue
tags = instance.get('tags', [])
for ctrl, pod_count in controllers_map.iteritems():
_tags = tags[:] # copy base tags
_tags.append('kube_replication_controller:{0}'.format(ctrl))
self.publish_gauge(self, NAMESPACE + '.pods.running', pod_count, _tags)