def json_hook(self, obj):
# Not a pod (hook is called for all objects)
if 'metadata' not in obj or 'status' not in obj:
return obj
# Quick exit for running/pending containers
pod_phase = obj.get('status', {}).get('phase')
if pod_phase in ["Running", "Pending"]:
return obj
# Filter out expired terminated pods, based on container finishedAt time
expired = True
for ctr in obj['status'].get('containerStatuses', []):
if "terminated" not in ctr.get("state", {}):
expired = False
break
finishedTime = ctr["state"]["terminated"].get("finishedAt")
if not finishedTime:
expired = False
break
if parse_rfc3339(finishedTime) > self.cutoff_date:
expired = False
break
if not expired:
return obj
# We are ignoring this pod
self.expired_count += 1
return None
def test_pod_expiration(monkeypatch, aggregator, tagger):
check = KubeletCheck('kubelet', None, {}, [{}])
check.pod_list_url = "dummyurl"
# Fixtures contains four pods:
# - dd-agent-ntepl old but running
# - hello1-1550504220-ljnzx succeeded and old enough to expire
# - hello5-1550509440-rlgvf succeeded but not old enough
# - hello8-1550505780-kdnjx has one old container and a recent container, don't expire
monkeypatch.setattr(check, 'perform_kubelet_query',
mock.Mock(return_value=MockStreamResponse('pods_expired.json')))
monkeypatch.setattr(check, '_compute_pod_expiration_datetime', mock.Mock(
return_value=parse_rfc3339("2019-02-18T16:00:06Z")
))
attrs = {'is_excluded.return_value': False}
check.pod_list_utils = mock.Mock(**attrs)
pod_list = check.retrieve_pod_list()
assert pod_list['expired_count'] == 1
expected_names = ['dd-agent-ntepl', 'hello5-1550509440-rlgvf', 'hello8-1550505780-kdnjx']
collected_names = [p['metadata']['name'] for p in pod_list['items']]
assert collected_names == expected_names
# Test .pods.expired gauge is submitted
check._report_container_state_metrics(pod_list, ["custom:tag"])
aggregator.assert_metric("kubernetes.pods.expired", value=1, tags=["custom:tag"])
