def json_hook(self, obj): # Not a pod (hook is called for all objects) if 'metadata' not in obj or 'status' not in obj: return obj # Quick exit for running/pending containers pod_phase = obj.get('status', {}).get('phase') if pod_phase in ["Running", "Pending"]: return obj # Filter out expired terminated pods, based on container finishedAt time expired = True for ctr in obj['status'].get('containerStatuses', []): if "terminated" not in ctr.get("state", {}): expired = False break finishedTime = ctr["state"]["terminated"].get("finishedAt") if not finishedTime: expired = False break if parse_rfc3339(finishedTime) > self.cutoff_date: expired = False break if not expired: return obj # We are ignoring this pod self.expired_count += 1 return None
def test_pod_expiration(monkeypatch, aggregator, tagger): check = KubeletCheck('kubelet', None, {}, [{}]) check.pod_list_url = "dummyurl" # Fixtures contains four pods: # - dd-agent-ntepl old but running # - hello1-1550504220-ljnzx succeeded and old enough to expire # - hello5-1550509440-rlgvf succeeded but not old enough # - hello8-1550505780-kdnjx has one old container and a recent container, don't expire monkeypatch.setattr(check, 'perform_kubelet_query', mock.Mock(return_value=MockStreamResponse('pods_expired.json'))) monkeypatch.setattr(check, '_compute_pod_expiration_datetime', mock.Mock( return_value=parse_rfc3339("2019-02-18T16:00:06Z") )) attrs = {'is_excluded.return_value': False} check.pod_list_utils = mock.Mock(**attrs) pod_list = check.retrieve_pod_list() assert pod_list['expired_count'] == 1 expected_names = ['dd-agent-ntepl', 'hello5-1550509440-rlgvf', 'hello8-1550505780-kdnjx'] collected_names = [p['metadata']['name'] for p in pod_list['items']] assert collected_names == expected_names # Test .pods.expired gauge is submitted check._report_container_state_metrics(pod_list, ["custom:tag"]) aggregator.assert_metric("kubernetes.pods.expired", value=1, tags=["custom:tag"])