def test_report_pods_running_none_ids(monkeypatch, tagger): # Make sure the method is resilient to inconsistent podlists podlist = json.loads(mock_from_file('pods.json')) podlist["items"][0]['metadata']['uid'] = None podlist["items"][1]['status']['containerStatuses'][0]['containerID'] = None check = KubeletCheck('kubelet', None, {}, [{}]) monkeypatch.setattr(check, 'retrieve_pod_list', mock.Mock(return_value=podlist)) monkeypatch.setattr(check, 'gauge', mock.Mock()) pod_list = check.retrieve_pod_list() check._report_pods_running(pod_list, []) calls = [ mock.call('kubernetes.pods.running', 1, ["pod_name:fluentd-gcp-v2.0.10-9q9t4"]), mock.call( 'kubernetes.containers.running', 2, [ "kube_container_name:prometheus-to-sd-exporter", "kube_deployment:fluentd-gcp-v2.0.10" ], ), ] check.gauge.assert_has_calls(calls, any_order=True)
def test_report_container_requests_limits(monkeypatch, tagger): check = KubeletCheck('kubelet', None, {}, [{}]) monkeypatch.setattr( check, 'retrieve_pod_list', mock.Mock(return_value=json.loads( mock_from_file('pods_requests_limits.json')))) monkeypatch.setattr(check, 'gauge', mock.Mock()) attrs = {'is_excluded.return_value': False} check.pod_list_utils = mock.Mock(**attrs) pod_list = check.retrieve_pod_list() tags = ['kube_container_name:cassandra'] check._report_container_spec_metrics(pod_list, tags) calls = [ mock.call('kubernetes.cpu.requests', 0.5, ['pod_name:cassandra-0'] + tags), mock.call('kubernetes.memory.requests', 1073741824.0, ['pod_name:cassandra-0'] + tags), mock.call('kubernetes.ephemeral-storage.requests', 0.5, ['pod_name:cassandra-0'] + tags), mock.call('kubernetes.cpu.limits', 0.5, ['pod_name:cassandra-0'] + tags), mock.call('kubernetes.memory.limits', 1073741824.0, ['pod_name:cassandra-0'] + tags), mock.call('kubernetes.ephemeral-storage.limits', 2147483648.0, ['pod_name:cassandra-0'] + tags), ] check.gauge.assert_has_calls(calls, any_order=True)
def test_pod_expiration(monkeypatch, aggregator, tagger): check = KubeletCheck('kubelet', None, {}, [{}]) check.pod_list_url = "dummyurl" # Fixtures contains four pods: # - dd-agent-ntepl old but running # - hello1-1550504220-ljnzx succeeded and old enough to expire # - hello5-1550509440-rlgvf succeeded but not old enough # - hello8-1550505780-kdnjx has one old container and a recent container, don't expire monkeypatch.setattr(check, 'perform_kubelet_query', mock.Mock(return_value=MockStreamResponse('pods_expired.json'))) monkeypatch.setattr(check, '_compute_pod_expiration_datetime', mock.Mock( return_value=parse_rfc3339("2019-02-18T16:00:06Z") )) attrs = {'is_excluded.return_value': False} check.pod_list_utils = mock.Mock(**attrs) pod_list = check.retrieve_pod_list() assert pod_list['expired_count'] == 1 expected_names = ['dd-agent-ntepl', 'hello5-1550509440-rlgvf', 'hello8-1550505780-kdnjx'] collected_names = [p['metadata']['name'] for p in pod_list['items']] assert collected_names == expected_names # Test .pods.expired gauge is submitted check._report_container_state_metrics(pod_list, ["custom:tag"]) aggregator.assert_metric("kubernetes.pods.expired", value=1, tags=["custom:tag"])
def test_report_container_spec_metrics(monkeypatch): check = KubeletCheck('kubelet', None, {}, [{}]) monkeypatch.setattr(check, 'retrieve_pod_list', mock.Mock(return_value=json.loads(mock_from_file('pods.json')))) monkeypatch.setattr(check, 'gauge', mock.Mock()) attrs = {'is_excluded.return_value': False} check.container_filter = mock.Mock(**attrs) pod_list = check.retrieve_pod_list() instance_tags = ["one:1", "two:2"] with mock.patch("datadog_checks.kubelet.kubelet.get_tags", side_effect=mocked_get_tags): check._report_container_spec_metrics(pod_list, instance_tags) calls = [ mock.call('kubernetes.cpu.requests', 0.1, ['pod_name:fluentd-gcp-v2.0.10-9q9t4'] + instance_tags), mock.call('kubernetes.memory.requests', 209715200.0, ['pod_name:fluentd-gcp-v2.0.10-9q9t4'] + instance_tags), mock.call('kubernetes.memory.limits', 314572800.0, ['pod_name:fluentd-gcp-v2.0.10-9q9t4'] + instance_tags), mock.call('kubernetes.cpu.requests', 0.1, instance_tags), mock.call('kubernetes.cpu.requests', 0.1, instance_tags), mock.call('kubernetes.memory.requests', 134217728.0, instance_tags), mock.call('kubernetes.cpu.limits', 0.25, instance_tags), mock.call('kubernetes.memory.limits', 536870912.0, instance_tags), mock.call('kubernetes.cpu.requests', 0.1, ["pod_name=demo-app-success-c485bc67b-klj45"] + instance_tags), ] check.gauge.assert_has_calls(calls, any_order=True)
def test_get_pod_by_metric_label(monkeypatch): check = KubeletCheck('kubelet', None, {}, [{}]) monkeypatch.setattr( check, 'retrieve_pod_list', mock.Mock(return_value=json.loads(mock_from_file('pods.json')))) check.pod_list = check.retrieve_pod_list() assert len(check.pod_list) == 4 kube_proxy = check._get_pod_by_metric_label([ Label("container_name", value="POD"), Label( "id", value="/kubepods/burstable/" "pod260c2b1d43b094af6d6b4ccba082c2db/" "0bce0ef7e6cd073e8f9cec3027e1c0057ce1baddce98113d742b816726a95ab1" ), ]) fluentd = check._get_pod_by_metric_label([ Label("container_name", value="POD"), Label( "id", value="/kubepods/burstable/" "pod2edfd4d9-10ce-11e8-bd5a-42010af00137/" "7990c0e549a1a578b1313475540afc53c91081c32e735564da6244ddf0b86030" ), ]) assert kube_proxy["metadata"][ "name"] == "kube-proxy-gke-haissam-default-pool-be5066f1-wnvn" assert fluentd["metadata"]["name"] == "fluentd-gcp-v2.0.10-9q9t4"
def test_report_pods_running(monkeypatch, tagger): check = KubeletCheck('kubelet', None, {}, [{}]) monkeypatch.setattr(check, 'retrieve_pod_list', mock.Mock(return_value=json.loads(mock_from_file('pods.json')))) monkeypatch.setattr(check, 'gauge', mock.Mock()) pod_list = check.retrieve_pod_list() check._report_pods_running(pod_list, []) calls = [ mock.call('kubernetes.pods.running', 1, ["pod_name:fluentd-gcp-v2.0.10-9q9t4"]), mock.call('kubernetes.pods.running', 1, ["pod_name:fluentd-gcp-v2.0.10-p13r3"]), mock.call('kubernetes.pods.running', 1, ['pod_name:demo-app-success-c485bc67b-klj45']), mock.call('kubernetes.containers.running', 2, [ "kube_container_name:fluentd-gcp", "kube_deployment:fluentd-gcp-v2.0.10" ]), mock.call('kubernetes.containers.running', 2, [ "kube_container_name:prometheus-to-sd-exporter", "kube_deployment:fluentd-gcp-v2.0.10" ]), mock.call('kubernetes.containers.running', 1, ['pod_name:demo-app-success-c485bc67b-klj45']), ] check.gauge.assert_has_calls(calls, any_order=True) # Make sure non running container/pods are not sent bad_calls = [ mock.call('kubernetes.pods.running', 1, ['pod_name:dd-agent-q6hpw']), mock.call('kubernetes.containers.running', 1, ['pod_name:dd-agent-q6hpw']), ] for c in bad_calls: assert c not in check.gauge.mock_calls
def test_report_container_spec_metrics(monkeypatch, tagger): check = KubeletCheck('kubelet', None, {}, [{}]) monkeypatch.setattr(check, 'retrieve_pod_list', mock.Mock(return_value=json.loads(mock_from_file('pods.json')))) monkeypatch.setattr(check, 'gauge', mock.Mock()) attrs = {'is_excluded.return_value': False} check.pod_list_utils = mock.Mock(**attrs) pod_list = check.retrieve_pod_list() instance_tags = ["one:1", "two:2"] check._report_container_spec_metrics(pod_list, instance_tags) calls = [ mock.call('kubernetes.cpu.requests', 0.1, [ 'kube_container_name:fluentd-gcp', 'kube_deployment:fluentd-gcp-v2.0.10' ] + instance_tags), mock.call('kubernetes.memory.requests', 209715200.0, [ 'kube_container_name:fluentd-gcp', 'kube_deployment:fluentd-gcp-v2.0.10' ] + instance_tags), mock.call('kubernetes.memory.limits', 314572800.0, [ 'kube_container_name:fluentd-gcp', 'kube_deployment:fluentd-gcp-v2.0.10' ] + instance_tags), mock.call('kubernetes.cpu.requests', 0.1, instance_tags), mock.call('kubernetes.cpu.requests', 0.1, instance_tags), mock.call('kubernetes.memory.requests', 134217728.0, instance_tags), mock.call('kubernetes.cpu.limits', 0.25, instance_tags), mock.call('kubernetes.memory.limits', 536870912.0, instance_tags), mock.call('kubernetes.cpu.requests', 0.1, ["pod_name:demo-app-success-c485bc67b-klj45"] + instance_tags), ] if any(map(lambda e: 'pod_name:pi-kff76' in e, [x[0][2] for x in check.gauge.call_args_list])): raise AssertionError("kubernetes.cpu.requests was submitted for a non-running pod") check.gauge.assert_has_calls(calls, any_order=True)
def test_report_pods_running(monkeypatch): check = KubeletCheck('kubelet', None, {}, [{}]) monkeypatch.setattr( check, 'retrieve_pod_list', mock.Mock(return_value=json.loads(mock_from_file('pods.json')))) monkeypatch.setattr(check, 'gauge', mock.Mock()) pod_list = check.retrieve_pod_list() with mock.patch("datadog_checks.kubelet.kubelet.get_tags", side_effect=mocked_get_tags): check._report_pods_running(pod_list, []) calls = [ mock.call('kubernetes.pods.running', 1, ["pod_name:fluentd-gcp-v2.0.10-9q9t4"]), mock.call('kubernetes.pods.running', 1, ["pod_name:fluentd-gcp-v2.0.10-fkeuj"]), mock.call('kubernetes.containers.running', 2, [ "kube_container_name:fluentd-gcp", "kube_deployment:fluentd-gcp-v2.0.10" ]), mock.call('kubernetes.containers.running', 2, [ "kube_container_name:prometheus-to-sd-exporter", "kube_deployment:fluentd-gcp-v2.0.10" ]), ] check.gauge.assert_has_calls(calls, any_order=True)
def test_create_pod_tags_by_pvc(monkeypatch, tagger): check = KubeletCheck('kubelet', {}, [{}]) monkeypatch.setattr( check, 'retrieve_pod_list', mock.Mock(return_value=json.loads(mock_from_file('pods.json')))) pod_list = check.retrieve_pod_list() pod_tags_by_pvc = check._create_pod_tags_by_pvc(pod_list) expected_result = { 'default/www-web-2': { 'kube_namespace:default', 'kube_service:nginx', 'kube_stateful_set:web', 'namespace:default', }, 'default/www2-web-3': { 'kube_namespace:default', 'kube_service:nginx', 'kube_stateful_set:web', 'namespace:default', }, } assert pod_tags_by_pvc == expected_result # Test empty case empty = defaultdict(set) pod_tags_by_pvc = check._create_pod_tags_by_pvc({}) assert pod_tags_by_pvc == empty
def test_is_static_pending_pod(monkeypatch): check = KubeletCheck('kubelet', None, {}, [{}]) monkeypatch.setattr( check, 'retrieve_pod_list', mock.Mock(return_value=json.loads(mock_from_file('pods.json')))) check.pod_list = check.retrieve_pod_list() assert len(check.pod_list) == 4 static_pod = check._get_pod_by_metric_label([ Label("container_name", value="POD"), Label( "id", value="/kubepods/burstable/" "pod260c2b1d43b094af6d6b4ccba082c2db/" "0bce0ef7e6cd073e8f9cec3027e1c0057ce1baddce98113d742b816726a95ab1" ), ]) api_pod = check._get_pod_by_metric_label([ Label("container_name", value="POD"), Label( "id", value="/kubepods/burstable/" "pod2edfd4d9-10ce-11e8-bd5a-42010af00137/" "7990c0e549a1a578b1313475540afc53c91081c32e735564da6244ddf0b86030" ), ]) assert check._is_static_pending_pod(static_pod) is True assert check._is_static_pending_pod(api_pod) is False
def test_is_pod_host_networked(monkeypatch): check = KubeletCheck('kubelet', None, {}, [{}]) monkeypatch.setattr(check, 'retrieve_pod_list', mock.Mock(return_value=json.loads(mock_from_file('pods.json')))) check.pod_list = check.retrieve_pod_list() assert len(check.pod_list) == 4 assert check._is_pod_host_networked("not-here") is False assert check._is_pod_host_networked('260c2b1d43b094af6d6b4ccba082c2db') is True assert check._is_pod_host_networked('2edfd4d9-10ce-11e8-bd5a-42010af00137') is False
def test_retrieve_pod_list_success(monkeypatch): check = KubeletCheck('kubelet', {}, [{}]) check.pod_list_url = "dummyurl" monkeypatch.setattr(check, 'perform_kubelet_query', mock.Mock(return_value=MockStreamResponse('pod_list_raw.dat'))) monkeypatch.setattr(check, '_compute_pod_expiration_datetime', mock.Mock(return_value=None)) retrieved = check.retrieve_pod_list() expected = json.loads(mock_from_file("pod_list_raw.json")) assert json.dumps(retrieved, sort_keys=True) == json.dumps(expected, sort_keys=True)
def test_retrieved_pod_list_failure(monkeypatch): def mock_perform_kubelet_query(s): raise Exception("network error") check = KubeletCheck('kubelet', None, {}, [{}]) check.pod_list_url = "dummyurl" monkeypatch.setattr(check, 'perform_kubelet_query', mock_perform_kubelet_query) retrieved = check.retrieve_pod_list() assert retrieved is None
def test_report_container_state_metrics(monkeypatch): check = KubeletCheck('kubelet', None, {}, [{}]) check.pod_list_url = "dummyurl" monkeypatch.setattr( check, 'perform_kubelet_query', mock.Mock(return_value=MockStreamResponse('pods_crashed.json'))) monkeypatch.setattr(check, '_compute_pod_expiration_datetime', mock.Mock(return_value=None)) monkeypatch.setattr(check, 'gauge', mock.Mock()) attrs = {'is_excluded.return_value': False} check.pod_list_utils = mock.Mock(**attrs) pod_list = check.retrieve_pod_list() instance_tags = ["one:1", "two:2"] with mock.patch("datadog_checks.kubelet.kubelet.get_tags", side_effect=mocked_get_tags): check._report_container_state_metrics(pod_list, instance_tags) calls = [ mock.call('kubernetes.containers.last_state.terminated', 1, [ 'kube_container_name:fluentd-gcp', 'kube_deployment:fluentd-gcp-v2.0.10' ] + instance_tags + ['reason:OOMKilled']), mock.call('kubernetes.containers.state.waiting', 1, [ 'kube_container_name:prometheus-to-sd-exporter', 'kube_deployment:fluentd-gcp-v2.0.10' ] + instance_tags + ['reason:CrashLoopBackOff']), mock.call('kubernetes.containers.restarts', 1, [ 'kube_container_name:fluentd-gcp', 'kube_deployment:fluentd-gcp-v2.0.10' ] + instance_tags), mock.call('kubernetes.containers.restarts', 0, [ 'kube_container_name:prometheus-to-sd-exporter', 'kube_deployment:fluentd-gcp-v2.0.10' ] + instance_tags), ] check.gauge.assert_has_calls(calls, any_order=True) container_state_gauges = [ x[0][2] for x in check.gauge.call_args_list if x[0][0].startswith('kubernetes.containers.state') ] if any(map(lambda e: 'reason:TransientReason' in e, container_state_gauges)): raise AssertionError( 'kubernetes.containers.state.* was submitted with a transient reason' ) if any( map(lambda e: not any(x for x in e if x.startswith('reason:')), container_state_gauges)): raise AssertionError( 'kubernetes.containers.state.* was submitted without a reason')
def test_retrieve_pod_list_success(monkeypatch): class MockResponse: def __init__(self, json_data): self.json_data = json_data def json(self): return (json.loads(self.json_data)) check = KubeletCheck('kubelet', None, {}, [{}]) check.pod_list_url = "dummyurl" monkeypatch.setattr(check, 'perform_kubelet_query', mock.Mock(return_value=MockResponse(mock_from_file('pod_list_raw.dat')))) retrieved = check.retrieve_pod_list() expected = json.loads(mock_from_file("pod_list_raw.json")) assert (json.dumps(retrieved, sort_keys=True) == json.dumps(expected, sort_keys=True))