def test_report_pods_running_none_ids(monkeypatch, tagger): # Make sure the method is resilient to inconsistent podlists podlist = json.loads(mock_from_file('pods.json')) podlist["items"][0]['metadata']['uid'] = None podlist["items"][1]['status']['containerStatuses'][0]['containerID'] = None check = KubeletCheck('kubelet', None, {}, [{}]) monkeypatch.setattr(check, 'retrieve_pod_list', mock.Mock(return_value=podlist)) monkeypatch.setattr(check, 'gauge', mock.Mock()) pod_list = check.retrieve_pod_list() check._report_pods_running(pod_list, []) calls = [ mock.call('kubernetes.pods.running', 1, [ 'cluster_name:stubbed-cluster-name', "pod_name:fluentd-gcp-v2.0.10-9q9t4" ]), mock.call( 'kubernetes.containers.running', 2, [ 'cluster_name:stubbed-cluster-name', "kube_container_name:prometheus-to-sd-exporter", "kube_deployment:fluentd-gcp-v2.0.10" ], ), ] check.gauge.assert_has_calls(calls, any_order=True)
def test_report_pods_running(monkeypatch, tagger): check = KubeletCheck('kubelet', None, {}, [{}]) monkeypatch.setattr(check, 'retrieve_pod_list', mock.Mock(return_value=json.loads(mock_from_file('pods.json')))) monkeypatch.setattr(check, 'gauge', mock.Mock()) pod_list = check.retrieve_pod_list() check._report_pods_running(pod_list, []) calls = [ mock.call('kubernetes.pods.running', 1, ['cluster_name:stubbed-cluster-name', "pod_name:fluentd-gcp-v2.0.10-9q9t4"]), mock.call('kubernetes.pods.running', 1, ['cluster_name:stubbed-cluster-name', "pod_name:fluentd-gcp-v2.0.10-p13r3"]), mock.call('kubernetes.pods.running', 1, ['cluster_name:stubbed-cluster-name', 'pod_name:demo-app-success-c485bc67b-klj45']), mock.call( 'kubernetes.containers.running', 2, ['cluster_name:stubbed-cluster-name', "kube_container_name:fluentd-gcp", "kube_deployment:fluentd-gcp-v2.0.10"], ), mock.call( 'kubernetes.containers.running', 2, ['cluster_name:stubbed-cluster-name', "kube_container_name:prometheus-to-sd-exporter", "kube_deployment:fluentd-gcp-v2.0.10"], ), mock.call('kubernetes.containers.running', 1, ['cluster_name:stubbed-cluster-name', 'pod_name:demo-app-success-c485bc67b-klj45']), ] check.gauge.assert_has_calls(calls, any_order=True) # Make sure non running container/pods are not sent bad_calls = [ mock.call('kubernetes.pods.running', 1, ['cluster_name:stubbed-cluster-name', 'pod_name:dd-agent-q6hpw']), mock.call('kubernetes.containers.running', 1, ['cluster_name:stubbed-cluster-name', 'pod_name:dd-agent-q6hpw']), ] for c in bad_calls: assert c not in check.gauge.mock_calls
def test_retrieve_pod_list_success(monkeypatch): check = KubeletCheck('kubelet', None, {}, [{}]) check.pod_list_url = "dummyurl" monkeypatch.setattr(check, 'perform_kubelet_query', mock.Mock(return_value=MockStreamResponse('pod_list_raw.dat'))) monkeypatch.setattr(check, '_compute_pod_expiration_datetime', mock.Mock(return_value=None)) retrieved = check.retrieve_pod_list() expected = json.loads(mock_from_file("pod_list_raw.json")) assert json.dumps(retrieved, sort_keys=True) == json.dumps(expected, sort_keys=True)
def test_retrieved_pod_list_failure(monkeypatch): def mock_perform_kubelet_query(s, stream=False): raise Exception("network error") check = KubeletCheck('kubelet', None, {}, [{}]) check.pod_list_url = "dummyurl" monkeypatch.setattr(check, 'perform_kubelet_query', mock_perform_kubelet_query) retrieved = check.retrieve_pod_list() assert retrieved is None
def test_pod_expiration(monkeypatch, aggregator, tagger): check = KubeletCheck('kubelet', None, {}, [{}]) check.pod_list_url = "dummyurl" # Fixtures contains four pods: # - dd-agent-ntepl old but running # - hello1-1550504220-ljnzx succeeded and old enough to expire # - hello5-1550509440-rlgvf succeeded but not old enough # - hello8-1550505780-kdnjx has one old container and a recent container, don't expire monkeypatch.setattr(check, 'perform_kubelet_query', mock.Mock(return_value=MockStreamResponse('pods_expired.json'))) monkeypatch.setattr( check, '_compute_pod_expiration_datetime', mock.Mock(return_value=parse_rfc3339("2019-02-18T16:00:06Z")) ) attrs = {'is_excluded.return_value': False} check.pod_list_utils = mock.Mock(**attrs) pod_list = check.retrieve_pod_list() assert pod_list['expired_count'] == 1 expected_names = ['dd-agent-ntepl', 'hello5-1550509440-rlgvf', 'hello8-1550505780-kdnjx'] collected_names = [p['metadata']['name'] for p in pod_list['items']] assert collected_names == expected_names # Test .pods.expired gauge is submitted check._report_container_state_metrics(pod_list, ["custom:tag"]) aggregator.assert_metric("kubernetes.pods.expired", value=1, tags=["custom:tag", 'cluster_name:stubbed-cluster-name']) # Ensure we can iterate twice over the podlist check._report_pods_running(pod_list, []) aggregator.assert_metric("kubernetes.pods.running", value=1, tags=["pod_name:dd-agent-ntepl", 'cluster_name:stubbed-cluster-name']) aggregator.assert_metric("kubernetes.containers.running", value=1, tags=["pod_name:dd-agent-ntepl", 'cluster_name:stubbed-cluster-name'])
def test_report_node_metrics(monkeypatch): check = KubeletCheck('kubelet', None, {}, [{}]) mock_resp = mock.Mock(status_code=200, raise_for_status=mock.Mock()) mock_resp.json = mock.Mock(return_value={'num_cores': 4, 'memory_capacity': 512}) monkeypatch.setattr(check, '_retrieve_node_spec', mock.Mock(return_value=mock_resp)) monkeypatch.setattr(check, 'gauge', mock.Mock()) check._report_node_metrics(['foo:bar']) calls = [ mock.call('kubernetes.cpu.capacity', 4.0, ['foo:bar', 'cluster_name:stubbed-cluster-name']), mock.call('kubernetes.memory.capacity', 512.0, ['foo:bar', 'cluster_name:stubbed-cluster-name']), ] check.gauge.assert_has_calls(calls, any_order=False)
def test_kubelet_default_options(): check = KubeletCheck('kubelet', None, {}, [{}]) assert check.cadvisor_scraper_config['namespace'] == 'kubernetes' assert check.kubelet_scraper_config['namespace'] == 'kubernetes' assert isinstance(check.cadvisor_scraper_config, dict) assert isinstance(check.kubelet_scraper_config, dict)
def mock_kubelet_check(monkeypatch, instances): """ Returns a check that uses mocked data for responses from prometheus endpoints, pod list, and node spec. """ check = KubeletCheck('kubelet', None, {}, instances) monkeypatch.setattr(check, 'retrieve_pod_list', mock.Mock(return_value=json.loads(mock_from_file('pods.json')))) mock_resp = mock.Mock(status_code=200, raise_for_status=mock.Mock(), json=mock.Mock(return_value=NODE_SPEC)) monkeypatch.setattr(check, '_retrieve_node_spec', mock.Mock(return_value=mock_resp)) monkeypatch.setattr(check, '_perform_kubelet_check', mock.Mock(return_value=None)) monkeypatch.setattr(check, '_compute_pod_expiration_datetime', mock.Mock(return_value=None)) def mocked_poll(*args, **kwargs): scraper_config = args[0] prometheus_url = scraper_config['prometheus_url'] if prometheus_url.endswith('/metrics/cadvisor'): # Mock response for "/metrics/cadvisor" content = mock_from_file('cadvisor_metrics.txt') elif prometheus_url.endswith('/metrics'): # Mock response for "/metrics" content = mock_from_file('kubelet_metrics.txt') else: raise Exception("Must be a valid endpoint") attrs = {'close.return_value': True, 'iter_lines.return_value': content.split('\n'), 'content': content} return mock.Mock(headers={'Content-Type': 'text/plain'}, **attrs) monkeypatch.setattr(check, 'poll', mock.Mock(side_effect=mocked_poll)) return check
def test_cadvisor_default_options(): check = KubeletCheck('kubelet', None, {}, [{}]) cadvisor_scraper_config = check.cadvisor_scraper_config assert check.fs_usage_bytes == {} assert check.mem_usage_bytes == {} assert cadvisor_scraper_config['namespace'] == 'kubernetes' assert cadvisor_scraper_config['metrics_mapper'] == {}
def test_compute_pod_expiration_datetime(monkeypatch): # Invalid input with mock.patch("stackstate_checks.kubelet.kubelet.get_config", return_value="") as p: assert KubeletCheck._compute_pod_expiration_datetime() is None p.assert_called_with("kubernetes_pod_expiration_duration") with mock.patch("stackstate_checks.kubelet.kubelet.get_config", return_value="invalid"): assert KubeletCheck._compute_pod_expiration_datetime() is None # Disabled with mock.patch("stackstate_checks.kubelet.kubelet.get_config", return_value="0"): assert KubeletCheck._compute_pod_expiration_datetime() is None # Set to 15 minutes with mock.patch("stackstate_checks.kubelet.kubelet.get_config", return_value="900"): expire = KubeletCheck._compute_pod_expiration_datetime() assert expire is not None now = datetime.utcnow().replace(tzinfo=UTC) assert abs((now - expire).seconds - 60 * 15) < 2
def test_perform_kubelet_check(monkeypatch): check = KubeletCheck('kubelet', None, {}, [{}]) check.kube_health_url = "http://127.0.0.1:10255/healthz" check.kubelet_credentials = KubeletCredentials({}) monkeypatch.setattr(check, 'service_check', mock.Mock()) instance_tags = ["one:1"] get = MockResponse() with mock.patch("requests.get", side_effect=get): check._perform_kubelet_check(instance_tags) get.assert_has_calls( [ mock.call( 'http://127.0.0.1:10255/healthz', cert=None, headers=None, params={'verbose': True}, stream=False, timeout=10, verify=None, ) ] ) calls = [mock.call('kubernetes.kubelet.check', 0, tags=instance_tags + ['cluster_name:stubbed-cluster-name'])] check.service_check.assert_has_calls(calls)
def test_report_node_metrics_kubernetes1_18(monkeypatch, aggregator): check = KubeletCheck('kubelet', None, {}, [{}]) check.kubelet_credentials = KubeletCredentials({'verify_tls': 'false'}) check.node_spec_url = "http://localhost:10255/spec" get = mock.MagicMock(status_code=404, iter_lines=lambda **kwargs: "Error Code") get.raise_for_status.side_effect = requests.HTTPError('error') with mock.patch('requests.get', return_value=get): check._report_node_metrics(['foo:bar']) aggregator.assert_all_metrics_covered()
def test_report_container_state_metrics(monkeypatch, tagger): check = KubeletCheck('kubelet', None, {}, [{}]) check.pod_list_url = "dummyurl" monkeypatch.setattr(check, 'perform_kubelet_query', mock.Mock(return_value=MockStreamResponse('pods_crashed.json'))) monkeypatch.setattr(check, '_compute_pod_expiration_datetime', mock.Mock(return_value=None)) monkeypatch.setattr(check, 'gauge', mock.Mock()) attrs = {'is_excluded.return_value': False} check.pod_list_utils = mock.Mock(**attrs) pod_list = check.retrieve_pod_list() instance_tags = ["one:1", "two:2"] check._report_container_state_metrics(pod_list, instance_tags) calls = [ mock.call( 'kubernetes.containers.last_state.terminated', 1, instance_tags + ['kube_container_name:fluentd-gcp', 'kube_deployment:fluentd-gcp-v2.0.10', 'cluster_name:stubbed-cluster-name'] + ['reason:OOMKilled'], ), mock.call( 'kubernetes.containers.state.waiting', 1, instance_tags + ['kube_container_name:prometheus-to-sd-exporter', 'kube_deployment:fluentd-gcp-v2.0.10', 'cluster_name:stubbed-cluster-name'] + ['reason:CrashLoopBackOff'], ), mock.call( 'kubernetes.containers.restarts', 1, instance_tags + ['kube_container_name:fluentd-gcp', 'kube_deployment:fluentd-gcp-v2.0.10', 'cluster_name:stubbed-cluster-name'], ), mock.call( 'kubernetes.containers.restarts', 0, instance_tags + ['kube_container_name:prometheus-to-sd-exporter', 'kube_deployment:fluentd-gcp-v2.0.10', 'cluster_name:stubbed-cluster-name'], ), ] check.gauge.assert_has_calls(calls, any_order=True) container_state_gauges = [ x[0][2] for x in check.gauge.call_args_list if x[0][0].startswith('kubernetes.containers.state') ] if any(map(lambda e: 'reason:TransientReason' in e, container_state_gauges)): raise AssertionError('kubernetes.containers.state.* was submitted with a transient reason') if any(map(lambda e: not any(x for x in e if x.startswith('reason:')), container_state_gauges)): raise AssertionError('kubernetes.containers.state.* was submitted without a reason')
def test_silent_tls_warning(caplog, monkeypatch, aggregator): check = KubeletCheck('kubelet', None, {}, [{}]) check.kube_health_url = "https://example.com/" check.kubelet_credentials = KubeletCredentials({'verify_tls': 'false'}) with caplog.at_level(logging.DEBUG): check._perform_kubelet_check([]) expected_message = 'An unverified HTTPS request is being made to https://example.com/' for _, _, message in caplog.record_tuples: assert message != expected_message
def test_kubelet_check_cadvisor(monkeypatch, aggregator, tagger): instance_with_tag = {"tags": ["instance:tag"], "cadvisor_port": 4194} cadvisor_url = "http://valid:port/url" check = KubeletCheck('kubelet', None, {}, [instance_with_tag]) monkeypatch.setattr( check, 'retrieve_pod_list', mock.Mock( return_value=json.loads(mock_from_file('pods_list_1.2.json')))) mock_resp = mock.Mock(status_code=200, raise_for_status=mock.Mock()) mock_resp.json = mock.Mock(return_value=NODE_SPEC) monkeypatch.setattr(check, '_retrieve_node_spec', mock.Mock(return_value=mock_resp)) monkeypatch.setattr(check, '_perform_kubelet_check', mock.Mock(return_value=None)) monkeypatch.setattr( check, '_retrieve_cadvisor_metrics', mock.Mock( return_value=json.loads(mock_from_file('cadvisor_1.2.json')))) monkeypatch.setattr(check, 'detect_cadvisor', mock.Mock(return_value=cadvisor_url)) monkeypatch.setattr(check, 'process', mock.Mock(return_value=None)) check.check(instance_with_tag) assert check.cadvisor_legacy_url == cadvisor_url check.retrieve_pod_list.assert_called_once() check._retrieve_node_spec.assert_called_once() check._retrieve_cadvisor_metrics.assert_called_once() check._perform_kubelet_check.assert_called_once() # called twice so pct metrics are guaranteed to be there check.check(instance_with_tag) for metric in EXPECTED_METRICS_COMMON: aggregator.assert_metric(metric) aggregator.assert_metric_has_tag(metric, "instance:tag") for metric in EXPECTED_METRICS_CADVISOR: aggregator.assert_metric(metric) aggregator.assert_metric_has_tag(metric, "instance:tag") assert aggregator.metrics_asserted_pct == 100.0
def test_report_container_spec_metrics(monkeypatch, tagger): check = KubeletCheck('kubelet', None, {}, [{}]) monkeypatch.setattr(check, 'retrieve_pod_list', mock.Mock(return_value=json.loads(mock_from_file('pods.json')))) monkeypatch.setattr(check, 'gauge', mock.Mock()) attrs = {'is_excluded.return_value': False} check.pod_list_utils = mock.Mock(**attrs) pod_list = check.retrieve_pod_list() instance_tags = ["one:1", "two:2"] check._report_container_spec_metrics(pod_list, instance_tags) calls = [ mock.call( 'kubernetes.cpu.requests', 0.1, instance_tags + ['kube_container_name:fluentd-gcp', 'kube_deployment:fluentd-gcp-v2.0.10', 'cluster_name:stubbed-cluster-name'], ), mock.call( 'kubernetes.memory.requests', 209715200.0, instance_tags + ['kube_container_name:fluentd-gcp', 'kube_deployment:fluentd-gcp-v2.0.10', 'cluster_name:stubbed-cluster-name'], ), mock.call( 'kubernetes.memory.limits', 314572800.0, instance_tags + ['kube_container_name:fluentd-gcp', 'kube_deployment:fluentd-gcp-v2.0.10', 'cluster_name:stubbed-cluster-name'], ), mock.call('kubernetes.cpu.requests', 0.1, instance_tags + ['cluster_name:stubbed-cluster-name']), mock.call('kubernetes.cpu.requests', 0.1, instance_tags + ['cluster_name:stubbed-cluster-name']), mock.call('kubernetes.memory.requests', 134217728.0, instance_tags + ['cluster_name:stubbed-cluster-name']), mock.call('kubernetes.cpu.limits', 0.25, instance_tags + ['cluster_name:stubbed-cluster-name']), mock.call('kubernetes.memory.limits', 536870912.0, instance_tags + ['cluster_name:stubbed-cluster-name']), mock.call('kubernetes.cpu.requests', 0.1, instance_tags + ["pod_name:demo-app-success-c485bc67b-klj45", 'cluster_name:stubbed-cluster-name']), ] if any(map(lambda e: 'pod_name:pi-kff76' in e, [x[0][2] for x in check.gauge.call_args_list])): raise AssertionError("kubernetes.cpu.requests was submitted for a non-running pod") check.gauge.assert_has_calls(calls, any_order=True)
def test_check_instance_key(): check = KubeletCheck('kubelet', None, {}, [{}]) assert check.get_instance_key({}) == AgentIntegrationInstance('kubelet', 'stubbed-cluster-name')
def test_detect_cadvisor_404(mock_request): mock_request.head('http://kubelet:4192/api/v1.3/subcontainers/', status_code=404) with pytest.raises(HTTPError): url = KubeletCheck.detect_cadvisor("http://kubelet:10250", 4192) assert url == ""
def test_detect_cadvisor_port_zero(): with pytest.raises(ValueError): url = KubeletCheck.detect_cadvisor("http://kubelet:10250", 0) assert url == ""
def check(): return KubeletCheck('kubelet', None, {}, [{}])
def test_bad_config(): with pytest.raises(Exception): KubeletCheck('kubelet', None, {}, [{}, {}])
def test_parse_quantity(): for raw, res in iteritems(QUANTITIES): assert KubeletCheck.parse_quantity(raw) == res
def test_detect_cadvisor_nominal(mock_request): mock_request.head('http://kubelet:4192/api/v1.3/subcontainers/', text='{}') url = KubeletCheck.detect_cadvisor("http://kubelet:10250", 4192) assert url == "http://kubelet:4192/api/v1.3/subcontainers/"