def test_count_all_nodes(aggregator): my_mocks = consul_mocks._get_consul_mocks() consul_check = ConsulCheck(common.CHECK_NAME, {}, [consul_mocks.MOCK_CONFIG]) consul_mocks.mock_check(consul_check, my_mocks) consul_check.check(None) aggregator.assert_metric('consul.catalog.total_nodes', value=2, tags=['consul_datacenter:dc1'])
def test_service_checks_disable_service_tag(aggregator): consul_check = ConsulCheck(common.CHECK_NAME, {}, [consul_mocks.MOCK_CONFIG_DISABLE_SERVICE_TAG]) my_mocks = consul_mocks._get_consul_mocks() my_mocks['consul_request'] = consul_mocks.mock_get_health_check consul_mocks.mock_check(consul_check, my_mocks) consul_check.check(None) expected_tags = [ 'consul_datacenter:dc1', 'check:server-loadbalancer', 'consul_service_id:server-loadbalancer', 'consul_service:server-loadbalancer', ] aggregator.assert_service_check('consul.check', status=ConsulCheck.CRITICAL, tags=expected_tags, count=1) expected_tags = [ 'consul_datacenter:dc1', 'check:server-api', 'consul_service_id:server-loadbalancer', 'consul_service:server-loadbalancer', ] aggregator.assert_service_check('consul.check', status=ConsulCheck.OK, tags=expected_tags, count=1) expected_tags = [ 'consul_datacenter:dc1', 'check:server-api', 'consul_service:server-loadbalancer' ] aggregator.assert_service_check('consul.check', status=ConsulCheck.OK, tags=expected_tags, count=1) expected_tags = [ 'consul_datacenter:dc1', 'check:server-api', 'consul_service_id:server-loadbalancer' ] aggregator.assert_service_check('consul.check', status=ConsulCheck.OK, tags=expected_tags, count=1) expected_tags = [ 'consul_datacenter:dc1', 'check:server-status-empty', 'consul_service_id:server-empty', 'consul_service:server-empty', ] aggregator.assert_service_check('consul.check', status=ConsulCheck.UNKNOWN, tags=expected_tags, count=1) aggregator.assert_service_check('consul.check', count=5)
def test_prometheus_endpoint(aggregator, dd_environment, instance_prometheus, caplog): consul_check = ConsulCheck(common.CHECK_NAME, {}, [instance_prometheus]) common_tags = instance_prometheus['tags'] if common.PROMETHEUS_ENDPOINT_AVAILABLE: consul_check.check(instance_prometheus) aggregator.assert_service_check( 'consul.prometheus.health', tags=common_tags + [ 'endpoint:{}/v1/agent/metrics?format=prometheus'.format( common.URL) ], ) for metric in common.PROMETHEUS_METRICS: aggregator.assert_metric(metric, tags=common_tags, count=1) aggregator.assert_metric('consul.peers', value=3, count=1) aggregator.assert_all_metrics_covered() else: caplog.at_level(logging.WARNING) consul_check.check(instance_prometheus) assert ( "does not support the prometheus endpoint. " "Update Consul or set back `use_prometheus_endpoint` to false to remove this warning." in caplog.text)
def test_network_latency_checks(aggregator): consul_check = ConsulCheck( common.CHECK_NAME, {}, [consul_mocks.MOCK_CONFIG_NETWORK_LATENCY_CHECKS]) my_mocks = consul_mocks._get_consul_mocks() consul_mocks.mock_check(consul_check, my_mocks) # We start out as the leader, and stay that way consul_check._last_known_leader = consul_mocks.mock_get_cluster_leader_A() consul_check.check(None) latency = [] for m_name, metrics in aggregator._metrics.items(): if m_name.startswith('consul.net.'): latency.extend(metrics) latency.sort() # Make sure we have the expected number of metrics assert 19 == len(latency) # Only 3 dc-latency metrics since we only do source = self dc = [m for m in latency if '.dc.latency.' in m[0]] assert 3 == len(dc) assert 1.6746410750238774 == dc[0][2] # 16 latency metrics, 2 nodes * 8 metrics each node = [m for m in latency if '.node.latency.' in m[0]] assert 16 == len(node) assert 0.26577747932995816 == node[0][2]
def test_service_checks(aggregator): consul_check = ConsulCheck(common.CHECK_NAME, {}, {}) my_mocks = consul_mocks._get_consul_mocks() my_mocks['consul_request'] = consul_mocks.mock_get_health_check consul_mocks.mock_check(consul_check, my_mocks) consul_check.check(consul_mocks.MOCK_CONFIG) expected_tags = [ "consul_datacenter:dc1", "check:server-loadbalancer", "consul_service_id:server-loadbalancer", "service:server-loadbalancer", ] aggregator.assert_service_check('consul.check', status=ConsulCheck.CRITICAL, tags=expected_tags, count=1) expected_tags = [ "consul_datacenter:dc1", "check:server-api", "consul_service_id:server-loadbalancer", "service:server-loadbalancer", ] aggregator.assert_service_check('consul.check', status=ConsulCheck.OK, tags=expected_tags, count=1) expected_tags = [ "consul_datacenter:dc1", "check:server-api", "service:server-loadbalancer" ] aggregator.assert_service_check('consul.check', status=ConsulCheck.OK, tags=expected_tags, count=1) expected_tags = [ "consul_datacenter:dc1", "check:server-api", "consul_service_id:server-loadbalancer" ] aggregator.assert_service_check('consul.check', status=ConsulCheck.OK, tags=expected_tags, count=1) expected_tags = [ "consul_datacenter:dc1", "check:server-status-empty", "consul_service_id:server-empty", "service:server-empty", ] aggregator.assert_service_check('consul.check', status=ConsulCheck.UNKNOWN, tags=expected_tags, count=1) aggregator.assert_service_check('consul.check', count=5)
def test_get_nodes_with_service_critical(aggregator): consul_check = ConsulCheck(common.CHECK_NAME, {}, [consul_mocks.MOCK_CONFIG]) my_mocks = consul_mocks._get_consul_mocks() my_mocks[ 'get_nodes_with_service'] = consul_mocks.mock_get_nodes_with_service_critical consul_mocks.mock_check(consul_check, my_mocks) consul_check.check(None) expected_tags = [ 'consul_datacenter:dc1', 'consul_service_id:service-1', 'consul_service-1_service_tag:active', 'consul_service-1_service_tag:standby', 'consul_service_tag:active', 'consul_service_tag:standby', ] aggregator.assert_metric('consul.catalog.nodes_up', value=4, tags=expected_tags) aggregator.assert_metric('consul.catalog.nodes_passing', value=0, tags=expected_tags) aggregator.assert_metric('consul.catalog.nodes_warning', value=0, tags=expected_tags) aggregator.assert_metric('consul.catalog.nodes_critical', value=4, tags=expected_tags) expected_tags = ['consul_datacenter:dc1', 'consul_node_id:node-1'] aggregator.assert_metric('consul.catalog.services_up', value=24, tags=expected_tags) aggregator.assert_metric('consul.catalog.services_passing', value=0, tags=expected_tags) aggregator.assert_metric('consul.catalog.services_warning', value=0, tags=expected_tags) aggregator.assert_metric('consul.catalog.services_critical', value=24, tags=expected_tags) expected_tags = [ 'consul_datacenter:dc1', 'consul_service-1_service_tag:standby', 'consul_service_id:service-1', 'consul_node_id:node-1', 'consul_status:critical', ] aggregator.assert_metric('consul.catalog.services_count', value=3, tags=expected_tags) expected_tags.append('consul_service-1_service_tag:active') aggregator.assert_metric('consul.catalog.services_count', value=1, tags=expected_tags)
def test_cull_services_list(): consul_check = ConsulCheck(common.CHECK_NAME, {}, [{}]) my_mocks = consul_mocks._get_consul_mocks() consul_mocks.mock_check(consul_check, my_mocks) consul_check.check(consul_mocks.MOCK_CONFIG_LEADER_CHECK) # Pad num_services to kick in truncation logic num_services = consul_check.MAX_SERVICES + 20 # Max services parameter (from consul.yaml) set to be bigger than MAX_SERVICES and smaller than total of services max_services = num_services - 10 # Big whitelist services = consul_mocks.mock_get_n_services_in_cluster(num_services) whitelist = ['service_{}'.format(k) for k in range(num_services)] assert len(consul_check._cull_services_list(services, whitelist)) == consul_check.MAX_SERVICES # Big whitelist with max_services assert len(consul_check._cull_services_list(services, whitelist, max_services)) == max_services # Whitelist < MAX_SERVICES should spit out the whitelist whitelist = ['service_{}'.format(k) for k in range(consul_check.MAX_SERVICES - 1)] assert set(consul_check._cull_services_list(services, whitelist)) == set(whitelist) # Whitelist < max_services param should spit out the whitelist whitelist = ['service_{}'.format(k) for k in range(max_services - 1)] assert set(consul_check._cull_services_list(services, whitelist, max_services)) == set(whitelist) # No whitelist, still triggers truncation whitelist = [] assert len(consul_check._cull_services_list(services, whitelist)) == consul_check.MAX_SERVICES # No whitelist with max_services set, also triggers truncation whitelist = [] assert len(consul_check._cull_services_list(services, whitelist, max_services)) == max_services # Num. services < MAX_SERVICES should be no-op in absence of whitelist num_services = consul_check.MAX_SERVICES - 1 services = consul_mocks.mock_get_n_services_in_cluster(num_services) assert len(consul_check._cull_services_list(services, whitelist)) == num_services # Num. services < MAX_SERVICES should spit out only the whitelist when one is defined whitelist = ['service_1', 'service_2', 'service_3'] assert set(consul_check._cull_services_list(services, whitelist)) == set(whitelist) # Num. services < max_services (from consul.yaml) should be no-op in absence of whitelist num_services = max_services - 1 whitelist = [] services = consul_mocks.mock_get_n_services_in_cluster(num_services) assert len(consul_check._cull_services_list(services, whitelist, max_services)) == num_services # Num. services < max_services should spit out only the whitelist when one is defined whitelist = ['service_1', 'service_2', 'service_3'] assert set(consul_check._cull_services_list(services, whitelist, max_services)) == set(whitelist)
def test_single_node_install(aggregator, instance_single_node_install, dd_environment): consul_check = ConsulCheck(common.CHECK_NAME, {}, {}) consul_check.check(instance_single_node_install) for m in METRICS: aggregator.assert_metric(m, at_least=1) aggregator.assert_metric('consul.peers', value=3) aggregator.assert_service_check('consul.check') aggregator.assert_service_check('consul.up', tags=['consul_datacenter:dc1', 'consul_url:{}'.format(common.URL)])
def test_prometheus_endpoint(aggregator, dd_environment, instance_prometheus, caplog): consul_check = ConsulCheck(common.CHECK_NAME, {}, [instance_prometheus]) common_tags = instance_prometheus['tags'] if common.PROMETHEUS_ENDPOINT_AVAILABLE: consul_check.check(instance_prometheus) aggregator.assert_service_check( 'consul.prometheus.health', tags=common_tags + [ 'endpoint:{}/v1/agent/metrics?format=prometheus'.format( common.URL) ], ) for metric in common.PROMETHEUS_METRICS: aggregator.assert_metric(metric, tags=common_tags, count=1) aggregator.assert_metric('consul.peers', value=3, count=1) aggregator.assert_metric_has_tag_prefix( 'consul.raft.replication.appendEntries.logs', 'peer_id', count=2) for hist_suffix in ['count', 'sum', 'quantile']: aggregator.assert_metric_has_tag( 'consul.http.request.{}'.format(hist_suffix), 'method:GET', at_least=0) for metric in common.PROMETHEUS_HIST_METRICS: for tag in common_tags: aggregator.assert_metric_has_tag(metric + hist_suffix, tag, at_least=1) aggregator.assert_all_metrics_covered() # Some of the metrics documented in the metadata.csv were sent through DogStatsD as `timer` as well. # We end up with some of the prometheus metrics having a different in-app type. # Example with `consul.raft.commitTime.count`: # * It is a rate when submitting metrics with DogStatsD # * It is a rate when submitting metrics with OpenMetricsBaseCheck # TODO: solve conflict # aggregator.assert_metrics_using_metadata(get_metadata_metrics(), check_submission_type=True) else: caplog.at_level(logging.WARNING) consul_check.check(instance_prometheus) assert ( "does not support the prometheus endpoint. " "Update Consul or set back `use_prometheus_endpoint` to false to remove this warning." in caplog.text)
def test_config(test_case, extra_config, expected_http_kwargs): instance = extra_config check = ConsulCheck(common.CHECK_NAME, {}, instances=[instance]) with mock.patch('datadog_checks.base.utils.http.requests') as r: r.get.return_value = mock.MagicMock(status_code=200) check.check(None) http_wargs = dict( auth=mock.ANY, cert=mock.ANY, headers=mock.ANY, proxies=mock.ANY, timeout=mock.ANY, verify=mock.ANY ) http_wargs.update(expected_http_kwargs) r.get.assert_called_with('/v1/status/leader', **http_wargs)
def test_check(aggregator, instance, dd_environment): """ Testing Consul Integration """ consul_check = ConsulCheck(common.CHECK_NAME, {}, {}) consul_check.check(instance) for m in METRICS: aggregator.assert_metric(m, at_least=0) aggregator.assert_metric('consul.peers', value=3) aggregator.assert_service_check('consul.check') aggregator.assert_service_check('consul.up', tags=['consul_datacenter:dc1', 'consul_url:{}'.format(common.URL)])
def test_acl_forbidden(instance_bad_token, dd_environment): """ Testing Consul Integration with wrong ACL token """ consul_check = ConsulCheck(common.CHECK_NAME, {}, [instance_bad_token]) got_error_403 = False try: consul_check.check(None) except HTTPError as e: if e.response.status_code == 403: got_error_403 = True assert got_error_403
def test_get_peers_in_cluster(aggregator): my_mocks = consul_mocks._get_consul_mocks() consul_check = ConsulCheck(common.CHECK_NAME, {}, [consul_mocks.MOCK_CONFIG]) consul_mocks.mock_check(consul_check, my_mocks) consul_check.check(None) # When node is leader aggregator.assert_metric('consul.peers', value=3, tags=['consul_datacenter:dc1', 'mode:leader']) my_mocks['_get_cluster_leader'] = consul_mocks.mock_get_cluster_leader_B consul_mocks.mock_check(consul_check, my_mocks) consul_check.check(None) aggregator.assert_metric('consul.peers', value=3, tags=['consul_datacenter:dc1', 'mode:follower'])
def test_new_leader_event(aggregator): consul_check = ConsulCheck(common.CHECK_NAME, {}, [consul_mocks.MOCK_CONFIG_LEADER_CHECK]) my_mocks = consul_mocks._get_consul_mocks() my_mocks['_get_cluster_leader'] = consul_mocks.mock_get_cluster_leader_B consul_mocks.mock_check(consul_check, my_mocks) consul_check._last_known_leader = 'My Old Leader' consul_check.check(None) assert len(aggregator.events) == 1 event = aggregator.events[0] assert event['event_type'] == 'consul.new_leader' assert 'prev_consul_leader:My Old Leader' in event['tags'] assert 'curr_consul_leader:My New Leader' in event['tags']
def test_acl_forbidden(aggregator, consul_cluster): """ Testing Consul Integration with wrong ACL token """ consul_check = ConsulCheck(common.CHECK_NAME, {}, {}) got_error_403 = False try: consul_check.check(CONFIG_INTEGRATION_WRONG_TOKEN) except HTTPError as e: if e.response.status_code == 403: got_error_403 = True assert got_error_403
def test_integration(aggregator, consul_cluster): """ Testing Consul Integration """ consul_check = ConsulCheck(common.CHECK_NAME, {}, {}) consul_check.check(CONFIG_INTEGRATION) for m in METRICS: aggregator.assert_metric(m, at_least=0) aggregator.assert_metric('consul.peers', value=3) aggregator.assert_service_check('consul.check') aggregator.assert_service_check('consul.up')
def test_version_metadata(aggregator, instance, dd_environment, datadog_agent): consul_check = ConsulCheck(common.CHECK_NAME, {}, [instance]) consul_check.check_id = 'test:123' consul_check.check(instance) raw_version = common.CONSUL_VERSION.lstrip( 'v') # some versions contain `v` prefix major, minor, patch = raw_version.split('.') version_metadata = { 'version.scheme': 'semver', 'version.major': major, 'version.minor': minor, 'version.patch': patch, 'version.raw': raw_version, } datadog_agent.assert_metadata('test:123', version_metadata)
def test_prometheus_endpoint(aggregator, dd_environment, instance_prometheus, caplog): consul_check = ConsulCheck(common.CHECK_NAME, {}, [instance_prometheus]) common_tags = instance_prometheus['tags'] if common.PROMETHEUS_ENDPOINT_AVAILABLE: consul_check.check(instance_prometheus) aggregator.assert_service_check( 'consul.prometheus.health', tags=common_tags + [ 'endpoint:{}/v1/agent/metrics?format=prometheus'.format( common.URL) ], ) for metric in common.PROMETHEUS_METRICS: aggregator.assert_metric(metric, tags=common_tags, count=1) aggregator.assert_metric('consul.peers', value=3, count=1) aggregator.assert_metric_has_tag_prefix( 'consul.raft.replication.appendEntries.logs', 'peer_id', count=2) for hist_suffix in ['count', 'sum', 'quantile']: aggregator.assert_metric_has_tag( 'consul.http.request.{}'.format(hist_suffix), 'method:GET', at_least=0) for metric in common.PROMETHEUS_HIST_METRICS: for tag in common_tags: aggregator.assert_metric_has_tag(metric + hist_suffix, tag, at_least=1) aggregator.assert_all_metrics_covered() else: caplog.at_level(logging.WARNING) consul_check.check(instance_prometheus) assert ( "does not support the prometheus endpoint. " "Update Consul or set back `use_prometheus_endpoint` to false to remove this warning." in caplog.text)
def test_single_node_install(aggregator, instance_single_node_install, dd_environment): consul_check = ConsulCheck(common.CHECK_NAME, {}, [instance_single_node_install]) consul_check.check(instance_single_node_install) for m in METRICS: aggregator.assert_metric(m, at_least=1) aggregator.assert_metric('consul.peers', value=3) aggregator.assert_service_check('consul.check') aggregator.assert_service_check( 'consul.up', tags=['consul_datacenter:dc1', 'consul_url:{}'.format(common.URL)]) aggregator.assert_metrics_using_metadata(get_metadata_metrics(), check_submission_type=True) aggregator.assert_all_metrics_covered()
def test_check(aggregator, instance, dd_environment): """ Testing Consul Integration """ consul_check = ConsulCheck(common.CHECK_NAME, {}, [instance]) consul_check.check(instance) for m in METRICS + MULTI_NODE_METRICS: aggregator.assert_metric(m, at_least=0) aggregator.assert_metric('consul.peers', value=3) aggregator.assert_service_check('consul.check') aggregator.assert_service_check( 'consul.up', tags=['consul_datacenter:dc1', 'consul_url:{}'.format(common.URL)]) aggregator.assert_metrics_using_metadata(get_metadata_metrics(), check_submission_type=True) aggregator.assert_all_metrics_covered()
def test_get_nodes_with_service_warning(aggregator): consul_check = ConsulCheck(common.CHECK_NAME, {}, [consul_mocks.MOCK_CONFIG]) my_mocks = consul_mocks._get_consul_mocks() my_mocks[ 'get_nodes_with_service'] = consul_mocks.mock_get_nodes_with_service_warning consul_mocks.mock_check(consul_check, my_mocks) consul_check.check(None) expected_tags = [ 'consul_datacenter:dc1', 'consul_service_id:service-1', 'consul_service-1_service_tag:az-us-east-1a', 'consul_service_tag:az-us-east-1a', ] aggregator.assert_metric('consul.catalog.nodes_up', value=1, tags=expected_tags) aggregator.assert_metric('consul.catalog.nodes_passing', value=0, tags=expected_tags) aggregator.assert_metric('consul.catalog.nodes_warning', value=1, tags=expected_tags) aggregator.assert_metric('consul.catalog.nodes_critical', value=0, tags=expected_tags) expected_tags = ['consul_datacenter:dc1', 'consul_node_id:node-1'] aggregator.assert_metric('consul.catalog.services_up', value=6, tags=expected_tags) aggregator.assert_metric('consul.catalog.services_passing', value=0, tags=expected_tags) aggregator.assert_metric('consul.catalog.services_warning', value=6, tags=expected_tags) aggregator.assert_metric('consul.catalog.services_critical', value=0, tags=expected_tags)
def test_self_leader_event(aggregator): consul_check = ConsulCheck(common.CHECK_NAME, {}, [consul_mocks.MOCK_CONFIG_SELF_LEADER_CHECK]) my_mocks = consul_mocks._get_consul_mocks() instance_hash = hash_mutable(consul_mocks.MOCK_CONFIG_SELF_LEADER_CHECK) consul_check._instance_states[ instance_hash].last_known_leader = 'My Old Leader' our_url = consul_mocks.mock_get_cluster_leader_A(None) other_url = consul_mocks.mock_get_cluster_leader_B(None) # We become the leader my_mocks['_get_cluster_leader'] = consul_mocks.mock_get_cluster_leader_A consul_mocks.mock_check(consul_check, my_mocks) consul_check.check(consul_mocks.MOCK_CONFIG_SELF_LEADER_CHECK) assert len(aggregator.events) == 1 assert our_url == consul_check._instance_states[ instance_hash].last_known_leader event = aggregator.events[0] assert event['event_type'] == 'consul.new_leader' assert 'prev_consul_leader:My Old Leader' in event['tags'] assert 'curr_consul_leader:{}'.format(our_url) in event['tags'] # We are already the leader, no new events aggregator.reset() consul_check.check(consul_mocks.MOCK_CONFIG_SELF_LEADER_CHECK) assert len(aggregator.events) == 0 # We lose the leader, no new events my_mocks['_get_cluster_leader'] = consul_mocks.mock_get_cluster_leader_B consul_mocks.mock_check(consul_check, my_mocks) aggregator.reset() consul_check.check(consul_mocks.MOCK_CONFIG_SELF_LEADER_CHECK) assert len(aggregator.events) == 0 assert other_url == consul_check._instance_states[ instance_hash].last_known_leader # We regain the leadership my_mocks['_get_cluster_leader'] = consul_mocks.mock_get_cluster_leader_A consul_mocks.mock_check(consul_check, my_mocks) aggregator.reset() consul_check.check(consul_mocks.MOCK_CONFIG_SELF_LEADER_CHECK) assert len(aggregator.events) == 1 assert our_url == consul_check._instance_states[ instance_hash].last_known_leader event = aggregator.events[0] assert event['event_type'] == 'consul.new_leader' assert 'prev_consul_leader:{}'.format(other_url) in event['tags'] assert 'curr_consul_leader:{}'.format(our_url) in event['tags']