def test_count_all_nodes(aggregator):
my_mocks = consul_mocks._get_consul_mocks()
consul_check = ConsulCheck(common.CHECK_NAME, {}, [consul_mocks.MOCK_CONFIG])
consul_mocks.mock_check(consul_check, my_mocks)
consul_check.check(None)
aggregator.assert_metric('consul.catalog.total_nodes', value=2, tags=['consul_datacenter:dc1'])
def test_service_checks_disable_service_tag(aggregator):
consul_check = ConsulCheck(common.CHECK_NAME, {},
[consul_mocks.MOCK_CONFIG_DISABLE_SERVICE_TAG])
my_mocks = consul_mocks._get_consul_mocks()
my_mocks['consul_request'] = consul_mocks.mock_get_health_check
consul_mocks.mock_check(consul_check, my_mocks)
consul_check.check(None)
expected_tags = [
'consul_datacenter:dc1',
'check:server-loadbalancer',
'consul_service_id:server-loadbalancer',
'consul_service:server-loadbalancer',
]
aggregator.assert_service_check('consul.check',
status=ConsulCheck.CRITICAL,
tags=expected_tags,
count=1)
expected_tags = [
'consul_datacenter:dc1',
'check:server-api',
'consul_service_id:server-loadbalancer',
'consul_service:server-loadbalancer',
]
aggregator.assert_service_check('consul.check',
status=ConsulCheck.OK,
tags=expected_tags,
count=1)
expected_tags = [
'consul_datacenter:dc1', 'check:server-api',
'consul_service:server-loadbalancer'
]
aggregator.assert_service_check('consul.check',
status=ConsulCheck.OK,
tags=expected_tags,
count=1)
expected_tags = [
'consul_datacenter:dc1', 'check:server-api',
'consul_service_id:server-loadbalancer'
]
aggregator.assert_service_check('consul.check',
status=ConsulCheck.OK,
tags=expected_tags,
count=1)
expected_tags = [
'consul_datacenter:dc1',
'check:server-status-empty',
'consul_service_id:server-empty',
'consul_service:server-empty',
]
aggregator.assert_service_check('consul.check',
status=ConsulCheck.UNKNOWN,
tags=expected_tags,
count=1)
aggregator.assert_service_check('consul.check', count=5)
def test_prometheus_endpoint(aggregator, dd_environment, instance_prometheus,
caplog):
consul_check = ConsulCheck(common.CHECK_NAME, {}, [instance_prometheus])
common_tags = instance_prometheus['tags']
if common.PROMETHEUS_ENDPOINT_AVAILABLE:
consul_check.check(instance_prometheus)
aggregator.assert_service_check(
'consul.prometheus.health',
tags=common_tags + [
'endpoint:{}/v1/agent/metrics?format=prometheus'.format(
common.URL)
],
)
for metric in common.PROMETHEUS_METRICS:
aggregator.assert_metric(metric, tags=common_tags, count=1)
aggregator.assert_metric('consul.peers', value=3, count=1)
aggregator.assert_all_metrics_covered()
else:
caplog.at_level(logging.WARNING)
consul_check.check(instance_prometheus)
assert (
"does not support the prometheus endpoint. "
"Update Consul or set back `use_prometheus_endpoint` to false to remove this warning."
in caplog.text)
def test_network_latency_checks(aggregator):
consul_check = ConsulCheck(
common.CHECK_NAME, {},
[consul_mocks.MOCK_CONFIG_NETWORK_LATENCY_CHECKS])
my_mocks = consul_mocks._get_consul_mocks()
consul_mocks.mock_check(consul_check, my_mocks)
# We start out as the leader, and stay that way
consul_check._last_known_leader = consul_mocks.mock_get_cluster_leader_A()
consul_check.check(None)
latency = []
for m_name, metrics in aggregator._metrics.items():
if m_name.startswith('consul.net.'):
latency.extend(metrics)
latency.sort()
# Make sure we have the expected number of metrics
assert 19 == len(latency)
# Only 3 dc-latency metrics since we only do source = self
dc = [m for m in latency if '.dc.latency.' in m[0]]
assert 3 == len(dc)
assert 1.6746410750238774 == dc[0][2]
# 16 latency metrics, 2 nodes * 8 metrics each
node = [m for m in latency if '.node.latency.' in m[0]]
assert 16 == len(node)
assert 0.26577747932995816 == node[0][2]
def test_service_checks(aggregator):
consul_check = ConsulCheck(common.CHECK_NAME, {}, {})
my_mocks = consul_mocks._get_consul_mocks()
my_mocks['consul_request'] = consul_mocks.mock_get_health_check
consul_mocks.mock_check(consul_check, my_mocks)
consul_check.check(consul_mocks.MOCK_CONFIG)
expected_tags = [
"consul_datacenter:dc1",
"check:server-loadbalancer",
"consul_service_id:server-loadbalancer",
"service:server-loadbalancer",
]
aggregator.assert_service_check('consul.check',
status=ConsulCheck.CRITICAL,
tags=expected_tags,
count=1)
expected_tags = [
"consul_datacenter:dc1",
"check:server-api",
"consul_service_id:server-loadbalancer",
"service:server-loadbalancer",
]
aggregator.assert_service_check('consul.check',
status=ConsulCheck.OK,
tags=expected_tags,
count=1)
expected_tags = [
"consul_datacenter:dc1", "check:server-api",
"service:server-loadbalancer"
]
aggregator.assert_service_check('consul.check',
status=ConsulCheck.OK,
tags=expected_tags,
count=1)
expected_tags = [
"consul_datacenter:dc1", "check:server-api",
"consul_service_id:server-loadbalancer"
]
aggregator.assert_service_check('consul.check',
status=ConsulCheck.OK,
tags=expected_tags,
count=1)
expected_tags = [
"consul_datacenter:dc1",
"check:server-status-empty",
"consul_service_id:server-empty",
"service:server-empty",
]
aggregator.assert_service_check('consul.check',
status=ConsulCheck.UNKNOWN,
tags=expected_tags,
count=1)
aggregator.assert_service_check('consul.check', count=5)
def test_get_nodes_with_service_critical(aggregator):
consul_check = ConsulCheck(common.CHECK_NAME, {},
[consul_mocks.MOCK_CONFIG])
my_mocks = consul_mocks._get_consul_mocks()
my_mocks[
'get_nodes_with_service'] = consul_mocks.mock_get_nodes_with_service_critical
consul_mocks.mock_check(consul_check, my_mocks)
consul_check.check(None)
expected_tags = [
'consul_datacenter:dc1',
'consul_service_id:service-1',
'consul_service-1_service_tag:active',
'consul_service-1_service_tag:standby',
'consul_service_tag:active',
'consul_service_tag:standby',
]
aggregator.assert_metric('consul.catalog.nodes_up',
value=4,
tags=expected_tags)
aggregator.assert_metric('consul.catalog.nodes_passing',
value=0,
tags=expected_tags)
aggregator.assert_metric('consul.catalog.nodes_warning',
value=0,
tags=expected_tags)
aggregator.assert_metric('consul.catalog.nodes_critical',
value=4,
tags=expected_tags)
expected_tags = ['consul_datacenter:dc1', 'consul_node_id:node-1']
aggregator.assert_metric('consul.catalog.services_up',
value=24,
tags=expected_tags)
aggregator.assert_metric('consul.catalog.services_passing',
value=0,
tags=expected_tags)
aggregator.assert_metric('consul.catalog.services_warning',
value=0,
tags=expected_tags)
aggregator.assert_metric('consul.catalog.services_critical',
value=24,
tags=expected_tags)
expected_tags = [
'consul_datacenter:dc1',
'consul_service-1_service_tag:standby',
'consul_service_id:service-1',
'consul_node_id:node-1',
'consul_status:critical',
]
aggregator.assert_metric('consul.catalog.services_count',
value=3,
tags=expected_tags)
expected_tags.append('consul_service-1_service_tag:active')
aggregator.assert_metric('consul.catalog.services_count',
value=1,
tags=expected_tags)
def test_cull_services_list():
consul_check = ConsulCheck(common.CHECK_NAME, {}, [{}])
my_mocks = consul_mocks._get_consul_mocks()
consul_mocks.mock_check(consul_check, my_mocks)
consul_check.check(consul_mocks.MOCK_CONFIG_LEADER_CHECK)
# Pad num_services to kick in truncation logic
num_services = consul_check.MAX_SERVICES + 20
# Max services parameter (from consul.yaml) set to be bigger than MAX_SERVICES and smaller than total of services
max_services = num_services - 10
# Big whitelist
services = consul_mocks.mock_get_n_services_in_cluster(num_services)
whitelist = ['service_{}'.format(k) for k in range(num_services)]
assert len(consul_check._cull_services_list(services, whitelist)) == consul_check.MAX_SERVICES
# Big whitelist with max_services
assert len(consul_check._cull_services_list(services, whitelist, max_services)) == max_services
# Whitelist < MAX_SERVICES should spit out the whitelist
whitelist = ['service_{}'.format(k) for k in range(consul_check.MAX_SERVICES - 1)]
assert set(consul_check._cull_services_list(services, whitelist)) == set(whitelist)
# Whitelist < max_services param should spit out the whitelist
whitelist = ['service_{}'.format(k) for k in range(max_services - 1)]
assert set(consul_check._cull_services_list(services, whitelist, max_services)) == set(whitelist)
# No whitelist, still triggers truncation
whitelist = []
assert len(consul_check._cull_services_list(services, whitelist)) == consul_check.MAX_SERVICES
# No whitelist with max_services set, also triggers truncation
whitelist = []
assert len(consul_check._cull_services_list(services, whitelist, max_services)) == max_services
# Num. services < MAX_SERVICES should be no-op in absence of whitelist
num_services = consul_check.MAX_SERVICES - 1
services = consul_mocks.mock_get_n_services_in_cluster(num_services)
assert len(consul_check._cull_services_list(services, whitelist)) == num_services
# Num. services < MAX_SERVICES should spit out only the whitelist when one is defined
whitelist = ['service_1', 'service_2', 'service_3']
assert set(consul_check._cull_services_list(services, whitelist)) == set(whitelist)
# Num. services < max_services (from consul.yaml) should be no-op in absence of whitelist
num_services = max_services - 1
whitelist = []
services = consul_mocks.mock_get_n_services_in_cluster(num_services)
assert len(consul_check._cull_services_list(services, whitelist, max_services)) == num_services
# Num. services < max_services should spit out only the whitelist when one is defined
whitelist = ['service_1', 'service_2', 'service_3']
assert set(consul_check._cull_services_list(services, whitelist, max_services)) == set(whitelist)
def test_single_node_install(aggregator, instance_single_node_install, dd_environment):
consul_check = ConsulCheck(common.CHECK_NAME, {}, {})
consul_check.check(instance_single_node_install)
for m in METRICS:
aggregator.assert_metric(m, at_least=1)
aggregator.assert_metric('consul.peers', value=3)
aggregator.assert_service_check('consul.check')
aggregator.assert_service_check('consul.up', tags=['consul_datacenter:dc1', 'consul_url:{}'.format(common.URL)])
def test_prometheus_endpoint(aggregator, dd_environment, instance_prometheus,
caplog):
consul_check = ConsulCheck(common.CHECK_NAME, {}, [instance_prometheus])
common_tags = instance_prometheus['tags']
if common.PROMETHEUS_ENDPOINT_AVAILABLE:
consul_check.check(instance_prometheus)
aggregator.assert_service_check(
'consul.prometheus.health',
tags=common_tags + [
'endpoint:{}/v1/agent/metrics?format=prometheus'.format(
common.URL)
],
)
for metric in common.PROMETHEUS_METRICS:
aggregator.assert_metric(metric, tags=common_tags, count=1)
aggregator.assert_metric('consul.peers', value=3, count=1)
aggregator.assert_metric_has_tag_prefix(
'consul.raft.replication.appendEntries.logs', 'peer_id', count=2)
for hist_suffix in ['count', 'sum', 'quantile']:
aggregator.assert_metric_has_tag(
'consul.http.request.{}'.format(hist_suffix),
'method:GET',
at_least=0)
for metric in common.PROMETHEUS_HIST_METRICS:
for tag in common_tags:
aggregator.assert_metric_has_tag(metric + hist_suffix,
tag,
at_least=1)
aggregator.assert_all_metrics_covered()
# Some of the metrics documented in the metadata.csv were sent through DogStatsD as `timer` as well.
# We end up with some of the prometheus metrics having a different in-app type.
# Example with `consul.raft.commitTime.count`:
# * It is a rate when submitting metrics with DogStatsD
# * It is a rate when submitting metrics with OpenMetricsBaseCheck
# TODO: solve conflict
# aggregator.assert_metrics_using_metadata(get_metadata_metrics(), check_submission_type=True)
else:
caplog.at_level(logging.WARNING)
consul_check.check(instance_prometheus)
assert (
"does not support the prometheus endpoint. "
"Update Consul or set back `use_prometheus_endpoint` to false to remove this warning."
in caplog.text)
def test_config(test_case, extra_config, expected_http_kwargs):
instance = extra_config
check = ConsulCheck(common.CHECK_NAME, {}, instances=[instance])
with mock.patch('datadog_checks.base.utils.http.requests') as r:
r.get.return_value = mock.MagicMock(status_code=200)
check.check(None)
http_wargs = dict(
auth=mock.ANY, cert=mock.ANY, headers=mock.ANY, proxies=mock.ANY, timeout=mock.ANY, verify=mock.ANY
)
http_wargs.update(expected_http_kwargs)
r.get.assert_called_with('/v1/status/leader', **http_wargs)
def test_check(aggregator, instance, dd_environment):
"""
Testing Consul Integration
"""
consul_check = ConsulCheck(common.CHECK_NAME, {}, {})
consul_check.check(instance)
for m in METRICS:
aggregator.assert_metric(m, at_least=0)
aggregator.assert_metric('consul.peers', value=3)
aggregator.assert_service_check('consul.check')
aggregator.assert_service_check('consul.up', tags=['consul_datacenter:dc1', 'consul_url:{}'.format(common.URL)])
def test_acl_forbidden(instance_bad_token, dd_environment):
"""
Testing Consul Integration with wrong ACL token
"""
consul_check = ConsulCheck(common.CHECK_NAME, {}, [instance_bad_token])
got_error_403 = False
try:
consul_check.check(None)
except HTTPError as e:
if e.response.status_code == 403:
got_error_403 = True
assert got_error_403
def test_get_peers_in_cluster(aggregator):
my_mocks = consul_mocks._get_consul_mocks()
consul_check = ConsulCheck(common.CHECK_NAME, {}, [consul_mocks.MOCK_CONFIG])
consul_mocks.mock_check(consul_check, my_mocks)
consul_check.check(None)
# When node is leader
aggregator.assert_metric('consul.peers', value=3, tags=['consul_datacenter:dc1', 'mode:leader'])
my_mocks['_get_cluster_leader'] = consul_mocks.mock_get_cluster_leader_B
consul_mocks.mock_check(consul_check, my_mocks)
consul_check.check(None)
aggregator.assert_metric('consul.peers', value=3, tags=['consul_datacenter:dc1', 'mode:follower'])
def test_new_leader_event(aggregator):
consul_check = ConsulCheck(common.CHECK_NAME, {}, [consul_mocks.MOCK_CONFIG_LEADER_CHECK])
my_mocks = consul_mocks._get_consul_mocks()
my_mocks['_get_cluster_leader'] = consul_mocks.mock_get_cluster_leader_B
consul_mocks.mock_check(consul_check, my_mocks)
consul_check._last_known_leader = 'My Old Leader'
consul_check.check(None)
assert len(aggregator.events) == 1
event = aggregator.events[0]
assert event['event_type'] == 'consul.new_leader'
assert 'prev_consul_leader:My Old Leader' in event['tags']
assert 'curr_consul_leader:My New Leader' in event['tags']
def test_acl_forbidden(aggregator, consul_cluster):
"""
Testing Consul Integration with wrong ACL token
"""
consul_check = ConsulCheck(common.CHECK_NAME, {}, {})
got_error_403 = False
try:
consul_check.check(CONFIG_INTEGRATION_WRONG_TOKEN)
except HTTPError as e:
if e.response.status_code == 403:
got_error_403 = True
assert got_error_403
def test_integration(aggregator, consul_cluster):
"""
Testing Consul Integration
"""
consul_check = ConsulCheck(common.CHECK_NAME, {}, {})
consul_check.check(CONFIG_INTEGRATION)
for m in METRICS:
aggregator.assert_metric(m, at_least=0)
aggregator.assert_metric('consul.peers', value=3)
aggregator.assert_service_check('consul.check')
aggregator.assert_service_check('consul.up')
def test_version_metadata(aggregator, instance, dd_environment, datadog_agent):
consul_check = ConsulCheck(common.CHECK_NAME, {}, [instance])
consul_check.check_id = 'test:123'
consul_check.check(instance)
raw_version = common.CONSUL_VERSION.lstrip(
'v') # some versions contain `v` prefix
major, minor, patch = raw_version.split('.')
version_metadata = {
'version.scheme': 'semver',
'version.major': major,
'version.minor': minor,
'version.patch': patch,
'version.raw': raw_version,
}
datadog_agent.assert_metadata('test:123', version_metadata)
def test_prometheus_endpoint(aggregator, dd_environment, instance_prometheus,
caplog):
consul_check = ConsulCheck(common.CHECK_NAME, {}, [instance_prometheus])
common_tags = instance_prometheus['tags']
if common.PROMETHEUS_ENDPOINT_AVAILABLE:
consul_check.check(instance_prometheus)
aggregator.assert_service_check(
'consul.prometheus.health',
tags=common_tags + [
'endpoint:{}/v1/agent/metrics?format=prometheus'.format(
common.URL)
],
)
for metric in common.PROMETHEUS_METRICS:
aggregator.assert_metric(metric, tags=common_tags, count=1)
aggregator.assert_metric('consul.peers', value=3, count=1)
aggregator.assert_metric_has_tag_prefix(
'consul.raft.replication.appendEntries.logs', 'peer_id', count=2)
for hist_suffix in ['count', 'sum', 'quantile']:
aggregator.assert_metric_has_tag(
'consul.http.request.{}'.format(hist_suffix),
'method:GET',
at_least=0)
for metric in common.PROMETHEUS_HIST_METRICS:
for tag in common_tags:
aggregator.assert_metric_has_tag(metric + hist_suffix,
tag,
at_least=1)
aggregator.assert_all_metrics_covered()
else:
caplog.at_level(logging.WARNING)
consul_check.check(instance_prometheus)
assert (
"does not support the prometheus endpoint. "
"Update Consul or set back `use_prometheus_endpoint` to false to remove this warning."
in caplog.text)
def test_single_node_install(aggregator, instance_single_node_install,
dd_environment):
consul_check = ConsulCheck(common.CHECK_NAME, {},
[instance_single_node_install])
consul_check.check(instance_single_node_install)
for m in METRICS:
aggregator.assert_metric(m, at_least=1)
aggregator.assert_metric('consul.peers', value=3)
aggregator.assert_service_check('consul.check')
aggregator.assert_service_check(
'consul.up',
tags=['consul_datacenter:dc1', 'consul_url:{}'.format(common.URL)])
aggregator.assert_metrics_using_metadata(get_metadata_metrics(),
check_submission_type=True)
aggregator.assert_all_metrics_covered()
def test_check(aggregator, instance, dd_environment):
"""
Testing Consul Integration
"""
consul_check = ConsulCheck(common.CHECK_NAME, {}, [instance])
consul_check.check(instance)
for m in METRICS + MULTI_NODE_METRICS:
aggregator.assert_metric(m, at_least=0)
aggregator.assert_metric('consul.peers', value=3)
aggregator.assert_service_check('consul.check')
aggregator.assert_service_check(
'consul.up',
tags=['consul_datacenter:dc1', 'consul_url:{}'.format(common.URL)])
aggregator.assert_metrics_using_metadata(get_metadata_metrics(),
check_submission_type=True)
aggregator.assert_all_metrics_covered()
def test_get_nodes_with_service_warning(aggregator):
consul_check = ConsulCheck(common.CHECK_NAME, {},
[consul_mocks.MOCK_CONFIG])
my_mocks = consul_mocks._get_consul_mocks()
my_mocks[
'get_nodes_with_service'] = consul_mocks.mock_get_nodes_with_service_warning
consul_mocks.mock_check(consul_check, my_mocks)
consul_check.check(None)
expected_tags = [
'consul_datacenter:dc1',
'consul_service_id:service-1',
'consul_service-1_service_tag:az-us-east-1a',
'consul_service_tag:az-us-east-1a',
]
aggregator.assert_metric('consul.catalog.nodes_up',
value=1,
tags=expected_tags)
aggregator.assert_metric('consul.catalog.nodes_passing',
value=0,
tags=expected_tags)
aggregator.assert_metric('consul.catalog.nodes_warning',
value=1,
tags=expected_tags)
aggregator.assert_metric('consul.catalog.nodes_critical',
value=0,
tags=expected_tags)
expected_tags = ['consul_datacenter:dc1', 'consul_node_id:node-1']
aggregator.assert_metric('consul.catalog.services_up',
value=6,
tags=expected_tags)
aggregator.assert_metric('consul.catalog.services_passing',
value=0,
tags=expected_tags)
aggregator.assert_metric('consul.catalog.services_warning',
value=6,
tags=expected_tags)
aggregator.assert_metric('consul.catalog.services_critical',
value=0,
tags=expected_tags)
def test_self_leader_event(aggregator):
consul_check = ConsulCheck(common.CHECK_NAME, {},
[consul_mocks.MOCK_CONFIG_SELF_LEADER_CHECK])
my_mocks = consul_mocks._get_consul_mocks()
instance_hash = hash_mutable(consul_mocks.MOCK_CONFIG_SELF_LEADER_CHECK)
consul_check._instance_states[
instance_hash].last_known_leader = 'My Old Leader'
our_url = consul_mocks.mock_get_cluster_leader_A(None)
other_url = consul_mocks.mock_get_cluster_leader_B(None)
# We become the leader
my_mocks['_get_cluster_leader'] = consul_mocks.mock_get_cluster_leader_A
consul_mocks.mock_check(consul_check, my_mocks)
consul_check.check(consul_mocks.MOCK_CONFIG_SELF_LEADER_CHECK)
assert len(aggregator.events) == 1
assert our_url == consul_check._instance_states[
instance_hash].last_known_leader
event = aggregator.events[0]
assert event['event_type'] == 'consul.new_leader'
assert 'prev_consul_leader:My Old Leader' in event['tags']
assert 'curr_consul_leader:{}'.format(our_url) in event['tags']
# We are already the leader, no new events
aggregator.reset()
consul_check.check(consul_mocks.MOCK_CONFIG_SELF_LEADER_CHECK)
assert len(aggregator.events) == 0
# We lose the leader, no new events
my_mocks['_get_cluster_leader'] = consul_mocks.mock_get_cluster_leader_B
consul_mocks.mock_check(consul_check, my_mocks)
aggregator.reset()
consul_check.check(consul_mocks.MOCK_CONFIG_SELF_LEADER_CHECK)
assert len(aggregator.events) == 0
assert other_url == consul_check._instance_states[
instance_hash].last_known_leader
# We regain the leadership
my_mocks['_get_cluster_leader'] = consul_mocks.mock_get_cluster_leader_A
consul_mocks.mock_check(consul_check, my_mocks)
aggregator.reset()
consul_check.check(consul_mocks.MOCK_CONFIG_SELF_LEADER_CHECK)
assert len(aggregator.events) == 1
assert our_url == consul_check._instance_states[
instance_hash].last_known_leader
event = aggregator.events[0]
assert event['event_type'] == 'consul.new_leader'
assert 'prev_consul_leader:{}'.format(other_url) in event['tags']
assert 'curr_consul_leader:{}'.format(our_url) in event['tags']
