def test_pshard_metrics(aggregator, spin_up_elastic):
""" Tests that the pshard related metrics are forwarded and that the
document count for primary indexes is twice smaller as the global
document count when "number_of_replicas" is set to 1 """
elastic_latency = 10
config = {
'url': URL,
'pshard_stats': True,
'username': USER,
'password': PASSWORD
}
requests.put(URL + '/_settings',
data='{"index": {"number_of_replicas": 1}}')
requests.put(URL + '/testindex/testtype/2',
data='{"name": "Jane Doe", "age": 27}')
requests.put(URL + '/testindex/testtype/1',
data='{"name": "John Doe", "age": 42}')
time.sleep(elastic_latency)
elastic_check = ESCheck(CHECK_NAME, {}, {})
elastic_check.check(config)
pshard_stats_metrics = dict(ESCheck.PRIMARY_SHARD_METRICS)
if get_es_version() >= [1, 0, 0]:
pshard_stats_metrics.update(ESCheck.PRIMARY_SHARD_METRICS_POST_1_0)
for m_name, desc in pshard_stats_metrics.iteritems():
if desc[0] == "gauge":
aggregator.assert_metric(m_name, count=1, tags=[])
# Our pshard metrics are getting sent, let's check that they're accurate
# Note: please make sure you don't install Maven on the CI for future
# elastic search CI integrations. It would make the line below fail :/
aggregator.assert_metric('elasticsearch.primaries.docs.count')
def test_node_name_as_host(dd_environment, instance_normalize_hostname, aggregator, node_tags):
elastic_check = ESCheck('elastic', {}, instances=[instance_normalize_hostname])
elastic_check.check(None)
node_name = node_tags[-1].split(':')[1]
for m_name, _ in iteritems(STATS_METRICS):
aggregator.assert_metric(m_name, count=1, tags=node_tags, hostname=node_name)
def test_cat_allocation_metrics(dd_environment, aggregator, instance, cluster_tags):
instance['cat_allocation_stats'] = True
elastic_check = ESCheck('elastic', {}, instances=[instance])
elastic_check.check(None)
for m_name in CAT_ALLOCATION_METRICS:
aggregator.assert_metric(m_name)
def test_jvm_gc_rate_metrics(dd_environment, instance, aggregator, cluster_tags, node_tags, slm_tags):
instance['gc_collectors_as_rate'] = True
check = ESCheck('elastic', {}, instances=[instance])
check.check(instance)
for metric in JVM_RATES:
aggregator.assert_metric(metric, at_least=1, tags=node_tags)
_test_check(check, instance, aggregator, cluster_tags, node_tags, slm_tags)
def test_check(benchmark):
elastic_check = ESCheck(CHECK_NAME, {}, {})
for _ in range(3):
try:
elastic_check.check(CONFIG)
except Exception:
time.sleep(1)
benchmark(elastic_check.check, CONFIG)
def test_index_metrics(dd_environment, aggregator, instance, cluster_tags):
instance['index_stats'] = True
elastic_check = ESCheck('elastic', {}, instances=[instance])
es_version = elastic_check._get_es_version()
if es_version < [1, 0, 0]:
pytest.skip("Index metrics are only tested in version 1.0.0+")
elastic_check.check(None)
for m_name in index_stats_for_version(es_version):
aggregator.assert_metric(m_name, tags=cluster_tags + ['index_name:testindex'])
def test_url_join(aggregator):
elastic_check = ESCheck(CHECK_NAME, {}, {})
adm_forwarder_joined_url = elastic_check._join_url(
"https://localhost:9444/elasticsearch-admin",
"/stats",
admin_forwarder=True)
assert adm_forwarder_joined_url == "https://localhost:9444/elasticsearch-admin/stats"
joined_url = elastic_check._join_url(
"https://localhost:9444/elasticsearch-admin", "/stats")
assert joined_url == "https://localhost:9444/stats"
def test_disable_cluster_tag(dd_environment, instance, aggregator, new_cluster_tags):
disable_instance = deepcopy(instance)
disable_instance['disable_legacy_cluster_tag'] = True
elastic_check = ESCheck('elastic', {}, instances=[disable_instance])
elastic_check.check(None)
es_version = elastic_check._get_es_version()
# cluster stats
expected_metrics = health_stats_for_version(es_version)
expected_metrics.update(CLUSTER_PENDING_TASKS)
for m_name in expected_metrics:
aggregator.assert_metric(m_name, at_least=1, tags=new_cluster_tags)
def test_check_slm_stats(dd_environment, instance, aggregator, cluster_tags, node_tags, slm_tags):
slm_instance = deepcopy(instance)
slm_instance['slm_stats'] = True
elastic_check = ESCheck('elastic', {}, instances=[slm_instance])
elastic_check.check(None)
_test_check(elastic_check, slm_instance, aggregator, cluster_tags, node_tags)
# SLM stats
slm_metrics = slm_stats_for_version(elastic_check._get_es_version())
for m_name in slm_metrics:
aggregator.assert_metric(m_name, at_least=1, tags=slm_tags)
def test__join_url():
instance = {
"url": "https://localhost:9444/elasticsearch-admin",
"admin_forwarder": True,
}
check = ESCheck('elastic', {}, instances=[instance])
adm_forwarder_joined_url = check._join_url("/stats", admin_forwarder=True)
assert adm_forwarder_joined_url == "https://localhost:9444/elasticsearch-admin/stats"
joined_url = check._join_url("/stats", admin_forwarder=False)
assert joined_url == "https://localhost:9444/stats"
def test_config_parser():
elastic_check = ESCheck(CHECK_NAME, {}, {})
instance = {
"username": "******",
"password": "******",
"is_external": "yes",
"url": "http://foo.bar",
"tags": ["a", "b:c"],
}
c = elastic_check.get_instance_config(instance)
assert c.username == "user"
assert c.password == "pass"
assert c.admin_forwarder is False
assert c.cluster_stats is True
assert c.url == "http://foo.bar"
assert c.tags == ["url:http://foo.bar", "a", "b:c"]
assert c.timeout == elastic_check.DEFAULT_TIMEOUT
assert c.service_check_tags == ["host:foo.bar", "port:None", "a", "b:c"]
instance = {"url": "http://192.168.42.42:12999", "timeout": 15}
c = elastic_check.get_instance_config(instance)
assert c.username is None
assert c.password is None
assert c.cluster_stats is False
assert c.url == "http://192.168.42.42:12999"
assert c.tags == ["url:http://192.168.42.42:12999"]
assert c.timeout == 15
assert c.service_check_tags == ["host:192.168.42.42", "port:12999"]
instance = {
"username": "******",
"password": "******",
"url": "https://foo.bar:9200",
"ssl_verify": "true",
"ssl_cert": "/path/to/cert.pem",
"ssl_key": "/path/to/cert.key",
"admin_forwarder": "1"
}
c = elastic_check.get_instance_config(instance)
assert c.username == "user"
assert c.password == "pass"
assert c.admin_forwarder is True
assert c.cluster_stats is False
assert c.url == "https://foo.bar:9200"
assert c.tags == ["url:https://foo.bar:9200"]
assert c.timeout == elastic_check.DEFAULT_TIMEOUT
assert c.service_check_tags == ["host:foo.bar", "port:9200"]
assert c.ssl_verify == "true"
assert c.ssl_cert == "/path/to/cert.pem"
assert c.ssl_key == "/path/to/cert.key"
def test_custom_queries_one_invalid(dd_environment, dd_run_check, instance, aggregator):
custom_queries = [
{
# Wrong endpoint
'endpoint': '/_nodes2',
'data_path': '_nodes',
'columns': [
{
'value_path': 'total',
'name': 'elasticsearch.custom.metric2',
},
],
},
{
# Good endpoint
'endpoint': '/_nodes',
'data_path': '_nodes',
'columns': [
{
'value_path': 'total',
'name': 'elasticsearch.custom.metric',
},
],
},
]
instance = deepcopy(instance)
instance['custom_queries'] = custom_queries
check = ESCheck('elastic', {}, instances=[instance])
dd_run_check(check)
aggregator.assert_metric('elasticsearch.custom.metric', metric_type=aggregator.GAUGE)
def test_custom_queries_non_existent_tags(caplog, dd_environment, dd_run_check, instance, aggregator, cluster_tags):
custom_queries = [
{
'endpoint': '/_nodes',
'data_path': '_nodes',
'columns': [
{
'value_path': 'total',
'name': 'elasticsearch.custom.metric',
},
{
'value_path': 'totals', # nonexistent elasticsearch metric as tag
'name': 'nonexistent_tag',
'type': 'tag',
},
],
},
]
instance = deepcopy(instance)
instance['custom_queries'] = custom_queries
check = ESCheck('elastic', {}, instances=[instance])
caplog.clear()
with caplog.at_level(logging.DEBUG):
dd_run_check(check)
aggregator.assert_metric('elasticsearch.custom.metric', count=1, tags=cluster_tags)
assert 'Dynamic tag is null: _nodes.total -> nonexistent_tag' in caplog.text
def test_health_event(dd_environment, aggregator):
dummy_tags = ['elastique:recherche']
instance = {'url': URL, 'username': USER, 'password': PASSWORD, 'tags': dummy_tags}
elastic_check = ESCheck('elastic', {}, instances=[instance])
es_version = elastic_check._get_es_version()
# Should be yellow at first
requests.put(URL + '/_settings', data='{"index": {"number_of_replicas": 100}')
elastic_check.check(None)
if es_version < [2, 0, 0]:
assert len(aggregator.events) == 1
assert sorted(aggregator.events[0]['tags']) == sorted(set(['url:{}'.format(URL)] + dummy_tags + CLUSTER_TAG))
else:
aggregator.assert_service_check('elasticsearch.cluster_health')
def test_index_metrics(aggregator, spin_up_elastic):
# Tests that index level metrics are forwarded
config = {
'url': URL,
'index_stats': True,
'username': USER,
'password': PASSWORD
}
elastic_check = ESCheck(CHECK_NAME, {}, {})
index_metrics = dict(ESCheck.INDEX_STATS_METRICS)
elastic_check.check(config)
if get_es_version() >= [1, 0, 0]:
for m_name, desc in index_metrics.iteritems():
aggregator.assert_metric(m_name)
def test_pshard_metrics(dd_environment, aggregator):
instance = {'url': URL, 'pshard_stats': True, 'username': USER, 'password': PASSWORD}
elastic_check = ESCheck('elastic', {}, instances=[instance])
es_version = elastic_check._get_es_version()
elastic_check.check(None)
pshard_stats_metrics = pshard_stats_for_version(es_version)
for m_name, desc in iteritems(pshard_stats_metrics):
if desc[0] == 'gauge':
aggregator.assert_metric(m_name)
# Our pshard metrics are getting sent, let's check that they're accurate
# Note: please make sure you don't install Maven on the CI for future
# elastic search CI integrations. It would make the line below fail :/
aggregator.assert_metric('elasticsearch.primaries.docs.count')
def test__get_urls():
instance = {'url': URL}
elastic_check = ESCheck('elastic', {}, instances=[instance])
health_url, stats_url, pshard_stats_url, pending_tasks_url = elastic_check._get_urls(
[])
assert health_url == '/_cluster/health'
assert stats_url == '/_cluster/nodes/_local/stats?all=true'
assert pshard_stats_url == '/_stats'
assert pending_tasks_url is None
health_url, stats_url, pshard_stats_url, pending_tasks_url = elastic_check._get_urls(
[1, 0, 0])
assert health_url == '/_cluster/health'
assert stats_url == '/_nodes/_local/stats?all=true'
assert pshard_stats_url == '/_stats'
assert pending_tasks_url == '/_cluster/pending_tasks'
health_url, stats_url, pshard_stats_url, pending_tasks_url = elastic_check._get_urls(
[6, 0, 0])
assert health_url == '/_cluster/health'
assert stats_url == '/_nodes/_local/stats'
assert pshard_stats_url == '/_stats'
assert pending_tasks_url == '/_cluster/pending_tasks'
instance["cluster_stats"] = True
elastic_check = ESCheck('elastic', {}, instances=[instance])
health_url, stats_url, pshard_stats_url, pending_tasks_url = elastic_check._get_urls(
[])
assert health_url == '/_cluster/health'
assert stats_url == '/_cluster/nodes/stats?all=true'
assert pshard_stats_url == '/_stats'
assert pending_tasks_url is None
health_url, stats_url, pshard_stats_url, pending_tasks_url = elastic_check._get_urls(
[1, 0, 0])
assert health_url == '/_cluster/health'
assert stats_url == '/_nodes/stats?all=true'
assert pshard_stats_url == '/_stats'
assert pending_tasks_url == '/_cluster/pending_tasks'
health_url, stats_url, pshard_stats_url, pending_tasks_url = elastic_check._get_urls(
[6, 0, 0])
assert health_url == '/_cluster/health'
assert stats_url == '/_nodes/stats'
assert pshard_stats_url == '/_stats'
assert pending_tasks_url == '/_cluster/pending_tasks'
def test_aws_auth_url(instance, expected_aws_host, expected_aws_service):
check = ESCheck('elastic', {}, instances=[instance])
assert getattr(check.http.options.get('auth'), 'aws_host', None) == expected_aws_host
assert getattr(check.http.options.get('auth'), 'service', None) == expected_aws_service
# make sure class attribute HTTP_CONFIG_REMAPPER is not modified
assert 'aws_host' not in ESCheck.HTTP_CONFIG_REMAPPER
def test_custom_query_invalid_config(dd_environment, dd_run_check, instance,
aggregator, invalid_custom_queries):
instance = deepcopy(instance)
instance['custom_queries'] = invalid_custom_queries
check = ESCheck('elastic', {}, instances=[instance])
with pytest.raises(Exception):
dd_run_check(check)
def test_index_metrics(benchmark, dd_environment):
instance = {
'url': URL,
'index_stats': True,
'username': USER,
'password': PASSWORD
}
elastic_check = ESCheck('elastic', {}, instances=[instance])
benchmark(elastic_check.check, instance)
def test_index_metrics(benchmark):
config = {
'url': URL,
'index_stats': True,
'username': USER,
'password': PASSWORD
}
elastic_check = ESCheck(CHECK_NAME, {}, {})
benchmark(elastic_check.check, config)
def test_health_event(aggregator, spin_up_elastic):
dummy_tags = ['elastique:recherche']
config = {
'url': URL,
'username': USER,
'password': PASSWORD,
'tags': dummy_tags
}
elastic_check = ESCheck(CHECK_NAME, {}, {})
# Should be yellow at first
requests.put(URL + '/_settings',
data='{"index": {"number_of_replicas": 100}')
elastic_check.check(config)
if get_es_version() < [2, 0, 0]:
assert len(aggregator.events) == 1
assert sorted(aggregator.events[0]['tags']) == sorted(
set(['url:' + URL] + dummy_tags + CLUSTER_TAG))
else:
aggregator.assert_service_check('elasticsearch.cluster_health')
def test__get_urls(instance, url_fix):
elastic_check = ESCheck('elastic', {}, instances=[instance])
health_url, stats_url, pshard_stats_url, pending_tasks_url, slm_url = elastic_check._get_urls(
[])
assert health_url == '/_cluster/health'
assert stats_url == '/_cluster/nodes/' + url_fix + 'stats?all=true'
assert pshard_stats_url == '/_stats'
assert pending_tasks_url is None
assert slm_url is None
health_url, stats_url, pshard_stats_url, pending_tasks_url, slm_url = elastic_check._get_urls(
[1, 0, 0])
assert health_url == '/_cluster/health'
assert stats_url == '/_nodes/' + url_fix + 'stats?all=true'
assert pshard_stats_url == '/_stats'
assert pending_tasks_url == '/_cluster/pending_tasks'
assert slm_url is None
health_url, stats_url, pshard_stats_url, pending_tasks_url, slm_url = elastic_check._get_urls(
[6, 0, 0])
assert health_url == '/_cluster/health'
assert stats_url == '/_nodes/' + url_fix + 'stats'
assert pshard_stats_url == '/_stats'
assert pending_tasks_url == '/_cluster/pending_tasks'
assert slm_url is None
health_url, stats_url, pshard_stats_url, pending_tasks_url, slm_url = elastic_check._get_urls(
[7, 4, 0])
assert health_url == '/_cluster/health'
assert stats_url == '/_nodes/' + url_fix + 'stats'
assert pshard_stats_url == '/_stats'
assert pending_tasks_url == '/_cluster/pending_tasks'
assert slm_url == ('/_slm/policy'
if instance.get('slm_stats') is True else None)
def test_detailed_index_stats(dd_environment, aggregator):
instance = {
"url": URL,
"cluster_stats": True,
"pshard_stats": True,
"detailed_index_stats": True,
"tls_verify": False,
}
elastic_check = ESCheck('elastic', {}, instances=[instance])
es_version = elastic_check._get_es_version()
elastic_check.check(None)
pshard_stats_metrics = pshard_stats_for_version(es_version)
for m_name, desc in iteritems(pshard_stats_metrics):
if desc[0] == 'gauge' and desc[1].startswith('_all.'):
aggregator.assert_metric(m_name)
aggregator.assert_metric_has_tag('elasticsearch.primaries.docs.count', tag='index_name:_all')
aggregator.assert_metric_has_tag('elasticsearch.primaries.docs.count', tag='index_name:testindex')
aggregator.assert_metric_has_tag('elasticsearch.primaries.docs.count', tag='index_name:.testindex')
aggregator.assert_metrics_using_metadata(
get_metadata_metrics(),
check_metric_type=False,
exclude=[
"system.cpu.idle",
"system.load.1",
"system.load.15",
"system.load.5",
"system.mem.free",
"system.mem.total",
"system.mem.usable",
"system.mem.used",
"system.net.bytes_rcvd",
"system.net.bytes_sent",
"system.swap.free",
"system.swap.total",
"system.swap.used",
],
)
def test_pshard_metrics(benchmark):
elastic_latency = 10
config = {
'url': URL,
'pshard_stats': True,
'username': USER,
'password': PASSWORD
}
requests.put(URL + '/_settings',
data='{"index": {"number_of_replicas": 1}}')
requests.put(URL + '/testindex/testtype/2',
data='{"name": "Jane Doe", "age": 27}')
requests.put(URL + '/testindex/testtype/1',
data='{"name": "John Doe", "age": 42}')
time.sleep(elastic_latency)
elastic_check = ESCheck(CHECK_NAME, {}, {})
benchmark(elastic_check.check, config)
def test_custom_queries_valid_metrics(dd_environment, dd_run_check, instance, aggregator):
custom_queries = [
{
'endpoint': '/_nodes',
'data_path': '_nodes',
'columns': [
{
'value_path': 'total',
'name': 'elasticsearch.custom.metric',
},
{'value_path': 'total', 'name': 'elasticsearch.custom.metric2', 'type': 'monotonic_count'},
],
},
]
instance = deepcopy(instance)
instance['custom_queries'] = custom_queries
check = ESCheck('elastic', {}, instances=[instance])
dd_run_check(check)
aggregator.assert_metric('elasticsearch.custom.metric2', metric_type=aggregator.MONOTONIC_COUNT)
aggregator.assert_metric('elasticsearch.custom.metric', metric_type=aggregator.GAUGE)
def test_custom_queries_valid_tags(dd_environment, dd_run_check, instance, aggregator, cluster_tags):
custom_queries = [
{
'endpoint': '/_nodes',
'data_path': '_nodes',
'columns': [
{
'value_path': 'total',
'name': 'elasticsearch.custom.metric',
},
{'value_path': 'total', 'name': 'dynamic_tag', 'type': 'tag'},
],
'tags': ['custom_tag:1'],
},
]
instance = deepcopy(instance)
instance['custom_queries'] = custom_queries
check = ESCheck('elastic', {}, instances=[instance])
dd_run_check(check)
tags = cluster_tags + ['custom_tag:1'] + ['dynamic_tag:1']
aggregator.assert_metric('elasticsearch.custom.metric', metric_type=aggregator.GAUGE, tags=tags)
def test_custom_queries_with_payload(dd_environment, dd_run_check, instance, aggregator, cluster_tags):
custom_queries = [
{
'endpoint': '/_search',
'data_path': 'hits.total',
'payload': {"query": {"match": {"phrase": {"query": ""}}}},
'columns': [
{
'value_path': 'value',
'name': 'elasticsearch.custom.metric',
},
{'value_path': 'relation', 'name': 'dynamic_tag', 'type': 'tag'},
],
},
]
instance = deepcopy(instance)
instance['custom_queries'] = custom_queries
check = ESCheck('elastic', {}, instances=[instance])
dd_run_check(check)
tags = cluster_tags + ['dynamic_tag:eq']
aggregator.assert_metric('elasticsearch.custom.metric', metric_type=aggregator.GAUGE, tags=tags)
def test_custom_queries_non_existent_metrics(caplog, dd_environment, dd_run_check, instance, aggregator):
custom_queries = [
{
'endpoint': '/_nodes',
'data_path': '_nodes',
'columns': [
{
'value_path': 'totals', # nonexistent elasticsearch metric
'name': 'elasticsearch.custom.metric',
},
],
'tags': ['custom_tag:1'],
},
]
instance = deepcopy(instance)
instance['custom_queries'] = custom_queries
check = ESCheck('elastic', {}, instances=[instance])
caplog.clear()
with caplog.at_level(logging.DEBUG):
dd_run_check(check)
aggregator.assert_metric('elasticsearch.custom.metric', count=0)
assert 'Metric not found: _nodes.totals -> elasticsearch.custom.metric' in caplog.text
def test_aws_auth_no_url(instance, expected_aws_host, expected_aws_service):
with pytest.raises(ConfigurationError):
ESCheck('elastic', {}, instances=[instance])