def test_bad_config(self, aggregator):
instance = INSTANCES['invalid']
c = Vault(Vault.CHECK_NAME, {}, [instance])
with pytest.raises(Exception):
run_check(c)
aggregator.assert_service_check(Vault.SERVICE_CHECK_CONNECT, count=0)
def test_service_check_initialized_ok(self, aggregator):
instance = INSTANCES['main']
c = Vault(Vault.CHECK_NAME, {}, [instance])
run_check(c)
aggregator.assert_service_check(Vault.SERVICE_CHECK_INITIALIZED,
status=Vault.OK,
count=1)
def test_service_check_unsealed_ok(self, aggregator):
instance = INSTANCES['main']
c = Vault(Vault.CHECK_NAME, {}, [instance])
run_check(c)
aggregator.assert_service_check(Vault.SERVICE_CHECK_UNSEALED,
status=Vault.OK,
count=1)
def test_service_check_connect_ok(self, aggregator):
instance = INSTANCES['main']
c = Vault(Vault.CHECK_NAME, {}, [instance])
run_check(c)
aggregator.assert_service_check(Vault.SERVICE_CHECK_CONNECT,
status=Vault.OK,
count=1)
def test_service_check_initialized_ok_all_tags(self, aggregator):
instance = INSTANCES['main']
c = Vault(Vault.CHECK_NAME, {}, [instance])
config = c.get_config(instance)
# Keep a reference for use during mock
requests_get = requests.get
def mock_requests_get(url, *args, **kwargs):
if url == config['api_url'] + '/sys/leader':
return MockResponse({
'ha_enabled': False,
'is_self': True,
'leader_address': '',
'leader_cluster_address': ''
})
elif url == config['api_url'] + '/sys/health':
return MockResponse({
'cluster_id': '9e25ccdb-09ea-8bd8-0521-34cf3ef7a4cc',
'cluster_name': 'vault-cluster-f5f44063',
'initialized': True,
'replication_dr_mode': 'disabled',
'replication_performance_mode': 'disabled',
'sealed': False,
'server_time_utc': 1529357080,
'standby': False,
'version': '0.10.2',
})
return requests_get(url, *args, **kwargs)
with mock.patch('requests.get',
side_effect=mock_requests_get,
autospec=True):
c.check(instance)
expected_tags = [
'instance:foobar',
'is_leader:true',
'cluster_name:vault-cluster-f5f44063',
'vault_version:0.10.2',
]
aggregator.assert_service_check(Vault.SERVICE_CHECK_INITIALIZED,
status=Vault.OK,
tags=expected_tags,
count=1)
def test_bad_config(self, aggregator):
instance = INSTANCES['invalid']
c = Vault(Vault.CHECK_NAME, {}, [instance])
with pytest.raises(Exception, match='^Vault setting `api_url` is required$'):
run_check(c, extract_message=True)
aggregator.assert_service_check(Vault.SERVICE_CHECK_CONNECT, count=0)
def test_bad_config(self, aggregator, dd_run_check, use_openmetrics):
instance = {'use_openmetrics': use_openmetrics}
instance.update(INSTANCES['invalid'])
c = Vault(Vault.CHECK_NAME, {}, [instance])
with pytest.raises(Exception):
dd_run_check(c)
aggregator.assert_service_check(Vault.SERVICE_CHECK_CONNECT, count=0)
def test_unsupported_api_version_fallback(self, aggregator):
instance = INSTANCES['unsupported_api']
c = Vault(Vault.CHECK_NAME, {}, [instance])
assert not instance['api_url'].endswith(Vault.DEFAULT_API_VERSION)
run_check(c)
assert c._api_url.endswith(Vault.DEFAULT_API_VERSION)
aggregator.assert_service_check(Vault.SERVICE_CHECK_CONNECT, status=Vault.OK, count=1)
def test_replication_dr_mode_changed(self, aggregator, dd_run_check,
use_openmetrics):
instance = {'use_openmetrics': use_openmetrics}
instance.update(INSTANCES['main'])
c = Vault(Vault.CHECK_NAME, {}, [instance])
c.log.debug = mock.MagicMock()
# Keep a reference for use during mock
requests_get = requests.get
def mock_requests_get(url, *args, **kwargs):
if url == instance['api_url'] + '/sys/health':
if getattr(mock_requests_get, 'first_health_call', True):
mock_requests_get.first_health_call = False
replication_dr_mode = 'primary'
else:
replication_dr_mode = 'secondary'
return MockResponse(
json_data={
'cluster_id': '9e25ccdb-09ea-8bd8-0521-34cf3ef7a4cc',
'cluster_name': 'vault-cluster-f5f44063',
'initialized': False,
'replication_dr_mode': replication_dr_mode,
'replication_performance_mode': 'primary',
'sealed': False,
'server_time_utc': 1529357080,
'standby': True,
'performance_standby': False,
'version': '0.10.2',
},
status_code=200,
)
return requests_get(url, *args, **kwargs)
with mock.patch('requests.get',
side_effect=mock_requests_get,
autospec=True):
dd_run_check(c)
assert not c._replication_dr_secondary_mode
aggregator.assert_service_check(Vault.SERVICE_CHECK_CONNECT,
status=Vault.OK,
count=1)
aggregator.assert_metric('vault.is_leader', 1)
assert_all_metrics(aggregator)
aggregator.reset()
dd_run_check(c)
c.log.debug.assert_called_with(
"Detected vault in replication DR secondary mode, skipping Prometheus metric collection."
)
assert c._replication_dr_secondary_mode
aggregator.assert_metric('vault.is_leader', 1)
aggregator.assert_service_check(Vault.SERVICE_CHECK_CONNECT,
status=Vault.OK,
count=1)
assert_all_metrics(aggregator)
def test_service_check_connect_ok(self, aggregator, dd_run_check,
use_openmetrics):
instance = {'use_openmetrics': use_openmetrics}
instance.update(INSTANCES['main'])
c = Vault(Vault.CHECK_NAME, {}, [instance])
dd_run_check(c, dd_run_check)
aggregator.assert_service_check(Vault.SERVICE_CHECK_CONNECT,
status=Vault.OK,
count=1)
def test_service_check_initialized_ok(self, aggregator, dd_run_check,
use_openmetrics):
instance = {'use_openmetrics': use_openmetrics}
instance.update(INSTANCES['main'])
c = Vault(Vault.CHECK_NAME, {}, [instance])
dd_run_check(c)
aggregator.assert_service_check(Vault.SERVICE_CHECK_INITIALIZED,
status=Vault.OK,
count=1)
def test_service_check_500_fail(self, aggregator, global_tags):
instance = INSTANCES['main']
c = Vault(Vault.CHECK_NAME, {}, [instance])
with mock.patch('requests.get', return_value=MockResponse('', status_code=500)):
with pytest.raises(
Exception, match=r'^The Vault endpoint `{}.+?` returned 500$'.format(re.escape(instance['api_url'])),
):
run_check(c, extract_message=True)
aggregator.assert_service_check(Vault.SERVICE_CHECK_CONNECT, status=Vault.CRITICAL, tags=global_tags, count=1)
def test_sys_leader_non_standard_status_codes(self, aggregator):
instance = INSTANCES['main']
c = Vault(Vault.CHECK_NAME, {}, [instance])
config = c.get_config(instance)
# Keep a reference for use during mock
requests_get = requests.get
def mock_requests_get(url, *args, **kwargs):
if url == config['api_url'] + '/sys/leader':
return MockResponse({'errors': ["Vault is sealed"]},
status_code=503)
return requests_get(url, *args, **kwargs)
with mock.patch('requests.get',
side_effect=mock_requests_get,
autospec=True):
c.check(instance)
aggregator.assert_metric('vault.is_leader', count=0)
def test_is_leader_metric_false(self, aggregator):
instance = INSTANCES['main']
c = Vault(Vault.CHECK_NAME, {}, [instance])
config = c.get_config(instance)
# Keep a reference for use during mock
requests_get = requests.get
def mock_requests_get(url, *args, **kwargs):
if url == config['api_url'] + '/sys/leader':
return MockResponse(
{'ha_enabled': False, 'is_self': False, 'leader_address': 'bar', 'leader_cluster_address': ''}
)
return requests_get(url, *args, **kwargs)
with mock.patch('requests.get', side_effect=mock_requests_get, autospec=True):
c.check(instance)
aggregator.assert_metric('vault.is_leader', 0)
def test_route_transform(self, aggregator, no_token_instance, global_tags):
c = Vault(Vault.CHECK_NAME, {}, [no_token_instance])
c.parse_config()
content = (
'# HELP vault_route_rollback_sys_ vault_route_rollback_sys_\n'
'# TYPE vault_route_rollback_sys_ summary\n'
'vault_route_rollback_sys_{quantile="0.5"} 3\n'
'vault_route_rollback_sys_{quantile="0.9"} 3\n'
'vault_route_rollback_sys_{quantile="0.99"} 4\n'
'vault_route_rollback_sys_ 3.2827999591827393\n'
'vault_route_rollback_sys_ 1')
def iter_lines(**_):
for elt in content.split("\n"):
yield elt
with mock.patch('datadog_checks.base.utils.http.requests') as r:
r.get.return_value = mock.MagicMock(status_code=200,
content=content,
iter_lines=iter_lines)
c.process(c._scraper_config, c._metric_transformers)
for quantile in [0.5, 0.9, 0.99]:
quantile_tag = 'quantile:{}'.format(quantile)
aggregator.assert_metric(
'vault.vault.route.rollback.sys.quantile',
tags=global_tags + [quantile_tag])
aggregator.assert_metric('vault.route.rollback.quantile',
tags=global_tags +
[quantile_tag, 'mountpoint:sys'])
aggregator.assert_all_metrics_covered()
def test_unsupported_api_version_fallback(self, aggregator, dd_run_check,
use_openmetrics):
instance = {'use_openmetrics': use_openmetrics}
instance.update(INSTANCES['unsupported_api'])
c = Vault(Vault.CHECK_NAME, {}, [instance])
assert not instance['api_url'].endswith(DEFAULT_API_VERSION)
dd_run_check(c)
assert c._api_url.endswith(DEFAULT_API_VERSION)
aggregator.assert_service_check(Vault.SERVICE_CHECK_CONNECT,
status=Vault.OK,
count=1)
def test_disable_legacy_cluster_tag(self, aggregator, dd_run_check,
global_tags):
instance = {
'disable_legacy_cluster_tag': True,
'use_openmetrics': False
}
instance.update(INSTANCES['main'])
c = Vault(Vault.CHECK_NAME, {}, [instance])
# Keep a reference for use during mock
requests_get = requests.get
def mock_requests_get(url, *args, **kwargs):
if url == instance['api_url'] + '/sys/leader':
return MockResponse(
json_data={
'ha_enabled': False,
'is_self': True,
'leader_address': '',
'leader_cluster_address': ''
})
elif url == instance['api_url'] + '/sys/health':
return MockResponse(
json_data={
'cluster_id': '9e25ccdb-09ea-8bd8-0521-34cf3ef7a4cc',
'cluster_name': 'vault-cluster-f5f44063',
'initialized': True,
'replication_dr_mode': 'disabled',
'replication_performance_mode': 'disabled',
'sealed': False,
'server_time_utc': 1529357080,
'standby': False,
'version': '0.10.2',
})
return requests_get(url, *args, **kwargs)
with mock.patch('requests.get',
side_effect=mock_requests_get,
autospec=True):
dd_run_check(c)
expected_tags = [
'is_leader:true',
'vault_cluster:vault-cluster-f5f44063',
'vault_version:0.10.2',
]
expected_tags.extend(global_tags)
aggregator.assert_service_check(Vault.SERVICE_CHECK_INITIALIZED,
status=Vault.OK,
tags=expected_tags,
count=1)
def test_token_renewal(self, caplog, aggregator, instance, global_tags):
instance = instance()
instance['token_renewal_wait'] = 1
c = Vault(Vault.CHECK_NAME, {}, [instance])
renew_client_token = c.renew_client_token
run_check(c)
aggregator.assert_service_check(Vault.SERVICE_CHECK_CONNECT, status=Vault.OK, count=1, tags=global_tags)
aggregator.assert_service_check(Vault.SERVICE_CHECK_CONNECT, status=Vault.WARNING, count=0)
aggregator.assert_service_check(Vault.SERVICE_CHECK_CONNECT, status=Vault.CRITICAL, count=0)
assert 'Permission denied, refreshing the client token...' not in caplog.text
c.set_client_token('foo')
c.renew_client_token = lambda: None
aggregator.reset()
run_check(c)
aggregator.assert_service_check(Vault.SERVICE_CHECK_CONNECT, status=Vault.OK, count=0)
aggregator.assert_service_check(Vault.SERVICE_CHECK_CONNECT, status=Vault.WARNING, count=1, tags=global_tags)
aggregator.assert_service_check(Vault.SERVICE_CHECK_CONNECT, status=Vault.CRITICAL, count=0)
assert 'Permission denied, refreshing the client token...' in caplog.text
aggregator.reset()
with pytest.raises(Exception, match='^403 Client Error: Forbidden for url'):
run_check(c, extract_message=True)
aggregator.assert_service_check(Vault.SERVICE_CHECK_CONNECT, status=Vault.OK, count=0)
aggregator.assert_service_check(Vault.SERVICE_CHECK_CONNECT, status=Vault.WARNING, count=0)
aggregator.assert_service_check(Vault.SERVICE_CHECK_CONNECT, status=Vault.CRITICAL, count=1, tags=global_tags)
renew_client_token()
aggregator.reset()
run_check(c)
aggregator.assert_service_check(Vault.SERVICE_CHECK_CONNECT, status=Vault.OK, count=1, tags=global_tags)
aggregator.assert_service_check(Vault.SERVICE_CHECK_CONNECT, status=Vault.WARNING, count=0)
aggregator.assert_service_check(Vault.SERVICE_CHECK_CONNECT, status=Vault.CRITICAL, count=0)
def test_noauth_needed(self, aggregator, no_token_instance, global_tags):
c = Vault(Vault.CHECK_NAME, {}, [no_token_instance])
run_check(c, extract_message=True)
aggregator.assert_service_check(Vault.SERVICE_CHECK_CONNECT,
status=Vault.OK,
count=1,
tags=global_tags)
aggregator.assert_service_check(Vault.SERVICE_CHECK_CONNECT,
status=Vault.WARNING,
count=0)
aggregator.assert_service_check(Vault.SERVICE_CHECK_CONNECT,
status=Vault.CRITICAL,
count=0)
def test_ha_is_perf_standby(self, aggregator):
instance = INSTANCES['main']
c = Vault(Vault.CHECK_NAME, {}, [instance])
config = c.get_config(instance)
# Keep a reference for use during mock
requests_get = requests.get
def mock_requests_get(url, *args, **kwargs):
if url == config['api_url'] + '/sys/health':
status_code = 200 if kwargs.get(
'params', {}).get('perfstandbyok') else 473
return MockResponse(
{
'cluster_id': '9e25ccdb-09ea-8bd8-0521-34cf3ef7a4cc',
'cluster_name': 'vault-cluster-f5f44063',
'initialized': False,
'replication_dr_mode': 'disabled',
'replication_performance_mode': 'disabled',
'sealed': False,
'server_time_utc': 1529357080,
'standby': False,
'performance_standby': True,
'version': '0.10.2',
},
status_code,
)
return requests_get(url, *args, **kwargs)
with mock.patch('requests.get',
side_effect=mock_requests_get,
autospec=True):
c.check(instance)
aggregator.assert_metric('vault.is_leader', 0)
aggregator.assert_all_metrics_covered()
def test_service_check_connect_fail(self, aggregator):
instance = INSTANCES['bad_url']
c = Vault(Vault.CHECK_NAME, {}, [instance])
with pytest.raises(
Exception,
match=r'^Vault endpoint `{}.+?` timed out after 1\.0 seconds$'.format(re.escape(instance['api_url'])),
):
run_check(c, extract_message=True)
aggregator.assert_service_check(
Vault.SERVICE_CHECK_CONNECT,
status=Vault.CRITICAL,
tags=['instance:foobar', 'api_url:http://1.2.3.4:555/v1'],
count=1,
)
def test_replication_dr_mode_collect_secondary(self, aggregator,
dd_run_check,
use_openmetrics):
instance = {
'use_openmetrics': use_openmetrics,
'collect_secondary_dr': True
}
instance.update(INSTANCES['main'])
c = Vault(Vault.CHECK_NAME, {}, [instance])
c.log.debug = mock.MagicMock()
# Keep a reference for use during mock
requests_get = requests.get
def mock_requests_get(url, *args, **kwargs):
if url == instance['api_url'] + '/sys/health':
return MockResponse(
json_data={
'cluster_id': '9e25ccdb-09ea-8bd8-0521-34cf3ef7a4cc',
'cluster_name': 'vault-cluster-f5f44063',
'initialized': False,
'replication_dr_mode': 'secondary',
'replication_performance_mode': 'primary',
'sealed': False,
'server_time_utc': 1529357080,
'standby': True,
'performance_standby': False,
'version': '0.10.2',
},
status_code=200,
)
return requests_get(url, *args, **kwargs)
metric_collection = 'OpenMetrics' if use_openmetrics else 'Prometheus'
with mock.patch('requests.get',
side_effect=mock_requests_get,
autospec=True):
dd_run_check(c)
c.log.debug.assert_called_with(
"Detected vault in replication DR secondary mode but also detected that "
"`collect_secondary_dr` is enabled, %s metric collection will still occur."
% metric_collection)
aggregator.assert_metric('vault.is_leader', 1)
aggregator.assert_service_check(Vault.SERVICE_CHECK_CONNECT,
status=Vault.OK,
count=1)
assert_all_metrics(aggregator)
def test_no_token(self, aggregator, instance, global_tags):
instance = instance()
instance['no_token'] = True
c = Vault(Vault.CHECK_NAME, {}, [instance])
with pytest.raises(Exception,
match='^400 Client Error: Bad Request for url'):
run_check(c, extract_message=True)
aggregator.assert_service_check(Vault.SERVICE_CHECK_CONNECT,
status=Vault.OK,
count=0)
aggregator.assert_service_check(Vault.SERVICE_CHECK_CONNECT,
status=Vault.WARNING,
count=0)
aggregator.assert_service_check(Vault.SERVICE_CHECK_CONNECT,
status=Vault.CRITICAL,
count=1,
tags=global_tags)
def test_noauth_needed(self, aggregator, dd_run_check, no_token_instance,
global_tags, use_openmetrics):
no_token_instance['use_openmetrics'] = use_openmetrics
c = Vault(Vault.CHECK_NAME, {}, [no_token_instance])
dd_run_check(c, extract_message=True)
aggregator.assert_service_check(Vault.SERVICE_CHECK_CONNECT,
status=Vault.OK,
count=1,
tags=global_tags)
aggregator.assert_service_check(Vault.SERVICE_CHECK_CONNECT,
status=Vault.WARNING,
count=0)
aggregator.assert_service_check(Vault.SERVICE_CHECK_CONNECT,
status=Vault.CRITICAL,
count=0)
if use_openmetrics:
aggregator.assert_service_check('vault.openmetrics.health',
status=c.CRITICAL,
count=0)
def test_sys_leader_non_standard_status_codes(self, aggregator,
dd_run_check,
use_openmetrics):
instance = {'use_openmetrics': use_openmetrics}
instance.update(INSTANCES['main'])
c = Vault(Vault.CHECK_NAME, {}, [instance])
# Keep a reference for use during mock
requests_get = requests.get
def mock_requests_get(url, *args, **kwargs):
if url == instance['api_url'] + '/sys/leader':
return MockResponse(json_data={'errors': ["Vault is sealed"]},
status_code=503)
return requests_get(url, *args, **kwargs)
with mock.patch('requests.get',
side_effect=mock_requests_get,
autospec=True):
dd_run_check(c)
aggregator.assert_metric('vault.is_leader', count=0)
def test_sys_health_non_standard_status_codes(self, aggregator,
dd_run_check, status_code,
use_openmetrics):
instance = {'use_openmetrics': use_openmetrics}
instance.update(INSTANCES['main'])
c = Vault(Vault.CHECK_NAME, {}, [instance])
# Keep a reference for use during mock
requests_get = requests.get
def mock_requests_get(url, *args, **kwargs):
if url == instance['api_url'] + '/sys/health':
return MockResponse(
json_data={
'cluster_id': '9e25ccdb-09ea-8bd8-0521-34cf3ef7a4cc',
'cluster_name': 'vault-cluster-f5f44063',
'initialized': False,
'replication_dr_mode': 'disabled',
'replication_performance_mode': 'disabled',
'sealed': False,
'server_time_utc': 1529357080,
'standby': True,
'performance_standby': False,
'version': '0.10.2',
},
status_code=status_code,
)
return requests_get(url, *args, **kwargs)
with mock.patch('requests.get',
side_effect=mock_requests_get,
autospec=True):
dd_run_check(c)
aggregator.assert_metric('vault.is_leader', 1)
assert_all_metrics(aggregator)
def test_bad_config(self, aggregator):
instance = INSTANCES['invalid']
c = Vault(Vault.CHECK_NAME, {}, [instance])
c.check(instance)
aggregator.assert_service_check(Vault.SERVICE_CHECK_CONNECT, count=0)
def check():
return lambda inst: Vault('vault', {}, [inst])
def check():
check = Vault('vault', {}, [INSTANCES['main']])
return check
