def test_login_metrics(dd_run_check, aggregator, instance):
# type: (Callable[[SnowflakeCheck], None], AggregatorStub, Dict[str, Any]) -> None
expected_login_metrics = [('SNOWFLAKE_UI', 2, 6, 8),
('PYTHON_DRIVER', 0, 148, 148)]
with mock.patch(
'datadog_checks.snowflake.SnowflakeCheck.execute_query_raw',
return_value=expected_login_metrics):
check = SnowflakeCheck(CHECK_NAME, {}, [instance])
check._conn = mock.MagicMock()
check._query_manager.queries = [Query(queries.LoginMetrics)]
dd_run_check(check)
snowflake_tags = EXPECTED_TAGS + ['client_type:SNOWFLAKE_UI']
aggregator.assert_metric('snowflake.logins.fail.count',
value=2,
tags=snowflake_tags)
aggregator.assert_metric('snowflake.logins.success.count',
value=6,
tags=snowflake_tags)
aggregator.assert_metric('snowflake.logins.total',
value=8,
tags=snowflake_tags)
python_tags = EXPECTED_TAGS + ['client_type:PYTHON_DRIVER']
aggregator.assert_metric('snowflake.logins.fail.count',
value=0,
tags=python_tags)
aggregator.assert_metric('snowflake.logins.success.count',
value=148,
tags=python_tags)
aggregator.assert_metric('snowflake.logins.total',
value=148,
tags=python_tags)
def test_warehouse_load(dd_run_check, aggregator, instance):
# type: (Callable[[SnowflakeCheck], None], AggregatorStub, Dict[str, Any]) -> None
expected_wl_metrics = [
('COMPUTE_WH', Decimal('0.000446667'), Decimal('0E-9'),
Decimal('0E-9'), Decimal('0E-9')),
]
expected_tags = EXPECTED_TAGS + ['warehouse:COMPUTE_WH']
with mock.patch(
'datadog_checks.snowflake.SnowflakeCheck.execute_query_raw',
return_value=expected_wl_metrics):
check = SnowflakeCheck(CHECK_NAME, {}, [instance])
check._conn = mock.MagicMock()
check._query_manager.queries = [Query(queries.WarehouseLoad)]
dd_run_check(check)
aggregator.assert_metric('snowflake.query.executed',
value=0.000446667,
tags=expected_tags)
aggregator.assert_metric('snowflake.query.queued_overload',
value=0,
count=1,
tags=expected_tags)
aggregator.assert_metric('snowflake.query.queued_provision',
value=0,
count=1,
tags=expected_tags)
aggregator.assert_metric('snowflake.query.blocked',
value=0,
count=1,
tags=expected_tags)
def test_query_metrics(dd_run_check, aggregator, instance):
# type: (Callable[[SnowflakeCheck], None], AggregatorStub, Dict[str, Any]) -> None
expected_query_metrics = [
(
'USE',
'COMPUTE_WH',
'SNOWFLAKE',
None,
Decimal('4.333333'),
Decimal('24.555556'),
Decimal('0.000000'),
Decimal('0.000000'),
Decimal('0.000000'),
Decimal('0.000000'),
Decimal('0.000000'),
),
]
expected_tags = EXPECTED_TAGS + [
'warehouse:COMPUTE_WH', 'database:SNOWFLAKE', 'schema:None',
'query_type:USE'
]
with mock.patch(
'datadog_checks.snowflake.SnowflakeCheck.execute_query_raw',
return_value=expected_query_metrics):
check = SnowflakeCheck(CHECK_NAME, {}, [instance])
check._conn = mock.MagicMock()
check._query_manager.queries = [Query(queries.QueryHistory)]
dd_run_check(check)
aggregator.assert_metric('snowflake.query.execution_time',
value=4.333333,
tags=expected_tags)
aggregator.assert_metric('snowflake.query.compilation_time',
value=24.555556,
tags=expected_tags)
aggregator.assert_metric('snowflake.query.bytes_scanned',
value=0,
count=1,
tags=expected_tags)
aggregator.assert_metric('snowflake.query.bytes_written',
value=0,
count=1,
tags=expected_tags)
aggregator.assert_metric('snowflake.query.bytes_deleted',
value=0,
count=1,
tags=expected_tags)
aggregator.assert_metric('snowflake.query.bytes_spilled.local',
value=0,
count=1,
tags=expected_tags)
aggregator.assert_metric('snowflake.query.bytes_spilled.remote',
value=0,
count=1,
tags=expected_tags)
def test_emit_generic_and_non_generic_tags_by_default(instance):
instance = copy.deepcopy(instance)
instance['disable_generic_tags'] = False
check = SnowflakeCheck(CHECK_NAME, {}, [instance])
tags = EXPECTED_TAGS + [
'service_type:WAREHOUSE_METERING', 'service:COMPUTE_WH'
]
normalised_tags = tags + ['snowflake_service:COMPUTE_WH']
assert set(normalised_tags) == set(check._normalize_tags_type(tags))
def test_metric_group_exceptions(instance):
instance = copy.deepcopy(instance)
instance['metric_groups'] = ['fake.metric.group']
with pytest.raises(
Exception,
match='No valid metric_groups configured, please list at least one.'
):
check = SnowflakeCheck(CHECK_NAME, {}, [instance])
check.log = mock.MagicMock()
check.log.warning.assert_called_once_with(
"Invalid metric_groups found in snowflake conf.yaml: fake.metric.group"
)
def test_config():
# Test missing account
user_config = {
'username': '******',
'password': '******',
}
with pytest.raises(Exception, match='Must specify an account'):
SnowflakeCheck(CHECK_NAME, {}, [user_config])
# Test missing user and pass
account_config = {'account': 'TEST123'}
with pytest.raises(Exception, match='Must specify a user and password'):
SnowflakeCheck(CHECK_NAME, {}, [account_config])
def test_authenticator_option_pass(options):
instance = {
# Common configuration
'account': 'test_acct.us-central1.gcp',
'database': 'SNOWFLAKE',
'schema': 'ACCOUNT_USAGE',
'role': 'ACCOUNTADMIN',
'user': '******',
}
instance.update(options)
check = SnowflakeCheck(CHECK_NAME, {}, [instance])
check.load_configuration_models()
def test_invalid_auth(instance):
# Test oauth
oauth_inst = copy.deepcopy(instance)
oauth_inst['authenticator'] = 'oauth'
with pytest.raises(Exception,
match='If using OAuth, you must specify a token'):
SnowflakeCheck(CHECK_NAME, {}, [oauth_inst])
oauth_inst['authenticator'] = 'testauth'
with pytest.raises(
Exception,
match='The Authenticator method set is invalid: testauth'):
SnowflakeCheck(CHECK_NAME, {}, [oauth_inst])
def test_no_schema(dd_run_check, aggregator, instance):
config = copy.deepcopy(instance)
del config['schema']
config['login_timeout'] = 5
check = SnowflakeCheck(CHECK_NAME, {}, [config])
dd_run_check(check)
aggregator.assert_service_check('snowflake.can_connect',
SnowflakeCheck.CRITICAL)
def test_emits_critical_service_check_when_service_is_down(
dd_run_check, aggregator, instance):
config = copy.deepcopy(instance)
config['login_timeout'] = 5
check = SnowflakeCheck(CHECK_NAME, {}, [config])
dd_run_check(check)
aggregator.assert_service_check('snowflake.can_connect',
SnowflakeCheck.CRITICAL)
def test_invalid_oauth(oauth_instance):
# Test oauth without user
with pytest.raises(Exception, match='Must specify a user'):
SnowflakeCheck(CHECK_NAME, {}, [INVALID_CONFIG])
# Test oauth without token
no_token_config = copy.deepcopy(INVALID_CONFIG)
no_token_config['user'] = "******"
with pytest.raises(Exception,
match='If using OAuth, you must specify a token'):
SnowflakeCheck(CHECK_NAME, {}, [no_token_config])
oauth_inst = copy.deepcopy(oauth_instance)
oauth_inst['authenticator'] = 'testauth'
with pytest.raises(
Exception,
match='The Authenticator method set is invalid: testauth'):
SnowflakeCheck(CHECK_NAME, {}, [oauth_inst])
def test_version_metadata(dd_run_check, instance, datadog_agent):
expected_version = [('4.30.2', )]
version_metadata = {
'version.major': '4',
'version.minor': '30',
'version.patch': '2',
'version.raw': '4.30.2',
'version.scheme': 'semver',
}
with mock.patch(
'datadog_checks.snowflake.SnowflakeCheck.execute_query_raw',
return_value=expected_version):
check = SnowflakeCheck(CHECK_NAME, {}, [instance])
check.check_id = 'test:123'
check._conn = mock.MagicMock()
check._query_manager.queries = []
dd_run_check(check)
datadog_agent.assert_metadata('test:123', version_metadata)
def test_db_storage_metrics(dd_run_check, aggregator, instance):
# type: (Callable[[SnowflakeCheck], None], AggregatorStub, Dict[str, Any]) -> None
expected_db_storage_usage = [('SNOWFLAKE_DB', Decimal('133.000000'),
Decimal('9.100000'))]
expected_tags = EXPECTED_TAGS + ['database:SNOWFLAKE_DB']
with mock.patch(
'datadog_checks.snowflake.SnowflakeCheck.execute_query_raw',
return_value=expected_db_storage_usage):
check = SnowflakeCheck(CHECK_NAME, {}, [instance])
check._conn = mock.MagicMock()
check._query_manager.queries = [Query(queries.DatabaseStorageMetrics)]
dd_run_check(check)
aggregator.assert_metric('snowflake.storage.database.storage_bytes',
value=133.0,
tags=expected_tags)
aggregator.assert_metric('snowflake.storage.database.failsafe_bytes',
value=9.1,
tags=expected_tags)
def test_proxy_init_config_override(instance):
with mock.patch('datadog_checks.base.stubs.datadog_agent.get_config',
return_value=INVALID_PROXY):
init_config = {'proxy': PROXY_CONFIG}
check = SnowflakeCheck(CHECK_NAME, init_config, [instance])
assert check.http.options['proxies'] == {
'http': 'http_host',
'https': 'https_host'
}
assert check._proxies == {'http': 'http_host', 'https': 'https_host'}
assert check.http.no_proxy_uris == ['uri1', 'uri2', 'uri3', 'uri4']
def test_proxy_agent_config(instance):
with mock.patch('datadog_checks.base.stubs.datadog_agent.get_config',
return_value=PROXY_CONFIG):
check = SnowflakeCheck(CHECK_NAME, {}, [instance])
assert check.http.options['proxies'] == {
'http': 'http_host',
'https': 'https_host'
}
assert check._proxies == {'http': 'http_host', 'https': 'https_host'}
assert check.http.no_proxy_uris == ['uri1', 'uri2', 'uri3', 'uri4']
def test_no_metric_group(instance):
inst = copy.deepcopy(instance)
inst['metric_groups'] = []
with pytest.raises(
Exception,
match=
'No valid metric_groups or custom query configured, please list at least one.'
):
SnowflakeCheck(CHECK_NAME, {}, [inst])
inst['custom_queries'] = [
{
'query': "SELECT a,b from mytable where a='stuff' limit 1;",
'columns': [{}, {
'name': 'metric.b',
'type': 'gauge'
}],
},
]
SnowflakeCheck(CHECK_NAME, {}, [inst])
def test_warehouse_usage_metrics(dd_run_check, aggregator, instance):
# type: (Callable[[SnowflakeCheck], None], AggregatorStub, Dict[str, Any]) -> None
expected_wh_usage = [(
'COMPUTE_WH',
Decimal('13.000000000'),
Decimal('1.000000000'),
Decimal('0.870148056'),
Decimal('0.066934465846'),
Decimal('13.870148056'),
Decimal('1.066934465846'),
)]
expected_tags = EXPECTED_TAGS + ['warehouse:COMPUTE_WH']
with mock.patch(
'datadog_checks.snowflake.SnowflakeCheck.execute_query_raw',
return_value=expected_wh_usage):
check = SnowflakeCheck(CHECK_NAME, {}, [instance])
check._conn = mock.MagicMock()
check._query_manager.queries = [Query(queries.WarehouseCreditUsage)]
dd_run_check(check)
aggregator.assert_metric('snowflake.billing.warehouse.cloud_service.avg',
count=1,
tags=expected_tags)
aggregator.assert_metric('snowflake.billing.warehouse.total_credit.avg',
count=1,
tags=expected_tags)
aggregator.assert_metric(
'snowflake.billing.warehouse.virtual_warehouse.avg',
count=1,
tags=expected_tags)
aggregator.assert_metric('snowflake.billing.warehouse.cloud_service.sum',
count=1,
tags=expected_tags)
aggregator.assert_metric('snowflake.billing.warehouse.total_credit.sum',
count=1,
tags=expected_tags)
aggregator.assert_metric(
'snowflake.billing.warehouse.virtual_warehouse.sum',
count=1,
tags=expected_tags)
def test_additional_metric_groups(instance):
instance = copy.deepcopy(instance)
instance['metric_groups'] = ['snowflake.logins', 'snowflake.data_transfer']
check = SnowflakeCheck(CHECK_NAME, {}, [instance])
assert check.config.metric_groups == [
'snowflake.logins', 'snowflake.data_transfer'
]
assert check.metric_queries == [
queries.LoginMetrics,
queries.DataTransferHistory,
]
def test_default_auth(instance):
check = SnowflakeCheck(CHECK_NAME, {}, [instance])
check._conn = mock.MagicMock()
check._query_manager = mock.MagicMock()
with mock.patch('datadog_checks.snowflake.check.sf') as sf:
check.check(instance)
sf.connect.assert_called_with(
user='******',
password='******',
account='test_acct.us-central1.gcp',
database='SNOWFLAKE',
schema='ACCOUNT_USAGE',
warehouse=None,
role='ACCOUNTADMIN',
passcode_in_password=False,
passcode=None,
client_prefetch_threads=4,
login_timeout=60,
ocsp_response_cache_filename=None,
authenticator='snowflake',
token=None,
client_session_keep_alive=False,
proxy_host=None,
proxy_port=None,
proxy_user=None,
proxy_password=None,
)
def test_oauth_auth(instance):
# Test oauth
oauth_inst = copy.deepcopy(instance)
oauth_inst['authenticator'] = 'oauth'
oauth_inst['token'] = 'testtoken'
with mock.patch('datadog_checks.snowflake.check.sf') as sf:
check = SnowflakeCheck(CHECK_NAME, {}, [oauth_inst])
check._conn = mock.MagicMock()
check._query_manager = mock.MagicMock()
check.check(oauth_inst)
sf.connect.assert_called_with(
user='******',
password='******',
account='test_acct.us-central1.gcp',
database='SNOWFLAKE',
schema='ACCOUNT_USAGE',
warehouse=None,
role='ACCOUNTADMIN',
passcode_in_password=False,
passcode=None,
client_prefetch_threads=4,
login_timeout=60,
ocsp_response_cache_filename=None,
authenticator='oauth',
token='testtoken',
client_session_keep_alive=False,
)
def test_storage_metrics(dd_run_check, aggregator, instance):
# type: (Callable[[SnowflakeCheck], None], AggregatorStub, Dict[str, Any]) -> None
expected_storage = [(Decimal('0.000000'), Decimal('1206.000000'),
Decimal('19.200000'))]
with mock.patch(
'datadog_checks.snowflake.SnowflakeCheck.execute_query_raw',
return_value=expected_storage):
check = SnowflakeCheck(CHECK_NAME, {}, [instance])
check._conn = mock.MagicMock()
check._query_manager.queries = [Query(queries.StorageUsageMetrics)]
dd_run_check(check)
aggregator.assert_metric('snowflake.storage.storage_bytes.total',
value=0.0,
tags=EXPECTED_TAGS)
aggregator.assert_metric('snowflake.storage.stage_bytes.total',
value=1206.0,
tags=EXPECTED_TAGS)
aggregator.assert_metric('snowflake.storage.failsafe_bytes.total',
value=19.2,
tags=EXPECTED_TAGS)
def test_aggregate_last_24_hours_queries(aggregate_last_24_hours,
expected_query):
inst = {
'metric_groups': ['snowflake.replication'],
'username': '******',
'password': '******',
'account': 'account',
'role': 'role',
}
inst['aggregate_last_24_hours'] = aggregate_last_24_hours
check = SnowflakeCheck(CHECK_NAME, {}, [inst])
# Only one query configured
assert check.metric_queries[0]['query'] == expected_query
def test_credit_usage_metrics(dd_run_check, aggregator, instance):
# type: (AggregatorStub, Dict[str, Any]) -> None
expected_credit_usage = [(
'WAREHOUSE_METERING',
'COMPUTE_WH',
Decimal('12.000000000'),
Decimal('1.000000000'),
Decimal('0.80397000'),
Decimal('0.066997500000'),
Decimal('12.803970000'),
Decimal('1.066997500000'),
)]
expected_tags = EXPECTED_TAGS + [
'service_type:WAREHOUSE_METERING', 'service:COMPUTE_WH'
]
with mock.patch(
'datadog_checks.snowflake.SnowflakeCheck.execute_query_raw',
return_value=expected_credit_usage):
check = SnowflakeCheck(CHECK_NAME, {}, [instance])
check._conn = mock.MagicMock()
check._query_manager.queries = [queries.CreditUsage]
dd_run_check(check)
aggregator.assert_metric('snowflake.billing.cloud_service.sum',
count=1,
tags=expected_tags)
aggregator.assert_metric('snowflake.billing.cloud_service.avg',
count=1,
tags=expected_tags)
aggregator.assert_metric('snowflake.billing.total_credit.sum', count=1)
aggregator.assert_metric('snowflake.billing.total_credit.avg', count=1)
aggregator.assert_metric('snowflake.billing.virtual_warehouse.sum',
count=1)
aggregator.assert_metric('snowflake.billing.virtual_warehouse.avg',
count=1)
def test_token_path(dd_run_check, aggregator):
instance = {
'username': '******',
'account': 'account',
'role': 'ACCOUNTADMIN',
'authenticator': 'oauth',
'token_path': '/path/to/token',
}
default_args = {
'user': '******',
'password': None,
'account': 'account',
'database': 'SNOWFLAKE',
'schema': 'ACCOUNT_USAGE',
'warehouse': None,
'role': 'ACCOUNTADMIN',
'passcode_in_password': False,
'passcode': None,
'client_prefetch_threads': 4,
'login_timeout': 60,
'ocsp_response_cache_filename': None,
'authenticator': 'oauth',
'client_session_keep_alive': False,
'private_key': None,
'proxy_host': None,
'proxy_port': None,
'proxy_user': None,
'proxy_password': None,
}
tokens = ['mytoken1', 'mytoken2', 'mytoken3']
check = SnowflakeCheck(CHECK_NAME, {}, [instance])
with mock.patch(
'datadog_checks.snowflake.check.open',
side_effect=[
mock.mock_open(read_data=tok).return_value for tok in tokens
],
), mock.patch('datadog_checks.snowflake.check.sf') as sf:
dd_run_check(check)
sf.connect.assert_called_once_with(token='mytoken1', **default_args)
dd_run_check(check)
sf.connect.assert_called_with(token='mytoken2', **default_args)
dd_run_check(check)
sf.connect.assert_called_with(token='mytoken3', **default_args)
def test_default_metric_groups(instance):
check = SnowflakeCheck(CHECK_NAME, {}, [instance])
assert check.config.metric_groups == [
'snowflake.query',
'snowflake.billing',
'snowflake.storage',
'snowflake.logins',
]
assert check.metric_queries == [
queries.WarehouseLoad,
queries.QueryHistory,
queries.CreditUsage,
queries.WarehouseCreditUsage,
queries.StorageUsageMetrics,
queries.LoginMetrics,
]
def test_key_auth(dd_run_check, instance):
# Key auth
inst = copy.deepcopy(instance)
inst['private_key_path'] = os.path.join(os.path.dirname(__file__), 'keys',
'rsa_key_example.p8')
check = SnowflakeCheck(CHECK_NAME, {}, [inst])
# Checking size instead of the read key
read_key = check.read_key()
assert len(read_key) == 1216
with mock.patch('datadog_checks.snowflake.check.sf') as sf:
check = SnowflakeCheck(CHECK_NAME, {}, [inst])
dd_run_check(check)
sf.connect.assert_called_with(
user='******',
password='******',
account='test_acct.us-central1.gcp',
database='SNOWFLAKE',
schema='ACCOUNT_USAGE',
warehouse=None,
role='ACCOUNTADMIN',
passcode_in_password=False,
passcode=None,
client_prefetch_threads=4,
login_timeout=3,
ocsp_response_cache_filename=None,
authenticator='snowflake',
token=None,
private_key=read_key,
client_session_keep_alive=False,
proxy_host=None,
proxy_port=None,
proxy_user=None,
proxy_password=None,
)
inst['private_key_path'] = os.path.join(os.path.dirname(__file__), 'keys',
'wrong_key.p8')
check = SnowflakeCheck(CHECK_NAME, {}, [inst])
with pytest.raises(FileNotFoundError):
check.read_key()
# Read key protected by a passphrase
inst['private_key_path'] = os.path.join(os.path.dirname(__file__), 'keys',
'rsa_key_pass_example.p8')
inst['private_key_password'] = '******'
check = SnowflakeCheck(CHECK_NAME, {}, [inst])
assert len(check.read_key()) == 1218
def test_default_authentication(instance):
# Test default auth
check = SnowflakeCheck(CHECK_NAME, {}, [instance])
assert check.config.authenticator == 'snowflake'
def test_mixed_metric_group(instance):
instance = copy.deepcopy(instance)
instance['metric_groups'] = ['fake.metric.group', 'snowflake.logins']
check = SnowflakeCheck(CHECK_NAME, {}, [instance])
assert check.metric_queries == [queries.LoginMetrics]
