def test_handles_invalid_slack_alerts_and_skips_if_condition_is_false(
self,
mock_execution,
get_connection,
mock_slack_client,
):
mock_execution.return_value = [(5, )]
loader = whale_loader.WhaleLoader()
loader.init(self.loader_config)
loader.load(self.record)
sql_query = f"select count(distinct id) from {self.schema}.{self.table} where id is null"
alert_message = f"Nulls found in {self.schema}.{self.table} column id."
slack_alerts = [
SlackAlert(
condition=">0",
message=alert_message,
channels=["#data-monitoring", "@bob"],
),
SlackAlert(
condition="not a valid condition",
message=alert_message,
channels=["@jack"],
),
SlackAlert(
condition=">2",
message=alert_message,
channels=[],
),
]
self.add_metrics_and_alerts_to_markdown("null-ids", sql_query,
slack_alerts)
mock_slack_client.return_value = {"message": {"text": alert_message}}
extractor = MetricRunner()
extractor.init(self.extractor_config)
results = extractor.extract()
calls = [
call(channel="#data-monitoring", text=alert_message),
call(channel="@bob", text=alert_message),
]
mock_slack_client.assert_has_calls(calls, any_order=True)
def add_metrics(extractors: list, conf: ConfigTree, connection):
conf.put(f"{METRIC_RUNNER_SCOPE}.{MetricRunner.DATABASE_KEY}", connection.name)
conf.put(
f"{METRIC_RUNNER_SCOPE}.{SQL_ALCHEMY_ENGINE_SCOPE}.{SQLAlchemyEngine.CONN_STRING_KEY}",
connection.conn_string,
)
conf.put(
f"{METRIC_RUNNER_SCOPE}.{SQL_ALCHEMY_ENGINE_SCOPE}.{SQLAlchemyEngine.CREDENTIALS_PATH_KEY}",
connection.key_path,
)
extractors.append(MetricRunner())
return extractors, conf
def test_sends_sql_query_to_sql_alchemy(self, mock_execution,
get_connection,
mock_slack_client) -> None:
mock_execution.return_value = [(0, )]
mock_slack_client.return_value = {"message": {"text": "Placeholder"}}
loader = whale_loader.WhaleLoader()
loader.init(self.loader_config)
loader.load(self.record)
sql_query = f"select count(distinct id) from {self.schema}.{self.table} where id is null"
slack_alerts = [
SlackAlert(
condition=">0",
message=f"Nulls found in {self.schema}.{self.table} column id.",
channels=["#data-monitoring", "@bob"],
),
]
self.add_metrics_and_alerts_to_markdown("null-ids", sql_query,
slack_alerts)
extractor = MetricRunner()
extractor.init(self.extractor_config)
extractor.extract()
mock_execution.assert_called_once_with(f" | {sql_query}",
is_dict_return_enabled=False)
from whale.extractor.glue_extractor import GlueExtractor
from whale.extractor.snowflake_metadata_extractor import SnowflakeMetadataExtractor
from whale.extractor.metric_runner import MetricRunner
from whale.engine.sql_alchemy_engine import SQLAlchemyEngine
from databuilder.extractor.sql_alchemy_extractor import SQLAlchemyExtractor
from databuilder.extractor.hive_table_metadata_extractor import (
HiveTableMetadataExtractor, )
from databuilder.extractor.postgres_metadata_extractor import PostgresMetadataExtractor
from databuilder.extractor.redshift_metadata_extractor import RedshiftMetadataExtractor
BUILD_SCRIPT_TEMPLATE = """source {venv_path}/bin/activate \
&& {python_binary} {build_script_path}"""
SQL_ALCHEMY_SCOPE = SQLAlchemyExtractor().get_scope()
SQL_ALCHEMY_ENGINE_SCOPE = SQLAlchemyEngine().get_scope()
METRIC_RUNNER_SCOPE = MetricRunner().get_scope()
def get_sql_alchemy_conn_string_key(scope):
conn_string_key = f"{scope}.{SQL_ALCHEMY_SCOPE}.{SQLAlchemyExtractor.CONN_STRING}"
return conn_string_key
def add_metrics(extractors: list, conf: ConfigTree, connection):
conf.put(f"{METRIC_RUNNER_SCOPE}.{MetricRunner.DATABASE_KEY}",
connection.name)
conf.put(
f"{METRIC_RUNNER_SCOPE}.{SQL_ALCHEMY_ENGINE_SCOPE}.{SQLAlchemyEngine.CONN_STRING_KEY}",
connection.conn_string,
)
conf.put(