def test_sends_sql_query_to_sql_alchemy(self, mock_execution,
get_connection,
mock_slack_client) -> None:
mock_execution.return_value = [(0, )]
mock_slack_client.return_value = {"message": {"text": "Placeholder"}}
loader = whale_loader.WhaleLoader()
loader.init(self.loader_config)
loader.load(self.record)
sql_query = f"select count(distinct id) from {self.schema}.{self.table} where id is null"
slack_alerts = [
SlackAlert(
condition=">0",
message=f"Nulls found in {self.schema}.{self.table} column id.",
channels=["#data-monitoring", "@bob"],
),
]
self.add_metrics_and_alerts_to_markdown("null-ids", sql_query,
slack_alerts)
extractor = UGCRunner()
extractor.init(self.extractor_config)
extractor.extract()
mock_execution.assert_called_once_with(f" | {sql_query}",
is_dict_return_enabled=False)
def test_load_index_metadata(patched_config):
index_metadata = IndexMetadata(
name="mock_index",
columns=["mock_column_1", "mock_column_2"],
)
record = TableIndexesMetadata(
database="mock_database",
cluster="mock_catalog",
schema="mock_schema",
table="mock_table",
indexes=[index_metadata],
)
loader = whale_loader.WhaleLoader()
loader.init(patched_config)
loader.load(record)
loader.close()
file_path = os.path.join(
patched_config.get("base_directory"),
"mock_database/mock_catalog.mock_schema.mock_table.md",
)
with open(file_path, "r") as f:
written_record = f.read()
assert "mock_schema" in written_record
assert "mock_table" in written_record
assert "mock_catalog" in written_record
assert "mock_database" in written_record
assert "mock_index" in written_record
def test_handles_invalid_slack_alerts_and_skips_if_condition_is_false(
self,
mock_execution,
get_connection,
mock_slack_client,
):
mock_execution.return_value = [(5, )]
loader = whale_loader.WhaleLoader()
loader.init(self.loader_config)
loader.load(self.record)
sql_query = f"select count(distinct id) from {self.schema}.{self.table} where id is null"
alert_message = f"Nulls found in {self.schema}.{self.table} column id."
slack_alerts = [
SlackAlert(
condition=">0",
message=alert_message,
channels=["#data-monitoring", "@bob"],
),
SlackAlert(
condition="not a valid condition",
message=alert_message,
channels=["@jack"],
),
SlackAlert(
condition=">2",
message=alert_message,
channels=[],
),
]
self.add_metrics_and_alerts_to_markdown("null-ids", sql_query,
slack_alerts)
mock_slack_client.return_value = {"message": {"text": alert_message}}
extractor = UGCRunner()
extractor.init(self.extractor_config)
results = extractor.extract()
calls = [
call(channel="#data-monitoring", text=alert_message),
call(channel="@bob", text=alert_message),
]
mock_slack_client.assert_has_calls(calls, any_order=True)
def test_load_no_catalog(patched_config):
record = TableMetadata(
database='mock_database',
cluster=None,
schema='mock_schema',
name='mock_table',
markdown_blob='Test',
)
loader = whale_loader.WhaleLoader()
loader.init(patched_config)
loader.load(record)
loader.close()
file_path = os.path.join(patched_config.get("base_directory"),
'mock_database/mock_schema.mock_table.md')
with open(file_path, 'r') as f:
written_record = f.read()
assert record.markdown_blob in written_record
def test_load_no_catalog(patched_config):
record = TableMetadata(
database="mock_database",
cluster=None,
schema="mock_schema",
name="mock_table",
)
loader = whale_loader.WhaleLoader()
loader.init(patched_config)
loader.load(record)
loader.close()
file_path = os.path.join(patched_config.get("base_directory"),
"mock_database/mock_schema.mock_table.md")
with open(file_path, "r") as f:
written_record = f.read()
assert "mock_schema" in written_record
assert "mock_table" in written_record
assert "mock_database" in written_record
def test_load_catalog_specified(patched_config):
record = TableMetadata(
database="mock_database",
cluster="mock_catalog",
schema="mock_schema",
name="mock_table",
markdown_blob="Test",
)
loader = whale_loader.WhaleLoader()
loader.init(patched_config)
loader.load(record)
loader.close()
file_path = os.path.join(
patched_config.get("base_directory"),
"mock_database/mock_catalog.mock_schema.mock_table.md",
)
with open(file_path, "r") as f:
written_record = f.read()
assert record.markdown_blob in written_record