def test_dataset_dest_and_table_dest_are_required_together_eventually(
self, attr):
task = BigQueryStreamingInsert(**{attr: "some-value"})
with pytest.raises(ValueError) as exc:
task.run(records=[])
assert attr in str(exc.value)
assert "must be provided" in str(exc.value)
def test_creds_are_pulled_from_secret_at_runtime(self, monkeypatch):
task = BigQueryStreamingInsert(
dataset_id="id",
table="table",
credentials_secret="GOOGLE_APPLICATION_CREDENTIALS",
)
client_util = MagicMock()
monkeypatch.setattr("prefect.tasks.gcp.bigquery.get_bigquery_client",
client_util)
with prefect.context(secrets=dict(GOOGLE_APPLICATION_CREDENTIALS=42)):
task.run(records=[])
assert client_util.call_args[1]["credentials"] == 42
def test_project_is_pulled_from_creds_and_can_be_overriden_at_anytime(
self, monkeypatch):
task = BigQueryStreamingInsert(
dataset_id="id",
table="table",
credentials_secret="GOOGLE_APPLICATION_CREDENTIALS",
)
task_proj = BigQueryStreamingInsert(
dataset_id="id",
table="table",
project="test-init",
credentials_secret="GOOGLE_APPLICATION_CREDENTIALS",
)
client_util = MagicMock()
monkeypatch.setattr("prefect.tasks.gcp.bigquery.get_bigquery_client",
client_util)
with prefect.context(secrets=dict(GOOGLE_APPLICATION_CREDENTIALS={})):
task.run(records=[])
task_proj.run(records=[])
task_proj.run(records=[], project="run-time")
x, y, z = client_util.call_args_list
assert x[1]["project"] is None ## will be pulled from credentials
assert y[1]["project"] == "test-init" ## pulled from init
assert z[1]["project"] == "run-time" ## pulled from run kwarg