def test_change_schema():
conf = BQTestKitConfig({DEFAULT_LOCATION: "EU"})
project = Project("test_project", bq_client=None, bqtk_config=conf)
ds = Dataset("dataset_foo",
project=project,
bq_client=None,
bqtk_config=conf)
table = Table("table_bar",
from_dataset=ds,
bq_client=None,
bqtk_config=conf)
table = table.with_schema(from_="""
[
{
"description": "field 1",
"mode": "REQUIRED",
"name": "f1",
"type": "STRING"
}
]""")
assert table.schema == [
SchemaField("f1",
field_type="STRING",
mode="REQUIRED",
description="field 1")
]
def test_full_constructor():
conf = BQTestKitConfig({DEFAULT_LOCATION: "EU"})
project = Project("test_project", bq_client=None, bqtk_config=conf)
ds = Dataset("dataset_foo",
project=project,
bq_client=None,
bqtk_config=conf)
schema = [SchemaField("f1", field_type="INT64")]
table = Table("table_bar",
from_dataset=ds,
alias="bar",
bq_client=None,
bqtk_config=conf,
resource_strategy=Noop(),
isolate_with=lambda x: x.name + "bar",
partition_type=IngestionTime(),
schema=schema,
exists_ok=True)
assert table.dataset.project.name == "test_project"
assert table.dataset.name == "dataset_foo"
assert table.name == "table_bar"
assert isinstance(table.resource_strategy, Noop)
assert table.alias == "bar"
assert table.isolate_func is not None
assert table.isolate_func(table) == "table_barbar"
assert len(table.create_options) == 1
assert isinstance(table.partition_type, IngestionTime)
assert table.schema == schema
def test_change_clean_and_keep():
conf = BQTestKitConfig({DEFAULT_LOCATION: "EU"})
project = Project("test_project", bq_client=None, bqtk_config=conf)
ds = Dataset("dataset_foo",
project=project,
bq_client=None,
bqtk_config=conf)
table = Table("table_bar",
from_dataset=ds,
bq_client=None,
bqtk_config=conf)
table = table.clean_and_keep()
assert isinstance(table.resource_strategy, CleanBeforeAndKeepAfter)
def test_invalid_json_schema():
conf = BQTestKitConfig({DEFAULT_LOCATION: "EU"})
project = Project("test_project", bq_client=None, bqtk_config=conf)
ds = Dataset("dataset_foo",
project=project,
bq_client=None,
bqtk_config=conf)
table = Table("table_bar",
from_dataset=ds,
bq_client=None,
bqtk_config=conf)
with pytest.raises(JSONDecodeError):
table.with_schema(from_="this is not a json")
def test_invalid_change_partition_by():
conf = BQTestKitConfig({DEFAULT_LOCATION: "EU"})
project = Project("test_project", bq_client=None, bqtk_config=conf)
ds = Dataset("dataset_foo",
project=project,
bq_client=None,
bqtk_config=conf)
table = Table("table_bar",
from_dataset=ds,
bq_client=None,
bqtk_config=conf)
with pytest.raises(Exception):
table.partition_by("zoo")
def test_json_loader():
conf = BQTestKitConfig({DEFAULT_LOCATION: "EU"})
project = Project("test_project", bq_client=None, bqtk_config=conf)
ds = Dataset("dataset_foo",
project=project,
bq_client=None,
bqtk_config=conf)
table = Table("table_bar",
from_dataset=ds,
bq_client=None,
bqtk_config=conf)
pfl = PackageFileLoader(
"tests/ut/bq_test_kit/resource_loaders/resources/package_file_test_resource.txt"
)
assert isinstance(table.json_loader(from_=pfl), JsonDataLoader)
def test_change_alias():
conf = BQTestKitConfig({DEFAULT_LOCATION: "EU"})
project = Project("test_project", bq_client=None, bqtk_config=conf)
ds = Dataset("dataset_foo",
project=project,
bq_client=None,
bqtk_config=conf)
table = Table("table_bar",
from_dataset=ds,
bq_client=None,
bqtk_config=conf)
table = table.with_alias("bar")
assert table.alias == "bar"
table = table.with_alias(None)
assert table.alias is None
def test_global_dict():
pfl = PackageFileLoader("tests/ut/bq_test_kit/bq_dsl/resources/dummy_query.sql")
conf = BQTestKitConfig({DEFAULT_LOCATION: "EU"})
bq_tpl = BQQueryTemplate(from_=pfl, bqtk_config=conf, bq_client=None, interpolators=[DummyInterpolator()])
project = Project("test_project", bq_client=None, bqtk_config=conf)
ds = Dataset("dataset_foo", project=project, bq_client=None,
bqtk_config=conf)
table = Table("table_bar", from_dataset=ds, bq_client=None, bqtk_config=conf)
table_with_alias = Table("table_bar", from_dataset=ds, bq_client=None, bqtk_config=conf).with_alias("table_foobar")
bq_tpl = bq_tpl.with_global_dict({"test_project": "unknown"})
assert bq_tpl.global_dict == {"test_project": "unknown"}
bq_tpl = bq_tpl.update_global_dict([table, ds, project, table_with_alias])
assert bq_tpl.global_dict == {
"test_project": project.fqdn(),
"dataset_foo": ds.fqdn(),
"dataset_foo_table_bar": table.fqdn(),
"table_foobar": table_with_alias.fqdn()
}
bq_tpl = bq_tpl.update_global_dict({"dataset_foo_table_bar": "my_override",
"new_complex_key": {"new_key": "new_value"}})
assert bq_tpl.global_dict == {
"test_project": project.fqdn(),
"dataset_foo": ds.fqdn(),
"dataset_foo_table_bar": "my_override",
"table_foobar": table_with_alias.fqdn(),
"new_complex_key": {"new_key": "new_value"}
}
def test_change_partition_by():
conf = BQTestKitConfig({DEFAULT_LOCATION: "EU"})
project = Project("test_project", bq_client=None, bqtk_config=conf)
ds = Dataset("dataset_foo",
project=project,
bq_client=None,
bqtk_config=conf)
table = Table("table_bar",
from_dataset=ds,
bq_client=None,
bqtk_config=conf)
table = table.partition_by(
Range(on_field="zoo", start=0, end=10000, interval=10))
assert isinstance(table.partition_type, Range)
assert table.partition_type.field == "zoo"
assert table.partition_type.start == 0
assert table.partition_type.end == 10000
assert table.partition_type.interval == 10
def test_change_create_options():
conf = BQTestKitConfig({
DEFAULT_LOCATION: "EU"
}).with_test_context("context")
project = Project("test_project", bq_client=None, bqtk_config=conf)
ds = Dataset("dataset_foo",
project=project,
bq_client=None,
bqtk_config=conf)
table = Table("table_bar",
from_dataset=ds,
bq_client=None,
bqtk_config=conf)
create_options = {"exists_ok": True, "timeout": 10}
table = table.with_create_options(**create_options)
assert table.create_options == create_options
table = table.with_create_options()
assert table.create_options == {}
def test_change_isolate():
conf = BQTestKitConfig({
DEFAULT_LOCATION: "EU"
}).with_test_context("context")
project = Project("test_project", bq_client=None, bqtk_config=conf)
ds = Dataset("dataset_foo",
project=project,
bq_client=None,
bqtk_config=conf)
table = Table("table_bar",
from_dataset=ds,
bq_client=None,
bqtk_config=conf)
table = table.isolate()
assert table.isolate_func(table) == f"{table.name}_context"
assert table.fqdn() == (f"{table.dataset.project.name}."
f"{table.dataset.name}."
f"{table.name}_context")
def test_default_constructor():
conf = BQTestKitConfig({DEFAULT_LOCATION: "EU"})
project = Project("test_project", bq_client=None, bqtk_config=conf)
ds = Dataset("dataset_foo",
project=project,
bq_client=None,
bqtk_config=conf)
table = Table("table_bar",
from_dataset=ds,
bq_client=None,
bqtk_config=conf)
assert table.dataset.project.name == "test_project"
assert table.dataset.name == "dataset_foo"
assert table.name == "table_bar"
assert isinstance(table.resource_strategy, CleanAfter)
assert table.alias is None
assert table.isolate_func(table) == "table_bar"
assert table.create_options == {}
assert table.schema == []
assert isinstance(table.partition_type, NoPartition)
def test_change_destination():
pfl = PackageFileLoader("tests/ut/bq_test_kit/bq_dsl/resources/dummy_query.sql")
conf = BQTestKitConfig({DEFAULT_LOCATION: "EU"})
bq_tpl = BQQueryTemplate(from_=pfl, bqtk_config=conf, bq_client=None)
project = Project("test_project", bq_client=None, bqtk_config=conf)
ds = Dataset("dataset_foo", project=project, bq_client=None,
bqtk_config=conf)
table = Table("table_bar", from_dataset=ds, bq_client=None, bqtk_config=conf)
assert bq_tpl.job_config.destination is None
bq_tpl = bq_tpl.with_destination(table)
assert bq_tpl.job_config.destination.table_id == "table_bar"
def test_invalid_change_schema():
conf = BQTestKitConfig({DEFAULT_LOCATION: "EU"})
project = Project("test_project", bq_client=None, bqtk_config=conf)
ds = Dataset("dataset_foo",
project=project,
bq_client=None,
bqtk_config=conf)
table = Table("table_bar",
from_dataset=ds,
bq_client=None,
bqtk_config=conf)
with pytest.raises(InvalidInstanceException):
table.with_schema(from_=[1, 2, 3])
with pytest.raises(InvalidInstanceException):
table.with_schema(from_=1)