def test_table_dsl():
conf = BQTestKitConfig({DEFAULT_LOCATION: "EU"})
project = Project("test_project", bq_client=None, bqtk_config=conf)
ds = Dataset("dataset_foo",
project=project,
bq_client=None,
bqtk_config=conf)
project.datasets = [ds]
t = ds.table("table_foo")
assert isinstance(t, Table)
assert t.dataset.table("table_foo") == t
assert t.dataset.name == "dataset_foo"
assert t.dataset.project.name == "test_project"
assert t.dataset.project == t.project
assert t.dataset == t.dataset.project.dataset("dataset_foo")
t = t.dataset.table("table_bar")
assert len(t.dataset.tables) == 2
assert t.dataset.tables[0].dataset == t.dataset.tables[1].dataset
assert t.dataset.tables[0].name == "table_foo"
assert t.dataset.tables[1].name == "table_bar"
assert t.dataset.name == "dataset_foo"
assert t.dataset.project.name == "test_project"
assert t.dataset.project == t.project
t = t.table("table_foobar")
assert len(t.dataset.tables) == 3
assert t.dataset.tables[0].dataset == t.dataset.tables[
1].dataset == t.dataset.tables[2].dataset
assert t.dataset.tables[0].name == "table_foo"
assert t.dataset.tables[1].name == "table_bar"
assert t.dataset.tables[2].name == "table_foobar"
assert t.dataset.name == "dataset_foo"
assert t.dataset.project.name == "test_project"
assert t.dataset.project == t.project
def test_global_dict():
pfl = PackageFileLoader("tests/ut/bq_test_kit/bq_dsl/resources/dummy_query.sql")
conf = BQTestKitConfig({DEFAULT_LOCATION: "EU"})
bq_tpl = BQQueryTemplate(from_=pfl, bqtk_config=conf, bq_client=None, interpolators=[DummyInterpolator()])
project = Project("test_project", bq_client=None, bqtk_config=conf)
ds = Dataset("dataset_foo", project=project, bq_client=None,
bqtk_config=conf)
table = Table("table_bar", from_dataset=ds, bq_client=None, bqtk_config=conf)
table_with_alias = Table("table_bar", from_dataset=ds, bq_client=None, bqtk_config=conf).with_alias("table_foobar")
bq_tpl = bq_tpl.with_global_dict({"test_project": "unknown"})
assert bq_tpl.global_dict == {"test_project": "unknown"}
bq_tpl = bq_tpl.update_global_dict([table, ds, project, table_with_alias])
assert bq_tpl.global_dict == {
"test_project": project.fqdn(),
"dataset_foo": ds.fqdn(),
"dataset_foo_table_bar": table.fqdn(),
"table_foobar": table_with_alias.fqdn()
}
bq_tpl = bq_tpl.update_global_dict({"dataset_foo_table_bar": "my_override",
"new_complex_key": {"new_key": "new_value"}})
assert bq_tpl.global_dict == {
"test_project": project.fqdn(),
"dataset_foo": ds.fqdn(),
"dataset_foo_table_bar": "my_override",
"table_foobar": table_with_alias.fqdn(),
"new_complex_key": {"new_key": "new_value"}
}
def test_change_noop():
conf = BQTestKitConfig({DEFAULT_LOCATION: "EU"})
project = Project("test_project", bq_client=None, bqtk_config=conf)
ds = Dataset("dataset_foo",
project=project,
bq_client=None,
bqtk_config=conf)
ds = ds.noop()
assert ds.project.name == "test_project"
assert ds.name == "dataset_foo"
assert ds.isolate_func(ds) == "dataset_foo"
assert ds.location == "EU"
assert ds.create_options == {}
assert isinstance(ds.resource_strategy, Noop)
def test_default_constructor():
conf = BQTestKitConfig({DEFAULT_LOCATION: "EU"})
project = Project("test_project", bq_client=None, bqtk_config=conf)
ds = Dataset("dataset_foo",
project=project,
bq_client=None,
bqtk_config=conf)
assert ds.project.name == "test_project"
assert ds.name == "dataset_foo"
assert isinstance(ds.resource_strategy, CleanAfter)
assert ds.isolate_func(ds) == "dataset_foo"
assert ds.location == "EU"
assert ds.create_options == {}
assert ds.fqdn() == f"{ds.project.name}.{ds.name}"
def test_change_schema():
conf = BQTestKitConfig({DEFAULT_LOCATION: "EU"})
project = Project("test_project", bq_client=None, bqtk_config=conf)
ds = Dataset("dataset_foo",
project=project,
bq_client=None,
bqtk_config=conf)
table = Table("table_bar",
from_dataset=ds,
bq_client=None,
bqtk_config=conf)
table = table.with_schema(from_="""
[
{
"description": "field 1",
"mode": "REQUIRED",
"name": "f1",
"type": "STRING"
}
]""")
assert table.schema == [
SchemaField("f1",
field_type="STRING",
mode="REQUIRED",
description="field 1")
]
def test_full_constructor():
conf = BQTestKitConfig({DEFAULT_LOCATION: "EU"})
project = Project("test_project", bq_client=None, bqtk_config=conf)
ds = Dataset("dataset_foo",
project=project,
bq_client=None,
bqtk_config=conf)
schema = [SchemaField("f1", field_type="INT64")]
table = Table("table_bar",
from_dataset=ds,
alias="bar",
bq_client=None,
bqtk_config=conf,
resource_strategy=Noop(),
isolate_with=lambda x: x.name + "bar",
partition_type=IngestionTime(),
schema=schema,
exists_ok=True)
assert table.dataset.project.name == "test_project"
assert table.dataset.name == "dataset_foo"
assert table.name == "table_bar"
assert isinstance(table.resource_strategy, Noop)
assert table.alias == "bar"
assert table.isolate_func is not None
assert table.isolate_func(table) == "table_barbar"
assert len(table.create_options) == 1
assert isinstance(table.partition_type, IngestionTime)
assert table.schema == schema
def test_full_constructor():
conf = BQTestKitConfig({DEFAULT_LOCATION: "EU"})
project = Project("test_project", bq_client=None, bqtk_config=conf)
ds = Dataset("dataset_foo",
project=project,
bq_client=None,
bqtk_config=conf,
resource_strategy=Noop(),
isolate_with=lambda x: x.name + 'bar',
location="US",
exists_ok=True)
assert ds.project.name == "test_project"
assert ds.name == "dataset_foo"
assert isinstance(ds.resource_strategy, Noop)
assert ds.isolate_func is not None
assert ds.isolate_func(ds) == ds.name + 'bar'
assert ds.location == "US"
assert len(ds.create_options) == 1
def test_change_create_options():
conf = BQTestKitConfig({
DEFAULT_LOCATION: "EU"
}).with_test_context("context")
project = Project("test_project", bq_client=None, bqtk_config=conf)
ds = Dataset("dataset_foo",
project=project,
bq_client=None,
bqtk_config=conf)
create_options = {"exists_ok": True, "timeout": 10}
ds = ds.with_create_options(**create_options)
assert ds.project.name == "test_project"
assert ds.name == "dataset_foo"
assert isinstance(ds.resource_strategy, CleanAfter)
assert ds.isolate_func(ds) == "dataset_foo"
assert ds.location == "EU"
assert ds.create_options == create_options
ds = ds.with_create_options()
assert ds.create_options == {}
def test_change_destination():
pfl = PackageFileLoader("tests/ut/bq_test_kit/bq_dsl/resources/dummy_query.sql")
conf = BQTestKitConfig({DEFAULT_LOCATION: "EU"})
bq_tpl = BQQueryTemplate(from_=pfl, bqtk_config=conf, bq_client=None)
project = Project("test_project", bq_client=None, bqtk_config=conf)
ds = Dataset("dataset_foo", project=project, bq_client=None,
bqtk_config=conf)
table = Table("table_bar", from_dataset=ds, bq_client=None, bqtk_config=conf)
assert bq_tpl.job_config.destination is None
bq_tpl = bq_tpl.with_destination(table)
assert bq_tpl.job_config.destination.table_id == "table_bar"
def test_change_clean_and_keep():
conf = BQTestKitConfig({DEFAULT_LOCATION: "EU"})
project = Project("test_project", bq_client=None, bqtk_config=conf)
ds = Dataset("dataset_foo",
project=project,
bq_client=None,
bqtk_config=conf)
table = Table("table_bar",
from_dataset=ds,
bq_client=None,
bqtk_config=conf)
table = table.clean_and_keep()
assert isinstance(table.resource_strategy, CleanBeforeAndKeepAfter)
def test_invalid_json_schema():
conf = BQTestKitConfig({DEFAULT_LOCATION: "EU"})
project = Project("test_project", bq_client=None, bqtk_config=conf)
ds = Dataset("dataset_foo",
project=project,
bq_client=None,
bqtk_config=conf)
table = Table("table_bar",
from_dataset=ds,
bq_client=None,
bqtk_config=conf)
with pytest.raises(JSONDecodeError):
table.with_schema(from_="this is not a json")
def test_invalid_change_partition_by():
conf = BQTestKitConfig({DEFAULT_LOCATION: "EU"})
project = Project("test_project", bq_client=None, bqtk_config=conf)
ds = Dataset("dataset_foo",
project=project,
bq_client=None,
bqtk_config=conf)
table = Table("table_bar",
from_dataset=ds,
bq_client=None,
bqtk_config=conf)
with pytest.raises(Exception):
table.partition_by("zoo")
def test_change_alias():
conf = BQTestKitConfig({DEFAULT_LOCATION: "EU"})
project = Project("test_project", bq_client=None, bqtk_config=conf)
ds = Dataset("dataset_foo",
project=project,
bq_client=None,
bqtk_config=conf)
table = Table("table_bar",
from_dataset=ds,
bq_client=None,
bqtk_config=conf)
table = table.with_alias("bar")
assert table.alias == "bar"
table = table.with_alias(None)
assert table.alias is None
def test_json_loader():
conf = BQTestKitConfig({DEFAULT_LOCATION: "EU"})
project = Project("test_project", bq_client=None, bqtk_config=conf)
ds = Dataset("dataset_foo",
project=project,
bq_client=None,
bqtk_config=conf)
table = Table("table_bar",
from_dataset=ds,
bq_client=None,
bqtk_config=conf)
pfl = PackageFileLoader(
"tests/ut/bq_test_kit/resource_loaders/resources/package_file_test_resource.txt"
)
assert isinstance(table.json_loader(from_=pfl), JsonDataLoader)
def test_invalid_change_schema():
conf = BQTestKitConfig({DEFAULT_LOCATION: "EU"})
project = Project("test_project", bq_client=None, bqtk_config=conf)
ds = Dataset("dataset_foo",
project=project,
bq_client=None,
bqtk_config=conf)
table = Table("table_bar",
from_dataset=ds,
bq_client=None,
bqtk_config=conf)
with pytest.raises(InvalidInstanceException):
table.with_schema(from_=[1, 2, 3])
with pytest.raises(InvalidInstanceException):
table.with_schema(from_=1)
def test_change_isolate():
conf = BQTestKitConfig({
DEFAULT_LOCATION: "EU"
}).with_test_context("context")
project = Project("test_project", bq_client=None, bqtk_config=conf)
ds = Dataset("dataset_foo",
project=project,
bq_client=None,
bqtk_config=conf)
table = Table("table_bar",
from_dataset=ds,
bq_client=None,
bqtk_config=conf)
table = table.isolate()
assert table.isolate_func(table) == f"{table.name}_context"
assert table.fqdn() == (f"{table.dataset.project.name}."
f"{table.dataset.name}."
f"{table.name}_context")
def test_change_create_options():
conf = BQTestKitConfig({
DEFAULT_LOCATION: "EU"
}).with_test_context("context")
project = Project("test_project", bq_client=None, bqtk_config=conf)
ds = Dataset("dataset_foo",
project=project,
bq_client=None,
bqtk_config=conf)
table = Table("table_bar",
from_dataset=ds,
bq_client=None,
bqtk_config=conf)
create_options = {"exists_ok": True, "timeout": 10}
table = table.with_create_options(**create_options)
assert table.create_options == create_options
table = table.with_create_options()
assert table.create_options == {}
def test_change_partition_by():
conf = BQTestKitConfig({DEFAULT_LOCATION: "EU"})
project = Project("test_project", bq_client=None, bqtk_config=conf)
ds = Dataset("dataset_foo",
project=project,
bq_client=None,
bqtk_config=conf)
table = Table("table_bar",
from_dataset=ds,
bq_client=None,
bqtk_config=conf)
table = table.partition_by(
Range(on_field="zoo", start=0, end=10000, interval=10))
assert isinstance(table.partition_type, Range)
assert table.partition_type.field == "zoo"
assert table.partition_type.start == 0
assert table.partition_type.end == 10000
assert table.partition_type.interval == 10
def test_default_constructor():
conf = BQTestKitConfig({DEFAULT_LOCATION: "EU"})
project = Project("test_project", bq_client=None, bqtk_config=conf)
ds = Dataset("dataset_foo",
project=project,
bq_client=None,
bqtk_config=conf)
table = Table("table_bar",
from_dataset=ds,
bq_client=None,
bqtk_config=conf)
assert table.dataset.project.name == "test_project"
assert table.dataset.name == "dataset_foo"
assert table.name == "table_bar"
assert isinstance(table.resource_strategy, CleanAfter)
assert table.alias is None
assert table.isolate_func(table) == "table_bar"
assert table.create_options == {}
assert table.schema == []
assert isinstance(table.partition_type, NoPartition)