def test_add_tasks(self):
dag = Dag("bqetl_test_dag", "daily", self.default_args)
query_file = (TEST_DIR / "data" / "test_sql" /
"moz-fx-data-test-project" / "test" /
"incremental_query_v1" / "query.sql")
scheduling = {
"dag_name": "bqetl_test_dag",
"default_args": {
"owner": "*****@*****.**"
},
"task_name": "custom_task_name",
}
metadata = Metadata("test", "test", ["*****@*****.**"], {},
scheduling)
task1 = Task.of_query(query_file)
task2 = Task.of_query(query_file, metadata)
assert dag.tasks == []
dag.add_tasks([task1, task2])
assert len(dag.tasks) == 2
def from_dict(cls, d):
"""
Parse DAG configurations from a dict and create new instances.
Expected dict format:
{
"bqetl_dag_name1": {
"schedule_interval": string,
"default_args": {
"owner": string,
"start_date": "YYYY-MM-DD",
...
}
},
"bqetl_dag_name2": {
"schedule_interval": string,
"default_args": {
"owner": string,
"start_date": "YYYY-MM-DD",
...
}
},
...
}
"""
if d is None:
return cls([])
dags = [Dag.from_dict({k: v}) for k, v in d.items()]
return cls(dags)
def test_dag_instantiation(self):
dag = Dag("bqetl_test_dag", "daily", self.default_args)
assert dag.name == "bqetl_test_dag"
assert dag.schedule_interval == "daily"
assert dag.tasks == []
assert dag.default_args == self.default_args
def test_public_data_json_dag_add_task(self):
public_json_dag = PublicDataJsonDag("bqetl_public_data_json_dag",
"daily", self.default_args)
assert len(public_json_dag.tasks) == 0
query_file = (TEST_DIR / "data" / "test_sql" /
"moz-fx-data-test-project" / "test" /
"incremental_query_v1" / "query.sql")
task = Task.of_query(query_file)
dag = Dag(
"bqetl_events",
"0 1 * * *",
DagDefaultArgs("*****@*****.**", "2020-01-01"),
[task],
)
public_json_dag.add_export_tasks([task], DagCollection([dag]))
assert len(public_json_dag.tasks) == 1
assert (public_json_dag.tasks[0].task_name ==
"export_public_data_json_test__incremental_query__v1")
assert public_json_dag.tasks[
0].dag_name == "bqetl_public_data_json_dag"
assert len(public_json_dag.tasks[0].dependencies) == 1
def test_add_tasks(self):
dag = Dag("test_dag", "daily", {})
query_file = (
TEST_DIR
/ "data"
/ "test_sql"
/ "test"
/ "incremental_query_v1"
/ "query.sql"
)
tasks = [Task.of_query(query_file), Task.of_query(query_file)]
assert dag.tasks == []
dag.add_tasks(tasks)
assert len(dag.tasks) == 2
def test_add_tasks_duplicate_names(self):
dag = Dag("bqetl_test_dag", "daily", self.default_args)
query_file = (TEST_DIR / "data" / "test_sql" / "test" /
"incremental_query_v1" / "query.sql")
scheduling = {
"dag_name": "bqetl_test_dag",
"default_args": {
"owner": "*****@*****.**"
},
"task_name": "duplicate_task_name",
}
metadata = Metadata("test", "test", ["*****@*****.**"], {},
scheduling)
task = Task.of_query(query_file, metadata)
with pytest.raises(ValueError):
dag.add_tasks([task, task])
def test_schedule_interval_format(self):
assert Dag("bqetl_test_dag", "daily", self.default_args)
assert Dag("bqetl_test_dag", "weekly", self.default_args)
assert Dag("bqetl_test_dag", "once", self.default_args)
with pytest.raises(ValueError):
assert Dag("bqetl_test_dag", "never", self.default_args)
assert Dag("bqetl_test_dag", "* * * * *", self.default_args)
assert Dag("bqetl_test_dag", "1 * * * *", self.default_args)
with pytest.raises(TypeError):
assert Dag("bqetl_test_dag", 323, self.default_args)
def test_from_dict(self):
dag = Dag.from_dict(
{
"test_dag": {
"schedule_interval": "daily",
"default_args": {
"owner": "*****@*****.**",
"param": "test_param",
},
}
}
)
assert dag.name == "test_dag"
assert dag.schedule_interval == "daily"
assert dag.default_args == {"owner": "*****@*****.**", "param": "test_param"}
def test_from_dict(self):
dag = Dag.from_dict({
"bqetl_test_dag": {
"schedule_interval": "daily",
"default_args": {
"start_date": "2020-05-12",
"owner": "*****@*****.**",
"email": ["*****@*****.**"],
},
}
})
assert dag.name == "bqetl_test_dag"
assert dag.schedule_interval == "daily"
assert dag.default_args.owner == "*****@*****.**"
assert dag.default_args.email == ["*****@*****.**"]
assert dag.default_args.depends_on_past is False
def from_dict(cls, d):
"""
Parse DAG configurations from a dict and create new instances.
Expected dict format:
{
"dag_name1": {
"schedule_interval": string,
"default_args": dict
},
"dag_name2": {
"schedule_interval": string,
"default_args": dict
},
...
}
"""
dags = [Dag.from_dict({k: v}) for k, v in d.items()]
return cls(dags)
def test_from_empty_dict(self):
with pytest.raises(DagParseException):
Dag.from_dict({})
def test_invalid_dag_name(self):
with pytest.raises(ValueError):
Dag("test_dag", "daily", self.default_args)
def test_from_dict_without_scheduling_interval(self):
with pytest.raises(DagParseException):
Dag.from_dict({"bqetl_test_dag": {}})
def test_from_dict_multiple_dags(self):
with pytest.raises(DagParseException):
Dag.from_dict({"bqetl_test_dag1": {}, "bqetl_test_dag2": {}})
def test_private_dag_name(self):
dag = Dag("private_bqetl_test_dag", "daily", self.default_args)
assert dag.name == "private_bqetl_test_dag"