def from_dict(cls, d):
"""
Parse DAG configurations from a dict and create new instances.
Expected dict format:
{
"bqetl_dag_name1": {
"schedule_interval": string,
"default_args": {
"owner": string,
"start_date": "YYYY-MM-DD",
...
}
},
"bqetl_dag_name2": {
"schedule_interval": string,
"default_args": {
"owner": string,
"start_date": "YYYY-MM-DD",
...
}
},
...
}
"""
if d is None:
return cls([])
dags = [Dag.from_dict({k: v}) for k, v in d.items()]
return cls(dags)
def test_from_dict(self):
dag = Dag.from_dict(
{
"test_dag": {
"schedule_interval": "daily",
"default_args": {
"owner": "*****@*****.**",
"param": "test_param",
},
}
}
)
assert dag.name == "test_dag"
assert dag.schedule_interval == "daily"
assert dag.default_args == {"owner": "*****@*****.**", "param": "test_param"}
def test_from_dict(self):
dag = Dag.from_dict({
"bqetl_test_dag": {
"schedule_interval": "daily",
"default_args": {
"start_date": "2020-05-12",
"owner": "*****@*****.**",
"email": ["*****@*****.**"],
},
}
})
assert dag.name == "bqetl_test_dag"
assert dag.schedule_interval == "daily"
assert dag.default_args.owner == "*****@*****.**"
assert dag.default_args.email == ["*****@*****.**"]
assert dag.default_args.depends_on_past is False
def from_dict(cls, d):
"""
Parse DAG configurations from a dict and create new instances.
Expected dict format:
{
"dag_name1": {
"schedule_interval": string,
"default_args": dict
},
"dag_name2": {
"schedule_interval": string,
"default_args": dict
},
...
}
"""
dags = [Dag.from_dict({k: v}) for k, v in d.items()]
return cls(dags)
def test_from_empty_dict(self):
with pytest.raises(DagParseException):
Dag.from_dict({})
def test_from_dict_without_scheduling_interval(self):
with pytest.raises(DagParseException):
Dag.from_dict({"bqetl_test_dag": {}})
def test_from_dict_multiple_dags(self):
with pytest.raises(DagParseException):
Dag.from_dict({"bqetl_test_dag1": {}, "bqetl_test_dag2": {}})