Exemplo n.º 1
0
    def test_add_tasks(self):
        dag = Dag("bqetl_test_dag", "daily", self.default_args)

        query_file = (TEST_DIR / "data" / "test_sql" /
                      "moz-fx-data-test-project" / "test" /
                      "incremental_query_v1" / "query.sql")

        scheduling = {
            "dag_name": "bqetl_test_dag",
            "default_args": {
                "owner": "*****@*****.**"
            },
            "task_name": "custom_task_name",
        }

        metadata = Metadata("test", "test", ["*****@*****.**"], {},
                            scheduling)

        task1 = Task.of_query(query_file)
        task2 = Task.of_query(query_file, metadata)

        assert dag.tasks == []

        dag.add_tasks([task1, task2])

        assert len(dag.tasks) == 2
Exemplo n.º 2
0
    def from_dict(cls, d):
        """
        Parse DAG configurations from a dict and create new instances.

        Expected dict format:
        {
            "bqetl_dag_name1": {
                "schedule_interval": string,
                "default_args": {
                    "owner": string,
                    "start_date": "YYYY-MM-DD",
                    ...
                }
            },
            "bqetl_dag_name2": {
                "schedule_interval": string,
                "default_args": {
                    "owner": string,
                    "start_date": "YYYY-MM-DD",
                    ...
                }
            },
            ...
        }
        """
        if d is None:
            return cls([])

        dags = [Dag.from_dict({k: v}) for k, v in d.items()]
        return cls(dags)
Exemplo n.º 3
0
    def test_dag_instantiation(self):
        dag = Dag("bqetl_test_dag", "daily", self.default_args)

        assert dag.name == "bqetl_test_dag"
        assert dag.schedule_interval == "daily"
        assert dag.tasks == []
        assert dag.default_args == self.default_args
Exemplo n.º 4
0
    def test_public_data_json_dag_add_task(self):
        public_json_dag = PublicDataJsonDag("bqetl_public_data_json_dag",
                                            "daily", self.default_args)

        assert len(public_json_dag.tasks) == 0

        query_file = (TEST_DIR / "data" / "test_sql" /
                      "moz-fx-data-test-project" / "test" /
                      "incremental_query_v1" / "query.sql")

        task = Task.of_query(query_file)
        dag = Dag(
            "bqetl_events",
            "0 1 * * *",
            DagDefaultArgs("*****@*****.**", "2020-01-01"),
            [task],
        )
        public_json_dag.add_export_tasks([task], DagCollection([dag]))

        assert len(public_json_dag.tasks) == 1
        assert (public_json_dag.tasks[0].task_name ==
                "export_public_data_json_test__incremental_query__v1")
        assert public_json_dag.tasks[
            0].dag_name == "bqetl_public_data_json_dag"
        assert len(public_json_dag.tasks[0].dependencies) == 1
Exemplo n.º 5
0
    def test_add_tasks(self):
        dag = Dag("test_dag", "daily", {})

        query_file = (
            TEST_DIR
            / "data"
            / "test_sql"
            / "test"
            / "incremental_query_v1"
            / "query.sql"
        )

        tasks = [Task.of_query(query_file), Task.of_query(query_file)]

        assert dag.tasks == []

        dag.add_tasks(tasks)

        assert len(dag.tasks) == 2
Exemplo n.º 6
0
    def test_add_tasks_duplicate_names(self):
        dag = Dag("bqetl_test_dag", "daily", self.default_args)

        query_file = (TEST_DIR / "data" / "test_sql" / "test" /
                      "incremental_query_v1" / "query.sql")

        scheduling = {
            "dag_name": "bqetl_test_dag",
            "default_args": {
                "owner": "*****@*****.**"
            },
            "task_name": "duplicate_task_name",
        }

        metadata = Metadata("test", "test", ["*****@*****.**"], {},
                            scheduling)
        task = Task.of_query(query_file, metadata)

        with pytest.raises(ValueError):
            dag.add_tasks([task, task])
Exemplo n.º 7
0
    def test_schedule_interval_format(self):
        assert Dag("bqetl_test_dag", "daily", self.default_args)
        assert Dag("bqetl_test_dag", "weekly", self.default_args)
        assert Dag("bqetl_test_dag", "once", self.default_args)

        with pytest.raises(ValueError):
            assert Dag("bqetl_test_dag", "never", self.default_args)

        assert Dag("bqetl_test_dag", "* * * * *", self.default_args)
        assert Dag("bqetl_test_dag", "1 * * * *", self.default_args)

        with pytest.raises(TypeError):
            assert Dag("bqetl_test_dag", 323, self.default_args)
Exemplo n.º 8
0
    def test_from_dict(self):
        dag = Dag.from_dict(
            {
                "test_dag": {
                    "schedule_interval": "daily",
                    "default_args": {
                        "owner": "*****@*****.**",
                        "param": "test_param",
                    },
                }
            }
        )

        assert dag.name == "test_dag"
        assert dag.schedule_interval == "daily"
        assert dag.default_args == {"owner": "*****@*****.**", "param": "test_param"}
Exemplo n.º 9
0
    def test_from_dict(self):
        dag = Dag.from_dict({
            "bqetl_test_dag": {
                "schedule_interval": "daily",
                "default_args": {
                    "start_date": "2020-05-12",
                    "owner": "*****@*****.**",
                    "email": ["*****@*****.**"],
                },
            }
        })

        assert dag.name == "bqetl_test_dag"
        assert dag.schedule_interval == "daily"
        assert dag.default_args.owner == "*****@*****.**"
        assert dag.default_args.email == ["*****@*****.**"]
        assert dag.default_args.depends_on_past is False
Exemplo n.º 10
0
    def from_dict(cls, d):
        """
        Parse DAG configurations from a dict and create new instances.

        Expected dict format:
        {
            "dag_name1": {
                "schedule_interval": string,
                "default_args": dict
            },
            "dag_name2": {
                "schedule_interval": string,
                "default_args": dict
            },
            ...
        }
        """
        dags = [Dag.from_dict({k: v}) for k, v in d.items()]
        return cls(dags)
Exemplo n.º 11
0
 def test_from_empty_dict(self):
     with pytest.raises(DagParseException):
         Dag.from_dict({})
Exemplo n.º 12
0
 def test_invalid_dag_name(self):
     with pytest.raises(ValueError):
         Dag("test_dag", "daily", self.default_args)
Exemplo n.º 13
0
 def test_from_dict_without_scheduling_interval(self):
     with pytest.raises(DagParseException):
         Dag.from_dict({"bqetl_test_dag": {}})
Exemplo n.º 14
0
 def test_from_dict_multiple_dags(self):
     with pytest.raises(DagParseException):
         Dag.from_dict({"bqetl_test_dag1": {}, "bqetl_test_dag2": {}})
Exemplo n.º 15
0
    def test_private_dag_name(self):
        dag = Dag("private_bqetl_test_dag", "daily", self.default_args)

        assert dag.name == "private_bqetl_test_dag"