Exemplo n.º 1
0
    def test_bulk_complete_correctly_interfaced(self):
        class BulkCompleteDailyTask(luigi.Task):
            d = luigi.DateParameter()

            @classmethod
            def bulk_complete(self, parameter_tuples):
                return parameter_tuples[:-2]

            def output(self):
                raise RuntimeError(
                    "Shouldn't get called while resolving deps via bulk_complete"
                )

        task = RangeDaily(now=datetime_to_epoch(datetime.datetime(2015, 12,
                                                                  1)),
                          of='BulkCompleteDailyTask',
                          start=datetime.date(2015, 11, 1),
                          stop=datetime.date(2015, 12, 1))

        expected = [
            'BulkCompleteDailyTask(d=2015-11-29)',
            'BulkCompleteDailyTask(d=2015-11-30)',
        ]

        actual = [t.task_id for t in task.requires()]
        self.assertEqual(actual, expected)
Exemplo n.º 2
0
    def test_bulk_complete_of_params(self):
        class BulkCompleteDailyTask(luigi.Task):
            non_positional_arbitrary_argument = luigi.Parameter(default="whatever", positional=False, significant=False)
            d = luigi.DateParameter()
            arbitrary_argument = luigi.BoolParameter()

            @classmethod
            def bulk_complete(cls, parameter_tuples):
                ptuples = list(parameter_tuples)
                for t in map(cls, ptuples):
                    assert t.arbitrary_argument
                return ptuples[:-2]

            def output(self):
                raise RuntimeError("Shouldn't get called while resolving deps via bulk_complete")

        task = RangeDaily(now=datetime_to_epoch(datetime.datetime(2015, 12, 1)),
                          of=BulkCompleteDailyTask,
                          of_params=dict(arbitrary_argument=True),
                          start=datetime.date(2015, 11, 1),
                          stop=datetime.date(2015, 12, 1))
        expected = [
            'BulkCompleteDailyTask(d=2015-11-29, arbitrary_argument=True)',
            'BulkCompleteDailyTask(d=2015-11-30, arbitrary_argument=True)',
        ]

        actual = [str(t) for t in task.requires()]
        self.assertEqual(actual, expected)
Exemplo n.º 3
0
    def test_param_name_with_inferred_fs(self):
        class MyTask(luigi.Task):
            some_non_range_param = luigi.Parameter(default='woo')
            date_param = luigi.DateParameter()

            def output(self):
                return MockTarget(self.date_param.strftime('/n2000y01a05n/%Y_%m-_-%daww/21mm%Hdara21/ooo'))

        range_task = RangeDaily(now=datetime_to_epoch(datetime.datetime(2015, 12, 2)),
                                of=MyTask,
                                start=datetime.date(2015, 12, 1),
                                stop=datetime.date(2015, 12, 2),
                                param_name='date_param')
        expected_task = MyTask('woo', datetime.date(2015, 12, 1))
        self.assertEqual(expected_task, list(range_task._requires())[0])
Exemplo n.º 4
0
    def test_param_name_with_inferred_fs(self):
        class MyTask(luigi.Task):
            some_non_range_param = luigi.Parameter(default='woo')
            date_param = luigi.DateParameter()

            def output(self):
                return MockTarget(self.date_param.strftime('/n2000y01a05n/%Y_%m-_-%daww/21mm%Hdara21/ooo'))

        range_task = RangeDaily(now=datetime_to_epoch(datetime.datetime(2015, 12, 2)),
                                of=MyTask,
                                start=datetime.date(2015, 12, 1),
                                stop=datetime.date(2015, 12, 2),
                                param_name='date_param')
        expected_task = MyTask('woo', datetime.date(2015, 12, 1))
        self.assertEqual(expected_task, list(range_task._requires())[0])
Exemplo n.º 5
0
    def test_bulk_complete(self, mock_connect):
        mock_cursor = MockPostgresCursor(
            ['DummyPostgresImporter(date=2015-01-03)'])
        mock_connect.return_value.cursor.return_value = mock_cursor

        task = RangeDaily(of='DummyPostgresImporter',
                          start=datetime.date(2015, 1, 2),
                          now=datetime_to_epoch(datetime.datetime(2015, 1, 7)))
        actual = [t.task_id for t in task.requires()]

        self.assertEqual(actual, [
            'DummyPostgresImporter(date=2015-01-02)',
            'DummyPostgresImporter(date=2015-01-04)',
            'DummyPostgresImporter(date=2015-01-05)',
            'DummyPostgresImporter(date=2015-01-06)',
        ])
        self.assertFalse(task.complete())
Exemplo n.º 6
0
    def test_bulk_complete(self, mock_connect):
        mock_cursor = MockPostgresCursor(
            ['DummyPostgresQuery_2015_01_03_838e32a989'])
        mock_connect.return_value.cursor.return_value = mock_cursor

        task = RangeDaily(of=DummyPostgresQuery,
                          start=datetime.date(2015, 1, 2),
                          now=datetime_to_epoch(datetime.datetime(2015, 1, 7)))
        actual = [t.task_id for t in task.requires()]

        self.assertEqual(actual, [
            'DummyPostgresQuery_2015_01_02_3a0ec498ed',
            'DummyPostgresQuery_2015_01_04_9c1d42ff62',
            'DummyPostgresQuery_2015_01_05_0f90e52357',
            'DummyPostgresQuery_2015_01_06_f91a47ec40',
        ])
        self.assertFalse(task.complete())
Exemplo n.º 7
0
    def test_bulk_complete(self, mock_connect):
        mock_cursor = MockPostgresCursor([
            'DummyPostgresImporter(date=2015-01-03)'
        ])
        mock_connect.return_value.cursor.return_value = mock_cursor

        task = RangeDaily(of='DummyPostgresImporter',
                          start=datetime.date(2015, 1, 2),
                          now=datetime_to_epoch(datetime.datetime(2015, 1, 7)))
        actual = [t.task_id for t in task.requires()]

        self.assertEqual(actual, [
            'DummyPostgresImporter(date=2015-01-02)',
            'DummyPostgresImporter(date=2015-01-04)',
            'DummyPostgresImporter(date=2015-01-05)',
            'DummyPostgresImporter(date=2015-01-06)',
        ])
        self.assertFalse(task.complete())
Exemplo n.º 8
0
    def test_bulk_complete(self, mock_connect):
        mock_cursor = MockPostgresCursor([
            DummyPostgresImporter(date=datetime.datetime(2015, 1, 3)).task_id
        ])
        mock_connect.return_value.cursor.return_value = mock_cursor

        task = RangeDaily(of=DummyPostgresImporter,
                          start=datetime.date(2015, 1, 2),
                          now=datetime_to_epoch(datetime.datetime(2015, 1, 7)))
        actual = sorted([t.task_id for t in task.requires()])

        self.assertEqual(actual, sorted([
            DummyPostgresImporter(date=datetime.datetime(2015, 1, 2)).task_id,
            DummyPostgresImporter(date=datetime.datetime(2015, 1, 4)).task_id,
            DummyPostgresImporter(date=datetime.datetime(2015, 1, 5)).task_id,
            DummyPostgresImporter(date=datetime.datetime(2015, 1, 6)).task_id,
        ]))
        self.assertFalse(task.complete())
Exemplo n.º 9
0
    def test_bulk_complete(self, mock_connect):
        mock_cursor = MockPostgresCursor([
            'DummyPostgresQuery_2015_01_03_838e32a989'
        ])
        mock_connect.return_value.cursor.return_value = mock_cursor

        task = RangeDaily(of=DummyPostgresQuery,
                          start=datetime.date(2015, 1, 2),
                          now=datetime_to_epoch(datetime.datetime(2015, 1, 7)))
        actual = [t.task_id for t in task.requires()]

        self.assertEqual(actual, [
            'DummyPostgresQuery_2015_01_02_3a0ec498ed',
            'DummyPostgresQuery_2015_01_04_9c1d42ff62',
            'DummyPostgresQuery_2015_01_05_0f90e52357',
            'DummyPostgresQuery_2015_01_06_f91a47ec40',
        ])
        self.assertFalse(task.complete())
Exemplo n.º 10
0
    def test_bulk_complete(self, mock_connect):
        mock_cursor = MockPostgresCursor([
            DummyPostgresImporter(date=datetime.datetime(2015, 1, 3)).task_id
        ])
        mock_connect.return_value.cursor.return_value = mock_cursor

        task = RangeDaily(of=DummyPostgresImporter,
                          start=datetime.date(2015, 1, 2),
                          now=datetime_to_epoch(datetime.datetime(2015, 1, 7)))
        actual = sorted([t.task_id for t in task.requires()])

        self.assertEqual(actual, sorted([
            DummyPostgresImporter(date=datetime.datetime(2015, 1, 2)).task_id,
            DummyPostgresImporter(date=datetime.datetime(2015, 1, 4)).task_id,
            DummyPostgresImporter(date=datetime.datetime(2015, 1, 5)).task_id,
            DummyPostgresImporter(date=datetime.datetime(2015, 1, 6)).task_id,
        ]))
        self.assertFalse(task.complete())
Exemplo n.º 11
0
    def test_missing_tasks_correctly_required(self):
        class SomeDailyTask(luigi.Task):
            d = luigi.DateParameter()

            def output(self):
                return MockTarget(self.d.strftime('/data/2014/p/v/z/%Y_/_%m-_-%doctor/20/ZOOO'))

        task = RangeDaily(now=datetime_to_epoch(datetime.datetime(2016, 4, 1)),
                          of=SomeDailyTask,
                          start=datetime.date(2014, 3, 20),
                          task_limit=3,
                          days_back=3 * 365)
        expected = [
            'SomeDailyTask(d=2014-03-20)',
            'SomeDailyTask(d=2014-03-22)',
            'SomeDailyTask(d=2014-03-25)',
        ]
        actual = [t.task_id for t in task.requires()]
        self.assertEqual(actual, expected)
Exemplo n.º 12
0
    def test_of_param_distinction(self):
        class MyTask(luigi.Task):
            arbitrary_param = luigi.Parameter(default='foo')
            arbitrary_integer_param = luigi.IntParameter(default=10)
            date_param = luigi.DateParameter()

            def complete(self):
                return False

        range_task_1 = RangeDaily(now=datetime_to_epoch(datetime.datetime(2015, 12, 2)),
                                  of=MyTask,
                                  start=datetime.date(2015, 12, 1),
                                  stop=datetime.date(2015, 12, 2))
        range_task_2 = RangeDaily(now=datetime_to_epoch(datetime.datetime(2015, 12, 2)),
                                  of=MyTask,
                                  of_params=dict(arbitrary_param="bar", abitrary_integer_param=2),
                                  start=datetime.date(2015, 12, 1),
                                  stop=datetime.date(2015, 12, 2))
        self.assertNotEqual(range_task_1.task_id, range_task_2.task_id)
Exemplo n.º 13
0
    def test_missing_tasks_correctly_required(self):
        class SomeDailyTask(luigi.Task):
            d = luigi.DateParameter()

            def output(self):
                return MockTarget(self.d.strftime('/data/2014/p/v/z/%Y_/_%m-_-%doctor/20/ZOOO'))

        task = RangeDaily(now=datetime_to_epoch(datetime.datetime(2016, 4, 1)),
                          of=SomeDailyTask,
                          start=datetime.date(2014, 3, 20),
                          task_limit=3,
                          days_back=3 * 365)
        expected = [
            'SomeDailyTask(d=2014-03-20)',
            'SomeDailyTask(d=2014-03-22)',
            'SomeDailyTask(d=2014-03-25)',
        ]
        actual = [str(t) for t in task.requires()]
        self.assertEqual(actual, expected)
Exemplo n.º 14
0
    def test_bulk_complete_correctly_interfaced(self):
        class BulkCompleteDailyTask(luigi.Task):
            d = luigi.DateParameter()

            @classmethod
            def bulk_complete(self, parameter_tuples):
                return list(parameter_tuples)[:-2]

            def output(self):
                raise RuntimeError("Shouldn't get called while resolving deps via bulk_complete")

        task = RangeDaily(now=datetime_to_epoch(datetime.datetime(2015, 12, 1)),
                          of=BulkCompleteDailyTask,
                          start=datetime.date(2015, 11, 1),
                          stop=datetime.date(2015, 12, 1))

        expected = [
            'BulkCompleteDailyTask(d=2015-11-29)',
            'BulkCompleteDailyTask(d=2015-11-30)',
        ]

        actual = [t.task_id for t in task.requires()]
        self.assertEqual(actual, expected)