Beispiel #1
0
    def test_bulk_complete_of_params(self):
        class BulkCompleteByMinutesTask(luigi.Task):
            non_positional_arbitrary_argument = luigi.Parameter(default="whatever", positional=False, significant=False)
            dh = luigi.DateMinuteParameter()
            arbitrary_argument = luigi.BoolParameter()

            @classmethod
            def bulk_complete(cls, parameter_tuples):
                for t in map(cls, parameter_tuples):
                    assert t.arbitrary_argument
                return parameter_tuples[:-2]

            def output(self):
                raise RuntimeError("Shouldn't get called while resolving deps via bulk_complete")

        task = RangeByMinutes(now=datetime_to_epoch(datetime.datetime(2015, 12, 1)),
                              of=BulkCompleteByMinutesTask,
                              of_params=dict(arbitrary_argument=True),
                              start=datetime.datetime(2015, 11, 1),
                              stop=datetime.datetime(2015, 12, 1),
                              minutes_interval=5)

        expected = [
            'BulkCompleteByMinutesTask(dh=2015-11-30T2350, arbitrary_argument=True)',
            'BulkCompleteByMinutesTask(dh=2015-11-30T2355, arbitrary_argument=True)',
        ]

        actual = [str(t) for t in task.requires()]
        self.assertEqual(actual, expected)
Beispiel #2
0
 def test_missing_directory(self):
     task = RangeByMinutes(now=datetime_to_epoch(
                        datetime.datetime(2014, 3, 21, 0, 0)),
                        of=TaskMinutesC,
                        start=datetime.datetime(2014, 3, 20, 23, 11),
                        stop=datetime.datetime(2014, 3, 20, 23, 21),
                        minutes_interval=5)
     self.assertFalse(task.complete())
     expected = [
         'TaskMinutesC(dm=2014-03-20T2315)',
         'TaskMinutesC(dm=2014-03-20T2320)']
     self.assertEqual([str(t) for t in task.requires()], expected)
Beispiel #3
0
 def test_missing_wrapper_tasks_correctly_required(self):
     expected_wrapper = [
         'CommonWrapperTaskMinutes(dm=2014-03-20T2300)',
         'CommonWrapperTaskMinutes(dm=2014-03-20T2305)',
         'CommonWrapperTaskMinutes(dm=2014-03-20T2310)',
         'CommonWrapperTaskMinutes(dm=2014-03-20T2315)']
     task = RangeByMinutes(
         now=datetime_to_epoch(datetime.datetime(2040, 4, 1, 0, 0, 0)),
         of=CommonWrapperTaskMinutes,
         start=datetime.datetime(2014, 3, 20, 23, 0, 0),
         stop=datetime.datetime(2014, 3, 20, 23, 20, 0),
         minutes_back=30 * 365 * 24 * 60,
         minutes_interval=5)
     actual = [str(t) for t in task.requires()]
     self.assertEqual(actual, expected_wrapper)
Beispiel #4
0
    def test_bulk_complete_correctly_interfaced(self):
        class BulkCompleteByMinutesTask(luigi.Task):
            dh = luigi.DateMinuteParameter()

            @classmethod
            def bulk_complete(cls, parameter_tuples):
                return parameter_tuples[:-2]

            def output(self):
                raise RuntimeError("Shouldn't get called while resolving deps via bulk_complete")

        task = RangeByMinutes(now=datetime_to_epoch(datetime.datetime(2015, 12, 1)),
                              of=BulkCompleteByMinutesTask,
                              start=datetime.datetime(2015, 11, 1),
                              stop=datetime.datetime(2015, 12, 1),
                              minutes_interval=5)

        expected = [
            'BulkCompleteByMinutesTask(dh=2015-11-30T2350)',
            'BulkCompleteByMinutesTask(dh=2015-11-30T2355)',
        ]

        actual = [str(t) for t in task.requires()]
        self.assertEqual(actual, expected)
Beispiel #5
0
    def test_missing_tasks_correctly_required(self):
        expected_tasks = [
            'SomeByMinutesTask(d=2016-03-31T0000)',
            'SomeByMinutesTask(d=2016-03-31T0005)',
            'SomeByMinutesTask(d=2016-03-31T0010)']

        class SomeByMinutesTask(luigi.Task):
            d = luigi.DateMinuteParameter()

            def output(self):
                return MockTarget(self.d.strftime('/data/2014/p/v/z/%Y_/_%m-_-%doctor/20/%HZ%MOOO'))

        for task_path in task_a_paths:
            MockTarget(task_path)
        # this test takes a few seconds. Since stop is not defined,
        # finite_datetimes constitute many years to consider
        task = RangeByMinutes(now=datetime_to_epoch(datetime.datetime(2016, 4, 1)),
                              of=SomeByMinutesTask,
                              start=datetime.datetime(2014, 3, 20, 17),
                              task_limit=3,
                              minutes_back=24 * 60,
                              minutes_interval=5)
        actual = [str(t) for t in task.requires()]
        self.assertEqual(actual, expected_tasks)