def test_bulk_complete_of_params(self): class BulkCompleteByMinutesTask(luigi.Task): non_positional_arbitrary_argument = luigi.Parameter(default="whatever", positional=False, significant=False) dh = luigi.DateMinuteParameter() arbitrary_argument = luigi.BoolParameter() @classmethod def bulk_complete(cls, parameter_tuples): for t in map(cls, parameter_tuples): assert t.arbitrary_argument return parameter_tuples[:-2] def output(self): raise RuntimeError("Shouldn't get called while resolving deps via bulk_complete") task = RangeByMinutes(now=datetime_to_epoch(datetime.datetime(2015, 12, 1)), of=BulkCompleteByMinutesTask, of_params=dict(arbitrary_argument=True), start=datetime.datetime(2015, 11, 1), stop=datetime.datetime(2015, 12, 1), minutes_interval=5) expected = [ 'BulkCompleteByMinutesTask(dh=2015-11-30T2350, arbitrary_argument=True)', 'BulkCompleteByMinutesTask(dh=2015-11-30T2355, arbitrary_argument=True)', ] actual = [str(t) for t in task.requires()] self.assertEqual(actual, expected)
def test_missing_directory(self): task = RangeByMinutes(now=datetime_to_epoch( datetime.datetime(2014, 3, 21, 0, 0)), of=TaskMinutesC, start=datetime.datetime(2014, 3, 20, 23, 11), stop=datetime.datetime(2014, 3, 20, 23, 21), minutes_interval=5) self.assertFalse(task.complete()) expected = [ 'TaskMinutesC(dm=2014-03-20T2315)', 'TaskMinutesC(dm=2014-03-20T2320)'] self.assertEqual([str(t) for t in task.requires()], expected)
def test_missing_wrapper_tasks_correctly_required(self): expected_wrapper = [ 'CommonWrapperTaskMinutes(dm=2014-03-20T2300)', 'CommonWrapperTaskMinutes(dm=2014-03-20T2305)', 'CommonWrapperTaskMinutes(dm=2014-03-20T2310)', 'CommonWrapperTaskMinutes(dm=2014-03-20T2315)'] task = RangeByMinutes( now=datetime_to_epoch(datetime.datetime(2040, 4, 1, 0, 0, 0)), of=CommonWrapperTaskMinutes, start=datetime.datetime(2014, 3, 20, 23, 0, 0), stop=datetime.datetime(2014, 3, 20, 23, 20, 0), minutes_back=30 * 365 * 24 * 60, minutes_interval=5) actual = [str(t) for t in task.requires()] self.assertEqual(actual, expected_wrapper)
def test_bulk_complete_correctly_interfaced(self): class BulkCompleteByMinutesTask(luigi.Task): dh = luigi.DateMinuteParameter() @classmethod def bulk_complete(cls, parameter_tuples): return parameter_tuples[:-2] def output(self): raise RuntimeError("Shouldn't get called while resolving deps via bulk_complete") task = RangeByMinutes(now=datetime_to_epoch(datetime.datetime(2015, 12, 1)), of=BulkCompleteByMinutesTask, start=datetime.datetime(2015, 11, 1), stop=datetime.datetime(2015, 12, 1), minutes_interval=5) expected = [ 'BulkCompleteByMinutesTask(dh=2015-11-30T2350)', 'BulkCompleteByMinutesTask(dh=2015-11-30T2355)', ] actual = [str(t) for t in task.requires()] self.assertEqual(actual, expected)
def test_missing_tasks_correctly_required(self): expected_tasks = [ 'SomeByMinutesTask(d=2016-03-31T0000)', 'SomeByMinutesTask(d=2016-03-31T0005)', 'SomeByMinutesTask(d=2016-03-31T0010)'] class SomeByMinutesTask(luigi.Task): d = luigi.DateMinuteParameter() def output(self): return MockTarget(self.d.strftime('/data/2014/p/v/z/%Y_/_%m-_-%doctor/20/%HZ%MOOO')) for task_path in task_a_paths: MockTarget(task_path) # this test takes a few seconds. Since stop is not defined, # finite_datetimes constitute many years to consider task = RangeByMinutes(now=datetime_to_epoch(datetime.datetime(2016, 4, 1)), of=SomeByMinutesTask, start=datetime.datetime(2014, 3, 20, 17), task_limit=3, minutes_back=24 * 60, minutes_interval=5) actual = [str(t) for t in task.requires()] self.assertEqual(actual, expected_tasks)