def test_bulk_complete_of_params(self):
class BulkCompleteByMinutesTask(luigi.Task):
non_positional_arbitrary_argument = luigi.Parameter(default="whatever", positional=False, significant=False)
dh = luigi.DateMinuteParameter()
arbitrary_argument = luigi.BoolParameter()
@classmethod
def bulk_complete(cls, parameter_tuples):
for t in map(cls, parameter_tuples):
assert t.arbitrary_argument
return parameter_tuples[:-2]
def output(self):
raise RuntimeError("Shouldn't get called while resolving deps via bulk_complete")
task = RangeByMinutes(now=datetime_to_epoch(datetime.datetime(2015, 12, 1)),
of=BulkCompleteByMinutesTask,
of_params=dict(arbitrary_argument=True),
start=datetime.datetime(2015, 11, 1),
stop=datetime.datetime(2015, 12, 1),
minutes_interval=5)
expected = [
'BulkCompleteByMinutesTask(dh=2015-11-30T2350, arbitrary_argument=True)',
'BulkCompleteByMinutesTask(dh=2015-11-30T2355, arbitrary_argument=True)',
]
actual = [str(t) for t in task.requires()]
self.assertEqual(actual, expected)
def test_missing_directory(self):
task = RangeByMinutes(now=datetime_to_epoch(
datetime.datetime(2014, 3, 21, 0, 0)),
of=TaskMinutesC,
start=datetime.datetime(2014, 3, 20, 23, 11),
stop=datetime.datetime(2014, 3, 20, 23, 21),
minutes_interval=5)
self.assertFalse(task.complete())
expected = [
'TaskMinutesC(dm=2014-03-20T2315)',
'TaskMinutesC(dm=2014-03-20T2320)']
self.assertEqual([str(t) for t in task.requires()], expected)
def test_missing_wrapper_tasks_correctly_required(self):
expected_wrapper = [
'CommonWrapperTaskMinutes(dm=2014-03-20T2300)',
'CommonWrapperTaskMinutes(dm=2014-03-20T2305)',
'CommonWrapperTaskMinutes(dm=2014-03-20T2310)',
'CommonWrapperTaskMinutes(dm=2014-03-20T2315)']
task = RangeByMinutes(
now=datetime_to_epoch(datetime.datetime(2040, 4, 1, 0, 0, 0)),
of=CommonWrapperTaskMinutes,
start=datetime.datetime(2014, 3, 20, 23, 0, 0),
stop=datetime.datetime(2014, 3, 20, 23, 20, 0),
minutes_back=30 * 365 * 24 * 60,
minutes_interval=5)
actual = [str(t) for t in task.requires()]
self.assertEqual(actual, expected_wrapper)
def test_bulk_complete_correctly_interfaced(self):
class BulkCompleteByMinutesTask(luigi.Task):
dh = luigi.DateMinuteParameter()
@classmethod
def bulk_complete(cls, parameter_tuples):
return parameter_tuples[:-2]
def output(self):
raise RuntimeError("Shouldn't get called while resolving deps via bulk_complete")
task = RangeByMinutes(now=datetime_to_epoch(datetime.datetime(2015, 12, 1)),
of=BulkCompleteByMinutesTask,
start=datetime.datetime(2015, 11, 1),
stop=datetime.datetime(2015, 12, 1),
minutes_interval=5)
expected = [
'BulkCompleteByMinutesTask(dh=2015-11-30T2350)',
'BulkCompleteByMinutesTask(dh=2015-11-30T2355)',
]
actual = [str(t) for t in task.requires()]
self.assertEqual(actual, expected)
def test_missing_tasks_correctly_required(self):
expected_tasks = [
'SomeByMinutesTask(d=2016-03-31T0000)',
'SomeByMinutesTask(d=2016-03-31T0005)',
'SomeByMinutesTask(d=2016-03-31T0010)']
class SomeByMinutesTask(luigi.Task):
d = luigi.DateMinuteParameter()
def output(self):
return MockTarget(self.d.strftime('/data/2014/p/v/z/%Y_/_%m-_-%doctor/20/%HZ%MOOO'))
for task_path in task_a_paths:
MockTarget(task_path)
# this test takes a few seconds. Since stop is not defined,
# finite_datetimes constitute many years to consider
task = RangeByMinutes(now=datetime_to_epoch(datetime.datetime(2016, 4, 1)),
of=SomeByMinutesTask,
start=datetime.datetime(2014, 3, 20, 17),
task_limit=3,
minutes_back=24 * 60,
minutes_interval=5)
actual = [str(t) for t in task.requires()]
self.assertEqual(actual, expected_tasks)