def test_bulk_complete_correctly_interfaced(self):
class BulkCompleteHourlyTask(luigi.Task):
dh = luigi.DateHourParameter()
@classmethod
def bulk_complete(cls, parameter_tuples):
return parameter_tuples[:-2]
def output(self):
raise RuntimeError(
"Shouldn't get called while resolving deps via bulk_complete"
)
task = RangeHourly(now=datetime_to_epoch(datetime.datetime(
2015, 12, 1)),
of='BulkCompleteHourlyTask',
start=datetime.datetime(2015, 11, 1),
stop=datetime.datetime(2015, 12, 1))
expected = [
'BulkCompleteHourlyTask(dh=2015-11-30T22)',
'BulkCompleteHourlyTask(dh=2015-11-30T23)',
]
actual = [t.task_id for t in task.requires()]
self.assertEqual(actual, expected)
def test_bulk_complete_of_params(self):
class BulkCompleteHourlyTask(luigi.Task):
non_positional_arbitrary_argument = luigi.Parameter(default="whatever", positional=False, significant=False)
dh = luigi.DateHourParameter()
arbitrary_argument = luigi.BoolParameter()
@classmethod
def bulk_complete(cls, parameter_tuples):
for t in map(cls, parameter_tuples):
assert t.arbitrary_argument
return parameter_tuples[:-2]
def output(self):
raise RuntimeError("Shouldn't get called while resolving deps via bulk_complete")
task = RangeHourly(now=datetime_to_epoch(datetime.datetime(2015, 12, 1)),
of=BulkCompleteHourlyTask,
of_params=dict(arbitrary_argument=True),
start=datetime.datetime(2015, 11, 1),
stop=datetime.datetime(2015, 12, 1))
expected = [
'BulkCompleteHourlyTask(dh=2015-11-30T22, arbitrary_argument=True)',
'BulkCompleteHourlyTask(dh=2015-11-30T23, arbitrary_argument=True)',
]
actual = [str(t) for t in task.requires()]
self.assertEqual(actual, expected)
def test_missing_wrapper_tasks_correctly_required(self):
task = RangeHourly(now=datetime_to_epoch(datetime.datetime(2040, 4, 1)),
of='CommonWrapperTask',
start=datetime.datetime(2014, 3, 20, 23),
stop=datetime.datetime(2014, 3, 21, 6),
hours_back=30 * 365 * 24)
actual = [t.task_id for t in task.requires()]
self.assertEqual(actual, expected_wrapper)
def test_missing_wrapper_tasks_correctly_required(self):
task = RangeHourly(now=datetime_to_epoch(datetime.datetime(2040, 4, 1)),
of='CommonWrapperTask',
start=datetime.datetime(2014, 3, 20, 23),
stop=datetime.datetime(2014, 3, 21, 6),
hours_back=30 * 365 * 24)
actual = [t.task_id for t in task.requires()]
self.assertEqual(actual, expected_wrapper)
def test_missing_tasks_correctly_required(self):
task = RangeHourly(now=datetime_to_epoch(datetime.datetime(2040, 4, 1)),
of='TaskA',
start=datetime.datetime(2014, 3, 20, 17),
task_limit=3,
hours_back=30 * 365 * 24) # this test takes around a minute for me. Since stop is not defined, finite_datehours constitute many years to consider
actual = [t.task_id for t in task.requires()]
self.assertEqual(str(actual), str(expected_a))
self.assertEqual(actual, expected_a)
def test_missing_directory(self):
task = RangeHourly(now=datetime_to_epoch(datetime.datetime(2014, 4,
1)),
of=TaskC,
start=datetime.datetime(2014, 3, 20, 23),
stop=datetime.datetime(2014, 3, 21, 1))
self.assertFalse(task.complete())
expected = ['TaskC(dh=2014-03-20T23)', 'TaskC(dh=2014-03-21T00)']
self.assertEqual([str(t) for t in task.requires()], expected)
def test_missing_tasks_correctly_required(self):
task = RangeHourly(now=datetime_to_epoch(datetime.datetime(2040, 4, 1)),
of='TaskA',
start=datetime.datetime(2014, 3, 20, 17),
task_limit=3,
hours_back=30 * 365 * 24) # this test takes around a minute for me. Since stop is not defined, finite_datehours constitute many years to consider
actual = [t.task_id for t in task.requires()]
self.assertEqual(str(actual), str(expected_a))
self.assertEqual(actual, expected_a)
def test_missing_tasks_correctly_required(self):
for task_path in task_a_paths:
MockTarget(task_path)
task = RangeHourly(now=datetime_to_epoch(datetime.datetime(2016, 4, 1)),
of=TaskA,
start=datetime.datetime(2014, 3, 20, 17),
task_limit=3,
hours_back=3 * 365 * 24) # this test takes a few seconds. Since stop is not defined, finite_datetimes constitute many years to consider
actual = [t.task_id for t in task.requires()]
self.assertEqual(actual, expected_a)
def test_missing_tasks_correctly_required(self):
for task_path in task_a_paths:
MockFile(task_path)
task = RangeHourly(now=datetime_to_epoch(datetime.datetime(2016, 4, 1)),
of='TaskA',
start=datetime.datetime(2014, 3, 20, 17),
task_limit=3,
hours_back=3 * 365 * 24) # this test takes a few seconds. Since stop is not defined, finite_datetimes constitute many years to consider
actual = [t.task_id for t in task.requires()]
self.assertEqual(actual, expected_a)
def test_missing_directory(self):
task = RangeHourly(now=datetime_to_epoch(
datetime.datetime(2014, 4, 1)),
of=TaskC,
start=datetime.datetime(2014, 3, 20, 23),
stop=datetime.datetime(2014, 3, 21, 1))
self.assertFalse(task.complete())
expected = [
'TaskC(dh=2014-03-20T23)',
'TaskC(dh=2014-03-21T00)']
self.assertEqual([t.task_id for t in task.requires()], expected)
def _test_filesystems_and_globs(self, task_cls, expected):
actual = list(RangeHourly._get_filesystems_and_globs(task_cls))
self.assertEqual(len(actual), len(expected))
for (actual_filesystem,
actual_glob), (expected_filesystem,
expected_glob) in zip(actual, expected):
self.assertTrue(isinstance(actual_filesystem, expected_filesystem))
self.assertEqual(actual_glob, expected_glob)
def test_bulk_complete_correctly_interfaced(self):
class BulkCompleteHourlyTask(luigi.Task):
dh = luigi.DateHourParameter()
@classmethod
def bulk_complete(cls, parameter_tuples):
return parameter_tuples[:-2]
def output(self):
raise RuntimeError("Shouldn't get called while resolving deps via bulk_complete")
task = RangeHourly(now=datetime_to_epoch(datetime.datetime(2015, 12, 1)),
of=BulkCompleteHourlyTask,
start=datetime.datetime(2015, 11, 1),
stop=datetime.datetime(2015, 12, 1))
expected = [
'BulkCompleteHourlyTask(dh=2015-11-30T22)',
'BulkCompleteHourlyTask(dh=2015-11-30T23)',
]
actual = [t.task_id for t in task.requires()]
self.assertEqual(actual, expected)
def _test_filesystems_and_globs(self, task_cls, expected):
actual = list(RangeHourly._get_filesystems_and_globs(task_cls))
self.assertEqual(len(actual), len(expected))
for (actual_filesystem, actual_glob), (expected_filesystem, expected_glob) in zip(actual, expected):
self.assertTrue(isinstance(actual_filesystem, expected_filesystem))
self.assertEqual(actual_glob, expected_glob)
def requires(self):
return RangeHourly(of=self.dependency,
start=datetime.combine(self.date, time.min),
stop=datetime.combine(self.date + timedelta(days=1),
time.min))
