def test_bulk_complete_correctly_interfaced(self): class BulkCompleteHourlyTask(luigi.Task): dh = luigi.DateHourParameter() @classmethod def bulk_complete(cls, parameter_tuples): return parameter_tuples[:-2] def output(self): raise RuntimeError( "Shouldn't get called while resolving deps via bulk_complete" ) task = RangeHourly(now=datetime_to_epoch(datetime.datetime( 2015, 12, 1)), of='BulkCompleteHourlyTask', start=datetime.datetime(2015, 11, 1), stop=datetime.datetime(2015, 12, 1)) expected = [ 'BulkCompleteHourlyTask(dh=2015-11-30T22)', 'BulkCompleteHourlyTask(dh=2015-11-30T23)', ] actual = [t.task_id for t in task.requires()] self.assertEqual(actual, expected)
def test_bulk_complete_of_params(self): class BulkCompleteHourlyTask(luigi.Task): non_positional_arbitrary_argument = luigi.Parameter(default="whatever", positional=False, significant=False) dh = luigi.DateHourParameter() arbitrary_argument = luigi.BoolParameter() @classmethod def bulk_complete(cls, parameter_tuples): for t in map(cls, parameter_tuples): assert t.arbitrary_argument return parameter_tuples[:-2] def output(self): raise RuntimeError("Shouldn't get called while resolving deps via bulk_complete") task = RangeHourly(now=datetime_to_epoch(datetime.datetime(2015, 12, 1)), of=BulkCompleteHourlyTask, of_params=dict(arbitrary_argument=True), start=datetime.datetime(2015, 11, 1), stop=datetime.datetime(2015, 12, 1)) expected = [ 'BulkCompleteHourlyTask(dh=2015-11-30T22, arbitrary_argument=True)', 'BulkCompleteHourlyTask(dh=2015-11-30T23, arbitrary_argument=True)', ] actual = [str(t) for t in task.requires()] self.assertEqual(actual, expected)
def test_missing_wrapper_tasks_correctly_required(self): task = RangeHourly(now=datetime_to_epoch(datetime.datetime(2040, 4, 1)), of='CommonWrapperTask', start=datetime.datetime(2014, 3, 20, 23), stop=datetime.datetime(2014, 3, 21, 6), hours_back=30 * 365 * 24) actual = [t.task_id for t in task.requires()] self.assertEqual(actual, expected_wrapper)
def test_missing_tasks_correctly_required(self): task = RangeHourly(now=datetime_to_epoch(datetime.datetime(2040, 4, 1)), of='TaskA', start=datetime.datetime(2014, 3, 20, 17), task_limit=3, hours_back=30 * 365 * 24) # this test takes around a minute for me. Since stop is not defined, finite_datehours constitute many years to consider actual = [t.task_id for t in task.requires()] self.assertEqual(str(actual), str(expected_a)) self.assertEqual(actual, expected_a)
def test_missing_directory(self): task = RangeHourly(now=datetime_to_epoch(datetime.datetime(2014, 4, 1)), of=TaskC, start=datetime.datetime(2014, 3, 20, 23), stop=datetime.datetime(2014, 3, 21, 1)) self.assertFalse(task.complete()) expected = ['TaskC(dh=2014-03-20T23)', 'TaskC(dh=2014-03-21T00)'] self.assertEqual([str(t) for t in task.requires()], expected)
def test_missing_tasks_correctly_required(self): for task_path in task_a_paths: MockTarget(task_path) task = RangeHourly(now=datetime_to_epoch(datetime.datetime(2016, 4, 1)), of=TaskA, start=datetime.datetime(2014, 3, 20, 17), task_limit=3, hours_back=3 * 365 * 24) # this test takes a few seconds. Since stop is not defined, finite_datetimes constitute many years to consider actual = [t.task_id for t in task.requires()] self.assertEqual(actual, expected_a)
def test_missing_tasks_correctly_required(self): for task_path in task_a_paths: MockFile(task_path) task = RangeHourly(now=datetime_to_epoch(datetime.datetime(2016, 4, 1)), of='TaskA', start=datetime.datetime(2014, 3, 20, 17), task_limit=3, hours_back=3 * 365 * 24) # this test takes a few seconds. Since stop is not defined, finite_datetimes constitute many years to consider actual = [t.task_id for t in task.requires()] self.assertEqual(actual, expected_a)
def test_missing_directory(self): task = RangeHourly(now=datetime_to_epoch( datetime.datetime(2014, 4, 1)), of=TaskC, start=datetime.datetime(2014, 3, 20, 23), stop=datetime.datetime(2014, 3, 21, 1)) self.assertFalse(task.complete()) expected = [ 'TaskC(dh=2014-03-20T23)', 'TaskC(dh=2014-03-21T00)'] self.assertEqual([t.task_id for t in task.requires()], expected)
def _test_filesystems_and_globs(self, task_cls, expected): actual = list(RangeHourly._get_filesystems_and_globs(task_cls)) self.assertEqual(len(actual), len(expected)) for (actual_filesystem, actual_glob), (expected_filesystem, expected_glob) in zip(actual, expected): self.assertTrue(isinstance(actual_filesystem, expected_filesystem)) self.assertEqual(actual_glob, expected_glob)
def test_bulk_complete_correctly_interfaced(self): class BulkCompleteHourlyTask(luigi.Task): dh = luigi.DateHourParameter() @classmethod def bulk_complete(cls, parameter_tuples): return parameter_tuples[:-2] def output(self): raise RuntimeError("Shouldn't get called while resolving deps via bulk_complete") task = RangeHourly(now=datetime_to_epoch(datetime.datetime(2015, 12, 1)), of=BulkCompleteHourlyTask, start=datetime.datetime(2015, 11, 1), stop=datetime.datetime(2015, 12, 1)) expected = [ 'BulkCompleteHourlyTask(dh=2015-11-30T22)', 'BulkCompleteHourlyTask(dh=2015-11-30T23)', ] actual = [t.task_id for t in task.requires()] self.assertEqual(actual, expected)
def requires(self): return RangeHourly(of=self.dependency, start=datetime.combine(self.date, time.min), stop=datetime.combine(self.date + timedelta(days=1), time.min))