def test_bulk_complete_correctly_interfaced(self): class BulkCompleteDailyTask(luigi.Task): d = luigi.DateParameter() @classmethod def bulk_complete(self, parameter_tuples): return parameter_tuples[:-2] def output(self): raise RuntimeError( "Shouldn't get called while resolving deps via bulk_complete" ) task = RangeDaily(now=datetime_to_epoch(datetime.datetime(2015, 12, 1)), of='BulkCompleteDailyTask', start=datetime.date(2015, 11, 1), stop=datetime.date(2015, 12, 1)) expected = [ 'BulkCompleteDailyTask(d=2015-11-29)', 'BulkCompleteDailyTask(d=2015-11-30)', ] actual = [t.task_id for t in task.requires()] self.assertEqual(actual, expected)
def test_bulk_complete_of_params(self): class BulkCompleteDailyTask(luigi.Task): non_positional_arbitrary_argument = luigi.Parameter(default="whatever", positional=False, significant=False) d = luigi.DateParameter() arbitrary_argument = luigi.BoolParameter() @classmethod def bulk_complete(cls, parameter_tuples): ptuples = list(parameter_tuples) for t in map(cls, ptuples): assert t.arbitrary_argument return ptuples[:-2] def output(self): raise RuntimeError("Shouldn't get called while resolving deps via bulk_complete") task = RangeDaily(now=datetime_to_epoch(datetime.datetime(2015, 12, 1)), of=BulkCompleteDailyTask, of_params=dict(arbitrary_argument=True), start=datetime.date(2015, 11, 1), stop=datetime.date(2015, 12, 1)) expected = [ 'BulkCompleteDailyTask(d=2015-11-29, arbitrary_argument=True)', 'BulkCompleteDailyTask(d=2015-11-30, arbitrary_argument=True)', ] actual = [str(t) for t in task.requires()] self.assertEqual(actual, expected)
def test_param_name_with_inferred_fs(self): class MyTask(luigi.Task): some_non_range_param = luigi.Parameter(default='woo') date_param = luigi.DateParameter() def output(self): return MockTarget(self.date_param.strftime('/n2000y01a05n/%Y_%m-_-%daww/21mm%Hdara21/ooo')) range_task = RangeDaily(now=datetime_to_epoch(datetime.datetime(2015, 12, 2)), of=MyTask, start=datetime.date(2015, 12, 1), stop=datetime.date(2015, 12, 2), param_name='date_param') expected_task = MyTask('woo', datetime.date(2015, 12, 1)) self.assertEqual(expected_task, list(range_task._requires())[0])
def test_bulk_complete(self, mock_connect): mock_cursor = MockPostgresCursor( ['DummyPostgresImporter(date=2015-01-03)']) mock_connect.return_value.cursor.return_value = mock_cursor task = RangeDaily(of='DummyPostgresImporter', start=datetime.date(2015, 1, 2), now=datetime_to_epoch(datetime.datetime(2015, 1, 7))) actual = [t.task_id for t in task.requires()] self.assertEqual(actual, [ 'DummyPostgresImporter(date=2015-01-02)', 'DummyPostgresImporter(date=2015-01-04)', 'DummyPostgresImporter(date=2015-01-05)', 'DummyPostgresImporter(date=2015-01-06)', ]) self.assertFalse(task.complete())
def test_bulk_complete(self, mock_connect): mock_cursor = MockPostgresCursor( ['DummyPostgresQuery_2015_01_03_838e32a989']) mock_connect.return_value.cursor.return_value = mock_cursor task = RangeDaily(of=DummyPostgresQuery, start=datetime.date(2015, 1, 2), now=datetime_to_epoch(datetime.datetime(2015, 1, 7))) actual = [t.task_id for t in task.requires()] self.assertEqual(actual, [ 'DummyPostgresQuery_2015_01_02_3a0ec498ed', 'DummyPostgresQuery_2015_01_04_9c1d42ff62', 'DummyPostgresQuery_2015_01_05_0f90e52357', 'DummyPostgresQuery_2015_01_06_f91a47ec40', ]) self.assertFalse(task.complete())
def test_bulk_complete(self, mock_connect): mock_cursor = MockPostgresCursor([ 'DummyPostgresImporter(date=2015-01-03)' ]) mock_connect.return_value.cursor.return_value = mock_cursor task = RangeDaily(of='DummyPostgresImporter', start=datetime.date(2015, 1, 2), now=datetime_to_epoch(datetime.datetime(2015, 1, 7))) actual = [t.task_id for t in task.requires()] self.assertEqual(actual, [ 'DummyPostgresImporter(date=2015-01-02)', 'DummyPostgresImporter(date=2015-01-04)', 'DummyPostgresImporter(date=2015-01-05)', 'DummyPostgresImporter(date=2015-01-06)', ]) self.assertFalse(task.complete())
def test_bulk_complete(self, mock_connect): mock_cursor = MockPostgresCursor([ DummyPostgresImporter(date=datetime.datetime(2015, 1, 3)).task_id ]) mock_connect.return_value.cursor.return_value = mock_cursor task = RangeDaily(of=DummyPostgresImporter, start=datetime.date(2015, 1, 2), now=datetime_to_epoch(datetime.datetime(2015, 1, 7))) actual = sorted([t.task_id for t in task.requires()]) self.assertEqual(actual, sorted([ DummyPostgresImporter(date=datetime.datetime(2015, 1, 2)).task_id, DummyPostgresImporter(date=datetime.datetime(2015, 1, 4)).task_id, DummyPostgresImporter(date=datetime.datetime(2015, 1, 5)).task_id, DummyPostgresImporter(date=datetime.datetime(2015, 1, 6)).task_id, ])) self.assertFalse(task.complete())
def test_bulk_complete(self, mock_connect): mock_cursor = MockPostgresCursor([ 'DummyPostgresQuery_2015_01_03_838e32a989' ]) mock_connect.return_value.cursor.return_value = mock_cursor task = RangeDaily(of=DummyPostgresQuery, start=datetime.date(2015, 1, 2), now=datetime_to_epoch(datetime.datetime(2015, 1, 7))) actual = [t.task_id for t in task.requires()] self.assertEqual(actual, [ 'DummyPostgresQuery_2015_01_02_3a0ec498ed', 'DummyPostgresQuery_2015_01_04_9c1d42ff62', 'DummyPostgresQuery_2015_01_05_0f90e52357', 'DummyPostgresQuery_2015_01_06_f91a47ec40', ]) self.assertFalse(task.complete())
def test_missing_tasks_correctly_required(self): class SomeDailyTask(luigi.Task): d = luigi.DateParameter() def output(self): return MockTarget(self.d.strftime('/data/2014/p/v/z/%Y_/_%m-_-%doctor/20/ZOOO')) task = RangeDaily(now=datetime_to_epoch(datetime.datetime(2016, 4, 1)), of=SomeDailyTask, start=datetime.date(2014, 3, 20), task_limit=3, days_back=3 * 365) expected = [ 'SomeDailyTask(d=2014-03-20)', 'SomeDailyTask(d=2014-03-22)', 'SomeDailyTask(d=2014-03-25)', ] actual = [t.task_id for t in task.requires()] self.assertEqual(actual, expected)
def test_of_param_distinction(self): class MyTask(luigi.Task): arbitrary_param = luigi.Parameter(default='foo') arbitrary_integer_param = luigi.IntParameter(default=10) date_param = luigi.DateParameter() def complete(self): return False range_task_1 = RangeDaily(now=datetime_to_epoch(datetime.datetime(2015, 12, 2)), of=MyTask, start=datetime.date(2015, 12, 1), stop=datetime.date(2015, 12, 2)) range_task_2 = RangeDaily(now=datetime_to_epoch(datetime.datetime(2015, 12, 2)), of=MyTask, of_params=dict(arbitrary_param="bar", abitrary_integer_param=2), start=datetime.date(2015, 12, 1), stop=datetime.date(2015, 12, 2)) self.assertNotEqual(range_task_1.task_id, range_task_2.task_id)
def test_missing_tasks_correctly_required(self): class SomeDailyTask(luigi.Task): d = luigi.DateParameter() def output(self): return MockTarget(self.d.strftime('/data/2014/p/v/z/%Y_/_%m-_-%doctor/20/ZOOO')) task = RangeDaily(now=datetime_to_epoch(datetime.datetime(2016, 4, 1)), of=SomeDailyTask, start=datetime.date(2014, 3, 20), task_limit=3, days_back=3 * 365) expected = [ 'SomeDailyTask(d=2014-03-20)', 'SomeDailyTask(d=2014-03-22)', 'SomeDailyTask(d=2014-03-25)', ] actual = [str(t) for t in task.requires()] self.assertEqual(actual, expected)
def test_bulk_complete_correctly_interfaced(self): class BulkCompleteDailyTask(luigi.Task): d = luigi.DateParameter() @classmethod def bulk_complete(self, parameter_tuples): return list(parameter_tuples)[:-2] def output(self): raise RuntimeError("Shouldn't get called while resolving deps via bulk_complete") task = RangeDaily(now=datetime_to_epoch(datetime.datetime(2015, 12, 1)), of=BulkCompleteDailyTask, start=datetime.date(2015, 11, 1), stop=datetime.date(2015, 12, 1)) expected = [ 'BulkCompleteDailyTask(d=2015-11-29)', 'BulkCompleteDailyTask(d=2015-11-30)', ] actual = [t.task_id for t in task.requires()] self.assertEqual(actual, expected)