def test_bulk_complete_correctly_interfaced(self):
class BulkCompleteDailyTask(luigi.Task):
d = luigi.DateParameter()
@classmethod
def bulk_complete(self, parameter_tuples):
return parameter_tuples[:-2]
def output(self):
raise RuntimeError(
"Shouldn't get called while resolving deps via bulk_complete"
)
task = RangeDaily(now=datetime_to_epoch(datetime.datetime(2015, 12,
1)),
of='BulkCompleteDailyTask',
start=datetime.date(2015, 11, 1),
stop=datetime.date(2015, 12, 1))
expected = [
'BulkCompleteDailyTask(d=2015-11-29)',
'BulkCompleteDailyTask(d=2015-11-30)',
]
actual = [t.task_id for t in task.requires()]
self.assertEqual(actual, expected)
def test_bulk_complete_of_params(self):
class BulkCompleteDailyTask(luigi.Task):
non_positional_arbitrary_argument = luigi.Parameter(default="whatever", positional=False, significant=False)
d = luigi.DateParameter()
arbitrary_argument = luigi.BoolParameter()
@classmethod
def bulk_complete(cls, parameter_tuples):
ptuples = list(parameter_tuples)
for t in map(cls, ptuples):
assert t.arbitrary_argument
return ptuples[:-2]
def output(self):
raise RuntimeError("Shouldn't get called while resolving deps via bulk_complete")
task = RangeDaily(now=datetime_to_epoch(datetime.datetime(2015, 12, 1)),
of=BulkCompleteDailyTask,
of_params=dict(arbitrary_argument=True),
start=datetime.date(2015, 11, 1),
stop=datetime.date(2015, 12, 1))
expected = [
'BulkCompleteDailyTask(d=2015-11-29, arbitrary_argument=True)',
'BulkCompleteDailyTask(d=2015-11-30, arbitrary_argument=True)',
]
actual = [str(t) for t in task.requires()]
self.assertEqual(actual, expected)
def test_param_name_with_inferred_fs(self):
class MyTask(luigi.Task):
some_non_range_param = luigi.Parameter(default='woo')
date_param = luigi.DateParameter()
def output(self):
return MockTarget(self.date_param.strftime('/n2000y01a05n/%Y_%m-_-%daww/21mm%Hdara21/ooo'))
range_task = RangeDaily(now=datetime_to_epoch(datetime.datetime(2015, 12, 2)),
of=MyTask,
start=datetime.date(2015, 12, 1),
stop=datetime.date(2015, 12, 2),
param_name='date_param')
expected_task = MyTask('woo', datetime.date(2015, 12, 1))
self.assertEqual(expected_task, list(range_task._requires())[0])
def test_param_name_with_inferred_fs(self):
class MyTask(luigi.Task):
some_non_range_param = luigi.Parameter(default='woo')
date_param = luigi.DateParameter()
def output(self):
return MockTarget(self.date_param.strftime('/n2000y01a05n/%Y_%m-_-%daww/21mm%Hdara21/ooo'))
range_task = RangeDaily(now=datetime_to_epoch(datetime.datetime(2015, 12, 2)),
of=MyTask,
start=datetime.date(2015, 12, 1),
stop=datetime.date(2015, 12, 2),
param_name='date_param')
expected_task = MyTask('woo', datetime.date(2015, 12, 1))
self.assertEqual(expected_task, list(range_task._requires())[0])
def test_bulk_complete(self, mock_connect):
mock_cursor = MockPostgresCursor(
['DummyPostgresImporter(date=2015-01-03)'])
mock_connect.return_value.cursor.return_value = mock_cursor
task = RangeDaily(of='DummyPostgresImporter',
start=datetime.date(2015, 1, 2),
now=datetime_to_epoch(datetime.datetime(2015, 1, 7)))
actual = [t.task_id for t in task.requires()]
self.assertEqual(actual, [
'DummyPostgresImporter(date=2015-01-02)',
'DummyPostgresImporter(date=2015-01-04)',
'DummyPostgresImporter(date=2015-01-05)',
'DummyPostgresImporter(date=2015-01-06)',
])
self.assertFalse(task.complete())
def test_bulk_complete(self, mock_connect):
mock_cursor = MockPostgresCursor(
['DummyPostgresQuery_2015_01_03_838e32a989'])
mock_connect.return_value.cursor.return_value = mock_cursor
task = RangeDaily(of=DummyPostgresQuery,
start=datetime.date(2015, 1, 2),
now=datetime_to_epoch(datetime.datetime(2015, 1, 7)))
actual = [t.task_id for t in task.requires()]
self.assertEqual(actual, [
'DummyPostgresQuery_2015_01_02_3a0ec498ed',
'DummyPostgresQuery_2015_01_04_9c1d42ff62',
'DummyPostgresQuery_2015_01_05_0f90e52357',
'DummyPostgresQuery_2015_01_06_f91a47ec40',
])
self.assertFalse(task.complete())
def test_bulk_complete(self, mock_connect):
mock_cursor = MockPostgresCursor([
'DummyPostgresImporter(date=2015-01-03)'
])
mock_connect.return_value.cursor.return_value = mock_cursor
task = RangeDaily(of='DummyPostgresImporter',
start=datetime.date(2015, 1, 2),
now=datetime_to_epoch(datetime.datetime(2015, 1, 7)))
actual = [t.task_id for t in task.requires()]
self.assertEqual(actual, [
'DummyPostgresImporter(date=2015-01-02)',
'DummyPostgresImporter(date=2015-01-04)',
'DummyPostgresImporter(date=2015-01-05)',
'DummyPostgresImporter(date=2015-01-06)',
])
self.assertFalse(task.complete())
def test_bulk_complete(self, mock_connect):
mock_cursor = MockPostgresCursor([
DummyPostgresImporter(date=datetime.datetime(2015, 1, 3)).task_id
])
mock_connect.return_value.cursor.return_value = mock_cursor
task = RangeDaily(of=DummyPostgresImporter,
start=datetime.date(2015, 1, 2),
now=datetime_to_epoch(datetime.datetime(2015, 1, 7)))
actual = sorted([t.task_id for t in task.requires()])
self.assertEqual(actual, sorted([
DummyPostgresImporter(date=datetime.datetime(2015, 1, 2)).task_id,
DummyPostgresImporter(date=datetime.datetime(2015, 1, 4)).task_id,
DummyPostgresImporter(date=datetime.datetime(2015, 1, 5)).task_id,
DummyPostgresImporter(date=datetime.datetime(2015, 1, 6)).task_id,
]))
self.assertFalse(task.complete())
def test_bulk_complete(self, mock_connect):
mock_cursor = MockPostgresCursor([
'DummyPostgresQuery_2015_01_03_838e32a989'
])
mock_connect.return_value.cursor.return_value = mock_cursor
task = RangeDaily(of=DummyPostgresQuery,
start=datetime.date(2015, 1, 2),
now=datetime_to_epoch(datetime.datetime(2015, 1, 7)))
actual = [t.task_id for t in task.requires()]
self.assertEqual(actual, [
'DummyPostgresQuery_2015_01_02_3a0ec498ed',
'DummyPostgresQuery_2015_01_04_9c1d42ff62',
'DummyPostgresQuery_2015_01_05_0f90e52357',
'DummyPostgresQuery_2015_01_06_f91a47ec40',
])
self.assertFalse(task.complete())
def test_bulk_complete(self, mock_connect):
mock_cursor = MockPostgresCursor([
DummyPostgresImporter(date=datetime.datetime(2015, 1, 3)).task_id
])
mock_connect.return_value.cursor.return_value = mock_cursor
task = RangeDaily(of=DummyPostgresImporter,
start=datetime.date(2015, 1, 2),
now=datetime_to_epoch(datetime.datetime(2015, 1, 7)))
actual = sorted([t.task_id for t in task.requires()])
self.assertEqual(actual, sorted([
DummyPostgresImporter(date=datetime.datetime(2015, 1, 2)).task_id,
DummyPostgresImporter(date=datetime.datetime(2015, 1, 4)).task_id,
DummyPostgresImporter(date=datetime.datetime(2015, 1, 5)).task_id,
DummyPostgresImporter(date=datetime.datetime(2015, 1, 6)).task_id,
]))
self.assertFalse(task.complete())
def test_missing_tasks_correctly_required(self):
class SomeDailyTask(luigi.Task):
d = luigi.DateParameter()
def output(self):
return MockTarget(self.d.strftime('/data/2014/p/v/z/%Y_/_%m-_-%doctor/20/ZOOO'))
task = RangeDaily(now=datetime_to_epoch(datetime.datetime(2016, 4, 1)),
of=SomeDailyTask,
start=datetime.date(2014, 3, 20),
task_limit=3,
days_back=3 * 365)
expected = [
'SomeDailyTask(d=2014-03-20)',
'SomeDailyTask(d=2014-03-22)',
'SomeDailyTask(d=2014-03-25)',
]
actual = [t.task_id for t in task.requires()]
self.assertEqual(actual, expected)
def test_of_param_distinction(self):
class MyTask(luigi.Task):
arbitrary_param = luigi.Parameter(default='foo')
arbitrary_integer_param = luigi.IntParameter(default=10)
date_param = luigi.DateParameter()
def complete(self):
return False
range_task_1 = RangeDaily(now=datetime_to_epoch(datetime.datetime(2015, 12, 2)),
of=MyTask,
start=datetime.date(2015, 12, 1),
stop=datetime.date(2015, 12, 2))
range_task_2 = RangeDaily(now=datetime_to_epoch(datetime.datetime(2015, 12, 2)),
of=MyTask,
of_params=dict(arbitrary_param="bar", abitrary_integer_param=2),
start=datetime.date(2015, 12, 1),
stop=datetime.date(2015, 12, 2))
self.assertNotEqual(range_task_1.task_id, range_task_2.task_id)
def test_missing_tasks_correctly_required(self):
class SomeDailyTask(luigi.Task):
d = luigi.DateParameter()
def output(self):
return MockTarget(self.d.strftime('/data/2014/p/v/z/%Y_/_%m-_-%doctor/20/ZOOO'))
task = RangeDaily(now=datetime_to_epoch(datetime.datetime(2016, 4, 1)),
of=SomeDailyTask,
start=datetime.date(2014, 3, 20),
task_limit=3,
days_back=3 * 365)
expected = [
'SomeDailyTask(d=2014-03-20)',
'SomeDailyTask(d=2014-03-22)',
'SomeDailyTask(d=2014-03-25)',
]
actual = [str(t) for t in task.requires()]
self.assertEqual(actual, expected)
def test_bulk_complete_correctly_interfaced(self):
class BulkCompleteDailyTask(luigi.Task):
d = luigi.DateParameter()
@classmethod
def bulk_complete(self, parameter_tuples):
return list(parameter_tuples)[:-2]
def output(self):
raise RuntimeError("Shouldn't get called while resolving deps via bulk_complete")
task = RangeDaily(now=datetime_to_epoch(datetime.datetime(2015, 12, 1)),
of=BulkCompleteDailyTask,
start=datetime.date(2015, 11, 1),
stop=datetime.date(2015, 12, 1))
expected = [
'BulkCompleteDailyTask(d=2015-11-29)',
'BulkCompleteDailyTask(d=2015-11-30)',
]
actual = [t.task_id for t in task.requires()]
self.assertEqual(actual, expected)