def test_requires_with_overwrite(self):
kwargs = self._get_kwargs()
kwargs['overwrite'] = True
task = InsertToMysqlLastCountryPerCourseTask(**kwargs)
required_tasks = task.requires()
query_task = required_tasks['insert_source']
self.assertTrue(query_task.overwrite)
def requires(self):
yield (
LoadInternalReportingCountryTableHive(
mapreduce_engine=self.mapreduce_engine,
n_reduce_tasks=self.n_reduce_tasks,
source=self.source,
pattern=self.pattern,
interval=self.interval,
interval_start=self.interval_start,
interval_end=self.interval_end,
overwrite_n_days=self.overwrite_n_days,
geolocation_data=self.geolocation_data,
overwrite=self.overwrite,
),
InsertToMysqlLastCountryPerCourseTask(
mapreduce_engine=self.mapreduce_engine,
n_reduce_tasks=self.n_reduce_tasks,
source=self.source,
pattern=self.pattern,
interval=self.interval,
interval_start=self.interval_start,
interval_end=self.interval_end,
overwrite_n_days=self.overwrite_n_days,
geolocation_data=self.geolocation_data,
overwrite=self.overwrite,
),
)
def test_requires(self):
task = InsertToMysqlLastCountryPerCourseTask(**self._get_kwargs())
required_tasks = dict(task.requires())
self.assertEquals(len(required_tasks), 2)
self.assertEquals(required_tasks['credentials'].output().path,
's3://fake/credentials.json')
self.assertEquals(
required_tasks['insert_source'].output().path,
's3://fake/warehouse/course_enrollment_location_current')