def test_requires(self): task = InsertToMysqlCourseEnrollByCountryWorkflow(**self._get_kwargs()) required_tasks = task.requires() self.assertEquals(len(required_tasks), 2) self.assertEquals(required_tasks['credentials'].output().path, 's3://config/credentials/output-database.json') self.assertEquals(required_tasks['insert_source'].output().path, 's3://output/course_country/path')
def test_requires_with_overwrite(self): kwargs = self._get_kwargs() kwargs['overwrite'] = True print kwargs task = InsertToMysqlCourseEnrollByCountryWorkflow(**kwargs) required_tasks = task.requires() print required_tasks query_task = required_tasks['insert_source'] self.assertTrue(query_task.overwrite)
def requires(self): yield ( LoadInternalReportingCountryTableHive( mapreduce_engine=self.mapreduce_engine, n_reduce_tasks=self.n_reduce_tasks, source=self.source, pattern=self.pattern, interval=self.interval, interval_start=self.interval_start, interval_end=self.interval_end, overwrite_n_days=self.overwrite_n_days, geolocation_data=self.geolocation_data, overwrite=self.overwrite, ), InsertToMysqlCourseEnrollByCountryWorkflow( mapreduce_engine=self.mapreduce_engine, n_reduce_tasks=self.n_reduce_tasks, source=self.source, pattern=self.pattern, interval=self.interval, interval_start=self.interval_start, interval_end=self.interval_end, overwrite_n_days=self.overwrite_n_days, geolocation_data=self.geolocation_data, overwrite=self.overwrite, course_country_output=self.course_country_output, ), )