def test_query_with_date_interval(self):
task = LastCountryOfUserPartitionTask(**self._get_kwargs())
query = task.query()
expected_query = textwrap.dedent("""
USE default;
ALTER TABLE last_country_of_user ADD IF NOT EXISTS PARTITION (dt='2014-01-01');
""")
self.assertEquals(query, expected_query)
def test_query_with_date_interval(self):
task = LastCountryOfUserPartitionTask(**self._get_kwargs())
query = task.query()
expected_query = textwrap.dedent(
"""
USE default;
ALTER TABLE last_country_of_user ADD IF NOT EXISTS PARTITION (dt='2014-01-01');
"""
)
self.assertEquals(query, expected_query)
def requires(self):
return LastCountryOfUserPartitionTask(
mapreduce_engine=self.mapreduce_engine,
n_reduce_tasks=self.n_reduce_tasks,
source=self.source,
pattern=self.pattern,
interval=self.interval,
interval_start=self.interval_start,
interval_end=self.interval_end,
overwrite_n_days=self.overwrite_n_days,
geolocation_data=self.geolocation_data,
overwrite=self.overwrite,
)
def test_requires(self):
task = LastCountryOfUserPartitionTask(**self._get_kwargs())
required_task = list(task.requires())[0]
self.assertEquals(
required_task.output().path,
's3://fake/warehouse/last_country_of_user/dt=2014-01-01')
def test_overwrite(self):
kwargs = self._get_kwargs()
kwargs['overwrite'] = True
task = LastCountryOfUserPartitionTask(**kwargs)
self.assertFalse(task.complete())
def test_requires(self):
task = LastCountryOfUserPartitionTask(**self._get_kwargs())
required_task = list(task.requires())[0]
self.assertEquals(required_task.output().path, 's3://fake/warehouse/last_country_of_user/dt=2014-01-01')
def test_overwrite(self):
kwargs = self._get_kwargs()
kwargs['overwrite'] = True
task = LastCountryOfUserPartitionTask(**kwargs)
self.assertFalse(task.complete())