def sync_data(self):
table = self.TABLE
LOGGER.info('Syncing data for {}'.format(table))
url = self.get_url()
params = self.get_params(_next=None)
resources = self.sync_paginated(url, params)
if self.CACHE_RESULTS:
stream_cache.add(table, resources)
LOGGER.info('Added {} {}s to cache'.format(len(resources), table))
LOGGER.info('Reached end of stream, moving on.')
save_state(self.state)
return self.state
def sync_data(self):
table = self.TABLE
date = get_last_record_value_for_table(self.state, table)
if date is None:
date = get_config_start_date(self.config)
interval = timedelta(days=7)
all_resources = []
while date < datetime.now(pytz.utc):
res = self.sync_data_for_period(date, interval)
all_resources.extend(res)
date = date + interval
if self.CACHE_RESULTS:
stream_cache.add(table, all_resources)
LOGGER.info('Added {} {}s to cache'.format(len(all_resources), table))
return self.state