def fetch_data(self, storage):
self.logger.info("Fetching data from Ckan at {0}".format(self.url))
client = CkanHighlevelClient(
self.url, api_key=self.conf.get('api_key'))
for dataset in client.iter_datasets():
self.logger.info("Dataset: {0}".format(dataset.id))
storage.set_object('dataset', dataset.id, dataset.serialize())
for group in client.iter_groups():
self.logger.info("Group: {0}".format(group.name))
storage.set_object('group', group.name, group.serialize())
for organization in client.iter_organizations():
self.logger.info("Organization: {0}".format(organization.name))
storage.set_object('organization', organization.name,
organization.serialize())
def download_and_print_ckan_datasets(ckan_url):
"""
Download and print datasets from ckan
"""
client = CkanHighlevelClient(ckan_url)
logger = logging.getLogger('ckan_crawl_demo')
logger.info('Starting function')
total = len(client.list_datasets())
current_app.report_progress(None, 0, total)
for cnt, dataset in enumerate(client.iter_datasets()):
logger.debug(repr(dataset))
current_app.report_progress(None, cnt + 1, total)
return total
