def __init__(self, ppn, from_date, until_date, collection='ddd', record_type='artikel'):
self.ppn = ppn
items_per_page = 100
self.query = query_template.format(**locals())
log.info('Start transferring articles. Collection: {collection}; Record type: {record_type}; '
'PPN: {ppn}; From: {from_date}; Until: {until_date}.'.format(**locals()))
def run(self):
"""Start the transfer process of articles from kranten.delpher.nl to amcat.vu.nl
"""
for result_page in self.delpher_api.result_pages():
articles = [DelpherToAmcat.convert_article(article) for article in result_page]
self.upload_to_amcat(articles)
log.info('Done. Processed all articles in set {self.set_id}.'.format(**locals()))
def list_next_articles(self):
"""Retrieve next page of search results
"""
url = self.results_url()
try:
response = request.get(url)
except:
log.exception('Could not get results for url {url}'.format(**locals()))
return []
self.number_of_records = response['numberOfRecords']
log.info('Page {self.page} of article list retrieved. '
'{self.records_processed} of {self.number_of_records} articles processed.'.format(**locals()))
return response['records']
def setup_amcat(self, from_date, until_date):
"""Create Amcat API object and save configuration data for later use.
"""
amcat_api = AmcatAPI(amcat.host, amcat.username, amcat.password)
log.info('Setup Amcat API with host {0}, username {1}. Use project {2}'.format(amcat.host, amcat.username,
amcat.project))
now = datetime.now().replace(microsecond=0)
set_name = amcat.set_name_template.format(**locals())
try:
aset = amcat_api.create_set(project=amcat.project, name=set_name, provenance=amcat.data_provenance)
except:
log.exception('Could not create article set')
raise
log.info('Created article set in Amcat. ID: {0}'.format(aset['id']))
self.set_id = aset['id']
return amcat_api
