def registry_dispatcher_document(self, code, collection):
"""
This task receive a list of codes that should be queued for DOI registry
"""
articlemeta = ThriftClient(domain=ARTICLEMETA_THRIFTSERVER)
document = articlemeta.document(code, collection)
code = '_'.join([document.collection_acronym, document.publisher_id])
log_title = 'Reading document: %s' % code
logger.info(log_title)
xml_file_name = '%s.xml' % code
doi = document.doi or ''
doi_prefix = document.doi.split('/')[0] if doi else ''
now = datetime.now()
if SUGGEST_DOI_IDENTIFICATION is True and not doi:
doi_prefix = CROSSREF_PREFIX
doi = '/'.join([
CROSSREF_PREFIX, document.publisher_ahead_id
or document.publisher_id
])
depitem = Deposit(code=code,
pid=document.publisher_id,
issn=document.journal.scielo_issn,
volume=document.issue.volume,
number=document.issue.number,
issue_label=document.issue.label,
journal=document.journal.title,
journal_acronym=document.journal.acronym,
collection_acronym=document.collection_acronym,
xml_file_name=xml_file_name,
doi=doi,
publication_year=int(document.publication_date[0:4]),
prefix=doi_prefix,
has_submission_xml_valid_references=False,
submission_updated_at=now,
submission_status='waiting',
updated_at=now,
started_at=now)
with transactional_session() as session:
deposit = session.query(Deposit).filter_by(code=code).first()
if deposit:
logger.info(
'deposit already exists. it will be deleted and '
're-created: "%s"', code)
session.delete(deposit)
session.add(depitem)
logger.info('deposit successfuly created for "%s": %s', code,
repr(deposit))
chain(
triage_deposit.s(code).set(queue='dispatcher'),
load_xml_from_articlemeta.s().set(queue='dispatcher'),
prepare_document.s().set(queue='dispatcher'),
register_doi.s().set(queue='dispatcher'),
request_doi_status.s().set(queue='releaser')).delay()
def differential_mode(self):
art_meta = ThriftClient()
logger.info("Running with differential mode")
ind_ids = set()
art_ids = set()
# all ids in search index
logger.info("Loading Search Index ids.")
itens_query = []
if self.collection:
itens_query.append('in:%s' % self.collection)
if self.issn:
itens_query.append('issn:%s' % self.issn)
query = '*:*' if len(itens_query) == 0 else ' AND '.join(itens_query)
list_ids = json.loads(self.solr.select(
{'q': query, 'fl': 'id,scielo_processing_date', 'rows': 1000000}))['response']['docs']
for id in list_ids:
ind_ids.add('%s-%s' % (id['id'], id.get('scielo_processing_date', '1900-01-01')))
# all ids in articlemeta
logger.info("Loading ArticleMeta ids.")
for item in art_meta.documents(
collection=self.collection,
issn=self.issn,
only_identifiers=True
):
art_ids.add('%s-%s-%s' % (item.code, item.collection, item.processing_date))
# Ids to remove
if self.delete is True:
logger.info("Running remove records process.")
remove_ids = set([i[:27] for i in ind_ids]) - set([i[:27] for i in art_ids])
logger.info("Removing (%d) documents from search index." % len(remove_ids))
total_to_remove = len(remove_ids)
if total_to_remove > 0:
for ndx, to_remove_id in enumerate(remove_ids, 1):
logger.debug("Removing (%d/%d): %s" % (ndx, total_to_remove, to_remove_id))
self.solr.delete('id:%s' % to_remove_id, commit=False)
# Ids to include
logger.info("Running include records process.")
include_ids = art_ids - ind_ids
logger.info("Including (%d) documents to search index." % len(include_ids))
total_to_include = len(include_ids)
if total_to_include > 0:
for ndx, to_include_id in enumerate(include_ids, 1):
logger.debug("Including (%d/%d): %s" % (ndx, total_to_include, to_include_id))
code = to_include_id[:23]
collection = to_include_id[24: 27]
processing_date = to_include_id[:-11]
document = art_meta.document(code=code, collection=collection)
try:
xml = self.pipeline_to_xml(document)
self.solr.update(xml, commit=False)
except ValueError as e:
logger.error("ValueError: {0}".format(e))
logger.exception(e)
continue
except Exception as e:
logger.error("Error: {0}".format(e))
logger.exception(e)
continue
def load_xml_from_articlemeta(self, code):
articlemeta = ThriftClient(domain=ARTICLEMETA_THRIFTSERVER)
exc_log_title = ''
with transactional_session() as session:
deposit = session.query(Deposit).filter_by(code=code).first()
log_title = 'Loading XML document from ArticleMeta (%s)' % code
log_event(
session, {
'title': log_title,
'type': 'submission',
'status': 'info',
'deposit_code': code
})
try:
xml = articlemeta.document(deposit.pid,
deposit.collection_acronym,
fmt='xmlcrossref')
except Exception as exc:
logger.info('could not fetch Crossref XML for "%s": %s', code,
str(exc))
logger.exception(exc)
deposit.submission_status = 'error'
deposit.submission_updated_at = datetime.now()
deposit.updated_at = datetime.now()
log_title = 'Fail to load XML document from ArticleMeta (%s)' % code
log_event(
session, {
'title': log_title,
'body': str(exc),
'type': 'submission',
'status': 'error',
'deposit_code': code
})
exc_log_title = log_title
else:
deposit.submission_status = 'waiting'
deposit.submission_xml = xml
deposit.submission_updated_at = datetime.now()
deposit.updated_at = datetime.now()
log_title = 'XML Document loaded from ArticleMeta (%s)' % code
log_event(
session, {
'title': log_title,
'type': 'submission',
'status': 'success',
'deposit_code': code
})
if exc_log_title:
raise self.retry(exc=ComunicationError(exc_log_title))
return code