def _mint_pid(obj, dummy_eng):
d = Deposition(obj)
recjson = d.get_latest_sip(sealed=False).metadata
if 'recid' not in recjson:
raise Exception("'recid' not found in sip metadata.")
pid_text = None
pid = recjson.get(pid_field, None)
if not pid:
# No pid found in recjson, so create new pid with user supplied
# function.
pid_text = recjson[pid_field] = pid_creator(recjson)
else:
# Pid found - check if it should be minted
if existing_pid_checker and existing_pid_checker(pid, recjson):
pid_text = pid
# Create an assign pid internally - actually registration will happen
# asynchronously later.
if pid_text:
current_app.logger.info("Registering pid %s" % pid_text)
pid_obj = PersistentIdentifier.create(pid_store_type, pid_text)
if pid_obj is None:
pid_obj = PersistentIdentifier.get(pid_store_type, pid_text)
try:
pid_obj.assign("rec", recjson['recid'])
except Exception:
register_exception(alert_admin=True)
d.update()
def _mint_pid(obj, dummy_eng):
d = Deposition(obj)
recjson = d.get_latest_sip(sealed=False).metadata
if 'recid' not in recjson:
raise Exception("'recid' not found in sip metadata.")
pid_text = None
pid = recjson.get(pid_field, None)
if not pid:
# No pid found in recjson, so create new pid with user supplied
# function.
pid_text = recjson[pid_field] = pid_creator(recjson)
else:
# Pid found - check if it should be minted
if existing_pid_checker and existing_pid_checker(pid, recjson):
pid_text = pid
# Create an assign pid internally - actually registration will happen
# asynchronously later.
if pid_text:
current_app.logger.info("Registering pid %s" % pid_text)
pid_obj = PersistentIdentifier.create(pid_store_type, pid_text)
if pid_obj is None:
pid_obj = PersistentIdentifier.get(pid_store_type, pid_text)
try:
pid_obj.assign("rec", recjson['recid'])
except Exception:
register_exception(alert_admin=True)
d.update()
def openaire_delete_doi(recid):
"""
Delete DOI in DataCite
If it fails, it will retry every 10 minutes for 1 hour.
"""
doi_val = get_fieldvalues(recid, "0247_a")[0]
logger.debug("Found DOI %s in record %s" % (doi_val, recid))
pid = PersistentIdentifier.get("doi", doi_val)
if not pid:
logger.debug("DOI not locally managed.")
return
else:
logger.debug("DOI locally managed.")
if not pid.has_object("rec", recid):
raise Exception(
"DOI %s is not assigned to record %s." % (doi_val, recid))
if pid.is_registered():
logger.info("Inactivating DOI %s for record %s" % (doi_val, recid))
if not pid.delete():
m = "Failed to inactive DOI %s" % doi_val
logger.error(m)
if not openaire_delete_doi.request.is_eager:
raise openaire_delete_doi.retry(exc=Exception(m))
else:
logger.info("Successfully inactivated DOI %s." % doi_val)
def doi_badge(doi):
"""Generate a badge for a specific DOI."""
pid = PersistentIdentifier.get("doi", doi)
if pid is None:
return abort(404)
return badge(doi)
def doi_action(record):
if record.get('doi', None) is not None:
return json_error(400, 'Record already has a DOI.')
recid = record['recid']
doi = build_doi(recid)
current_app.logger.info("Registering pid %s" % doi)
pid = PersistentIdentifier.create(DOI_PID_TYPE, doi)
if pid is None:
pid = PersistentIdentifier.get(DOI_PID_TYPE, doi)
try:
pid.assign('rec', recid)
datacite_register.delay(recid)
except Exception, e:
register_exception(alert_admin=True)
return json_error(400, '%s' % e)
def reserve_recid(record, *args, **kwargs):
"""Reserve a new record id for the record and set it inside."""
if record.get('recid', None) is None:
pid = PersistentIdentifier.create('recid', pid_value=None,
pid_provider='invenio')
pid.reserve()
record['recid'] = int(pid.pid_value)
def openaire_register_doi(recid):
"""
Register a DOI for new publication
If it fails, it will retry every 10 minutes for 1 hour.
"""
doi_val = get_fieldvalues(recid, "0247_a")[0]
logger.debug("Found DOI %s in record %s" % (doi_val, recid))
pid = PersistentIdentifier.get("doi", doi_val)
if not pid:
logger.debug("DOI not locally managed.")
return
else:
logger.debug("DOI locally managed.")
if not pid.has_object("rec", recid):
raise Exception(
"DOI %s is not assigned to record %s." % (doi_val, recid))
if pid.is_new() or pid.is_reserved():
logger.info("Registering DOI %s for record %s" % (doi_val, recid))
url = "%s/record/%s" % (CFG_DATACITE_SITE_URL, recid)
doc = format_record(recid, DEPOSIT_DATACITE_OF)
if not pid.register(url=url, doc=doc):
m = "Failed to register DOI %s" % doi_val
logger.error(m + "\n%s\n%s" % (url, doc))
if not openaire_register_doi.request.is_eager:
raise openaire_register_doi.retry(exc=Exception(m))
else:
logger.info("Successfully registered DOI %s." % doi_val)
def openaire_delete_doi(recid):
"""
Delete DOI in DataCite
If it fails, it will retry every 10 minutes for 1 hour.
"""
doi_val = get_fieldvalues(recid, "0247_a")[0]
logger.debug("Found DOI %s in record %s" % (doi_val, recid))
pid = PersistentIdentifier.get("doi", doi_val)
if not pid:
logger.debug("DOI not locally managed.")
return
else:
logger.debug("DOI locally managed.")
if not pid.has_object("rec", recid):
raise Exception(
"DOI %s is not assigned to record %s." % (doi_val, recid))
if pid.is_registered():
logger.info("Inactivating DOI %s for record %s" % (doi_val, recid))
if not pid.delete():
m = "Failed to inactive DOI %s" % doi_val
logger.error(m)
if not openaire_delete_doi.request.is_eager:
raise openaire_delete_doi.retry(exc=Exception(m))
else:
logger.info("Successfully inactivated DOI %s." % doi_val)
def openaire_register_doi(recid):
"""
Register a DOI for new publication
If it fails, it will retry every 10 minutes for 1 hour.
"""
doi_val = get_fieldvalues(recid, "0247_a")[0]
logger.debug("Found DOI %s in record %s" % (doi_val, recid))
pid = PersistentIdentifier.get("doi", doi_val)
if not pid:
logger.debug("DOI not locally managed.")
return
else:
logger.debug("DOI locally managed.")
if not pid.has_object("rec", recid):
raise Exception(
"DOI %s is not assigned to record %s." % (doi_val, recid))
if pid.is_new() or pid.is_reserved():
logger.info("Registering DOI %s for record %s" % (doi_val, recid))
url = "%s/record/%s" % (CFG_DATACITE_SITE_URL, recid)
doc = format_record(recid, DEPOSIT_DATACITE_OF)
if not pid.register(url=url, doc=doc):
m = "Failed to register DOI %s" % doi_val
logger.error(m + "\n%s\n%s" % (url, doc))
if not openaire_register_doi.request.is_eager:
raise openaire_register_doi.retry(exc=Exception(m))
else:
logger.info("Successfully registered DOI %s." % doi_val)
def _retrieve_record_id_from_pids(obj, eng):
record = obj[1]
eng.log.info('Look for PIDs inside the current record')
if not hasattr(record, 'persistent_identifiers'):
raise UploaderWorkflowException(
step, msg="An 'persistent_identifiers' method is needed")
for pid_name, pid_values in record.persistent_identifiers:
eng.log.info("Found PID '%s' trying to match it", (pid_name, ))
matching_recids = set()
for possible_pid in pid_values:
eng.log.info("Looking for PID %s", (possible_pid, ))
pid = PersistentIdentifier.get(possible_pid.get('type'),
possible_pid.get('value'),
possible_pid.get('provider'))
if pid:
eng.log.info("PID found in the data base %s",
(pid.object_value, ))
matching_recids.add(pid.object_value)
if len(matching_recids) > 1:
raise UploaderWorkflowException(
step,
msg="Found multiple match in the database, %s "
"for '%s'" % (repr(matching_recids), pid_name))
elif matching_recids:
record['recid'] = matching_recids.pop()
eng.log.info(
'Finish looking for PIDs inside the current record')
break
eng.log.info('Finish looking for PIDs inside the current record')
def update_pidstore(record, *args, **kwargs):
"""Save each PID present in the record to the PID storage."""
if not hasattr(record, 'persistent_identifiers'):
return
for pid_name, pid_values in record.persistent_identifiers:
for pid_value in pid_values:
pid = PersistentIdentifier.get(pid_value.get('type'),
pid_value.get('value'),
pid_value.get('provider'))
if pid is None:
pid = PersistentIdentifier.create(pid_value.get('type'),
pid_value.get('value'),
pid_value.get('provider'))
if not pid.has_object('rec', record['recid']):
pid.assign('rec', record['recid'])
def update_pidstore(record, *args, **kwargs):
"""Save each PID present in the record to the PID storage."""
if not hasattr(record, 'persistent_identifiers'):
return
for pid_name, pid_values in record.persistent_identifiers:
for pid_value in pid_values:
pid = PersistentIdentifier.get(
pid_value.get('type'), pid_value.get('value'),
pid_value.get('provider'))
if pid is None:
pid = PersistentIdentifier.create(
pid_value.get('type'), pid_value.get('value'),
pid_value.get('provider'))
if not pid.has_object('rec', record['recid']):
pid.assign('rec', record['recid'])
def doi_badge(doi):
"""Generate a badge for a specific DOI."""
pid = PersistentIdentifier.get("doi", doi)
if pid is None:
return abort(404)
return badge(doi)
def _retrieve_record_id_from_pids(obj, eng):
record = obj[1]
eng.log.info('Look for PIDs inside the current record')
if not hasattr(record, 'persistent_identifiers'):
raise UploaderWorkflowException(
step, msg="An 'persistent_identifiers' method is needed")
for pid_name, pid_values in record.persistent_identifiers:
eng.log.info("Found PID '%s' trying to match it", (pid_name, ))
matching_recids = set()
for possible_pid in pid_values:
eng.log.info("Looking for PID %s", (possible_pid, ))
pid = PersistentIdentifier.get(
possible_pid.get('type'), possible_pid.get('value'),
possible_pid.get('provider'))
if pid:
eng.log.info("PID found in the data base %s",
(pid.object_value, ))
matching_recids.add(pid.object_value)
if len(matching_recids) > 1:
raise UploaderWorkflowException(
step, msg="Found multiple match in the database, %s "
"for '%s'" % (repr(matching_recids), pid_name))
elif matching_recids:
record['recid'] = matching_recids.pop()
eng.log.info(
'Finish looking for PIDs inside the current record')
break
eng.log.info('Finish looking for PIDs inside the current record')
def reserve_recid(record, *args, **kwargs):
"""Reserve a new record id for the record and set it inside."""
if record.get('recid', None) is None:
pid = PersistentIdentifier.create('recid',
pid_value=None,
pid_provider='invenio')
pid.reserve()
record['recid'] = int(pid.pid_value)
def doi_badge(doi):
"""Generate a badge for a specific DOI."""
pid = PersistentIdentifier.get("doi", doi)
if pid is None:
return abort(404)
style = request.args.get('style', None)
return badge(doi, style)
def doi_badge(doi):
"""Generate a badge for a specific DOI."""
pid = PersistentIdentifier.get("doi", doi)
if pid is None:
return abort(404)
style = request.args.get('style', None)
return badge(doi, style)
def setUp(self):
self.app.config['CFG_DATACITE_URL'] = 'https://mds.example.org/'
self.app.config['CFG_DATACITE_USERNAME'] = '******'
self.app.config['CFG_DATACITE_PASSWORD'] = '******'
self.app.config['CFG_DATACITE_DOI_PREFIX'] = "10.1234"
from invenio.modules.pidstore.models import PersistentIdentifier
self.pid = PersistentIdentifier.create("doi", "10.1234/invenio.1234")
if not self.pid:
raise RuntimeError("Dirty database state.")
def setUp(self):
self.app.config['CFG_DATACITE_URL'] = 'https://mds.example.org/'
self.app.config['CFG_DATACITE_USERNAME'] = '******'
self.app.config['CFG_DATACITE_PASSWORD'] = '******'
self.app.config['CFG_DATACITE_DOI_PREFIX'] = "10.1234"
from invenio.modules.pidstore.models import PersistentIdentifier
self.pid = PersistentIdentifier.create("doi", "10.1234/invenio.1234")
if not self.pid:
raise RuntimeError("Dirty database state.")
def test_sync_registered(self, get_record_patch):
self.patch_get_record(get_record_patch)
httpretty.register_uri(
httpretty.GET,
"https://mds.example.org/doi/10.1234/invenio.1234",
body='http://invenio-software.org/record/1234',
status=200)
from invenio.modules.pidstore.tasks import datacite_sync
from invenio.modules.pidstore.models import PersistentIdentifier
pid = PersistentIdentifier.get("doi", "10.1234/invenio.1234")
self.assertEqual(pid.status, self.app.config['PIDSTORE_STATUS_NEW'])
datacite_sync(1)
pid = PersistentIdentifier.get("doi", "10.1234/invenio.1234")
self.assertEqual(pid.status,
self.app.config['PIDSTORE_STATUS_REGISTERED'])
def _reserve_record_id(obj, eng):
record = obj[1]
eng.log.info('Reserve a recid for the new record')
try:
pid = PersistentIdentifier.create('recid', pid_value=None,
pid_provider='invenio')
record['recid'] = int(pid.pid_value)
pid.reserve()
eng.log.info("Finish reserving a recid '%s' for the new record",
(pid.pid_value, ))
except Exception as e:
raise UploaderWorkflowException(step, e.message)
def test_sync_registered(self, get_record_patch):
self.patch_get_record(get_record_patch)
httpretty.register_uri(
httpretty.GET,
"https://mds.example.org/doi/10.1234/invenio.1234",
body='http://invenio-software.org/record/1234',
status=200
)
from invenio.modules.pidstore.tasks import datacite_sync
from invenio.modules.pidstore.models import PersistentIdentifier
pid = PersistentIdentifier.get("doi", "10.1234/invenio.1234")
self.assertEqual(pid.status, self.app.config['PIDSTORE_STATUS_NEW'])
datacite_sync(1)
pid = PersistentIdentifier.get("doi", "10.1234/invenio.1234")
self.assertEqual(pid.status,
self.app.config['PIDSTORE_STATUS_REGISTERED'])
def _reserve_record_id(obj, eng):
record = obj[1]
eng.log.info('Reserve a recid for the new record')
try:
pid = PersistentIdentifier.create('recid', pid_value=None,
pid_provider='invenio')
record['recid'] = int(pid.pid_value)
pid.reserve()
eng.log.info("Finish reserving a recid '%s' for the new record",
(pid.pid_value, ))
except Exception as e:
raise UploaderWorkflowException(step, e.message)
def _update_pidstore(obj, eng):
record = obj[1]
eng.log.info('Look for PIDs inside the current record and register '
'them in the DB')
if not hasattr(record, 'persistent_identifiers'):
raise UploaderWorkflowException(
step, msg="An 'persistent_identifiers' method is needed")
eng.log.info("Found PIDs '%s'", (record.persistent_identifiers, ))
for pid_name, pid_values in record.persistent_identifiers:
eng.log.info("Found PID '%s'", (pid_name, ))
for pid_value in pid_values:
pid = PersistentIdentifier.get(pid_value.get('type'),
pid_value.get('value'),
pid_value.get('provider'))
if pid is None:
pid = PersistentIdentifier.create(
pid_value.get('type'), pid_value.get('value'),
pid_value.get('provider'))
if not pid.has_object('rec', record['recid']):
pid.assign('rec', record['recid'])
eng.log.info('Finish looking for PIDs inside the current record and '
'register them in the DB')
def _update_pidstore(obj, eng):
record = obj[1]
eng.log.info('Look for PIDs inside the current record and register '
'them in the DB')
if not hasattr(record, 'persistent_identifiers'):
raise UploaderWorkflowException(
step, msg="An 'persistent_identifiers' method is needed")
eng.log.info("Found PIDs '%s'", (record.persistent_identifiers, ))
for pid_name, pid_values in record.persistent_identifiers:
eng.log.info("Found PID '%s'", (pid_name, ))
for pid_value in pid_values:
pid = PersistentIdentifier.get(
pid_value.get('type'), pid_value.get('value'),
pid_value.get('provider'))
if pid is None:
pid = PersistentIdentifier.create(
pid_value.get('type'), pid_value.get('value'),
pid_value.get('provider'))
if not pid.has_object('rec', record['recid']):
pid.assign('rec', record['recid'])
eng.log.info('Finish looking for PIDs inside the current record and '
'register them in the DB')
def doi_consistency_check():
"""Check if all DOIs in records have been registered."""
from invenio.base.globals import cfg
from invenio.modules.records.api import get_record
from invenio.legacy.search_engine import perform_request_search
from invenio.modules.pidstore.models import PersistentIdentifier
from invenio.modules.pidstore.tasks import datacite_register
result = perform_request_search(p="")
for r in result:
doi = get_record(r).get('doi')
if doi and doi.startswith(cfg['CFG_DATACITE_DOI_PREFIX']):
pid = PersistentIdentifier.get("doi", doi)
if pid and pid.is_new():
if pid.has_object("rec", r):
datacite_register.delay(r)
