def create_pids(cls, dump, deposit):
"""Create a persistent identifiers."""
# Mark deposit deleted if recid is deleted.
recid = dump.recid_pid
# Create depid
depid = PersistentIdentifier.create(pid_type='depid',
pid_value=str(dump.depid),
object_type='rec',
object_uuid=deposit.id,
status=PIDStatus.REGISTERED)
if recid and recid.status == PIDStatus.DELETED:
depid.delete()
if RecordIdentifier.query.get(dump.depid) is None:
RecordIdentifier.insert(dump.depid)
# Pre-reserved recid.
if not recid and dump.recid:
if dump.has_pid:
# Published deposit without a recid (this is an upload which
# never got ingested so we set it back to draft status and
# reserves the reid).
pass
recid = PersistentIdentifier.create(pid_type='recid',
pid_value=str(dump.recid),
status=PIDStatus.RESERVED)
if RecordIdentifier.query.get(dump.recid) is None:
RecordIdentifier.insert(dump.recid)
return depid, recid
def weko_deposit_minter(record_uuid, data, recid=None):
"""Weko deposit."""
if not recid:
id_ = RecordIdentifier.next()
else:
if isinstance(recid, int):
RecordIdentifier.insert(recid)
id_ = recid
recid = PersistentIdentifier.create('recid',
str(id_),
object_type='rec',
object_uuid=record_uuid,
status=PIDStatus.REGISTERED)
data['recid'] = str(recid.pid_value)
# Create depid with same pid_value of the recid
depid = PersistentIdentifier.create(
'depid',
str(recid.pid_value),
object_type='rec',
object_uuid=record_uuid,
status=PIDStatus.REGISTERED,
)
data.update({
'_deposit': {
'id': depid.pid_value,
'status': 'draft',
},
})
return depid
def create_record_and_pid(data):
"""Create the deposit record metadata and persistent identifier.
:param data: Raw JSON dump of the deposit.
:type data: dict
:returns: A deposit object and its pid
:rtype: (`invenio_records.api.Record`,
`invenio_pidstore.models.PersistentIdentifier`)
"""
from invenio_records.api import Record
from invenio_pidstore.models import PersistentIdentifier, PIDStatus
deposit = Record.create(data=data)
created = arrow.get(data['_p']['created']).datetime
deposit.model.created = created.replace(tzinfo=None)
depid = deposit['_p']['id']
pid = PersistentIdentifier.create(pid_type='depid',
pid_value=str(depid),
object_type='rec',
object_uuid=str(deposit.id),
status=PIDStatus.REGISTERED)
if RecordIdentifier.query.get(int(depid)) is None:
RecordIdentifier.insert(int(depid))
deposit.commit()
return deposit, pid
def create_pids(cls, dump, deposit):
"""Create a persistent identifiers."""
# Mark deposit deleted if recid is deleted.
recid = dump.recid_pid
# Create depid
depid = PersistentIdentifier.create(
pid_type='depid',
pid_value=str(dump.depid),
object_type='rec',
object_uuid=deposit.id,
status=PIDStatus.REGISTERED
)
if recid and recid.status == PIDStatus.DELETED:
depid.delete()
if RecordIdentifier.query.get(dump.depid) is None:
RecordIdentifier.insert(dump.depid)
# Pre-reserved recid.
if not recid and dump.recid:
if dump.has_pid:
# Published deposit without a recid (this is an upload which
# never got ingested so we set it back to draft status and
# reserves the reid).
pass
recid = PersistentIdentifier.create(
pid_type='recid',
pid_value=str(dump.recid),
status=PIDStatus.RESERVED
)
if RecordIdentifier.query.get(dump.recid) is None:
RecordIdentifier.insert(dump.recid)
return depid, recid
def minter(pid_type, pid_field, record):
"""Mint the given PID for the given record."""
PersistentIdentifier.create(
pid_type=pid_type,
pid_value=record[pid_field],
object_type="rec",
object_uuid=record.id,
status=PIDStatus.REGISTERED,
)
RecordIdentifier.next()
def record_db(db):
"""Load records json."""
id_ = uuid.uuid4()
record = Record.create({'title': 'Test record', 'recid': 11782}, id_=id_)
PersistentIdentifier.create(
pid_type='recid',
pid_value='11782',
status=PIDStatus.REGISTERED,
object_type='rec',
object_uuid=id_,
)
RecordIdentifier.insert(11782)
db.session.commit()
return record
def create_record(cls, dump):
"""Create a new record from dump."""
# Reserve record identifier, create record and recid pid in one
# operation.
timestamp, data = dump.pop_first_revision()
record = Record.create(data)
record.model.created = timestamp.replace(tzinfo=None)
RecordIdentifier.insert(dump.recid)
PersistentIdentifier.create(pid_type='recid',
pid_value=str(dump.recid),
object_type='rec',
object_uuid=str(record.id),
status=PIDStatus.REGISTERED)
db.session.commit()
return cls.update_record(dump, record=record)
def create(cls, object_type=None, object_uuid=None, **kwargs):
"""Create a new record identifier."""
# Request next integer in recid sequence.
assert 'pid_value' not in kwargs
provider_url = current_app.config.get('RECORDS_ID_PROVIDER_ENDPOINT',
None)
if not provider_url:
# Don't query external service in DEBUG mode
kwargs['pid_value'] = str(RecordIdentifier.next())
else:
response = requests.get(provider_url,
headers={'User-Agent': 'cernsearch'})
if not response.ok or response.text.strip().startswith('[ERROR]'):
raise PersistentIdentifierError(response.text)
kwargs['pid_value'] = response.text
kwargs.setdefault('status', cls.default_status)
if object_type and object_uuid:
kwargs['status'] = PIDStatus.REGISTERED
return super(CERNSearchRecordIdProvider,
cls).create(object_type=object_type,
object_uuid=object_uuid,
**kwargs)
def create(cls, object_type=None, object_uuid=None, **kwargs):
"""Create a new record identifier.
Note: if the object_type and object_uuid values are passed, then the
PID status will be automatically setted to
:attr:`invenio_pidstore.models.PIDStatus.REGISTERED`.
:param object_type: The object type. (Default: None.)
:param object_uuid: The object identifier. (Default: None).
:param kwargs: You specify the pid_value.
"""
# PID value not already exists, generating a new one
if not kwargs.get('pid_value'):
kwargs['pid_value'] = str(RecordIdentifier.next())
kwargs.setdefault('status', cls.default_status)
if object_type and object_uuid:
kwargs['status'] = PIDStatus.REGISTERED
try:
# Try to retreive PID
return cls.get(kwargs['pid_value'], cls.pid_type)
except PIDDoesNotExistError:
# Set default status
kwargs.setdefault('status', cls.default_status)
# if record is registered, change PID status
if object_type and object_uuid:
kwargs['status'] = PIDStatus.REGISTERED
# Call base provider
return super(Provider, cls).create(
object_type=object_type, object_uuid=object_uuid, **kwargs
)
def weko_deposit_minter(record_uuid, data):
"""Weko deposit."""
id_ = RecordIdentifier.next()
recid = PersistentIdentifier.create('recid',
str(id_),
object_type='rec',
object_uuid=record_uuid,
status=PIDStatus.REGISTERED)
# Create depid with same pid_value of the recid
depid = PersistentIdentifier.create(
'depid',
str(recid.pid_value),
object_type='rec',
object_uuid=record_uuid,
status=PIDStatus.REGISTERED,
)
data.update({
'_deposit': {
'id': depid.pid_value,
'status': 'draft',
},
})
return depid
def create(cls, object_type=None, object_uuid=None, **kwargs):
"""Create a new record identifier."""
# Request next integer in recid sequence.
assert 'pid_value' not in kwargs
if current_app.config.get('DEBUG'):
# Don't query external service in DEBUG mode
kwargs['pid_value'] = str(RecordIdentifier.next())
else:
response = requests.get(
current_app.config['RECORDS_ID_PROVIDER_ENDPOINT'],
headers={
'User-Agent': 'cdslabs'
}).text
if response.strip().lower().startswith('[error]'):
raise PersistentIdentifierError(response)
kwargs['pid_value'] = response
kwargs.setdefault('status', cls.default_status)
if object_type and object_uuid:
kwargs['status'] = PIDStatus.REGISTERED
return super(CDSRecordIdProvider, cls).create(object_type=object_type,
object_uuid=object_uuid,
**kwargs)
def create_record(cls, dump):
"""Create a new record from dump."""
# Reserve record identifier, create record and recid pid in one
# operation.
timestamp, data = dump.pop_first_revision()
record = Record.create(data)
record.model.created = timestamp.replace(tzinfo=None)
RecordIdentifier.insert(dump.recid)
PersistentIdentifier.create(
pid_type='recid',
pid_value=str(dump.recid),
object_type='rec',
object_uuid=str(record.id),
status=PIDStatus.REGISTERED
)
db.session.commit()
return cls.update_record(dump, record=record)
def create(cls, object_type=None, object_uuid=None, **kwargs):
"""Create a new record identifier."""
if "pid_value" not in kwargs:
if current_app.config.get("LEGACY_PID_PROVIDER"):
kwargs["pid_value"] = get_next_pid_from_legacy()
RecordIdentifier.insert(kwargs["pid_value"])
else:
kwargs["pid_value"] = RecordIdentifier.next()
else:
RecordIdentifier.insert(kwargs["pid_value"])
kwargs.setdefault("status", cls.default_status)
if object_type and object_uuid:
kwargs["status"] = PIDStatus.REGISTERED
return super(InspireRecordIdProvider, cls).create(
object_type=object_type, object_uuid=object_uuid, **kwargs
)
def create(cls, object_type=None, object_uuid=None, **kwargs):
"""Create a new record identifier."""
# Request next integer in recid sequence.
if 'pid_value' not in kwargs:
if current_app.config.get('LEGACY_PID_PROVIDER'):
kwargs['pid_value'] = _get_next_pid_from_legacy()
RecordIdentifier.insert(kwargs['pid_value'])
else:
kwargs['pid_value'] = RecordIdentifier.next()
else:
RecordIdentifier.insert(kwargs['pid_value'])
kwargs.setdefault('status', cls.default_status)
if object_type and object_uuid:
kwargs['status'] = PIDStatus.REGISTERED
return super(InspireRecordIdProvider, cls).create(
object_type=object_type, object_uuid=object_uuid, **kwargs)
def record_db(db):
"""Load records json."""
id_ = uuid.uuid4()
record = Record.create({
'title': 'Test record',
'recid': 11782
}, id_=id_)
PersistentIdentifier.create(
pid_type='recid',
pid_value='11782',
status=PIDStatus.REGISTERED,
object_type='rec',
object_uuid=id_,
)
RecordIdentifier.insert(11782)
db.session.commit()
return record
def create(cls, object_type=None, object_uuid=None, **kwargs):
# Request next integer in recid sequence.
kwargs['pid_value'] = str(RecordIdentifier.next())
kwargs.setdefault('status', cls.default_status)
return super(RecordIdProvider,
cls).create(object_type=object_type,
object_uuid=object_uuid,
**kwargs)
def default_parent_minter(record_uuid, data, pid_type, object_type):
"""Basic RecordIdentifier-based minter for parent PIDs."""
parent_id = RecordIdentifier.next()
return PersistentIdentifier.create(
pid_type=pid_type,
pid_value=str(parent_id),
object_type=object_type,
status=PIDStatus.REGISTERED,
)
def zenodo_reserved_record_minter(record_uuid=None, data=None):
"""Reserve a recid."""
id_ = RecordIdentifier.next()
recid = PersistentIdentifier.create('recid',
str(id_),
status=PIDStatus.RESERVED)
data['recid'] = int(recid.pid_value)
return recid
def zenodo_reserved_record_minter(record_uuid=None, data=None):
"""Reserve a recid."""
id_ = RecordIdentifier.next()
recid = PersistentIdentifier.create(
'recid', id_, status=PIDStatus.RESERVED
)
data['recid'] = recid.pid_value
return recid
def test_record_resolution(app, db):
"""Test resolution of PIDs to records."""
# OK PID
pid_ok, record = create_record({'title': 'test'})
# Deleted PID
pid_del, record = create_record({'title': 'deleted'})
pid_del.delete()
# Missing object PID
pid_noobj = PersistentIdentifier.create(
'recid', str(RecordIdentifier.next()), status=PIDStatus.REGISTERED)
db.session.commit()
# Redirected PID
pid_red = PersistentIdentifier.create(
'recid', '101', status=PIDStatus.REGISTERED)
pid_red.redirect(pid_ok)
# Redirect PID - different endpoint
pid_doi = PersistentIdentifier.create(
'doi', '10.1234/foo', status=PIDStatus.REGISTERED)
pid_red_doi = PersistentIdentifier.create(
'recid', '102', status=PIDStatus.REGISTERED)
pid_red_doi.redirect(pid_doi)
db.session.commit()
headers = [('Accept', 'application/json')]
with app.test_client() as client:
# PID deleted
res = client.get(
url_for('invenio_records_rest.recid_item',
pid_value=pid_del.pid_value),
headers=headers)
assert res.status_code == 410
# PID missing object
res = client.get(
url_for('invenio_records_rest.recid_item',
pid_value=pid_noobj.pid_value),
headers=headers)
assert res.status_code == 500
# Redirected invalid endpoint
res = client.get(
url_for('invenio_records_rest.recid_item',
pid_value=pid_red_doi.pid_value),
headers=headers)
assert res.status_code == 500
# Redirected
res = client.get(
url_for('invenio_records_rest.recid_item',
pid_value=pid_red.pid_value),
headers=headers)
assert res.status_code == 301
def test_record_resolution(app, db):
"""Test resolution of PIDs to records."""
# OK PID
pid_ok, record = create_record({'title': 'test'})
# Deleted PID
pid_del, record = create_record({'title': 'deleted'})
pid_del.delete()
# Missing object PID
pid_noobj = PersistentIdentifier.create('recid',
str(RecordIdentifier.next()),
status=PIDStatus.REGISTERED)
db.session.commit()
# Redirected PID
pid_red = PersistentIdentifier.create('recid',
'101',
status=PIDStatus.REGISTERED)
pid_red.redirect(pid_ok)
# Redirect PID - different endpoint
pid_doi = PersistentIdentifier.create('doi',
'10.1234/foo',
status=PIDStatus.REGISTERED)
pid_red_doi = PersistentIdentifier.create('recid',
'102',
status=PIDStatus.REGISTERED)
pid_red_doi.redirect(pid_doi)
db.session.commit()
headers = [('Accept', 'application/json')]
with app.test_client() as client:
# PID deleted
res = client.get(url_for('invenio_records_rest.recid_item',
pid_value=pid_del.pid_value),
headers=headers)
assert res.status_code == 410
# PID missing object
res = client.get(url_for('invenio_records_rest.recid_item',
pid_value=pid_noobj.pid_value),
headers=headers)
assert res.status_code == 500
# Redirected invalid endpoint
res = client.get(url_for('invenio_records_rest.recid_item',
pid_value=pid_red_doi.pid_value),
headers=headers)
assert res.status_code == 500
# Redirected
res = client.get(url_for('invenio_records_rest.recid_item',
pid_value=pid_red.pid_value),
headers=headers)
assert res.status_code == 301
def create(cls, object_type=None, object_uuid=None, **kwargs):
"""Create a new record identifier."""
# Request next integer in recid sequence.
if 'pid_value' not in kwargs:
if current_app.config.get('LEGACY_PID_PROVIDER'):
kwargs['pid_value'] = _get_next_pid_from_legacy()
RecordIdentifier.insert(kwargs['pid_value'])
else:
kwargs['pid_value'] = RecordIdentifier.next()
else:
RecordIdentifier.insert(kwargs['pid_value'])
kwargs.setdefault('status', cls.default_status)
if object_type and object_uuid:
kwargs['status'] = PIDStatus.REGISTERED
return super(InspireRecordIdProvider,
cls).create(object_type=object_type,
object_uuid=object_uuid,
**kwargs)
def create(cls, object_type=None, object_uuid=None, **kwargs):
"""Create a new record identifier."""
# Request next integer in recid sequence.
if 'pid_value' not in kwargs:
kwargs['pid_value'] = str(RecordIdentifier.next())
kwargs.setdefault('status', cls.default_status)
if object_type and object_uuid:
kwargs['status'] = PIDStatus.REGISTERED
return super(SCOAP3RecordIdProvider, cls).create(
object_type=object_type, object_uuid=object_uuid, **kwargs)
def zenodo_deposit_minter(record_uuid, data):
"""Mint a deposit identifier."""
provider = ZenodoDepositProvider.create(
object_type='rec',
object_uuid=record_uuid,
pid_value=RecordIdentifier.next(),
)
data['_deposit'] = {
'id': provider.pid.pid_value,
'status': 'draft',
}
return provider.pid
def create(cls, object_type=None, object_uuid=None, **kwargs):
"""Create a new record identifier."""
# Request next integer in recid sequence.
if 'pid_value' not in kwargs:
kwargs['pid_value'] = str(RecordIdentifier.next())
kwargs.setdefault('status', cls.default_status)
if object_type and object_uuid:
kwargs['status'] = PIDStatus.REGISTERED
return super(SCOAP3RecordIdProvider,
cls).create(object_type=object_type,
object_uuid=object_uuid,
**kwargs)
def zenodo_concept_recid_minter(record_uuid=None, data=None):
"""Mint the Concept RECID.
Reserves the Concept RECID for the record.
"""
parent_id = RecordIdentifier.next()
conceptrecid = PersistentIdentifier.create(
pid_type='recid',
pid_value=str(parent_id),
status=PIDStatus.RESERVED,
)
data['conceptrecid'] = conceptrecid.pid_value
return conceptrecid
def zenodo_concept_recid_minter(record_uuid=None, data=None):
"""Mint the Concept RECID.
Reserves the Concept RECID for the record.
"""
parent_id = RecordIdentifier.next()
conceptrecid = PersistentIdentifier.create(
pid_type='recid',
pid_value=str(parent_id),
status=PIDStatus.RESERVED,
)
data['conceptrecid'] = conceptrecid.pid_value
return conceptrecid
def create(cls, object_type=None, object_uuid=None, **kwargs):
"""Create a new record identifier."""
pid_value = kwargs.get("pid_value")
if pid_value is None:
if current_app.config.get("LEGACY_PID_PROVIDER"):
kwargs["pid_value"] = get_next_pid_from_legacy()
LOGGER.info("Control number from legacy",
recid=kwargs["pid_value"])
RecordIdentifier.insert(kwargs["pid_value"])
else:
kwargs["pid_value"] = str(RecordIdentifier.next())
LOGGER.info("Control number from RecordIdentifier",
recid=kwargs["pid_value"])
else:
LOGGER.info("Control number provided", recid=kwargs["pid_value"])
RecordIdentifier.insert(kwargs["pid_value"])
kwargs.setdefault("status", cls.default_status)
if object_type and object_uuid:
kwargs["status"] = PIDStatus.REGISTERED
return super().create(object_type=object_type,
object_uuid=object_uuid,
**kwargs)
def _migrate_recid(d):
"""Migrate the recid information."""
depid = d['_n']['_deposit']['id']
pid = d['_n']['_deposit'].get('pid')
if pid:
d['_n']['recid'] = int(pid['value'])
else:
# Create a recid if we don't have one - try to reserve identical
# number.
try:
PersistentIdentifier.get('recid', depid)
id_ = str(RecordIdentifier.next())
except PIDDoesNotExistError:
id_ = str(depid)
PersistentIdentifier.create('recid', id_, status=PIDStatus.RESERVED)
d['_n']['recid'] = int(id_)
return d
def create(cls, object_type=None, object_uuid=None, **kwargs):
"""Create a new record identifier.
Note: if the object_type and object_uuid values are passed, then the
PID status will be automatically setted to
:attr:`invenio_pidstore.models.PIDStatus.REGISTERED`.
:param object_type: The object type. (Default: None.)
:param object_uuid: The object identifier. (Default: None).
:param kwargs: You specify the pid_value.
"""
# Request next integer in recid sequence.
assert 'pid_value' not in kwargs
kwargs['pid_value'] = str(RecordIdentifier.next())
kwargs.setdefault('status', cls.default_status)
if object_type and object_uuid:
kwargs['status'] = PIDStatus.REGISTERED
return super(DepositUUIDProvider, cls).create(
object_type=object_type, object_uuid=object_uuid, **kwargs)
def _migrate_recid(d):
"""Migrate the recid information."""
depid = d['_n']['_deposit']['id']
pid = d['_n']['_deposit'].get('pid')
if pid:
d['_n']['recid'] = int(pid['value'])
else:
# Create a recid if we don't have one - try to reserve identical
# number.
try:
PersistentIdentifier.get('recid', depid)
id_ = str(RecordIdentifier.next())
except PIDDoesNotExistError:
id_ = str(depid)
PersistentIdentifier.create('recid', id_, status=PIDStatus.RESERVED)
d['_n']['recid'] = int(id_)
return d
def zenodo_deposit_minter(record_uuid, data):
"""Mint deposit identifier."""
id_ = RecordIdentifier.next()
depid = PersistentIdentifier.create(
'depid',
str(id_),
object_type='rec',
object_uuid=record_uuid,
status=PIDStatus.REGISTERED,
)
# Reserve recid with same number.
PersistentIdentifier.create('recid',
depid.pid_value,
status=PIDStatus.RESERVED)
data.update({
'_deposit': {
'id': depid.pid_value,
'status': 'draft',
},
'recid': id_,
})
return depid
def zenodo_deposit_minter(record_uuid, data):
"""Mint deposit identifier."""
id_ = RecordIdentifier.next()
depid = PersistentIdentifier.create(
'depid',
str(id_),
object_type='rec',
object_uuid=record_uuid,
status=PIDStatus.REGISTERED,
)
# Reserve recid with same number.
PersistentIdentifier.create(
'recid', depid.pid_value,
status=PIDStatus.RESERVED
)
data.update({
'_deposit': {
'id': depid.pid_value,
'status': 'draft',
},
'recid': id_,
})
return depid
def test_record_identifier(app, db):
"""Test base provider."""
with app.app_context():
assert RecordIdentifier.next() == 1
assert RecordIdentifier.next() == 2
assert RecordIdentifier.max() == 2
# Mess up the sequence
with db.session.begin_nested():
obj = RecordIdentifier(recid=3)
db.session.add(obj)
assert RecordIdentifier.max() == 3
# This tests a particular problem on PostgreSQL which is using
# sequences to generate auto incrementing columns and doesn't deal
# nicely with having values inserted in the table.
assert RecordIdentifier.next() == 4
RecordIdentifier.insert(10)
assert RecordIdentifier.next() == 11
assert RecordIdentifier.max() == 11
RecordIdentifier.insert(7)
assert RecordIdentifier.max() == 11
assert RecordIdentifier.next() == 12
def get_loans_for_items(items,
location,
patron_ids=None,
librarian_id="",
n_loans=100):
"""Return random loans."""
loc_pid = location[Location.pid_field]
len_patron_ids = len(patron_ids) - 1
len_items = len(items) - 1
LEN_LOAN_STATUSES = len(LOAN_STATUSES) - 1
current_year = datetime.now().year
def _get_loanable_item(items):
"""Return an item that is loanable."""
item = items[randint(1, len_items)]
if item["status"] != "LOANABLE":
return _get_loanable_item(items)
return item
def _get_valid_status(item, items_on_loans):
"""Return valid loan status for the item to avoid inconsistencies."""
# cannot have 2 loans in the same item
if item[Item.pid_field] in items_on_loans:
status = LOAN_STATUSES[0]
else:
status = LOAN_STATUSES[randint(0, LEN_LOAN_STATUSES)]
return status
loans = []
items_on_loans = []
for i in range(1, n_loans):
item = _get_loanable_item(items)
status = _get_valid_status(item, items_on_loans)
patron_id = randint(1, len_patron_ids)
transaction_date = datetime(current_year, randint(1, 12),
randint(1, 28))
expire_date = transaction_date + timedelta(days=10)
start_date = transaction_date + timedelta(days=3)
end_date = transaction_date + timedelta(days=13)
loan = {
Document.pid_field:
"{}".format(ITEM_DOCUMENT_MAPPING[item[Item.pid_field]]),
Item.pid_field:
"{}".format(item[Item.pid_field]),
Loan.pid_field:
"{}".format(i),
"patron_pid":
"{}".format(patron_id),
"pickup_location_pid":
"{}".format(loc_pid),
"request_expire_date":
expire_date.strftime("%Y-%m-%d"),
"state":
"{}".format(status),
"start_date":
start_date.strftime("%Y-%m-%d"),
"end_date":
end_date.strftime("%Y-%m-%d"),
"transaction_date":
transaction_date.strftime("%Y-%m-%d"),
"transaction_location_pid":
"{}".format(loc_pid),
"transaction_user_pid":
"{}".format(librarian_id),
}
if status == "PENDING":
loan[Item.pid_field] = ""
else:
loan[Item.pid_field] = "{}".format(item[Item.pid_field])
items_on_loans.append(item[Item.pid_field])
loans.append(loan)
RecordIdentifier.insert(len(loans))
return loans
def create_files_and_sip(deposit, dep_pid):
"""Create deposit Bucket, Files and SIPs."""
from invenio_pidstore.errors import PIDDoesNotExistError
from invenio_pidstore.models import PersistentIdentifier, PIDStatus
from invenio_sipstore.errors import SIPUserDoesNotExist
from invenio_sipstore.models import SIP, RecordSIP, SIPFile
from invenio_files_rest.models import Bucket, FileInstance, ObjectVersion
from invenio_records_files.models import RecordsBuckets
from invenio_db import db
buc = Bucket.create()
recbuc = RecordsBuckets(record_id=deposit.id, bucket_id=buc.id)
db.session.add(recbuc)
deposit.setdefault('_deposit', dict())
deposit.setdefault('_buckets', dict(deposit=str(buc.id)))
deposit.setdefault('_files', list())
files = deposit.get('files', [])
sips = deposit.get('sips', [])
# Look for prereserved DOI (and recid)
if 'drafts' in deposit:
drafts = list(deposit['drafts'].items())
if len(drafts) != 1:
logger.exception('Deposit {dep_pid} has multiple drafts'.format(
dep_pid=dep_pid))
if len(drafts) == 1:
draft_type, draft = drafts[0]
draft_v = draft['values']
if 'prereserve_doi' in draft_v:
pre_recid = str(draft_v['prereserve_doi']['recid'])
pre_doi = str(draft_v['prereserve_doi']['doi'])
# If pre-reserve info available, try to reserve 'recid'
try:
pid = PersistentIdentifier.get(pid_type='recid',
pid_value=str(pre_recid))
except PIDDoesNotExistError:
# Reserve recid
pid = PersistentIdentifier.create(
pid_type='recid',
pid_value=str(pre_recid),
object_type='rec',
status=PIDStatus.RESERVED)
# If pre-reserve info available, try to reserve 'doi'
try:
pid = PersistentIdentifier.get(pid_type='doi',
pid_value=str(pre_doi))
except PIDDoesNotExistError:
# Reserve DOI
pid = PersistentIdentifier.create(
pid_type='doi',
pid_value=str(pre_doi),
object_type='rec',
status=PIDStatus.RESERVED)
if RecordIdentifier.query.get(int(pre_recid)) is None:
RecordIdentifier.insert(int(pre_recid))
# Store the path -> FileInstance mappings for SIPFile creation later
dep_file_instances = list()
for file_ in files:
size = file_['size']
key = file_['name']
# Warning: Assumes all checksums are MD5!
checksum = 'md5:{0}'.format(file_['checksum'])
fi = FileInstance.create()
fi.set_uri(file_['path'], size, checksum)
ov = ObjectVersion.create(buc, key, _file_id=fi.id)
ext = splitext(ov.key)[1].lower()
if ext.startswith('.'):
ext = ext[1:]
file_meta = dict(
bucket=str(ov.bucket.id),
key=ov.key,
checksum=ov.file.checksum,
size=ov.file.size,
version_id=str(ov.version_id),
type=ext,
)
deposit['_files'].append(file_meta)
dep_file_instances.append((file_['path'], fi))
# Get a recid from SIP information
recid = None
if sips:
recids = [int(sip['metadata']['recid']) for sip in sips]
if len(set(recids)) > 1:
logger.error('Multiple recids ({recids}) found in deposit {depid}'
' does not exists.'.format(recids=recids,
depid=dep_pid.pid_value))
raise DepositMultipleRecids(dep_pid.pid_value, list(set(recids)))
elif recids: # If only one recid
recid = recids[0]
for idx, sip in enumerate(sips):
agent = None
user_id = None
if sip['agents']:
agent = dict(
ip_address=empty_str_if_none(sip['agents'][0].get(
'ip_address', "")),
email=empty_str_if_none(sip['agents'][0].get(
'email_address', "")),
)
user_id = sip['agents'][0]['user_id']
if user_id == 0:
user_id = None
content = sip['package']
sip_format = 'marcxml'
try:
sip = SIP.create(sip_format, content, user_id=user_id, agent=agent)
except SIPUserDoesNotExist:
logger.exception('User ID {user_id} referred in deposit {depid} '
'does not exists.'.format(
user_id=user_id, depid=dep_pid.pid_value))
sip = SIP.create(sip_format, content, agent=agent)
# Attach recid to SIP
if recid:
try:
pid = PersistentIdentifier.get(pid_type='recid',
pid_value=str(recid))
record_sip = RecordSIP(sip_id=sip.id, pid_id=pid.id)
db.session.add(record_sip)
except PIDDoesNotExistError:
logger.exception('Record {recid} referred in '
'Deposit {depid} does not exists.'.format(
recid=recid, depid=dep_pid.pid_value))
if deposit['_p']['submitted'] == True:
logger.exception('Pair {recid}/{depid} was submitted,'
' (should it be unpublished?).'.format(
recid=recid, depid=dep_pid.pid_value))
else:
logger.exception(
'Pair {recid}/{depid} was not submitted.'.format(
recid=recid, depid=dep_pid.pid_value))
# Reserve recid
pid = PersistentIdentifier.create(pid_type='recid',
pid_value=str(recid),
object_type='rec',
status=PIDStatus.RESERVED)
if RecordIdentifier.query.get(int(recid)) is None:
RecordIdentifier.insert(int(recid))
if idx == 0:
for fp, fi in dep_file_instances:
sipf = SIPFile(sip_id=sip.id, filepath=fp, file_id=fi.id)
db.session.add(sipf)
deposit.commit()
return deposit
