def test_record_identifier(app, db):
"""Test base provider."""
with app.app_context():
assert RecordIdentifier.next() == 1
assert RecordIdentifier.next() == 2
assert RecordIdentifier.max() == 2
# Mess up the sequence
with db.session.begin_nested():
obj = RecordIdentifier(recid=3)
db.session.add(obj)
assert RecordIdentifier.max() == 3
# This tests a particular problem on PostgreSQL which is using
# sequences to generate auto incrementing columns and doesn't deal
# nicely with having values inserted in the table.
assert RecordIdentifier.next() == 4
RecordIdentifier.insert(10)
assert RecordIdentifier.next() == 11
assert RecordIdentifier.max() == 11
RecordIdentifier.insert(7)
assert RecordIdentifier.max() == 11
assert RecordIdentifier.next() == 12
def weko_deposit_minter(record_uuid, data, recid=None):
"""Weko deposit."""
if not recid:
id_ = RecordIdentifier.next()
else:
if isinstance(recid, int):
RecordIdentifier.insert(recid)
id_ = recid
recid = PersistentIdentifier.create('recid',
str(id_),
object_type='rec',
object_uuid=record_uuid,
status=PIDStatus.REGISTERED)
data['recid'] = str(recid.pid_value)
# Create depid with same pid_value of the recid
depid = PersistentIdentifier.create(
'depid',
str(recid.pid_value),
object_type='rec',
object_uuid=record_uuid,
status=PIDStatus.REGISTERED,
)
data.update({
'_deposit': {
'id': depid.pid_value,
'status': 'draft',
},
})
return depid
def create_pids(cls, dump, deposit):
"""Create a persistent identifiers."""
# Mark deposit deleted if recid is deleted.
recid = dump.recid_pid
# Create depid
depid = PersistentIdentifier.create(
pid_type='depid',
pid_value=str(dump.depid),
object_type='rec',
object_uuid=deposit.id,
status=PIDStatus.REGISTERED
)
if recid and recid.status == PIDStatus.DELETED:
depid.delete()
if RecordIdentifier.query.get(dump.depid) is None:
RecordIdentifier.insert(dump.depid)
# Pre-reserved recid.
if not recid and dump.recid:
if dump.has_pid:
# Published deposit without a recid (this is an upload which
# never got ingested so we set it back to draft status and
# reserves the reid).
pass
recid = PersistentIdentifier.create(
pid_type='recid',
pid_value=str(dump.recid),
status=PIDStatus.RESERVED
)
if RecordIdentifier.query.get(dump.recid) is None:
RecordIdentifier.insert(dump.recid)
return depid, recid
def create_record_and_pid(data):
"""Create the deposit record metadata and persistent identifier.
:param data: Raw JSON dump of the deposit.
:type data: dict
:returns: A deposit object and its pid
:rtype: (`invenio_records.api.Record`,
`invenio_pidstore.models.PersistentIdentifier`)
"""
from invenio_records.api import Record
from invenio_pidstore.models import PersistentIdentifier, PIDStatus
deposit = Record.create(data=data)
created = arrow.get(data['_p']['created']).datetime
deposit.model.created = created.replace(tzinfo=None)
depid = deposit['_p']['id']
pid = PersistentIdentifier.create(pid_type='depid',
pid_value=str(depid),
object_type='rec',
object_uuid=str(deposit.id),
status=PIDStatus.REGISTERED)
if RecordIdentifier.query.get(int(depid)) is None:
RecordIdentifier.insert(int(depid))
deposit.commit()
return deposit, pid
def create_pids(cls, dump, deposit):
"""Create a persistent identifiers."""
# Mark deposit deleted if recid is deleted.
recid = dump.recid_pid
# Create depid
depid = PersistentIdentifier.create(pid_type='depid',
pid_value=str(dump.depid),
object_type='rec',
object_uuid=deposit.id,
status=PIDStatus.REGISTERED)
if recid and recid.status == PIDStatus.DELETED:
depid.delete()
if RecordIdentifier.query.get(dump.depid) is None:
RecordIdentifier.insert(dump.depid)
# Pre-reserved recid.
if not recid and dump.recid:
if dump.has_pid:
# Published deposit without a recid (this is an upload which
# never got ingested so we set it back to draft status and
# reserves the reid).
pass
recid = PersistentIdentifier.create(pid_type='recid',
pid_value=str(dump.recid),
status=PIDStatus.RESERVED)
if RecordIdentifier.query.get(dump.recid) is None:
RecordIdentifier.insert(dump.recid)
return depid, recid
def record_db(db):
"""Load records json."""
id_ = uuid.uuid4()
record = Record.create({'title': 'Test record', 'recid': 11782}, id_=id_)
PersistentIdentifier.create(
pid_type='recid',
pid_value='11782',
status=PIDStatus.REGISTERED,
object_type='rec',
object_uuid=id_,
)
RecordIdentifier.insert(11782)
db.session.commit()
return record
def create_record(cls, dump):
"""Create a new record from dump."""
# Reserve record identifier, create record and recid pid in one
# operation.
timestamp, data = dump.pop_first_revision()
record = Record.create(data)
record.model.created = timestamp.replace(tzinfo=None)
RecordIdentifier.insert(dump.recid)
PersistentIdentifier.create(pid_type='recid',
pid_value=str(dump.recid),
object_type='rec',
object_uuid=str(record.id),
status=PIDStatus.REGISTERED)
db.session.commit()
return cls.update_record(dump, record=record)
def create(cls, object_type=None, object_uuid=None, **kwargs):
"""Create a new record identifier."""
# Request next integer in recid sequence.
if 'pid_value' not in kwargs:
if current_app.config.get('LEGACY_PID_PROVIDER'):
kwargs['pid_value'] = _get_next_pid_from_legacy()
RecordIdentifier.insert(kwargs['pid_value'])
else:
kwargs['pid_value'] = RecordIdentifier.next()
else:
RecordIdentifier.insert(kwargs['pid_value'])
kwargs.setdefault('status', cls.default_status)
if object_type and object_uuid:
kwargs['status'] = PIDStatus.REGISTERED
return super(InspireRecordIdProvider, cls).create(
object_type=object_type, object_uuid=object_uuid, **kwargs)
def create_record(cls, dump):
"""Create a new record from dump."""
# Reserve record identifier, create record and recid pid in one
# operation.
timestamp, data = dump.pop_first_revision()
record = Record.create(data)
record.model.created = timestamp.replace(tzinfo=None)
RecordIdentifier.insert(dump.recid)
PersistentIdentifier.create(
pid_type='recid',
pid_value=str(dump.recid),
object_type='rec',
object_uuid=str(record.id),
status=PIDStatus.REGISTERED
)
db.session.commit()
return cls.update_record(dump, record=record)
def record_db(db):
"""Load records json."""
id_ = uuid.uuid4()
record = Record.create({
'title': 'Test record',
'recid': 11782
}, id_=id_)
PersistentIdentifier.create(
pid_type='recid',
pid_value='11782',
status=PIDStatus.REGISTERED,
object_type='rec',
object_uuid=id_,
)
RecordIdentifier.insert(11782)
db.session.commit()
return record
def create(cls, object_type=None, object_uuid=None, **kwargs):
"""Create a new record identifier."""
if "pid_value" not in kwargs:
if current_app.config.get("LEGACY_PID_PROVIDER"):
kwargs["pid_value"] = get_next_pid_from_legacy()
RecordIdentifier.insert(kwargs["pid_value"])
else:
kwargs["pid_value"] = RecordIdentifier.next()
else:
RecordIdentifier.insert(kwargs["pid_value"])
kwargs.setdefault("status", cls.default_status)
if object_type and object_uuid:
kwargs["status"] = PIDStatus.REGISTERED
return super(InspireRecordIdProvider, cls).create(
object_type=object_type, object_uuid=object_uuid, **kwargs
)
def create(cls, object_type=None, object_uuid=None, **kwargs):
"""Create a new record identifier."""
# Request next integer in recid sequence.
if 'pid_value' not in kwargs:
if current_app.config.get('LEGACY_PID_PROVIDER'):
kwargs['pid_value'] = _get_next_pid_from_legacy()
RecordIdentifier.insert(kwargs['pid_value'])
else:
kwargs['pid_value'] = RecordIdentifier.next()
else:
RecordIdentifier.insert(kwargs['pid_value'])
kwargs.setdefault('status', cls.default_status)
if object_type and object_uuid:
kwargs['status'] = PIDStatus.REGISTERED
return super(InspireRecordIdProvider,
cls).create(object_type=object_type,
object_uuid=object_uuid,
**kwargs)
def create(cls, object_type=None, object_uuid=None, **kwargs):
"""Create a new record identifier."""
pid_value = kwargs.get("pid_value")
if pid_value is None:
if current_app.config.get("LEGACY_PID_PROVIDER"):
kwargs["pid_value"] = get_next_pid_from_legacy()
LOGGER.info("Control number from legacy",
recid=kwargs["pid_value"])
RecordIdentifier.insert(kwargs["pid_value"])
else:
kwargs["pid_value"] = str(RecordIdentifier.next())
LOGGER.info("Control number from RecordIdentifier",
recid=kwargs["pid_value"])
else:
LOGGER.info("Control number provided", recid=kwargs["pid_value"])
RecordIdentifier.insert(kwargs["pid_value"])
kwargs.setdefault("status", cls.default_status)
if object_type and object_uuid:
kwargs["status"] = PIDStatus.REGISTERED
return super().create(object_type=object_type,
object_uuid=object_uuid,
**kwargs)
def create_files_and_sip(deposit, dep_pid):
"""Create deposit Bucket, Files and SIPs."""
from invenio_pidstore.errors import PIDDoesNotExistError
from invenio_pidstore.models import PersistentIdentifier, PIDStatus
from invenio_sipstore.errors import SIPUserDoesNotExist
from invenio_sipstore.models import SIP, RecordSIP, SIPFile
from invenio_files_rest.models import Bucket, FileInstance, ObjectVersion
from invenio_records_files.models import RecordsBuckets
from invenio_db import db
buc = Bucket.create()
recbuc = RecordsBuckets(record_id=deposit.id, bucket_id=buc.id)
db.session.add(recbuc)
deposit.setdefault('_deposit', dict())
deposit.setdefault('_buckets', dict(deposit=str(buc.id)))
deposit.setdefault('_files', list())
files = deposit.get('files', [])
sips = deposit.get('sips', [])
# Look for prereserved DOI (and recid)
if 'drafts' in deposit:
drafts = list(deposit['drafts'].items())
if len(drafts) != 1:
logger.exception('Deposit {dep_pid} has multiple drafts'.format(
dep_pid=dep_pid))
if len(drafts) == 1:
draft_type, draft = drafts[0]
draft_v = draft['values']
if 'prereserve_doi' in draft_v:
pre_recid = str(draft_v['prereserve_doi']['recid'])
pre_doi = str(draft_v['prereserve_doi']['doi'])
# If pre-reserve info available, try to reserve 'recid'
try:
pid = PersistentIdentifier.get(pid_type='recid',
pid_value=str(pre_recid))
except PIDDoesNotExistError:
# Reserve recid
pid = PersistentIdentifier.create(
pid_type='recid',
pid_value=str(pre_recid),
object_type='rec',
status=PIDStatus.RESERVED)
# If pre-reserve info available, try to reserve 'doi'
try:
pid = PersistentIdentifier.get(pid_type='doi',
pid_value=str(pre_doi))
except PIDDoesNotExistError:
# Reserve DOI
pid = PersistentIdentifier.create(
pid_type='doi',
pid_value=str(pre_doi),
object_type='rec',
status=PIDStatus.RESERVED)
if RecordIdentifier.query.get(int(pre_recid)) is None:
RecordIdentifier.insert(int(pre_recid))
# Store the path -> FileInstance mappings for SIPFile creation later
dep_file_instances = list()
for file_ in files:
size = file_['size']
key = file_['name']
# Warning: Assumes all checksums are MD5!
checksum = 'md5:{0}'.format(file_['checksum'])
fi = FileInstance.create()
fi.set_uri(file_['path'], size, checksum)
ov = ObjectVersion.create(buc, key, _file_id=fi.id)
ext = splitext(ov.key)[1].lower()
if ext.startswith('.'):
ext = ext[1:]
file_meta = dict(
bucket=str(ov.bucket.id),
key=ov.key,
checksum=ov.file.checksum,
size=ov.file.size,
version_id=str(ov.version_id),
type=ext,
)
deposit['_files'].append(file_meta)
dep_file_instances.append((file_['path'], fi))
# Get a recid from SIP information
recid = None
if sips:
recids = [int(sip['metadata']['recid']) for sip in sips]
if len(set(recids)) > 1:
logger.error('Multiple recids ({recids}) found in deposit {depid}'
' does not exists.'.format(recids=recids,
depid=dep_pid.pid_value))
raise DepositMultipleRecids(dep_pid.pid_value, list(set(recids)))
elif recids: # If only one recid
recid = recids[0]
for idx, sip in enumerate(sips):
agent = None
user_id = None
if sip['agents']:
agent = dict(
ip_address=empty_str_if_none(sip['agents'][0].get(
'ip_address', "")),
email=empty_str_if_none(sip['agents'][0].get(
'email_address', "")),
)
user_id = sip['agents'][0]['user_id']
if user_id == 0:
user_id = None
content = sip['package']
sip_format = 'marcxml'
try:
sip = SIP.create(sip_format, content, user_id=user_id, agent=agent)
except SIPUserDoesNotExist:
logger.exception('User ID {user_id} referred in deposit {depid} '
'does not exists.'.format(
user_id=user_id, depid=dep_pid.pid_value))
sip = SIP.create(sip_format, content, agent=agent)
# Attach recid to SIP
if recid:
try:
pid = PersistentIdentifier.get(pid_type='recid',
pid_value=str(recid))
record_sip = RecordSIP(sip_id=sip.id, pid_id=pid.id)
db.session.add(record_sip)
except PIDDoesNotExistError:
logger.exception('Record {recid} referred in '
'Deposit {depid} does not exists.'.format(
recid=recid, depid=dep_pid.pid_value))
if deposit['_p']['submitted'] == True:
logger.exception('Pair {recid}/{depid} was submitted,'
' (should it be unpublished?).'.format(
recid=recid, depid=dep_pid.pid_value))
else:
logger.exception(
'Pair {recid}/{depid} was not submitted.'.format(
recid=recid, depid=dep_pid.pid_value))
# Reserve recid
pid = PersistentIdentifier.create(pid_type='recid',
pid_value=str(recid),
object_type='rec',
status=PIDStatus.RESERVED)
if RecordIdentifier.query.get(int(recid)) is None:
RecordIdentifier.insert(int(recid))
if idx == 0:
for fp, fi in dep_file_instances:
sipf = SIPFile(sip_id=sip.id, filepath=fp, file_id=fi.id)
db.session.add(sipf)
deposit.commit()
return deposit
def get_loans_for_items(items,
location,
patron_ids=None,
librarian_id="",
n_loans=100):
"""Return random loans."""
loc_pid = location[Location.pid_field]
len_patron_ids = len(patron_ids) - 1
len_items = len(items) - 1
LEN_LOAN_STATUSES = len(LOAN_STATUSES) - 1
current_year = datetime.now().year
def _get_loanable_item(items):
"""Return an item that is loanable."""
item = items[randint(1, len_items)]
if item["status"] != "LOANABLE":
return _get_loanable_item(items)
return item
def _get_valid_status(item, items_on_loans):
"""Return valid loan status for the item to avoid inconsistencies."""
# cannot have 2 loans in the same item
if item[Item.pid_field] in items_on_loans:
status = LOAN_STATUSES[0]
else:
status = LOAN_STATUSES[randint(0, LEN_LOAN_STATUSES)]
return status
loans = []
items_on_loans = []
for i in range(1, n_loans):
item = _get_loanable_item(items)
status = _get_valid_status(item, items_on_loans)
patron_id = randint(1, len_patron_ids)
transaction_date = datetime(current_year, randint(1, 12),
randint(1, 28))
expire_date = transaction_date + timedelta(days=10)
start_date = transaction_date + timedelta(days=3)
end_date = transaction_date + timedelta(days=13)
loan = {
Document.pid_field:
"{}".format(ITEM_DOCUMENT_MAPPING[item[Item.pid_field]]),
Item.pid_field:
"{}".format(item[Item.pid_field]),
Loan.pid_field:
"{}".format(i),
"patron_pid":
"{}".format(patron_id),
"pickup_location_pid":
"{}".format(loc_pid),
"request_expire_date":
expire_date.strftime("%Y-%m-%d"),
"state":
"{}".format(status),
"start_date":
start_date.strftime("%Y-%m-%d"),
"end_date":
end_date.strftime("%Y-%m-%d"),
"transaction_date":
transaction_date.strftime("%Y-%m-%d"),
"transaction_location_pid":
"{}".format(loc_pid),
"transaction_user_pid":
"{}".format(librarian_id),
}
if status == "PENDING":
loan[Item.pid_field] = ""
else:
loan[Item.pid_field] = "{}".format(item[Item.pid_field])
items_on_loans.append(item[Item.pid_field])
loans.append(loan)
RecordIdentifier.insert(len(loans))
return loans
