def ensure_document_exists(self, document_pid):
"""Ensure document exists or raise."""
from invenio_app_ils.documents.api import Document
try:
Document.get_record_by_pid(document_pid)
except PIDDoesNotExistError:
raise ItemDocumentNotFoundError(document_pid)
def test_journal_relation_from_publication_info(app):
"""Test journal-document relation from publication info field."""
document_data = {
"$schema": "https://127.0.0.1:5000/schemas/documents/document-v1.0.0.json",
"created_by": {"type": "script", "value": "test"},
"pid": "4321",
"legacy_recid": "1111",
"title": "Book: A Book",
"document_type": "BOOK",
"authors": [{"full_name": "Author Author"}],
"abstracts": [{"value": "This is an abstract"}],
"language": ["it"],
"publication_year": "2020",
"identifiers": [{"scheme": "ISBN", "value": "0123456789"}],
"cover_metadata": {"ISBN": "0123456789"},
"publication_info": [{"journal_issue": "issue"}],
"_migration": {
"has_journal": True,
"journal_record_legacy_recids": [
{
"recid": "1234",
"volume": None,
}
],
},
}
journal_data = {
"$schema": "https://127.0.0.1:5000/schemas/series/series-v1.0.0.json",
"pid": "serid-4",
"title": "Dispersion Forces",
"authors": ["Buhmann, Stefan Yoshi"],
"abstract": "This is a multipart monograph",
"mode_of_issuance": "SERIAL",
"legacy_recid": "1234",
}
record_uuid = mint_record_pid(
DOCUMENT_PID_TYPE, "pid", {"pid": document_data["pid"]}
)
document = Document.create(document_data, record_uuid)
record_uuid = mint_record_pid(
SERIES_PID_TYPE, "pid", {"pid": journal_data["pid"]}
)
journal = Series.create(journal_data, record_uuid)
legacy_recid_minter(journal["legacy_recid"], record_uuid)
db.session.commit()
ri = RecordIndexer()
ri.index(document)
ri.index(journal)
current_search.flush_and_refresh(index="*")
link_documents_and_serials()
document_rec = Document.get_record_by_pid(document["pid"])
assert "serial" in document_rec.relations
def create_multipart_volumes(pid, multipart_legacy_recid, migration_volumes):
"""Create multipart volume documents."""
volumes = {}
# Combine all volume data by volume number
click.echo('Creating volume for {}...'.format(multipart_legacy_recid))
for obj in migration_volumes:
volume_number = obj['volume']
if volume_number not in volumes:
volumes[volume_number] = {}
volume = volumes[volume_number]
for key in obj:
if key != 'volume':
if key in volume:
raise KeyError(
'Duplicate key "{}" for multipart {}'.format(
key, multipart_legacy_recid))
volume[key] = obj[key]
volume_numbers = iter(sorted(volumes.keys()))
# Re-use the current record for the first volume
# TODO review this - there are more cases of multiparts
first_volume = next(volume_numbers)
first = Document.get_record_by_pid(pid)
if 'title' in volumes[first_volume]:
first['title'] = volumes[first_volume]['title']
first['volume'] = first_volume
first['_migration']['multipart_legacy_recid'] = multipart_legacy_recid
# to be tested
if 'legacy_recid' in first:
del first['legacy_recid']
first.commit()
yield first
# Create new records for the rest
for number in volume_numbers:
temp = first.copy()
temp['title'] = volumes[number]['title']
temp['volume'] = number
record_uuid = uuid.uuid4()
provider = DocumentIdProvider.create(
object_type='rec',
object_uuid=record_uuid,
)
temp['pid'] = provider.pid.pid_value
record = Document.create(temp, record_uuid)
record.commit()
yield record
def get_document_by_legacy_recid(legacy_recid):
"""Search documents by its legacy recid."""
search = DocumentSearch().query('bool',
filter=[
Q('term', legacy_recid=legacy_recid),
])
result = search.execute()
hits_total = result.hits.total if lt_es7 else result.hits.total.value
if not result.hits or hits_total < 1:
click.secho(
'no document found with legacy recid {}'.format(legacy_recid),
fg='red')
raise DocumentMigrationError(
'no document found with legacy recid {}'.format(legacy_recid))
elif hits_total > 1:
click.secho(
'no document found with legacy recid {}'.format(legacy_recid),
fg='red')
raise DocumentMigrationError(
'found more than one document with recid {}'.format(legacy_recid))
else:
click.secho(
'! document found with legacy recid {}'.format(legacy_recid),
fg='green')
return Document.get_record_by_pid(result.hits[0].pid)
def test_on_document_update():
"""Test document resolvers."""
indexer = _get_mock()
pid = "docid-2"
document = Document.get_record_by_pid(pid)
DocumentIndexer().index(document)
referenced = _assert_origin(indexer, DOCUMENT_PID_TYPE, pid)
# should re-index document request
n_doc_req = 1 # from test data
_assert_contains(referenced, DOCUMENT_REQUEST_PID_TYPE)
# should re-index eitem
n_eitems = 1 # from test data
_assert_contains(referenced, EITEM_PID_TYPE)
# should re-index item
n_items = 1 # from test data
_assert_contains(referenced, ITEM_PID_TYPE)
# should re-index acq
n_acq = 1 # from test data
_assert_contains(referenced, ORDER_PID_TYPE)
# should re-index ill
n_ill = 1 # from test data
_assert_contains(referenced, BORROWING_REQUEST_PID_TYPE)
expected_total = n_doc_req + n_eitems + n_items + n_acq + n_ill
assert len(referenced) == expected_total
def migrate_ebl_links():
"""Migrate external links from documents."""
search = get_documents_with_ebl_eitems()
click.echo("Found {} documents with ebl links.".format(search.count()))
for hit in search.scan():
# make sure the document is in DB not only ES
document = Document.get_record_by_pid(hit.pid)
click.echo("Processing document {}...".format(document["pid"]))
# find the ebl identifier
ebl_id = next(
(x for x in document["alternative_identifiers"]
if x["scheme"] == "EBL"),
None,
)
for url in document["_migration"]["eitems_ebl"]:
if not ebl_id:
raise EItemMigrationError(
"Document {pid} has no EBL alternative identifier"
" while EBL ebook link was found".format(
pid=document["pid"]))
eitem = create_eitem(document["pid"], open_access=False)
eitem["urls"] = [{"value": "EBL", "login_required": True}]
eitem.commit()
EItemIndexer().index(eitem)
document["_migration"]["eitems_has_ebl"] = False
document.commit()
db.session.commit()
DocumentIndexer().index(document)
def get_document_by_legacy_recid(legacy_recid):
"""Search documents by its legacy recid."""
search = DocumentSearch().query(
"bool", filter=[Q("term", legacy_recid=legacy_recid)])
result = search.execute()
hits_total = result.hits.total.value
if hits_total < 1:
click.secho(
"no document found with legacy recid {}".format(legacy_recid),
fg="red",
)
raise DocumentMigrationError(
"no document found with legacy recid {}".format(legacy_recid))
elif hits_total > 1:
click.secho(
"no document found with legacy recid {}".format(legacy_recid),
fg="red",
)
raise DocumentMigrationError(
"found more than one document with recid {}".format(legacy_recid))
else:
click.secho(
"! document found with legacy recid {}".format(legacy_recid),
fg="green",
)
return Document.get_record_by_pid(result.hits[0].pid)
def create_record(cls, dump):
"""Create a new record from dump."""
record_uuid = uuid.uuid4()
try:
# with db.session.begin_nested():
provider = DocumentIdProvider.create(
object_type="rec",
object_uuid=record_uuid,
)
timestamp, json_data = dump.revisions[-1]
json_data["pid"] = provider.pid.pid_value
json_data = clean_created_by_field(json_data)
document = Document.create(json_data, record_uuid)
document.model.created = dump.created.replace(tzinfo=None)
document.model.updated = timestamp.replace(tzinfo=None)
document.commit()
db.session.commit()
return document
except IlsValidationError as e:
click.secho("Field: {}".format(e.errors[0].res["field"]), fg="red")
click.secho(e.original_exception.message, fg="red")
db.session.rollback()
raise e
def get_document(document_pid):
"""Return the Document record."""
document = Document.get_record_by_pid(document_pid)
return {
"pid": document.get("pid"),
"title": document.get("title"),
"authors": document.get("authors"),
}
def _restore(original):
"""Restore the record to the original metadata."""
doc = Document.get_record_by_pid(original["pid"])
doc.clear()
doc.update(original)
doc.commit()
db.session.commit()
return doc
def serialize_on_loan_book_information(loan):
"""Serialize loan information."""
location = Location.get_record_by_pid(loan["transaction_location_pid"])
document = Document.get_record_by_pid(loan["document_pid"])
return dict(barcode=loan["item"]["barcode"],
end_date=loan["end_date"],
library=location["name"],
location=location["address"],
title=document["title"])
def test_document_creation_refs(app):
"""Test creation of a document."""
d = dict(pid="a1bc",
title="Test title",
authors=[dict(full_name="John Doe")],
publication_year="2010")
doc = Document.create(d)
_assert_extra_fields(doc)
def serialize_loan_request_book_information(loan):
"""Serialize loan information."""
location = Location.get_record_by_pid(loan["transaction_location_pid"])
document = Document.get_record_by_pid(loan["document_pid"])
return dict(request_start_date=loan["start_date"],
request_end_date=loan["end_date"],
library=location["name"],
location=location["address"],
title=document["title"])
def test_no_identifiers_no_previous_cover(doc):
"""It should not have cover_metadata because there are no ISBNs."""
del doc["identifiers"]
doc.pop("cover_metadata", "") # delete if present
doc.commit()
db.session.commit()
pick_identifier_with_cover(app, record=doc)
doc = Document.get_record_by_pid(doc["pid"])
assert "cover_metadata" not in doc
def test_identifiers_deleted_with_previous_cover(doc):
"""It should remove the cover_metadata because there are no ISBNs."""
del doc["identifiers"]
doc["cover_metadata"] = dict(ISBN="112344343")
doc.commit()
db.session.commit()
pick_identifier_with_cover(app, record=doc)
doc = Document.get_record_by_pid(doc["pid"])
assert "cover_metadata" not in doc
def test_access_permissions(
client, json_headers, testdata, users, with_access
):
"""Test GET documents with `_access` ignoring `restricted`."""
# set the documents to have read access only by patron2. `_access` should
# be taken into account and take precedence over `restricted`.
indexer = RecordIndexer()
doc1 = Document.get_record_by_pid("docid-open-access")
doc2 = Document.get_record_by_pid("docid-closed-access")
for doc in [doc1, doc2]:
doc.update(dict(_access=dict(read=[users["patron2"].id])))
doc.commit()
db.session.commit()
indexer.index(doc)
current_search.flush_and_refresh(index="documents")
test_data = [
("anonymous", "docid-open-access", 401, 0),
("patron1", "docid-open-access", 403, 0),
("patron2", "docid-open-access", 200, 1), # should have access
("librarian", "docid-open-access", 200, 1),
("admin", "docid-open-access", 200, 1),
("anonymous", "docid-closed-access", 401, 0),
("patron1", "docid-closed-access", 403, 0),
("patron2", "docid-closed-access", 200, 1), # should have access
("librarian", "docid-closed-access", 200, 1),
("admin", "docid-closed-access", 200, 1),
]
for user, pid, status_code, n_hits in test_data:
# item endpoint
user_login(client, user, users)
url = url_for("invenio_records_rest.docid_item", pid_value=pid)
res = client.get(url, headers=json_headers)
assert res.status_code == status_code
# list endpoint
user_login(client, user, users)
url = url_for(
"invenio_records_rest.docid_list", q="pid:{}".format(pid)
)
res = client.get(url, headers=json_headers)
hits = json.loads(res.data.decode("utf-8"))
assert hits["hits"]["total"] == n_hits
def test_email_on_loan_checkout(client, app_with_mail, users, testdata,
loan_params):
"""Test that an email is sent on loan checkout."""
loan_data = testdata["loans"][1]
loan = Loan.get_record_by_pid(loan_data["pid"])
with app_with_mail.extensions["mail"].record_messages() as outbox:
user_login(client, "admin", users)
assert len(outbox) == 0
current_circulation.circulation.trigger(
loan, **dict(loan_params, trigger="checkout"))
assert len(outbox) == 1
msg = outbox[0]
doc = Document.get_record_by_pid(loan_data["document_pid"])
expected_subject = (
"""InvenioILS: your loan for "{0}" has started.""".format(
doc["title"]))
assert msg.subject == expected_subject
edition_year = " ({edition} - {year})".format(edition=doc["edition"],
year=doc["publication_year"])
full_title = "{title}, {author}{edition_year}".format(
title=doc["title"],
author=doc["authors"][0]["full_name"],
edition_year=edition_year,
)
literature_url = "{host}{path}".format(
host=current_app.config["SPA_HOST"],
path=current_app.config["SPA_PATHS"]["literature"] %
{"pid": doc["pid"]},
)
profile_url = "{host}{path}".format(
host=current_app.config["SPA_HOST"],
path=current_app.config["SPA_PATHS"]["profile"],
)
expected_body_plain = """Dear Patron One,
your loan for "{doc_full_title}" <{literature_url}> has started.
The due date is {loan_end_date}.
You can see your ongoing and past loans in your profile page <{profile_url}>.
Kind regards,
InvenioILS""".format(
doc_full_title=full_title,
literature_url=literature_url,
loan_end_date=loan_data["end_date"],
profile_url=profile_url,
)
assert msg.body == expected_body_plain
def _test_delete():
"""Test relation creation."""
rec1, rec2 = _choose_endpoints_and_do_request(
(client, json_headers, "DELETE"),
(first_pid, first_pid_type, second_pid, second_pid_type),
payload,
)
rec3 = Document.get_record_by_pid(third_pid)
rec3 = rec3.replace_refs()
_assert_record_relations(rec1, expected={"relations": {}})
_assert_record_relations(rec2, expected={"relations": {}})
_assert_record_relations(rec3, expected={"relations": {}})
def get_document(document_pid):
"""Return the Document record."""
document = Document.get_record_by_pid(document_pid)
authors = []
for author in document.get("authors", []):
if "full_name" in author:
authors.append(author["full_name"])
return {
"pid": document.get("pid"),
"title": document.get("title", ""),
"authors": authors,
}
def validate_serial_relation(serial, recids):
relations = serial.relations.get().get('serial', [])
if len(recids) != len(relations):
click.echo('[Serial {}] Incorrect number of children: {} '
'(expected {})'.format(serial['pid'], len(relations),
len(recids)))
for relation in relations:
child = Document.get_record_by_pid(relation['pid'],
pid_type=relation['pid_type'])
if 'legacy_recid' in child and child['legacy_recid'] not in recids:
click.echo('[Serial {}] Unexpected child with legacy '
'recid: {}'.format(serial['pid'],
child['legacy_recid']))
def test_with_identifiers_with_valid_cover(doc):
"""It should have the ISBN in cover_metadata."""
# return cover is valid
mocker.patch(
"cds_ils.literature.tasks.is_valid_cover", return_value=True
)
doc["identifiers"] = [dict(scheme="ISBN", value="valid-isbn")]
doc["cover_metadata"] = dict(ISBN="valid-isbn")
doc.commit()
db.session.commit()
pick_identifier_with_cover(app, record=doc)
doc = Document.get_record_by_pid(doc["pid"])
assert doc["cover_metadata"] == dict(ISBN="valid-isbn")
def validate_multipart_relation(multipart, volumes):
relations = multipart.relations.get().get('multipart_monograph', [])
titles = [volume['title'] for volume in volumes if 'title' in volume]
count = len(set(v['volume'] for v in volumes))
if count != len(relations):
click.echo('[Multipart {}] Incorrect number of volumes: {} '
'(expected {})'.format(multipart['pid'], len(relations),
count))
for relation in relations:
child = Document.get_record_by_pid(relation['pid'],
pid_type=relation['pid_type'])
if child['title']['title'] not in titles:
click.echo('[Multipart {}] Title "{}" does not exist in '
'migration data'.format(multipart['pid'],
child['title']['title']))
def test_document_update_refs(app):
"""Test update of a document."""
d = dict(pid="a1bc",
title="Test title",
authors=[dict(full_name="John Doe")],
publication_year="2010")
doc = Document.create(d)
del doc["circulation"]
del doc["relations"]
del doc["eitems"]
del doc["items"]
del doc["stock"]
doc.update(dict(title="Test title 2"))
_assert_extra_fields(doc)
assert doc["title"] == "Test title 2"
def test_document_creation_refs(app):
"""Test creation of a document."""
d = dict(
pid="a1bc",
created_by={
"type": "script",
"value": "demo"
},
title="Test title",
authors=[dict(full_name="John Doe")],
publication_year="2010",
document_type="BOOK",
)
doc = Document.create(d)
_assert_extra_fields(doc)
def test_add_new_identifier(doc):
"""It should change the cover_metadata to the new identifier."""
# return cover is not valid
mocker.patch(
"cds_ils.literature.tasks.is_valid_cover", return_value=True
)
# the ISBN has been changed to a not valid one
doc["identifiers"] = [dict(scheme="ISBN", value="valid-isbn")]
# it had a previously valid ISBN
doc["cover_metadata"] = dict(ISBN="123456789")
doc.commit()
db.session.commit()
pick_identifier_with_cover(app, record=doc)
doc = Document.get_record_by_pid(doc["pid"])
assert doc["cover_metadata"] == dict(ISBN="valid-isbn")
def test_with_invalid_identifiers_no_valid_cover(doc):
"""It should remove cover_metadata, the ISBN is not valid."""
# return cover is not valid
mocker.patch(
"cds_ils.literature.tasks.is_valid_cover", return_value=False
)
# the ISBN has been changed to a not valid one
doc["identifiers"] = [dict(scheme="ISBN", value="not-valid-isbn")]
# it had a previously valid ISBN
doc["cover_metadata"] = dict(ISBN="valid-isbn")
doc.commit()
db.session.commit()
pick_identifier_with_cover(app, record=doc)
doc = Document.get_record_by_pid(doc["pid"])
assert "cover_metadata" not in doc
def validate_multipart_relation(multipart, volumes):
relations = multipart.relations.get().get("multipart_monograph", [])
titles = [volume["title"] for volume in volumes if "title" in volume]
count = len(set(v["volume"] for v in volumes))
if count != len(relations):
click.echo(
"[Multipart {}] Incorrect number of volumes: {} "
"(expected {})".format(multipart["pid"], len(relations), count)
)
for relation in relations:
child = Document.get_record_by_pid(
relation["pid"], pid_type=relation["pid_type"]
)
if child["title"] not in titles:
click.echo(
'[Multipart {}] Title "{}" does not exist in '
"migration data".format(multipart["pid"], child["title"])
)
def validate_serial_relation(serial, recids):
relations = serial.relations.get().get("serial", [])
if len(recids) != len(relations):
click.echo(
"[Serial {}] Incorrect number of children: {} "
"(expected {})".format(
serial["pid"], len(relations), len(recids)
)
)
for relation in relations:
child = Document.get_record_by_pid(
relation["pid"], pid_type=relation["pid_type"]
)
if "legacy_recid" in child and child["legacy_recid"] not in recids:
click.echo(
"[Serial {}] Unexpected child with legacy "
"recid: {}".format(serial["pid"], child["legacy_recid"])
)
def test_document_resolvers(app, testdata):
"""Test item resolving from loan."""
doc_pid = testdata["documents"][0]["pid"]
document = Document.get_record_by_pid(doc_pid)
document = document.replace_refs()
# relations
assert "relations" in document
# circulation
assert "active_loans" in document["circulation"]
# item and eitems
assert document["items"]["total"] == 9 and document["items"]["hits"]
assert document["eitems"]["total"] == 3 and document["eitems"]["hits"]
# stock
mediums = set([item["medium"] for item in document["items"]["hits"]])
mediums.add("ELECTRONIC_VERSION")
assert set(document["stock"]["mediums"]) == mediums
def migrate_ezproxy_links():
"""Migrate external links from documents."""
search = get_documents_with_proxy_eitems()
click.echo("Found {} documents with ezproxy links.".format(search.count()))
for hit in search.scan():
# make sure the document is in DB not only ES
document = Document.get_record_by_pid(hit.pid)
click.echo("Processing document {}...".format(document["pid"]))
for url in document["_migration"]["eitems_external"]:
eitem = create_eitem(document["pid"], open_access=False)
url["login_required"] = True
eitem["urls"] = [url]
eitem.commit()
EItemIndexer().index(eitem)
document["_migration"]["eitems_has_proxy"] = False
document.commit()
db.session.commit()
DocumentIndexer().index(document)
