def ensure_document_exists(self, document_pid): """Ensure document exists or raise.""" from invenio_app_ils.documents.api import Document try: Document.get_record_by_pid(document_pid) except PIDDoesNotExistError: raise ItemDocumentNotFoundError(document_pid)
def test_journal_relation_from_publication_info(app): """Test journal-document relation from publication info field.""" document_data = { "$schema": "https://127.0.0.1:5000/schemas/documents/document-v1.0.0.json", "created_by": {"type": "script", "value": "test"}, "pid": "4321", "legacy_recid": "1111", "title": "Book: A Book", "document_type": "BOOK", "authors": [{"full_name": "Author Author"}], "abstracts": [{"value": "This is an abstract"}], "language": ["it"], "publication_year": "2020", "identifiers": [{"scheme": "ISBN", "value": "0123456789"}], "cover_metadata": {"ISBN": "0123456789"}, "publication_info": [{"journal_issue": "issue"}], "_migration": { "has_journal": True, "journal_record_legacy_recids": [ { "recid": "1234", "volume": None, } ], }, } journal_data = { "$schema": "https://127.0.0.1:5000/schemas/series/series-v1.0.0.json", "pid": "serid-4", "title": "Dispersion Forces", "authors": ["Buhmann, Stefan Yoshi"], "abstract": "This is a multipart monograph", "mode_of_issuance": "SERIAL", "legacy_recid": "1234", } record_uuid = mint_record_pid( DOCUMENT_PID_TYPE, "pid", {"pid": document_data["pid"]} ) document = Document.create(document_data, record_uuid) record_uuid = mint_record_pid( SERIES_PID_TYPE, "pid", {"pid": journal_data["pid"]} ) journal = Series.create(journal_data, record_uuid) legacy_recid_minter(journal["legacy_recid"], record_uuid) db.session.commit() ri = RecordIndexer() ri.index(document) ri.index(journal) current_search.flush_and_refresh(index="*") link_documents_and_serials() document_rec = Document.get_record_by_pid(document["pid"]) assert "serial" in document_rec.relations
def create_multipart_volumes(pid, multipart_legacy_recid, migration_volumes): """Create multipart volume documents.""" volumes = {} # Combine all volume data by volume number click.echo('Creating volume for {}...'.format(multipart_legacy_recid)) for obj in migration_volumes: volume_number = obj['volume'] if volume_number not in volumes: volumes[volume_number] = {} volume = volumes[volume_number] for key in obj: if key != 'volume': if key in volume: raise KeyError( 'Duplicate key "{}" for multipart {}'.format( key, multipart_legacy_recid)) volume[key] = obj[key] volume_numbers = iter(sorted(volumes.keys())) # Re-use the current record for the first volume # TODO review this - there are more cases of multiparts first_volume = next(volume_numbers) first = Document.get_record_by_pid(pid) if 'title' in volumes[first_volume]: first['title'] = volumes[first_volume]['title'] first['volume'] = first_volume first['_migration']['multipart_legacy_recid'] = multipart_legacy_recid # to be tested if 'legacy_recid' in first: del first['legacy_recid'] first.commit() yield first # Create new records for the rest for number in volume_numbers: temp = first.copy() temp['title'] = volumes[number]['title'] temp['volume'] = number record_uuid = uuid.uuid4() provider = DocumentIdProvider.create( object_type='rec', object_uuid=record_uuid, ) temp['pid'] = provider.pid.pid_value record = Document.create(temp, record_uuid) record.commit() yield record
def get_document_by_legacy_recid(legacy_recid): """Search documents by its legacy recid.""" search = DocumentSearch().query('bool', filter=[ Q('term', legacy_recid=legacy_recid), ]) result = search.execute() hits_total = result.hits.total if lt_es7 else result.hits.total.value if not result.hits or hits_total < 1: click.secho( 'no document found with legacy recid {}'.format(legacy_recid), fg='red') raise DocumentMigrationError( 'no document found with legacy recid {}'.format(legacy_recid)) elif hits_total > 1: click.secho( 'no document found with legacy recid {}'.format(legacy_recid), fg='red') raise DocumentMigrationError( 'found more than one document with recid {}'.format(legacy_recid)) else: click.secho( '! document found with legacy recid {}'.format(legacy_recid), fg='green') return Document.get_record_by_pid(result.hits[0].pid)
def test_on_document_update(): """Test document resolvers.""" indexer = _get_mock() pid = "docid-2" document = Document.get_record_by_pid(pid) DocumentIndexer().index(document) referenced = _assert_origin(indexer, DOCUMENT_PID_TYPE, pid) # should re-index document request n_doc_req = 1 # from test data _assert_contains(referenced, DOCUMENT_REQUEST_PID_TYPE) # should re-index eitem n_eitems = 1 # from test data _assert_contains(referenced, EITEM_PID_TYPE) # should re-index item n_items = 1 # from test data _assert_contains(referenced, ITEM_PID_TYPE) # should re-index acq n_acq = 1 # from test data _assert_contains(referenced, ORDER_PID_TYPE) # should re-index ill n_ill = 1 # from test data _assert_contains(referenced, BORROWING_REQUEST_PID_TYPE) expected_total = n_doc_req + n_eitems + n_items + n_acq + n_ill assert len(referenced) == expected_total
def migrate_ebl_links(): """Migrate external links from documents.""" search = get_documents_with_ebl_eitems() click.echo("Found {} documents with ebl links.".format(search.count())) for hit in search.scan(): # make sure the document is in DB not only ES document = Document.get_record_by_pid(hit.pid) click.echo("Processing document {}...".format(document["pid"])) # find the ebl identifier ebl_id = next( (x for x in document["alternative_identifiers"] if x["scheme"] == "EBL"), None, ) for url in document["_migration"]["eitems_ebl"]: if not ebl_id: raise EItemMigrationError( "Document {pid} has no EBL alternative identifier" " while EBL ebook link was found".format( pid=document["pid"])) eitem = create_eitem(document["pid"], open_access=False) eitem["urls"] = [{"value": "EBL", "login_required": True}] eitem.commit() EItemIndexer().index(eitem) document["_migration"]["eitems_has_ebl"] = False document.commit() db.session.commit() DocumentIndexer().index(document)
def get_document_by_legacy_recid(legacy_recid): """Search documents by its legacy recid.""" search = DocumentSearch().query( "bool", filter=[Q("term", legacy_recid=legacy_recid)]) result = search.execute() hits_total = result.hits.total.value if hits_total < 1: click.secho( "no document found with legacy recid {}".format(legacy_recid), fg="red", ) raise DocumentMigrationError( "no document found with legacy recid {}".format(legacy_recid)) elif hits_total > 1: click.secho( "no document found with legacy recid {}".format(legacy_recid), fg="red", ) raise DocumentMigrationError( "found more than one document with recid {}".format(legacy_recid)) else: click.secho( "! document found with legacy recid {}".format(legacy_recid), fg="green", ) return Document.get_record_by_pid(result.hits[0].pid)
def create_record(cls, dump): """Create a new record from dump.""" record_uuid = uuid.uuid4() try: # with db.session.begin_nested(): provider = DocumentIdProvider.create( object_type="rec", object_uuid=record_uuid, ) timestamp, json_data = dump.revisions[-1] json_data["pid"] = provider.pid.pid_value json_data = clean_created_by_field(json_data) document = Document.create(json_data, record_uuid) document.model.created = dump.created.replace(tzinfo=None) document.model.updated = timestamp.replace(tzinfo=None) document.commit() db.session.commit() return document except IlsValidationError as e: click.secho("Field: {}".format(e.errors[0].res["field"]), fg="red") click.secho(e.original_exception.message, fg="red") db.session.rollback() raise e
def get_document(document_pid): """Return the Document record.""" document = Document.get_record_by_pid(document_pid) return { "pid": document.get("pid"), "title": document.get("title"), "authors": document.get("authors"), }
def _restore(original): """Restore the record to the original metadata.""" doc = Document.get_record_by_pid(original["pid"]) doc.clear() doc.update(original) doc.commit() db.session.commit() return doc
def serialize_on_loan_book_information(loan): """Serialize loan information.""" location = Location.get_record_by_pid(loan["transaction_location_pid"]) document = Document.get_record_by_pid(loan["document_pid"]) return dict(barcode=loan["item"]["barcode"], end_date=loan["end_date"], library=location["name"], location=location["address"], title=document["title"])
def test_document_creation_refs(app): """Test creation of a document.""" d = dict(pid="a1bc", title="Test title", authors=[dict(full_name="John Doe")], publication_year="2010") doc = Document.create(d) _assert_extra_fields(doc)
def serialize_loan_request_book_information(loan): """Serialize loan information.""" location = Location.get_record_by_pid(loan["transaction_location_pid"]) document = Document.get_record_by_pid(loan["document_pid"]) return dict(request_start_date=loan["start_date"], request_end_date=loan["end_date"], library=location["name"], location=location["address"], title=document["title"])
def test_no_identifiers_no_previous_cover(doc): """It should not have cover_metadata because there are no ISBNs.""" del doc["identifiers"] doc.pop("cover_metadata", "") # delete if present doc.commit() db.session.commit() pick_identifier_with_cover(app, record=doc) doc = Document.get_record_by_pid(doc["pid"]) assert "cover_metadata" not in doc
def test_identifiers_deleted_with_previous_cover(doc): """It should remove the cover_metadata because there are no ISBNs.""" del doc["identifiers"] doc["cover_metadata"] = dict(ISBN="112344343") doc.commit() db.session.commit() pick_identifier_with_cover(app, record=doc) doc = Document.get_record_by_pid(doc["pid"]) assert "cover_metadata" not in doc
def test_access_permissions( client, json_headers, testdata, users, with_access ): """Test GET documents with `_access` ignoring `restricted`.""" # set the documents to have read access only by patron2. `_access` should # be taken into account and take precedence over `restricted`. indexer = RecordIndexer() doc1 = Document.get_record_by_pid("docid-open-access") doc2 = Document.get_record_by_pid("docid-closed-access") for doc in [doc1, doc2]: doc.update(dict(_access=dict(read=[users["patron2"].id]))) doc.commit() db.session.commit() indexer.index(doc) current_search.flush_and_refresh(index="documents") test_data = [ ("anonymous", "docid-open-access", 401, 0), ("patron1", "docid-open-access", 403, 0), ("patron2", "docid-open-access", 200, 1), # should have access ("librarian", "docid-open-access", 200, 1), ("admin", "docid-open-access", 200, 1), ("anonymous", "docid-closed-access", 401, 0), ("patron1", "docid-closed-access", 403, 0), ("patron2", "docid-closed-access", 200, 1), # should have access ("librarian", "docid-closed-access", 200, 1), ("admin", "docid-closed-access", 200, 1), ] for user, pid, status_code, n_hits in test_data: # item endpoint user_login(client, user, users) url = url_for("invenio_records_rest.docid_item", pid_value=pid) res = client.get(url, headers=json_headers) assert res.status_code == status_code # list endpoint user_login(client, user, users) url = url_for( "invenio_records_rest.docid_list", q="pid:{}".format(pid) ) res = client.get(url, headers=json_headers) hits = json.loads(res.data.decode("utf-8")) assert hits["hits"]["total"] == n_hits
def test_email_on_loan_checkout(client, app_with_mail, users, testdata, loan_params): """Test that an email is sent on loan checkout.""" loan_data = testdata["loans"][1] loan = Loan.get_record_by_pid(loan_data["pid"]) with app_with_mail.extensions["mail"].record_messages() as outbox: user_login(client, "admin", users) assert len(outbox) == 0 current_circulation.circulation.trigger( loan, **dict(loan_params, trigger="checkout")) assert len(outbox) == 1 msg = outbox[0] doc = Document.get_record_by_pid(loan_data["document_pid"]) expected_subject = ( """InvenioILS: your loan for "{0}" has started.""".format( doc["title"])) assert msg.subject == expected_subject edition_year = " ({edition} - {year})".format(edition=doc["edition"], year=doc["publication_year"]) full_title = "{title}, {author}{edition_year}".format( title=doc["title"], author=doc["authors"][0]["full_name"], edition_year=edition_year, ) literature_url = "{host}{path}".format( host=current_app.config["SPA_HOST"], path=current_app.config["SPA_PATHS"]["literature"] % {"pid": doc["pid"]}, ) profile_url = "{host}{path}".format( host=current_app.config["SPA_HOST"], path=current_app.config["SPA_PATHS"]["profile"], ) expected_body_plain = """Dear Patron One, your loan for "{doc_full_title}" <{literature_url}> has started. The due date is {loan_end_date}. You can see your ongoing and past loans in your profile page <{profile_url}>. Kind regards, InvenioILS""".format( doc_full_title=full_title, literature_url=literature_url, loan_end_date=loan_data["end_date"], profile_url=profile_url, ) assert msg.body == expected_body_plain
def _test_delete(): """Test relation creation.""" rec1, rec2 = _choose_endpoints_and_do_request( (client, json_headers, "DELETE"), (first_pid, first_pid_type, second_pid, second_pid_type), payload, ) rec3 = Document.get_record_by_pid(third_pid) rec3 = rec3.replace_refs() _assert_record_relations(rec1, expected={"relations": {}}) _assert_record_relations(rec2, expected={"relations": {}}) _assert_record_relations(rec3, expected={"relations": {}})
def get_document(document_pid): """Return the Document record.""" document = Document.get_record_by_pid(document_pid) authors = [] for author in document.get("authors", []): if "full_name" in author: authors.append(author["full_name"]) return { "pid": document.get("pid"), "title": document.get("title", ""), "authors": authors, }
def validate_serial_relation(serial, recids): relations = serial.relations.get().get('serial', []) if len(recids) != len(relations): click.echo('[Serial {}] Incorrect number of children: {} ' '(expected {})'.format(serial['pid'], len(relations), len(recids))) for relation in relations: child = Document.get_record_by_pid(relation['pid'], pid_type=relation['pid_type']) if 'legacy_recid' in child and child['legacy_recid'] not in recids: click.echo('[Serial {}] Unexpected child with legacy ' 'recid: {}'.format(serial['pid'], child['legacy_recid']))
def test_with_identifiers_with_valid_cover(doc): """It should have the ISBN in cover_metadata.""" # return cover is valid mocker.patch( "cds_ils.literature.tasks.is_valid_cover", return_value=True ) doc["identifiers"] = [dict(scheme="ISBN", value="valid-isbn")] doc["cover_metadata"] = dict(ISBN="valid-isbn") doc.commit() db.session.commit() pick_identifier_with_cover(app, record=doc) doc = Document.get_record_by_pid(doc["pid"]) assert doc["cover_metadata"] == dict(ISBN="valid-isbn")
def validate_multipart_relation(multipart, volumes): relations = multipart.relations.get().get('multipart_monograph', []) titles = [volume['title'] for volume in volumes if 'title' in volume] count = len(set(v['volume'] for v in volumes)) if count != len(relations): click.echo('[Multipart {}] Incorrect number of volumes: {} ' '(expected {})'.format(multipart['pid'], len(relations), count)) for relation in relations: child = Document.get_record_by_pid(relation['pid'], pid_type=relation['pid_type']) if child['title']['title'] not in titles: click.echo('[Multipart {}] Title "{}" does not exist in ' 'migration data'.format(multipart['pid'], child['title']['title']))
def test_document_update_refs(app): """Test update of a document.""" d = dict(pid="a1bc", title="Test title", authors=[dict(full_name="John Doe")], publication_year="2010") doc = Document.create(d) del doc["circulation"] del doc["relations"] del doc["eitems"] del doc["items"] del doc["stock"] doc.update(dict(title="Test title 2")) _assert_extra_fields(doc) assert doc["title"] == "Test title 2"
def test_document_creation_refs(app): """Test creation of a document.""" d = dict( pid="a1bc", created_by={ "type": "script", "value": "demo" }, title="Test title", authors=[dict(full_name="John Doe")], publication_year="2010", document_type="BOOK", ) doc = Document.create(d) _assert_extra_fields(doc)
def test_add_new_identifier(doc): """It should change the cover_metadata to the new identifier.""" # return cover is not valid mocker.patch( "cds_ils.literature.tasks.is_valid_cover", return_value=True ) # the ISBN has been changed to a not valid one doc["identifiers"] = [dict(scheme="ISBN", value="valid-isbn")] # it had a previously valid ISBN doc["cover_metadata"] = dict(ISBN="123456789") doc.commit() db.session.commit() pick_identifier_with_cover(app, record=doc) doc = Document.get_record_by_pid(doc["pid"]) assert doc["cover_metadata"] == dict(ISBN="valid-isbn")
def test_with_invalid_identifiers_no_valid_cover(doc): """It should remove cover_metadata, the ISBN is not valid.""" # return cover is not valid mocker.patch( "cds_ils.literature.tasks.is_valid_cover", return_value=False ) # the ISBN has been changed to a not valid one doc["identifiers"] = [dict(scheme="ISBN", value="not-valid-isbn")] # it had a previously valid ISBN doc["cover_metadata"] = dict(ISBN="valid-isbn") doc.commit() db.session.commit() pick_identifier_with_cover(app, record=doc) doc = Document.get_record_by_pid(doc["pid"]) assert "cover_metadata" not in doc
def validate_multipart_relation(multipart, volumes): relations = multipart.relations.get().get("multipart_monograph", []) titles = [volume["title"] for volume in volumes if "title" in volume] count = len(set(v["volume"] for v in volumes)) if count != len(relations): click.echo( "[Multipart {}] Incorrect number of volumes: {} " "(expected {})".format(multipart["pid"], len(relations), count) ) for relation in relations: child = Document.get_record_by_pid( relation["pid"], pid_type=relation["pid_type"] ) if child["title"] not in titles: click.echo( '[Multipart {}] Title "{}" does not exist in ' "migration data".format(multipart["pid"], child["title"]) )
def validate_serial_relation(serial, recids): relations = serial.relations.get().get("serial", []) if len(recids) != len(relations): click.echo( "[Serial {}] Incorrect number of children: {} " "(expected {})".format( serial["pid"], len(relations), len(recids) ) ) for relation in relations: child = Document.get_record_by_pid( relation["pid"], pid_type=relation["pid_type"] ) if "legacy_recid" in child and child["legacy_recid"] not in recids: click.echo( "[Serial {}] Unexpected child with legacy " "recid: {}".format(serial["pid"], child["legacy_recid"]) )
def test_document_resolvers(app, testdata): """Test item resolving from loan.""" doc_pid = testdata["documents"][0]["pid"] document = Document.get_record_by_pid(doc_pid) document = document.replace_refs() # relations assert "relations" in document # circulation assert "active_loans" in document["circulation"] # item and eitems assert document["items"]["total"] == 9 and document["items"]["hits"] assert document["eitems"]["total"] == 3 and document["eitems"]["hits"] # stock mediums = set([item["medium"] for item in document["items"]["hits"]]) mediums.add("ELECTRONIC_VERSION") assert set(document["stock"]["mediums"]) == mediums
def migrate_ezproxy_links(): """Migrate external links from documents.""" search = get_documents_with_proxy_eitems() click.echo("Found {} documents with ezproxy links.".format(search.count())) for hit in search.scan(): # make sure the document is in DB not only ES document = Document.get_record_by_pid(hit.pid) click.echo("Processing document {}...".format(document["pid"])) for url in document["_migration"]["eitems_external"]: eitem = create_eitem(document["pid"], open_access=False) url["login_required"] = True eitem["urls"] = [url] eitem.commit() EItemIndexer().index(eitem) document["_migration"]["eitems_has_proxy"] = False document.commit() db.session.commit() DocumentIndexer().index(document)