def test_related_add_same_language(app, testdata):
"""Test adding related languages."""
doc1 = Document.get_record_by_pid(testdata["documents"][0]["pid"])
doc2 = Document.get_record_by_pid(testdata["documents"][1]["pid"])
doc1.related.add_language(doc2)
with pytest.raises(RelatedRecordError):
doc1.related.add_language(doc2)
def test_related_remove_language(app, testdata):
"""Test adding related languages."""
doc1 = Document.get_record_by_pid(testdata["documents"][0]["pid"])
doc2 = Document.get_record_by_pid(testdata["documents"][1]["pid"])
assert len(doc1.related.languages) == 0
doc1.related.add_language(doc2)
assert len(doc1.related.languages) == 1
doc1.related.remove_language(doc2)
assert len(doc1.related.languages) == 0
def test_related_add_existing_child(related_record, testdata):
"""Add a related language to an existing relation graph."""
docs = testdata["documents"]
series = testdata["series"]
doc1 = Document.get_record_by_pid(docs[0]["pid"])
doc6 = Document.get_record_by_pid(docs[4]["pid"])
ser10 = Series.get_record_by_pid(series[1]["pid"])
# Should fail if trying to add a child that already has relations
with pytest.raises(RelatedRecordError):
ser10.related.add_language(doc1)
with pytest.raises(RelatedRecordError):
ser10.related.add_language(doc6)
def test_related_add_language(app, testdata):
"""Test adding related languages."""
doc1 = Document.get_record_by_pid(testdata["documents"][0]["pid"])
doc2 = Document.get_record_by_pid(testdata["documents"][1]["pid"])
doc1.related.add_language(doc2)
parent_languages = doc1.related.languages
child_languages = doc2.related.languages
assert len(parent_languages) == 1
assert len(child_languages) == 1
assert parent_languages[0] == doc2
assert child_languages[0] == doc1
def related_record(testdata):
"""An example of a record with several relations."""
docs = testdata["documents"]
series = testdata["series"]
doc1 = Document.get_record_by_pid(docs[0]["pid"])
doc2 = Document.get_record_by_pid(docs[1]["pid"])
doc3 = Document.get_record_by_pid(docs[2]["pid"])
doc4 = Document.get_record_by_pid(docs[3]["pid"])
ser5 = Series.get_record_by_pid(series[0]["pid"])
doc6 = Document.get_record_by_pid(docs[4]["pid"])
doc7 = Document.get_record_by_pid(docs[5]["pid"])
doc8 = Document.get_record_by_pid(docs[6]["pid"])
doc9 = Document.get_record_by_pid(docs[7]["pid"])
doc1.related.add_edition(doc2)
doc1.related.add_edition(doc3)
doc1.related.add_edition(doc4)
doc1.related.add_edition(ser5)
doc1.related.add_language(doc6)
doc6.related.add_edition(doc7)
doc6.related.add_edition(doc8)
doc1.related.add_language(doc9)
return doc1
def test_document_remove_keyword(app, testdata):
"""Test removing keywords from document record."""
document_pid = testdata["documents"][0]["pid"]
keywords = [
Keyword.get_record_by_pid(kw["pid"]) for kw in testdata["keywords"]
]
document = Document.get_record_by_pid(document_pid)
for keyword in keywords:
with pytest.raises(DocumentKeywordNotFoundError):
assert not document.remove_keyword(keyword)
assert len(document["keyword_pids"]) == 0
for keyword in keywords:
document.add_keyword(keyword)
assert len(document["keyword_pids"]) == len(keywords)
assert document.remove_keyword(keywords[1])
assert len(document["keyword_pids"]) == len(keywords) - 1
# repeat
with pytest.raises(DocumentKeywordNotFoundError):
document.remove_keyword(keywords[1])
assert len(document["keyword_pids"]) == len(keywords) - 1
assert document.remove_keyword(keywords[0])
assert len(document["keyword_pids"]) == len(keywords) - 2
def test_document_remove_tag(app, testdata):
"""Test removing tags from document record."""
document_pid = testdata["documents"][0]["pid"]
tags = [Tag.get_record_by_pid(kw["pid"]) for kw in testdata["tags"]]
document = Document.get_record_by_pid(document_pid)
for tag in tags:
with pytest.raises(DocumentTagNotFoundError):
assert not document.remove_tag(tag)
assert len(document["tag_pids"]) == 0
for tag in tags:
document.add_tag(tag)
assert len(document["tag_pids"]) == len(tags)
assert document.remove_tag(tags[1])
assert len(document["tag_pids"]) == len(tags) - 1
# repeat
with pytest.raises(DocumentTagNotFoundError):
document.remove_tag(tags[1])
assert len(document["tag_pids"]) == len(tags) - 1
assert document.remove_tag(tags[0])
assert len(document["tag_pids"]) == len(tags) - 2
def testdata(app, db, es_clear):
"""Create, index and return test data."""
indexer = RecordIndexer()
locations = load_json_from_datadir("locations.json")
for location in locations:
record = Location.create(location)
mint_record_pid(LOCATION_PID_TYPE, Location.pid_field, record)
record.commit()
db.session.commit()
indexer.index(record)
internal_locations = load_json_from_datadir("internal_locations.json")
iloc_records = []
for internal_location in internal_locations:
record = InternalLocation.create(internal_location)
mint_record_pid(INTERNAL_LOCATION_PID_TYPE, InternalLocation.pid_field,
record)
record.commit()
iloc_records.append(record)
db.session.commit()
indexer.index(record)
documents = load_json_from_datadir("documents.json")
for doc in documents:
record = Document.create(doc)
mint_record_pid(DOCUMENT_PID_TYPE, Document.pid_field, record)
record.commit()
db.session.commit()
indexer.index(record)
items = load_json_from_datadir("items.json")
for item in items:
record = Item.create(item)
mint_record_pid(ITEM_PID_TYPE, Item.pid_field, record)
record.commit()
db.session.commit()
indexer.index(record)
loans = load_json_from_datadir("loans.json")
for loan in loans:
record = Loan.create(loan)
mint_record_pid(CIRCULATION_LOAN_PID_TYPE, Loan.pid_field, record)
record.commit()
db.session.commit()
indexer.index(record)
# re-index item attached to the loan
index_record_after_loan_change(app, record)
# flush all indices after indexing, otherwise ES won't be ready for tests
current_search.flush_and_refresh(index=None)
return {
"locations": locations,
"documents": documents,
"items": items,
"loans": loans,
}
def create_multipart_volumes(pid, multipart_legacy_recid, migration_volumes):
"""Create multipart volume documents."""
volumes = {}
# Combine all volume data by volume number
for obj in migration_volumes:
volume_number = obj['volume']
if volume_number not in volumes:
volumes[volume_number] = {}
volume = volumes[volume_number]
for key in obj:
if key != 'volume':
if key in volume:
raise KeyError(
'Duplicate key "{}" for multipart {}'.format(
key, multipart_legacy_recid))
volume[key] = obj[key]
volume_numbers = iter(sorted(volumes.keys()))
# Re-use the current record for the first volume
first_volume = next(volume_numbers)
first = Document.get_record_by_pid(pid)
if 'title' in volumes[first_volume]:
first['title']['title'] = volumes[first_volume]['title']
first['volume'] = first_volume
first['_migration']['multipart_legacy_recid'] = multipart_legacy_recid
if 'legacy_recid' in first:
del first['legacy_recid']
first.commit()
yield first
# Create new records for the rest
for number in volume_numbers:
temp = first.copy()
temp['title']['title'] = volumes[number]['title']
temp['volume'] = number
record_uuid = uuid.uuid4()
provider = DocumentIdProvider.create(
object_type='rec',
object_uuid=record_uuid,
)
temp['pid'] = provider.pid.pid_value
record = Document.create(temp, record_uuid)
record.commit()
yield record
def serialize_on_loan_book_information(loan):
"""Serialize loan information."""
location = Location.get_record_by_pid(loan["transaction_location_pid"])
document = Document.get_record_by_pid(loan["document_pid"])
return dict(barcode=loan["item"]["barcode"],
end_date=loan["end_date"],
library=location["name"],
location=location["address"],
title=document["title"])
def serialize_loan_request_book_information(loan):
"""Serialize loan information."""
location = Location.get_record_by_pid(loan["transaction_location_pid"])
document = Document.get_record_by_pid(loan["document_pid"])
return dict(request_start_date=loan["start_date"],
request_end_date=loan["end_date"],
library=location["name"],
location=location["address"],
title=document["title"])
def test_related_add_existing_record(app, related_record, testdata):
"""Test adding an existing related record that was on a different node."""
# Test language
docs = testdata["documents"]
doc6 = Document.get_record_by_pid(docs[4]["pid"])
doc9 = Document.get_record_by_pid(docs[7]["pid"])
assert len(doc6.related.languages) == len(doc9.related.languages) == 2
with pytest.raises(RelatedRecordError):
doc6.related.add_language(doc9)
assert len(doc6.related.languages) == len(doc9.related.languages) == 2
# Test edition
doc3 = Document.get_record_by_pid(docs[2]["pid"])
doc4 = Document.get_record_by_pid(docs[3]["pid"])
assert len(doc3.related.editions) == len(doc4.related.editions) == 4
with pytest.raises(RelatedRecordError):
doc4.related.add_edition(doc3)
assert len(doc3.related.editions) == len(doc4.related.editions) == 4
def _test_delete():
"""Test relation creation."""
rec1, rec2 = _choose_endpoints_and_do_request(
(client, json_headers, "DELETE"),
(first_pid, first_pid_type, second_pid, second_pid_type),
payload,
)
rec3 = Document.get_record_by_pid(third_pid)
rec3 = rec3.replace_refs()
_assert_record_relations(rec1, expected={"relations": {}})
_assert_record_relations(rec2, expected={"relations": {}})
_assert_record_relations(rec3, expected={"relations": {}})
def test_related_add_multiple_children(app, testdata):
"""Test adding an existing related record that was on a different node."""
# Test language
docs = testdata["documents"]
series = testdata["series"]
doc1 = Document.get_record_by_pid(docs[0]["pid"])
doc2 = Document.get_record_by_pid(docs[1]["pid"])
ser3 = Series.get_record_by_pid(series[0]["pid"])
assert len(doc1.related.editions) == 0
assert len(doc2.related.editions) == 0
assert len(ser3.related.editions) == 0
doc1.related.add_edition(doc2)
doc1.commit()
doc1 = Document.get_record_by_pid(docs[0]["pid"])
doc2 = Document.get_record_by_pid(docs[1]["pid"])
ser3 = Series.get_record_by_pid(series[0]["pid"])
assert len(doc1.related.editions) == 1
assert len(doc2.related.editions) == 1
assert len(ser3.related.editions) == 0
doc1.related.add_edition(ser3)
doc1.commit()
doc1 = Document.get_record_by_pid(docs[0]["pid"])
doc2 = Document.get_record_by_pid(docs[1]["pid"])
ser3 = Series.get_record_by_pid(series[0]["pid"])
assert len(doc1.related.editions) == 2
assert len(doc2.related.editions) == 2
assert len(ser3.related.editions) == 2
def testdata(app, db, es_clear, system_user):
"""Create, index and return test data."""
indexer = RecordIndexer()
locations = load_json_from_datadir("locations.json")
for location in locations:
record = Location.create(location)
mint_record_pid(LOCATION_PID_TYPE, "pid", record)
record.commit()
db.session.commit()
indexer.index(record)
internal_locations = load_json_from_datadir("internal_locations.json")
for internal_location in internal_locations:
record = InternalLocation.create(internal_location)
mint_record_pid(INTERNAL_LOCATION_PID_TYPE, "pid", record)
record.commit()
db.session.commit()
indexer.index(record)
documents = load_json_from_datadir("documents.json")
for doc in documents:
record = Document.create(doc)
mint_record_pid(DOCUMENT_PID_TYPE, "pid", record)
record.commit()
db.session.commit()
indexer.index(record)
items = load_json_from_datadir("items.json")
for item in items:
record = Item.create(item)
mint_record_pid(ITEM_PID_TYPE, "pid", record)
record.commit()
db.session.commit()
indexer.index(record)
loans = load_json_from_datadir("loans.json")
for loan in loans:
record = Loan.create(loan)
mint_record_pid(CIRCULATION_LOAN_PID_TYPE, "pid", record)
record.commit()
db.session.commit()
indexer.index(record)
# flush all indices after indexing, otherwise ES won't be ready for tests
current_search.flush_and_refresh(index='*')
return {
"locations": locations,
"documents": documents,
"items": items,
"loans": loans,
}
def validate_serial_relation(serial, recids):
relations = serial.relations.get().get('serial', [])
if len(recids) != len(relations):
click.echo('[Serial {}] Incorrect number of children: {} '
'(expected {})'.format(serial['pid'], len(relations),
len(recids)))
for relation in relations:
child = Document.get_record_by_pid(relation['pid'],
pid_type=relation['pid_type'])
if 'legacy_recid' in child and child['legacy_recid'] not in recids:
click.echo('[Serial {}] Unexpected child with legacy '
'recid: {}'.format(serial['pid'],
child['legacy_recid']))
def test_related_add_editions_to_child(app, testdata):
"""Test adding related editions to a child."""
doc1 = Document.get_record_by_pid(testdata["documents"][0]["pid"])
doc2 = Document.get_record_by_pid(testdata["documents"][1]["pid"])
ser3 = Series.get_record_by_pid(testdata["series"][0]["pid"])
doc1.related.add_edition(doc2)
doc2.related.add_edition(ser3)
parent_editions = doc1.related.editions
child1_editions = doc2.related.editions
child2_editions = ser3.related.editions
assert len(parent_editions) == 2
assert len(child1_editions) == 2
assert len(child2_editions) == 2
assert parent_editions[0] == doc2
assert parent_editions[1] == ser3
assert child1_editions[0] == doc1
assert child1_editions[1] == ser3
assert child2_editions[0] == doc2
assert child2_editions[1] == doc1
def test_related_add_editions_to_parent(app, testdata):
"""Test adding related editions."""
doc1 = Document.get_record_by_pid(testdata["documents"][0]["pid"])
doc2 = Document.get_record_by_pid(testdata["documents"][1]["pid"])
child2 = Series.get_record_by_pid(testdata["series"][0]["pid"])
doc1.related.add_edition(doc2)
parent_editions = doc1.related.editions
child_editions = doc2.related.editions
assert len(parent_editions) == 1
assert len(child_editions) == 1
assert parent_editions[0] == doc2
assert child_editions[0] == doc1
doc1.related.add_edition(child2)
parent_editions = doc1.related.editions
child_editions = child2.related.editions
assert len(parent_editions) == 2
assert len(child_editions) == 2
assert parent_editions[0] == doc2
assert parent_editions[1] == child2
assert child_editions[0] == doc1
assert child_editions[1] == doc2
def validate_multipart_relation(multipart, volumes):
relations = multipart.relations.get().get('multipart_monograph', [])
titles = [volume['title'] for volume in volumes if 'title' in volume]
count = len(set(v['volume'] for v in volumes))
if count != len(relations):
click.echo('[Multipart {}] Incorrect number of volumes: {} '
'(expected {})'.format(multipart['pid'], len(relations),
count))
for relation in relations:
child = Document.get_record_by_pid(relation['pid'],
pid_type=relation['pid_type'])
if child['title']['title'] not in titles:
click.echo('[Multipart {}] Title "{}" does not exist in '
'migration data'.format(multipart['pid'],
child['title']['title']))
def document_resolver(loan_pid):
"""Resolve a Document given a Loan PID."""
try:
document_pid = get_field_value(Loan, loan_pid, "document_pid")
except KeyError:
return {}
try:
document = Document.get_record_by_pid(document_pid)
# add only some fields
obj = {}
include_document_keys = ["title", "pid", "circulation", "authors"]
for key in include_document_keys:
obj[key] = document[key]
except PIDDeletedError:
obj = {}
return obj
def test_related_complex_record(related_record, testdata):
"""Test the related record fixture."""
docs = testdata["documents"]
series = testdata["series"]
doc1 = Document.get_record_by_pid(docs[0]["pid"])
doc2 = Document.get_record_by_pid(docs[1]["pid"])
doc3 = Document.get_record_by_pid(docs[2]["pid"])
doc4 = Document.get_record_by_pid(docs[3]["pid"])
ser5 = Series.get_record_by_pid(series[0]["pid"])
doc6 = Document.get_record_by_pid(docs[4]["pid"])
doc7 = Document.get_record_by_pid(docs[5]["pid"])
doc8 = Document.get_record_by_pid(docs[6]["pid"])
doc9 = Document.get_record_by_pid(docs[7]["pid"])
assert len(doc9.related.editions) == 0
for rec in (doc1, doc2, doc3, doc4, ser5):
assert len(rec.related.editions) == 4
for rec in (doc6, doc7, doc8):
assert len(rec.related.editions) == 2
for rec in (doc1, doc6, doc9):
assert len(rec.related.languages) == 2
doc1.related.remove_language(doc6)
for rec in (doc1, doc9):
assert len(rec.related.languages) == 1
assert len(doc6.related.languages) == 0
for rec in (doc6, doc7, doc8):
assert len(rec.related.editions) == 2
ser5.related.remove_edition(doc3)
for rec in (doc1, doc2, doc4, ser5):
assert len(rec.related.editions) == 3
assert len(doc3.related.editions) == 0
def create_record(cls, dump):
"""Create a new record from dump."""
# Reserve record identifier, create record and recid pid in one
# operation.
timestamp, data = dump.latest
record = Record.create(data)
record_uuid = uuid.uuid4()
provider = DocumentIdProvider.create(
object_type='rec',
object_uuid=record_uuid,
)
timestamp, json_data = dump.rest[-1]
json_data['pid'] = provider.pid.pid_value
record.model.json = json_data
record.model.created = dump.created.replace(tzinfo=None)
record.model.updated = timestamp.replace(tzinfo=None)
document = Document.create(record.model.json, record_uuid)
document.commit()
db.session.commit()
return document
def test_document_add_keywords(app, testdata):
"""Test adding new keywords to document record."""
document_pid = testdata["documents"][0]["pid"]
keyword_data = testdata["keywords"]
keywords = [Keyword.get_record_by_pid(kw["pid"]) for kw in keyword_data]
document = Document.get_record_by_pid(document_pid)
assert len(document["keyword_pids"]) == 0
for i, keyword in enumerate(keywords):
document.add_keyword(keyword)
doc_name = document.replace_refs()["keywords"][i]["name"]
assert doc_name == keyword_data[i]["name"]
assert len(document["keyword_pids"]) == len(keywords)
# Shouldnt' add more keywords
for keyword in keywords:
document.add_keyword(keyword)
assert len(document["keyword_pids"]) == len(keywords)
def test_document_add_tags(app, testdata):
"""Test adding new tags to document record."""
document_pid = testdata["documents"][0]["pid"]
tag_data = testdata["tags"]
tags = [Tag.get_record_by_pid(kw["pid"]) for kw in tag_data]
document = Document.get_record_by_pid(document_pid)
assert len(document["tag_pids"]) == 0
for i, tag in enumerate(tags):
document.add_tag(tag)
doc_name = document.replace_refs()["tags"][i]["name"]
assert doc_name == tag_data[i]["name"]
assert len(document["tag_pids"]) == len(tags)
# Shouldnt' add more tags
for tag in tags:
document.add_tag(tag)
assert len(document["tag_pids"]) == len(tags)
def test_related_remove_parent(testdata):
"""Test remove parent."""
pid1 = testdata["documents"][0]["pid"]
pid2 = testdata["documents"][1]["pid"]
pid3 = testdata["documents"][2]["pid"]
doc1 = Document.get_record_by_pid(pid1)
doc2 = Document.get_record_by_pid(pid2)
doc3 = Document.get_record_by_pid(pid3)
doc1.related.add_edition(doc2)
doc1.related.add_edition(doc3)
doc1.commit()
doc1 = Document.get_record_by_pid(pid1)
doc2 = Document.get_record_by_pid(pid2)
doc3 = Document.get_record_by_pid(pid3)
assert len(doc1.related.editions) == 2
assert len(doc2.related.editions) == 2
assert len(doc3.related.editions) == 2
with pytest.raises(RelatedRecordError):
doc2.related.remove_edition(doc1)
doc2.commit()
def testdata(app, db, es_clear):
"""Create, index and return test data."""
indexer = RecordIndexer()
locations = load_json_from_datadir("locations.json")
for location in locations:
record = Location.create(location)
mint_record_pid(LOCATION_PID_TYPE, "pid", record)
record.commit()
db.session.commit()
indexer.index(record)
internal_locations = load_json_from_datadir("internal_locations.json")
for internal_location in internal_locations:
record = InternalLocation.create(internal_location)
mint_record_pid(INTERNAL_LOCATION_PID_TYPE, "pid", record)
record.commit()
db.session.commit()
indexer.index(record)
keywords = load_json_from_datadir("keywords.json")
for keyword in keywords:
record = Keyword.create(keyword)
mint_record_pid(KEYWORD_PID_TYPE, "pid", record)
record.commit()
db.session.commit()
indexer.index(record)
series_data = load_json_from_datadir("series.json")
for series in series_data:
record = Series.create(series)
mint_record_pid(SERIES_PID_TYPE, "pid", record)
record.commit()
db.session.commit()
indexer.index(record)
documents = load_json_from_datadir("documents.json")
for doc in documents:
record = Document.create(doc)
mint_record_pid(DOCUMENT_PID_TYPE, "pid", record)
record.commit()
db.session.commit()
indexer.index(record)
items = load_json_from_datadir("items.json")
for item in items:
record = Item.create(item)
mint_record_pid(ITEM_PID_TYPE, "pid", record)
record.commit()
db.session.commit()
indexer.index(record)
eitems = load_json_from_datadir("eitems.json")
for eitem in eitems:
record = EItem.create(eitem)
mint_record_pid(EITEM_PID_TYPE, "pid", record)
record.commit()
db.session.commit()
indexer.index(record)
loans = load_json_from_datadir("loans.json")
for loan in loans:
record = Loan.create(loan)
mint_record_pid(CIRCULATION_LOAN_PID_TYPE, "pid", record)
record.commit()
db.session.commit()
indexer.index(record)
# flush all indices after indexing, otherwise ES won't be ready for tests
current_search.flush_and_refresh(index='*')
return {
"locations": locations,
"internal_locations": internal_locations,
"documents": documents,
"items": items,
"loans": loans,
"keywords": keywords,
"series": series_data,
}
def index_document_after_loan_indexed(document_pid):
"""Index documentt to re-compute circulation information."""
if document_pid:
document = Document.get_record_by_pid(document_pid)
RecordIndexer().index(document)
def _test_create():
"""Test relation creation."""
rec1, rec2 = _choose_endpoints_and_do_request(
(client, json_headers, "POST"),
(first_pid, first_pid_type, second_pid, second_pid_type),
payload,
)
rec3 = Document.get_record_by_pid(third_pid)
rec3 = rec3.replace_refs()
_assert_record_relations(
rec1,
expected={
"relations": {
"language": [
{
"pid": second_pid,
"pid_type": second_pid_type,
"title": rec2["title"],
"languages": rec2["languages"],
"relation_type": "language",
},
{
"pid": third_pid,
"pid_type": third_pid_type,
"title": rec3["title"],
"relation_type": "language",
},
]
}
},
)
_assert_record_relations(
rec2,
expected={
"relations": {
"language": [
{
"pid": first_pid,
"pid_type": first_pid_type,
"title": rec1["title"],
"languages": rec1["languages"],
"edition": rec1["edition"],
"relation_type": "language",
},
{
"pid": third_pid,
"pid_type": third_pid_type,
"title": rec3["title"],
"relation_type": "language",
},
]
}
},
)
_assert_record_relations(
rec3,
expected={
"relations": {
"language": [
{
"pid": first_pid,
"pid_type": first_pid_type,
"title": rec1["title"],
"languages": rec1["languages"],
"edition": rec1["edition"],
"relation_type": "language",
},
{
"pid": second_pid,
"pid_type": second_pid_type,
"title": rec2["title"],
"languages": rec2["languages"],
"relation_type": "language",
},
]
}
},
)
def relations_resolver(document_pid):
"""Resolve record relations and add metadata."""
document = Document.get_record_by_pid(document_pid)
return document.relations.get()
def _document_for_item_resolver(document_pid):
"""Return the document for the given item."""
document = Document.get_record_by_pid(document_pid)
# delete circulation field when document is dereferenced inside item
del document["circulation"]
return document