def test_returns_empty_if_no_reference_or_record_field(get_records_mock):
schema = ReferenceItemSchemaV1()
dump = {}
expected = {}
record = LiteratureRecord(dump)
result = schema.dumps(record).data
assert expected == orjson.loads(result)
def test_revert_to_revision(inspire_app, clean_celery_session,
record_with_two_revisions):
user = create_user(role=Roles.cataloger.value)
record = LiteratureRecord.get_record_by_pid_value(111)
assert record["titles"][0]["title"] == "record rev1"
with inspire_app.test_client() as client:
login_user_via_session(client, email=user.email)
response = client.put(
"/api/editor/literature/111/revisions/revert",
content_type="application/json",
data=orjson.dumps({"revision_id": 0}),
)
assert response.status_code == 200
record = LiteratureRecord.get_record_by_pid_value(111)
assert record["titles"][0]["title"] == "record rev0"
def test_forces_collaborations_to_be_object_if_reference_not_linked(
get_records_mock):
schema = ReferenceItemSchemaV1()
dump = {"reference": {"collaborations": ["CMS", "LHCb"]}}
expected = {"collaborations": [{"value": "CMS"}, {"value": "LHCb"}]}
record = LiteratureRecord(dump)
result = schema.dumps(record).data
assert expected == json.loads(result)
def test_returns_non_empty_fields_if_some_fields_missing(get_records_mock):
schema = ReferenceItemSchemaV1()
dump = {"reference": {"label": "123", "control_number": 123}}
expected = {"label": "123", "control_number": 123}
record = LiteratureRecord(dump)
result = schema.dumps(record).data
assert expected == orjson.loads(result)
def test_update_record_update_citation_table(base_app, db, es):
data = faker.record("lit")
record = LiteratureRecord.create(data)
data2 = faker.record("lit")
record2 = LiteratureRecord.create(data2)
# Cannot use model.citations and model.references
# when updating record which is not commited,
# as they will return data from before the update
assert len(RecordCitations.query.all()) == 0
data = faker.record(
"lit", data=record, literature_citations=[record2["control_number"]]
)
record.update(data)
assert len(RecordCitations.query.all()) == 1
def test_literature_citation_count_property(base_app, db, es):
data = faker.record("lit")
record = InspireRecord.create(data)
data2 = faker.record("lit", literature_citations=[record["control_number"]])
record2 = LiteratureRecord.create(data2)
assert record.citation_count == 1
assert record2.citation_count == 0
def test_returns_none_as_name_if_empty_present(get_records_mock):
schema = AcceleratorExperimentSchemaV1()
dump = {}
expected = {"name": None}
record = LiteratureRecord(dump)
result = schema.dumps(record).data
assert expected == json.loads(result)
def test_create_record_from_db_depending_on_its_pid_type(base_app, db, es):
data = faker.record("lit")
record = InspireRecord.create(data)
assert type(record) == LiteratureRecord
assert record.pid_type == "lit"
record = LiteratureRecord.create(data)
assert type(record) == LiteratureRecord
assert record.pid_type == "lit"
def test_get_enhanced_es_data_do_not_change_original_record(inspire_app, datadir):
data = orjson.loads((datadir / "788797.json").read_text())
original_data = deepcopy(data)
record = LiteratureRecord.create(data=data)
record.get_enhanced_es_data()
assert sorted(record) == sorted(original_data)
def test_migrate_and_insert_record_blacklisted_pid(inspire_app,
override_config):
raw_record = (b"<record>"
b' <controlfield tag="001">12345</controlfield>'
b' <datafield tag="980" ind1=" " ind2=" ">'
b' <subfield code="a">HEP</subfield>'
b" </datafield>"
b"</record>")
with override_config(MIGRATION_PID_TYPE_BLACKLIST=["lit"]):
migrate_and_insert_record(raw_record)
with pytest.raises(PIDDoesNotExistError):
LiteratureRecord.get_record_by_pid_value("12345")
prod_record = LegacyRecordsMirror.query.filter(
LegacyRecordsMirror.recid == 12345).one()
assert prod_record.valid is False
def test_get_modified_references_returns_all_references_when_earliest_date_changed(
inspire_app, celery_app_with_context, celery_session_worker):
cited_data = {
"$schema": "http://localhost:5000/schemas/records/hep.json",
"titles": [{
"title": "Test a valid record"
}],
"document_type": ["article"],
"_collections": ["Literature"],
}
cited_record = LiteratureRecord.create(cited_data)
citing_data = {
"$schema":
"http://localhost:5000/schemas/records/hep.json",
"titles": [{
"title": "My Title"
}],
"document_type": ["article"],
"_collections": ["Literature"],
"preprint_date":
"2019-06-28",
"references": [{
"record": {
"$ref":
f"http://localhost:5000/api/literature/{cited_record['control_number']}"
}
}],
}
citing_record = LiteratureRecord.create(citing_data)
db.session.commit()
assert citing_record.get_modified_references() == [cited_record.id]
data_update = {
"preprint_date": "2018-06-28",
"control_number": citing_record["control_number"],
}
citing_data.update(data_update)
citing_record.update(citing_data)
db.session.commit()
assert citing_record.get_modified_references() == [cited_record.id]
def test_cds_sync_continues_when_some_records_fails(inspire_app_for_cds_sync):
expected_1203988_external_system_identifiers = [{
"value": "1273173",
"schema": "CDS"
}]
expected_1314109_external_system_identifiers = [{
"value": "1742265",
"schema": "CDS"
}]
expected_1314110_external_system_identifiers = [{
"value": "2003162",
"schema": "CDS"
}]
LiteratureRecord.create(
faker.record("lit", data={"control_number": 1203988}))
LiteratureRecord.create(
faker.record(
"lit",
data={
"control_number":
1314109,
"external_system_identifiers": [{
"schema": "CDS",
"value": "1742265"
}],
},
))
LiteratureRecord.create(
faker.record("lit", data={"control_number": 1314110}))
db.session.commit()
sync_identifiers("2020-07-01")
runs = CDSRun.query.all()
assert len(runs) == 1
assert runs[0].status == CDSRunStatus.FINISHED
assert runs[0].message == ""
assert (LiteratureRecord.get_record_by_pid_value(
"1203988")["external_system_identifiers"] ==
expected_1203988_external_system_identifiers)
assert (LiteratureRecord.get_record_by_pid_value(
"1314109")["external_system_identifiers"] ==
expected_1314109_external_system_identifiers)
assert (LiteratureRecord.get_record_by_pid_value(
"1314110")["external_system_identifiers"] ==
expected_1314110_external_system_identifiers)
def test_migrate_record_from_specified_date_only(inspire_app):
raw_record_1 = (
b"<record>"
b' <controlfield tag="001">666</controlfield>'
b' <datafield tag="245" ind1=" " ind2=" ">'
b' <subfield code="a">On the validity of INSPIRE records</subfield>'
b" </datafield>"
b' <datafield tag="980" ind1=" " ind2=" ">'
b' <subfield code="a">HEP</subfield>'
b" </datafield>"
b"</record>"
)
raw_record_2 = (
b"<record>"
b' <controlfield tag="001">667</controlfield>'
b' <datafield tag="245" ind1=" " ind2=" ">'
b' <subfield code="a">On the validity of INSPIRE records</subfield>'
b" </datafield>"
b' <datafield tag="980" ind1=" " ind2=" ">'
b' <subfield code="a">HEP</subfield>'
b" </datafield>"
b"</record>"
)
prod_record_1 = LegacyRecordsMirror.from_marcxml(raw_record_1)
prod_record_1.last_updated = datetime.datetime(2010, 1, 1)
db.session.merge(prod_record_1)
prod_record_2 = LegacyRecordsMirror.from_marcxml(raw_record_2)
prod_record_1.last_updated = datetime.datetime(2020, 1, 1)
db.session.merge(prod_record_2)
migrate_from_mirror(date_from="2015-01-01")
rec_1 = LegacyRecordsMirror.query.filter_by(recid=666).one()
rec_2 = LegacyRecordsMirror.query.filter_by(recid=667).one()
# Only record 2 should be valid as rec_1 migration should not run.
assert rec_1.valid is None
assert rec_2.valid is True
with pytest.raises(PIDDoesNotExistError):
LiteratureRecord.get_record_by_pid_value("666")
assert LiteratureRecord.get_record_by_pid_value("667")
def test_migrate_record_from_miror_steals_pids_from_deleted_records(inspire_app):
raw_record = (
b"<record>"
b' <controlfield tag="001">98765</controlfield>'
b' <datafield tag="024" ind1="7" ind2=" ">'
b' <subfield code="9">DOI</subfield>'
b' <subfield code="a">10.1000/a_doi</subfield>'
b" </datafield>"
b' <datafield tag="245" ind1=" " ind2=" ">'
b' <subfield code="a">A record to be merged</subfield>'
b" </datafield>"
b' <datafield tag="980" ind1=" " ind2=" ">'
b' <subfield code="a">HEP</subfield>'
b" </datafield>"
b"</record>"
)
migrate_and_insert_record(raw_record)
record = LiteratureRecord.get_record_by_pid_value("98765")
assert PersistentIdentifier.get("doi", "10.1000/a_doi").object_uuid == record.id
raw_record = (
b"<record>"
b' <controlfield tag="001">31415</controlfield>'
b' <datafield tag="024" ind1="7" ind2=" ">'
b' <subfield code="9">DOI</subfield>'
b' <subfield code="a">10.1000/a_doi</subfield>'
b" </datafield>"
b' <datafield tag="245" ind1=" " ind2=" ">'
b' <subfield code="a">A record that was merged</subfield>'
b" </datafield>"
b' <datafield tag="980" ind1=" " ind2=" ">'
b' <subfield code="a">HEP</subfield>'
b" </datafield>"
b' <datafield tag="981" ind1=" " ind2=" ">'
b' <subfield code="a">98765</subfield>'
b" </datafield>"
b"</record>"
)
migrate_and_insert_record(raw_record)
assert LiteratureRecord.get_record_by_pid_value("98765")
merged_record = LiteratureRecord.get_record_by_pid_value("31415")
assert (
PersistentIdentifier.get("doi", "10.1000/a_doi").object_uuid == merged_record.id
)
def test_conference_paper_get_updated_reference_conference_when_document_type_changes_to_other_conf_related(
inspire_app, clean_celery_session):
conference_1 = ConferencesRecord.create(faker.record("con"))
conference_1_control_number = conference_1["control_number"]
ref_1 = f"http://localhost:8000/api/conferences/{conference_1_control_number}"
expected_result = [conference_1.id]
data = {
"publication_info": [{
"conference_record": {
"$ref": ref_1
}
}],
"document_type": ["conference paper"],
}
record = LiteratureRecord.create(faker.record("lit", data))
record_control_number = record["control_number"]
db.session.commit()
expected_versions_len = 1
results = record.model.versions.all()
result_latest_version = results[-1].json
assert expected_versions_len == len(results)
assert result_latest_version == record
data = copy.deepcopy(dict(record))
data["document_type"] = ["proceedings"]
record.update(data)
db.session.commit()
record = LiteratureRecord.get_record_by_pid_value(record_control_number)
assert expected_result == sorted(
record.get_newest_linked_conferences_uuid())
expected_versions_len = 2
results = record.model.versions.all()
result_latest_version = results[-1].json
assert expected_versions_len == len(results)
assert result_latest_version == record
def test_lit_records_with_citations_updates(app, celery_app_with_context,
celery_session_worker,
retry_until_matched):
data = faker.record("lit")
rec = LiteratureRecord.create(data)
db.session.commit()
steps = [
{
"step": es.indices.refresh,
"args": ["records-hep"]
},
{
"step": LiteratureSearch.get_record_data_from_es,
"args": [rec],
"expected_result": {
"expected_key": "citation_count",
"expected_result": 0
},
},
]
retry_until_matched(steps)
citations = [rec["control_number"]]
data_2 = faker.record("lit", literature_citations=citations)
LiteratureRecord.create(data_2)
db.session.commit()
steps = [
{
"step": es.indices.refresh,
"args": ["records-hep"]
},
{
"step": LiteratureSearch.get_record_data_from_es,
"args": [rec],
"expected_result": {
"expected_key": "citation_count",
"expected_result": 1
},
},
]
retry_until_matched(steps)
def test_feature_flag_for_redirection_disables_redirection_when_turned_off(
inspire_app, override_config):
with override_config(FEATURE_FLAG_ENABLE_REDIRECTION_OF_PIDS=False):
record_1 = create_record("lit")
create_record("lit", data={"deleted_records": [record_1["self"]]})
record_1_from_db = LiteratureRecord.get_record_by_pid_value(
record_1["control_number"])
assert record_1_from_db.id == record_1.id
def test_returns_legacy_name_as_name(get_records_mock):
schema = AcceleratorExperimentSchemaV1()
dump = {"legacy_name": "Test"}
record = LiteratureRecord(dump)
expected = {"name": "Test"}
result = schema.dumps(record).data
assert expected == orjson.loads(result)
def test_migrate_from_mirror_doesnt_index_deleted_records(inspire_app):
record_fixture_path = pkg_resources.resource_filename(
__name__, os.path.join("fixtures", "dummy.xml"))
record_fixture_path_deleted = pkg_resources.resource_filename(
__name__, os.path.join("fixtures", "deleted_record.xml"))
migrate_from_file(record_fixture_path)
migrate_from_file(record_fixture_path_deleted)
current_search.flush_and_refresh("records-hep")
expected_record_lit_es_len = 1
record_lit_uuid = LiteratureRecord.get_uuid_from_pid_value(12345)
with pytest.raises(PIDDoesNotExistError):
LiteratureRecord.get_uuid_from_pid_value(1234)
record_lit_es = LiteratureSearch().get_record(
str(record_lit_uuid)).execute().hits
record_lit_es_len = len(record_lit_es)
assert expected_record_lit_es_len == record_lit_es_len
def test_hal_push_retry_on_stale_data_error(inspire_app):
data = faker.record("lit", with_control_number=True)
record = LiteratureRecord.create(data)
db.session.commit()
record["authors"] = [{"full_name": "Test Author"}]
record.update(dict(record))
db.session.commit()
with pytest.raises(StaleDataError):
hal_push(record["control_number"], 3)
def test_assign_from_an_author_to_another_that_is_not_stub(inspire_app):
cataloger = create_user(role="cataloger")
from_author = create_record("aut")
to_author = create_record("aut", data={"stub": False})
literature = create_record(
"lit",
data={
"authors": [
{
"full_name": "Urhan, Ahmet",
"record": {
"$ref": "http://localhost:5000/api/authors/17200"
},
},
{
"full_name": "Urhan, Harun",
"record": {
"$ref":
f"http://localhost:5000/api/authors/{from_author['control_number']}"
},
},
]
},
)
with inspire_app.test_client() as client:
login_user_via_session(client, email=cataloger.email)
response = client.post(
"/assign/author",
data=orjson.dumps({
"literature_recids": [literature["control_number"]],
"from_author_recid":
from_author["control_number"],
"to_author_recid":
to_author["control_number"],
}),
content_type="application/json",
)
response_status_code = response.status_code
assert response_status_code == 200
literature_after = LiteratureRecord.get_record_by_pid_value(
literature["control_number"])
literature_author = literature_after["authors"][1]
assert literature_author["record"] == {
"$ref":
f"http://localhost:5000/api/authors/{to_author['control_number']}"
}
assert literature_author["curated_relation"]
to_author_after = AuthorsRecord.get_record_by_pid_value(
to_author["control_number"])
assert not to_author_after["stub"]
def test_get_all_ids_by_update_date(inspire_app):
record = create_record("lit")
record2 = create_record("lit")
uuids = list(LiteratureRecord.get_recids_by_updated_datetime())
assert len(uuids) == 2
assert str(record.id) in uuids
assert str(record2.id) in uuids
uuids = list(InspireRecord.get_recids_by_updated_datetime())
assert len(uuids) == 0
new_date = datetime.strptime("2021-07-01", "%Y-%m-%d")
uuids = list(
LiteratureRecord.get_recids_by_updated_datetime(before=new_date))
assert len(uuids) == 0
uuids = list(
LiteratureRecord.get_recids_by_updated_datetime(after=new_date))
assert len(uuids) == 2
def test_create_or_update_when_redirection_enabled_updates_original_record(
inspire_app):
rec_to_delete = create_record("lit")
rec_which_deletes = create_record(
"lit", data={"deleted_records": [rec_to_delete["self"]]})
expected_title = {"title": "New title"}
data = dict(
LiteratureRecord.get_record_by_pid_value(
rec_to_delete["control_number"], original_record=True))
data["titles"][0] = expected_title
InspireRecord.create_or_update(data)
deleted_rec = LiteratureRecord.get_record_by_pid_value(
rec_to_delete["control_number"], original_record=True)
new_rec = LiteratureRecord.get_record_by_pid_value(
rec_which_deletes["control_number"])
assert deleted_rec["titles"][0] == expected_title
assert new_rec["titles"][0] == rec_which_deletes["titles"][0]
def test_redirect_wrong_new_pid_status(inspire_app):
record_1 = create_record("lit")
record_2 = create_record("lit")
record_2_pid = record_2.control_number_pid
record_2_pid.status = PIDStatus.DELETED
record_2_data = dict(record_2)
record_2_data["deleted_records"] = [record_1["self"]]
with pytest.raises(WrongRedirectionPidStatus):
record_2.update(record_2_data)
record_1_from_db = LiteratureRecord.get_record_by_pid_value(
record_1["control_number"])
record_2_from_db = LiteratureRecord.get_record_by_pid_value(
record_2["control_number"])
assert record_1_from_db.id == record_1.id
assert record_2_from_db.id == record_2.id
def test_redirect_and_delete_record_from_deleted_records_field(inspire_app):
record_to_delete = create_record("lit")
record = create_record("lit")
data = dict(record)
data["deleted_records"] = [record_to_delete["self"]]
record.update(data)
deleted_records = record["deleted_records"]
assert len(deleted_records) == 1
old_pid = record_to_delete.control_number_pid
assert old_pid.is_redirected()
record_redirected = LiteratureRecord.get_record_by_pid_value(
record_to_delete.control_number)
assert record_redirected.id == record.id
original_record = LiteratureRecord.get_record(record_to_delete.id,
with_deleted=True)
assert original_record["deleted"] is True
def test_lit_record_removed_from_es_when_deleted(inspire_app, clean_celery_session):
data = faker.record("lit")
rec = LiteratureRecord.create(data)
db.session.commit()
assert_record_in_es(rec["control_number"])
rec.delete()
db.session.commit()
assert_record_not_in_es(rec["control_number"])
def test_experiment_literature_table_is_not_updated_when_feature_flag_is_disabled(
update_function_mock, inspire_app):
institution = create_record("exp")
institution_control_number = institution["control_number"]
ref = f"http://localhost:8000/api/experiments/{institution_control_number}"
data = {
"accelerator_experiments": [{
"legacy_name": "LIGO",
"record": {
"$ref": ref
}
}]
}
record_data = faker.record("lit", data)
LiteratureRecord.create(record_data, disable_relations_update=True)
update_function_mock.assert_not_called()
LiteratureRecord.create(record_data, disable_relations_update=False)
update_function_mock.assert_called()
def test_add_and_remove_figs_and_docs_when_files_flag_disabled(
fsopen_mock, base_app, db, init_files_db, disable_files
):
expected_doc_filename = "document_file"
expected_fig_filename = "figure_file"
files = {
"documents": [
{
"url": "http://document_url.cern.ch/file.pdf",
"filename": "file_name.pdf",
"key": expected_doc_filename,
}
],
"figures": [
{"url": "http://figure_url.cern.ch/file.png", "key": expected_fig_filename}
],
}
data = faker.record("lit")
record = LiteratureRecord.create(data)
record.add_files(**files)
# added 2 records, should be 0
assert len(record.files.keys) == 0
with pytest.raises(KeyError):
record["_files"]
assert "documents" in record
assert "figures" in record
assert len(record["documents"]) == 1
assert len(record["figures"]) == 1
assert record["documents"][0]["key"] == expected_doc_filename
assert record["figures"][0]["key"] == expected_fig_filename
record.set_files(documents=files["documents"])
# when set only documents, figures should be removed
with pytest.raises(KeyError):
record["_files"]
assert "documents" in record
assert "figures" not in record
assert len(record.files.keys) == 0
assert len(record["documents"]) == 1
with pytest.raises(KeyError):
record["figures"]
assert record["documents"][0]["key"] == expected_doc_filename
def test_lit_record_appear_in_es_when_created(inspire_app, clean_celery_session):
data = faker.record("lit")
record = LiteratureRecord.create(data)
db.session.commit()
def assert_record():
current_search.flush_and_refresh("records-hep")
record_from_es = LiteratureSearch().get_record_data_from_es(record)
assert record_from_es["_ui_display"]
retry_until_pass(assert_record)
def test_get_modified_references_returns_no_references_when_non_impacting_metadata_changed(
app, clear_environment):
cited_data = {
"$schema": "http://localhost:5000/schemas/records/hep.json",
"titles": [{
"title": "Test a valid record"
}],
"document_type": ["article"],
"_collections": ["Literature"],
}
cited_record = LiteratureRecord.create(cited_data)
citing_data = {
"$schema":
"http://localhost:5000/schemas/records/hep.json",
"titles": [{
"title": "My title"
}],
"document_type": ["article"],
"_collections": ["Literature"],
"preprint_date":
"2019-06-28",
"references": [{
"record": {
"$ref":
f"http://localhost:5000/api/literature/{cited_record['control_number']}"
}
}],
}
citing_record = LiteratureRecord.create(citing_data)
db.session.commit()
assert citing_record.get_modified_references() == [cited_record.id]
data_update = {"titles": [{"title": "updated title"}]}
citing_data.update(data_update)
citing_record.update(citing_data)
db.session.commit()
assert citing_record.get_modified_references() == []
