def test_phonetic_blocks_keep_order_in_redis_based_on_timestamp(
base_app, db, es, redis
):
with freeze_time(datetime.datetime(2015, 8, 18, 8, 51, 50)):
author_data = {"authors": [{"full_name": "Ellis, John Richard"}]}
data = faker.record("lit", data=author_data)
InspireRecord.create(data)
with freeze_time(datetime.datetime(2015, 8, 18, 9, 51, 50)):
author_data2 = {"authors": [{"full_name": "Jimmy"}]}
data2 = faker.record("lit", data=author_data2)
LiteratureRecord.create(data2)
assert "ELj" == redis.zpopmin("author_phonetic_blocks")[0][0]
assert "JANY" == redis.zpopmin("author_phonetic_blocks")[0][0]
def test_lock_is_created_for_hal_push_task(mocked_lock, inspire_app, get_fixture):
record_json = orjson.loads(get_fixture("hal_preprod_record.json"))
record_data = faker.record("lit", data=record_json)
record = InspireRecord.create(record_data)
institute_json = orjson.loads(get_fixture("hal_preprod_institute.json"))
institute_data = faker.record("ins", data=institute_json)
InspireRecord.create(institute_data)
_hal_push(record)
recid = record["control_number"]
lock_name = f"hal:{recid}"
assert mocked_lock.mock_calls[0][1][0] == lock_name
assert mocked_lock.mock_calls[0][2] == {"blocking": True}
def test_get_domains(app):
app.config["HAL_DOMAIN_MAPPING"] = {
"Accelerators": "phys.phys.phys-acc-ph",
"Astrophysics": "phys.astr",
"Computing": "info",
"Data Analysis and Statistics": "phys.phys.phys-data-an",
"Experiment-HEP": "phys.hexp",
"Experiment-Nucl": "phys.nexp",
"General Physics": "phys.phys.phys-gen-ph",
"Gravitation and Cosmology": "phys.grqc",
"Instrumentation": "phys.phys.phys-ins-det",
"Lattice": "phys.hlat",
"Math and Math Physics": "phys.mphy",
"Other": "phys",
"Phenomenology-HEP": "phys.hphe",
"Theory-HEP": "phys.hthe",
"Theory-Nucl": "phys.nucl",
}
schema = load_schema("hep")
subschema = schema["properties"]["inspire_categories"]
data = {"inspire_categories": [{"term": "Experiment-HEP"}]}
assert validate(data["inspire_categories"], subschema) is None
record_data = faker.record("lit", data)
record = InspireRecord.create(record_data)
expected = ["phys.hexp"]
result = get_domains(record)
assert expected == result
record.delete()
def test_creating_record_with_id_provided_but_without_control_number_properly_mints_identifiers(
inspire_app, ):
record_data = {
"$schema": "https://inspirebeta.net/schemas/records/hep.json",
"arxiv_eprints": [{
"value": "2105.06728",
"categories": ["astro-ph.IM"]
}],
"_collections": ["Literature"],
"document_type": ["article"],
"titles": [{
"title": "Some title"
}],
}
id_ = uuid.uuid4()
rec = InspireRecord.create(data=record_data, id_=id_)
assert PersistentIdentifier.query.filter_by(object_uuid=id_).count() == 2
assert (PersistentIdentifier.query.filter_by(pid_type="arxiv",
pid_value="2105.06728",
object_uuid=id_).count() == 1)
cn_minted = PersistentIdentifier.query.filter_by(pid_type="lit",
object_uuid=id_).one()
assert cn_minted.pid_value == str(rec["control_number"])
def test_recalculate_references_recalculates_more_than_10_references(
inspire_app, clean_celery_session):
journal_data = faker.record("jou", with_control_number=True)
journal = InspireRecord.create(journal_data)
journal_record_reference = journal["self"]["$ref"]
literature_data = faker.record("lit")
literature_data.update({
"publication_info": [{
"journal_record": {
"$ref": journal_record_reference
}
}]
})
for i in range(11):
InspireRecord.create(literature_data)
db.session.commit()
def assert_all_records_in_es():
literature_records_from_es = list(LiteratureSearch().query_from_iq(
query_string=
f"publication_info.journal_record.$ref: {journal_record_reference}"
).scan())
journal_record_from_es = InspireSearch.get_record_data_from_es(journal)
assert len(literature_records_from_es) == 11 and journal_record_from_es
retry_until_pass(assert_all_records_in_es, retry_interval=5)
merged_journal_data = faker.record("jou", with_control_number=True)
merged_journal_data.update(
{"deleted_records": [{
"$ref": journal_record_reference
}]})
merged_journal_record = InspireRecord.create(merged_journal_data)
db.session.commit()
def assert_recalculate_references_task():
literature_records_from_es = list(LiteratureSearch().query_from_iq(
query_string=
f'publication_info.journal_record.$ref: {merged_journal_record["self"]["$ref"]}'
).scan())
assert len(literature_records_from_es) == 11
retry_until_pass(assert_recalculate_references_task, retry_interval=3)
def test_get_hal_id_map(app, get_fixture):
record_json = orjson.loads(get_fixture("_get_hal_id_map.json"))
record_data = faker.record("lit", data=record_json)
record = InspireRecord.create(record_data)
institute_json = orjson.loads(
get_fixture("_get_hal_id_map_institution.json"))
institute_data = faker.record("ins", data=institute_json)
institute_record = InspireRecord.create(institute_data)
result = _get_hal_id_map(record)
expected = {912490: "53946"}
assert result == expected
record.delete()
institute_record.delete()
def mock_job_create_and_update_time(date, data=None):
record_data = faker.record("job", data=data)
record = InspireRecord.create(record_data)
mock_datetime = mock.PropertyMock(return_value=date)
type(record).created = mock_datetime
type(record).updated = mock_datetime
record._index()
return record
def test_record_update_not_run_orcid_when_passed_parameter_to_disable_orcid(
orcid_mock, base_app, db
):
data1 = faker.record("lit")
data2 = faker.record("lit")
record1 = InspireRecord.create(data1, disable_orcid_push=True)
record1.update(data2, disable_orcid_push=True)
assert orcid_mock.call_count == 0
def test_record_update_runs_citation_recalculate_on_default(
citation_recalculate_mock, base_app, db
):
data1 = faker.record("lit")
data2 = faker.record("lit")
record1 = InspireRecord.create(data1)
record1.update(data2)
assert citation_recalculate_mock.call_count == 2
def test_record_update_skips_citation_recalculate_when_passed_parameter_to_skip(
citation_recalculate_mock, base_app, db
):
data1 = faker.record("lit")
data2 = faker.record("lit")
record1 = InspireRecord.create(data1, disable_citation_update=True)
record1.update(data2, disable_citation_update=True)
assert citation_recalculate_mock.call_count == 0
def test_update_relations_recalculate_citations_with_different_type_of_records_doesnt_throw_an_exception(
inspire_app, celery_app_with_context, celery_session_worker):
data_cited = faker.record("lit", with_control_number=True)
record_cited = InspireRecord.create(data_cited,
disable_relations_update=True)
db.session.commit()
record_cited_control_number = record_cited["control_number"]
data_citing = faker.record(
"lit",
literature_citations=[record_cited_control_number],
with_control_number=True,
)
record_citing = InspireRecord.create(data_citing,
disable_relations_update=True)
db.session.commit()
records = [
create_record_async("aut"),
create_record_async("job"),
create_record_async("jou"),
create_record_async("exp"),
create_record_async("con"),
create_record_async("dat"),
create_record_async("ins"),
]
uuids = [record.id
for record in records] + [record_cited.id, record_citing.id]
task = update_relations.delay(uuids)
results = task.get(timeout=5)
uuids = [str(uuid) for uuid in uuids]
assert results == uuids
result_record_cited = RecordCitations.query.filter_by(
cited_id=record_cited.id).one()
assert record_citing.id == result_record_cited.citer_id
record_cited = InspireRecord.get_record_by_pid_value(
record_cited_control_number, "lit")
record_cited_citation_count = 1
assert record_cited_citation_count == record_cited.citation_count
def test_update_relations(inspire_app, celery_app_with_context,
celery_session_worker):
conference_data = faker.record("con", with_control_number=True)
conference_record = InspireRecord.create(conference_data)
data_cited = faker.record("lit", with_control_number=True)
record_cited = InspireRecord.create(data_cited,
disable_relations_update=True)
db.session.commit()
record_cited_control_number = record_cited["control_number"]
conference_control_number = conference_record["control_number"]
conf_ref = f"http://localhost:8000/api/conferences/{conference_control_number}"
data = faker.record(
"lit",
literature_citations=[record_cited_control_number],
with_control_number=True,
)
data["publication_info"] = [{"conference_record": {"$ref": conf_ref}}]
data["document_type"] = ["conference paper"]
record = InspireRecord.create(data, disable_relations_update=True)
db.session.commit()
uuids = [record_cited.id, record.id]
task = update_relations.delay(uuids)
task.get(timeout=5)
result_record_cited = RecordCitations.query.filter_by(
cited_id=record_cited.id).one()
assert record.id == result_record_cited.citer_id
record_cited = InspireRecord.get_record_by_pid_value(
record_cited_control_number, "lit")
expected_record_cited_citation_count = 1
assert expected_record_cited_citation_count == record_cited.citation_count
conf_paper = ConferenceLiterature.query.filter_by(
conference_uuid=conference_record.id).one()
assert conf_paper.literature_uuid == record.id
def test_create_record_from_db_depending_on_its_pid_type(base_app, db, es):
data = faker.record("con")
record = InspireRecord.create(data)
assert type(record) == ConferencesRecord
assert record.pid_type == "con"
record = ConferencesRecord.create(data)
assert type(record) == ConferencesRecord
assert record.pid_type == "con"
def test_create_record_from_db_depending_on_its_pid_type(inspire_app):
data = faker.record("ins")
record = InspireRecord.create(data)
assert type(record) == InstitutionsRecord
assert record.pid_type == "ins"
record = InstitutionsRecord.create(data)
assert type(record) == InstitutionsRecord
assert record.pid_type == "ins"
def test_literature_citation_count_property(base_app, db, es):
data = faker.record("lit")
record = InspireRecord.create(data)
data2 = faker.record("lit", literature_citations=[record["control_number"]])
record2 = LiteratureRecord.create(data2)
assert record.citation_count == 1
assert record2.citation_count == 0
def test_create_record_from_db_depending_on_its_pid_type(inspire_app):
data = faker.record("jou")
record = InspireRecord.create(data)
assert type(record) == JournalsRecord
assert record.pid_type == "jou"
record = JournalsRecord.create(data)
assert type(record) == JournalsRecord
assert record.pid_type == "jou"
def test_create_record_from_db_depending_on_its_pid_type(inspire_app):
data = faker.record("exp")
record = InspireRecord.create(data)
assert isinstance(record, ExperimentsRecord)
assert record.pid_type == "exp"
record = ExperimentsRecord.create(data)
assert isinstance(record, ExperimentsRecord)
assert record.pid_type == "exp"
def test_create_record_from_db_depending_on_its_pid_type(base_app, db, es):
data = faker.record("exp")
record = InspireRecord.create(data)
assert type(record) == ExperimentsRecord
assert record.pid_type == "exp"
record = ExperimentsRecord.create(data)
assert type(record) == ExperimentsRecord
assert record.pid_type == "exp"
def test_create_record_from_db_depending_on_its_pid_type(inspire_app):
data = faker.record("con")
record = InspireRecord.create(data)
assert isinstance(record, ConferencesRecord)
assert record.pid_type == "con"
record = ConferencesRecord.create(data)
assert isinstance(record, ConferencesRecord)
assert record.pid_type == "con"
def test_create_record_from_db_depending_on_its_pid_type(inspire_app):
data = faker.record("aut")
record = InspireRecord.create(data)
assert isinstance(record, AuthorsRecord)
assert record.pid_type == "aut"
record = AuthorsRecord.create(data)
assert isinstance(record, AuthorsRecord)
assert record.pid_type == "aut"
def test_create_record_from_db_depending_on_its_pid_type(base_app, db, es):
data = faker.record("dat")
record = InspireRecord.create(data)
assert type(record) == DataRecord
assert record.pid_type == "dat"
record = DataRecord.create(data)
assert type(record) == DataRecord
assert record.pid_type == "dat"
def test_create_record_from_db_depending_on_its_pid_type(inspire_app):
data = faker.record("sem")
record = InspireRecord.create(data)
assert type(record) == SeminarsRecord
assert record.pid_type == "sem"
record = SeminarsRecord.create(data)
assert type(record) == SeminarsRecord
assert record.pid_type == "sem"
def test_recalculate_references_after_conference_record_merge(
inspire_app, clean_celery_session):
conference_data = faker.record("con", with_control_number=True)
conference = InspireRecord.create(conference_data)
conference_record_reference = conference["self"]["$ref"]
literature_data = faker.record("lit", with_control_number=True)
literature_data.update({
"publication_info": [{
"conference_record": {
"$ref": conference_record_reference
}
}]
})
literature = InspireRecord.create(literature_data)
db.session.commit()
def assert_all_records_in_es():
literature_record_from_es = InspireSearch.get_record_data_from_es(
literature)
conference_record_from_es = InspireSearch.get_record_data_from_es(
conference)
assert literature_record_from_es and conference_record_from_es
retry_until_pass(assert_all_records_in_es, retry_interval=3)
merged_conference_data = faker.record("con", with_control_number=True)
merged_conference_data.update(
{"deleted_records": [{
"$ref": conference_record_reference
}]})
merged_conference_record = InspireRecord.create(merged_conference_data)
db.session.commit()
def assert_recalculate_references_task():
literature_record_from_es = InspireSearch.get_record_data_from_es(
literature)
assert (literature_record_from_es["publication_info"][0]
["conference_record"]["$ref"] ==
merged_conference_record["self"]["$ref"])
retry_until_pass(assert_recalculate_references_task, retry_interval=3)
def test_unicode_data(inspire_app, get_fixture):
record_json = orjson.loads(get_fixture("hal_preprod_unicode_record.json"))
record_data = faker.record("lit", data=record_json)
record = InspireRecord.create(record_data)
institute_json = orjson.loads(
get_fixture("hal_preprod_unicode_institute.json"))
institute_data = faker.record("ins", data=institute_json)
InspireRecord.create(institute_data)
receipt = _hal_push(record)
assert receipt
assert receipt.parsed
updated_record = InspireRecord.get_record_by_pid_value(
record["control_number"], "lit")
assert (get_values_for_schema(
get_value(updated_record, "external_system_identifiers", []),
"HAL")[0] == receipt.id)
def create_record_async(record_type, data=None, skip_validation=False):
data = faker.record(
record_type,
data=data,
with_control_number=True,
skip_validation=skip_validation,
)
record = InspireRecord.create(data)
db.session.commit()
return record
def _create_record(record_type, data=None):
accepted_record_types = base_app.config["PID_TYPE_TO_INDEX"].keys()
if record_type not in accepted_record_types:
raise ValueError(f"{record_type} is not supported")
index = base_app.config["PID_TYPE_TO_INDEX"][record_type]
record_data = faker.record(record_type, data=data)
record = InspireRecord.create(record_data)
record._indexing = record._index()
es_clear.indices.refresh(index)
return record
def test_job_update_force_undelete_flag(inspire_app):
data = faker.record("con", with_control_number=True)
data_1 = faker.record("con", with_control_number=True)
record = InspireRecord.create(data)
record_1 = InspireRecord.create(data_1)
db.session.commit()
record.delete()
record_1.delete()
db.session.commit()
record["deleted"] = False
record.update(dict(record), force_undelete=True)
assert not record["deleted"]
record_1["deleted"] = False
with pytest.raises(ValidationError):
record_1.update(dict(record_1))
def test_literature_can_cite_only_existing_records(base_app, db, es):
data = faker.record("dat")
record = InspireRecord.create(data)
data2 = faker.record("lit", data_citations=[record["control_number"], 9999, 9998])
record2 = LiteratureRecord.create(data2)
assert len(record.model.citations) == 1
assert len(record.model.references) == 0
assert len(record2.model.citations) == 0
assert len(record2.model.references) == 1
assert len(RecordCitations.query.all()) == 1
def test_process_references_in_records_doesnt_call_get_modified_references_for_non_lit_records(
get_modified_references_mock, inspire_app
):
data = {
"$schema": "https://inspire/schemas/records/authors.json",
"_collections": ["Authors"],
"name": {"value": "Doe, John"},
}
record = InspireRecord.create(data)
process_references_in_records([record.id])
get_modified_references_mock.assert_not_called()
data = {
"$schema": "http://localhost:5000/schemas/records/hep.json",
"titles": [{"title": "Test a valid record"}],
"document_type": ["article"],
"_collections": ["Literature"],
}
record = InspireRecord.create(data)
process_references_in_records([record.id])
assert get_modified_references_mock.call_count == 1
def test_create_record_update_citation_table_for_literature_citation(inspire_app):
data = faker.record("dat")
record = DataRecord.create(data)
data2 = faker.record("lit", data_citations=[record["control_number"]])
record2 = InspireRecord.create(data2)
assert len(record.model.citations) == 1
assert len(record.model.references) == 0
assert len(record2.model.citations) == 0
assert len(record2.model.references) == 1
assert len(RecordCitations.query.all()) == 1
