def test_record_created_through_api_is_indexed(inspire_app,
celery_app_with_context,
celery_session_worker):
data = faker.record("aut")
token = AccessTokenFactory()
db.session.commit()
headers = {"Authorization": f"Bearer {token.access_token}"}
content_type = "application/json"
response = inspire_app.test_client().post("/api/authors",
json=data,
headers=headers,
content_type=content_type)
assert response.status_code == 201
current_search.flush_and_refresh("records-authors")
steps = [
{
"step": current_search.flush_and_refresh,
"args": ["records-authors"]
},
{
"step": es_search,
"args": ["records-authors"],
"expected_result": {
"expected_key": "hits.total.value",
"expected_result": 1,
},
},
]
retry_until_matched(steps)
def test_aut_record_update_when_changed(inspire_app, celery_app_with_context,
celery_session_worker):
data = faker.record("aut")
rec = AuthorsRecord.create(data)
db.session.commit()
expected_death_date = "1900-01-01"
data["death_date"] = expected_death_date
data["control_number"] = rec["control_number"]
rec.update(data)
db.session.commit()
steps = [
{
"step": current_search.flush_and_refresh,
"args": ["records-authors"]
},
{
"step": es_search,
"args": ["records-authors"],
"expected_result": {
"expected_key": "hits.total.value",
"expected_result": 1,
},
},
{
"step": es_search,
"args": ["records-authors"],
"expected_result": {
"expected_key": "hits.hits[0]._source.death_date",
"expected_result": expected_death_date,
},
},
]
retry_until_matched(steps)["hits"]["hits"]
def test_aut_record_appear_in_es_when_created(inspire_app,
celery_app_with_context,
celery_session_worker):
data = faker.record("aut")
rec = AuthorsRecord.create(data)
db.session.commit()
expected_id = str(rec.id)
steps = [
{
"step": current_search.flush_and_refresh,
"args": ["records-authors"]
},
{
"step": es_search,
"args": ["records-authors"],
"expected_result": {
"expected_key": "hits.total.value",
"expected_result": 1,
},
},
{
"step": es_search,
"args": ["records-authors"],
"expected_result": {
"expected_key": "hits.hits[0]._id",
"expected_result": expected_id,
},
},
]
retry_until_matched(steps)
def test_retry_until_matched_raises_last_error_on_timeout():
def _test_func():
raise ConnectionError
steps = [{"step": _test_func}]
with pytest.raises(ConnectionError):
retry_until_matched(steps, 1)
def test_experiment_record_updates_in_es_when_lit_rec_refers_to_it(
inspire_app, celery_app_with_context, celery_session_worker):
experiment_1 = ExperimentsRecord.create(faker.record("exp"))
experiment_1_control_number = experiment_1["control_number"]
ref_1 = f"http://localhost:8000/api/experiments/{experiment_1_control_number}"
db.session.commit()
expected_number_of_papers = 0
steps = [
{
"step": current_search.flush_and_refresh,
"args": ["records-experiments"]
},
{
"step": es_search,
"args": ["records-experiments"],
"expected_result": {
"expected_key": "hits.total.value",
"expected_result": 1,
},
},
{
"step": es_search,
"args": ["records-experiments"],
"expected_result": {
"expected_key": "hits.hits[0]._source.number_of_papers",
"expected_result": expected_number_of_papers,
},
},
]
retry_until_matched(steps)
data = {
"accelerator_experiments": [{
"legacy_name": "LIGO",
"record": {
"$ref": ref_1
}
}]
}
LiteratureRecord.create(faker.record("lit", data))
db.session.commit()
expected_number_of_papers = 1
steps = [
{
"step": current_search.flush_and_refresh,
"args": ["records-experiments"]
},
{
"step": es_search,
"args": ["records-experiments"],
"expected_result": {
"expected_key": "hits.hits[0]._source.number_of_papers",
"expected_result": expected_number_of_papers,
},
},
]
retry_until_matched(steps)
def test_lit_record_reindexes_references_when_earliest_date_changed(
inspire_app, celery_app_with_context, celery_session_worker):
data_cited_record = faker.record("lit")
cited_record = LiteratureRecord.create(data_cited_record)
db.session.commit()
citations = [cited_record["control_number"]]
data_citing_record = faker.record("lit",
literature_citations=citations,
data={"preprint_date": "2018-06-28"})
citing_record = LiteratureRecord.create(data_citing_record)
db.session.commit()
steps = [
{
"step": current_search.flush_and_refresh,
"args": ["records-hep"]
},
{
"step": LiteratureSearch.get_record_data_from_es,
"args": [cited_record],
"expected_result": {
"expected_key": "citations_by_year",
"expected_result": [{
"count": 1,
"year": 2018
}],
},
},
]
retry_until_matched(steps)
data_citing_record["preprint_date"] = "2019-06-28"
data_citing_record["control_number"] = citing_record["control_number"]
citing_record.update(data_citing_record)
db.session.commit()
current_search.flush_and_refresh("records-hep")
steps = [
{
"step": current_search.flush_and_refresh,
"args": ["records-hep"]
},
{
"step": LiteratureSearch.get_record_data_from_es,
"args": [cited_record],
"expected_result": {
"expected_key": "citations_by_year",
"expected_result": [{
"count": 1,
"year": 2019
}],
},
},
]
retry_until_matched(steps)
def test_literature_regression_changing_bai_in_record_reindex_records_which_are_citing_changed_one(
inspire_app, celery_app_with_context, celery_session_worker,
enable_self_citations):
data = {
"authors": [{
"full_name":
"Jean-Luc Picard",
"ids": [{
"schema": "INSPIRE BAI",
"value": "Jean.L.Picard.1"
}],
}]
}
data = faker.record("lit", data=data)
base_record = LiteratureRecord.create(data)
citer_data = faker.record(
"lit", literature_citations=[base_record["control_number"]])
citer = LiteratureRecord.create(citer_data)
db.session.commit()
steps = [
{
"step": current_search.flush_and_refresh,
"args": ["records-hep"]
},
{
"step": LiteratureSearch.get_record_data_from_es,
"args": [citer],
"expected_key": "referenced_authors_bais",
"expected_result": ["Jean.L.Picard.1"],
},
]
retry_until_matched(steps)
data = dict(base_record)
data["authors"][0]["ids"][0]["value"] = "Jean.L.Picard.2"
base_record.update(data)
db.session.commit()
steps = [
{
"step": current_search.flush_and_refresh,
"args": ["records-hep"]
},
{
"step": LiteratureSearch.get_record_data_from_es,
"args": [citer],
"expected_key": "referenced_authors_bais",
"expected_result": ["Jean.L.Picard.2"],
},
]
retry_until_matched(steps)
def test_signature_linked_by_disambiguation_has_correct_facet_author_name(
inspire_app, celery_app_with_context, celery_session_worker):
data = faker.record("lit")
data["authors"] = [{
"full_name": "Doe, John",
"uuid": "94fc2b0a-dc17-42c2-bae3-ca0024079e51"
}]
record = LiteratureRecord.create(data)
db.session.commit()
clusters = [{
"signatures": [{
"publication_id":
record["control_number"],
"signature_uuid":
"94fc2b0a-dc17-42c2-bae3-ca0024079e51",
}],
"authors": [],
}]
disambiguate_signatures(clusters)
author_pids = PersistentIdentifier.query.filter_by(pid_type="aut").all()
assert len(author_pids) == 1
pid_value = author_pids[0].pid_value
author = AuthorsRecord.get_record_by_pid_value(pid_value)
author_control_number = author.pop("control_number")
expected_facet_author_name = [f"{author_control_number}_John Doe"]
expected_record_ref = f"http://localhost:5000/api/authors/{pid_value}"
steps = [
{
"step": current_search.flush_and_refresh,
"args": ["records-hep"]
},
{
"step": es_search,
"args": ["records-hep"],
"expected_result": {
"expected_key": "hits.total.value",
"expected_result": 1,
},
},
{
"expected_key": "hits.hits[0]._source.facet_author_name",
"expected_result": expected_facet_author_name,
},
{
"expected_key": "hits.hits[0]._source.authors[0].record.$ref",
"expected_result": expected_record_ref,
},
]
retry_until_matched(steps)
def test_retry_until_matched_simple_step_with_expected_key():
steps = [{
"step": dict,
"kwargs": {
"a": 1,
"b": [{
"z": 2
}],
"c": 3
},
"expected_result": {
"expected_key": "b[0].z",
"expected_result": 2
},
}]
retry_until_matched(steps)
def test_retry_until_matched_actually_retries_all_steps():
def _test_func(l, p):
l.append(p)
return l
test_list = []
steps = [
{
"step": _test_func,
"args": [test_list, 1]
},
{
"expected_result": [1, 1, 1]
},
]
retry_until_matched(steps)
def assert_citation_count(cited_record, expected_count):
steps = [
{
"step": current_search.flush_and_refresh,
"args": ["records-hep"]
},
{
"step": LiteratureSearch.get_record_data_from_es,
"args": [cited_record],
"expected_result": {
"expected_key": "citation_count",
"expected_result": expected_count,
},
},
]
retry_until_matched(steps)
def test_retry_until_matched_multi_step_wrong_value():
steps = [{"step": add, "args": [1, 2], "expected_result": 4}]
with pytest.raises(AssertionError):
retry_until_matched(steps, timeout=1)
steps = [{
"step": dict,
"kwargs": {
"a": 1,
"b": [{
"z": 3
}],
"c": [3]
},
"expected_key": "b[0].z",
"expected_result": 2,
}]
with pytest.raises(AssertionError):
retry_until_matched(steps, timeout=1)
def test_retry_until_matched_multi_step_with_expected_key_simpler_definition():
steps = [
{
"step": dict,
"kwargs": {
"a": 1,
"b": [{
"z": 2
}],
"c": [3]
},
"expected_key": "b[0].z",
"expected_result": 2,
},
{
"expected_key": "c[0]",
"expected_result": 3
},
]
retry_until_matched(steps)
def assert_es_hits_count(expected_hits_count, additional_steps=None):
steps = [
{
"step": current_search.flush_and_refresh,
"args": ["records-hep"]
},
{
"step": es_search,
"args": ["records-hep"],
"expected_result": {
"expected_key": "hits.total.value",
"expected_result": expected_hits_count,
},
},
]
if additional_steps:
steps.extend(additional_steps)
return retry_until_matched(steps)
def test_retry_until_matched_simple_step_without_expected_key():
steps = [{"step": add, "args": [1, 2], "expected_result": 3}]
retry_until_matched(steps)
def test_retry_until_matched_raises_timeout_on_timeout_when_no_other_exceptions(
):
steps = [{"step": sleep, "args": [2]}, {"step": sleep, "args": [2]}]
with pytest.raises(TimeoutError):
retry_until_matched(steps, 1)
def test_indexer_updates_authors_papers_when_name_changes(
inspire_app, celery_app_with_context, celery_session_worker):
author_data = faker.record("aut")
author = AuthorsRecord.create(author_data)
db.session.commit()
current_search.flush_and_refresh("records-authors")
author_cn = author["control_number"]
lit_data = {
"authors": [{
"record": {
"$ref": f"https://labs.inspirehep.net/api/authors/{author_cn}"
},
"full_name": author["name"]["value"],
}]
}
lit_data = faker.record("lit", data=lit_data)
lit_1 = LiteratureRecord.create(lit_data)
db.session.commit()
expected_hits = 1
expected_facet_author_name_count = 1
expected_facet_author_name = f"{author['control_number']}_{author['name']['value']}"
steps = [
{
"step": current_search.flush_and_refresh,
"args": ["*"]
},
{
"step": es_search,
"args": ["records-hep"],
"expected_result": {
"expected_key": "hits.total.value",
"expected_result": expected_hits,
},
},
{
"expected_key": "hits.hits[0]._source.facet_author_name[0]",
"expected_result": expected_facet_author_name,
},
]
results = retry_until_matched(steps, timeout=45)
assert (len(results["hits"]["hits"][0]["_source"]["facet_author_name"]) ==
expected_facet_author_name_count)
data = dict(author)
data["name"]["value"] = "Some other name"
author.update(data)
db.session.commit()
expected_facet_author_name = f"{author['control_number']}_Some other name"
steps = [
{
"step": current_search.flush_and_refresh,
"args": ["*"]
},
{
"step": es_search,
"args": ["records-hep"],
"expected_result": {
"expected_key": "hits.total.value",
"expected_result": expected_hits,
},
},
{
"expected_key": "hits.hits[0]._source.facet_author_name[0]",
"expected_result": expected_facet_author_name,
},
]
results = retry_until_matched(steps, timeout=45)
assert (len(results["hits"]["hits"][0]["_source"]["facet_author_name"]) ==
expected_facet_author_name_count)
def test_conference_record_updates_in_es_when_lit_rec_reffers_to_it(
inspire_app, celery_app_with_context, celery_session_worker):
conference_1 = ConferencesRecord.create(faker.record("con"))
conference_1_control_number = conference_1["control_number"]
ref_1 = f"http://localhost:8000/api/conferences/{conference_1_control_number}"
db.session.commit()
expected_contributions_count = 0
steps = [
{
"step": current_search.flush_and_refresh,
"args": ["records-conferences"]
},
{
"step": es_search,
"args": ["records-conferences"],
"expected_result": {
"expected_key": "hits.total.value",
"expected_result": 1,
},
},
{
"step": es_search,
"args": ["records-conferences"],
"expected_result": {
"expected_key": "hits.hits[0]._source.number_of_contributions",
"expected_result": expected_contributions_count,
},
},
]
retry_until_matched(steps)
data = {
"publication_info": [{
"conference_record": {
"$ref": ref_1
}
}],
"document_type": ["conference paper"],
}
LiteratureRecord.create(faker.record("lit", data))
data = {
"publication_info": [{
"conference_record": {
"$ref": ref_1
},
"journal_title": "nice title"
}],
"document_type": ["proceedings"],
}
record2 = LiteratureRecord.create(faker.record("lit", data))
db.session.commit()
expected_contributions_count = 1
steps = [
{
"step": current_search.flush_and_refresh,
"args": ["records-conferences"]
},
{
"step": es_search,
"args": ["records-conferences"],
"expected_result": {
"expected_key": "hits.hits[0]._source.number_of_contributions",
"expected_result": expected_contributions_count,
},
},
]
retry_until_matched(steps)
expected_proceedings = [ProceedingInfoItemSchemaV1().dump(record2).data]
steps = [
{
"step": current_search.flush_and_refresh,
"args": ["records-conferences"]
},
{
"step": es_search,
"args": ["records-conferences"],
"expected_result": {
"expected_key": "hits.hits[0]._source.proceedings",
"expected_result": expected_proceedings,
},
},
]
retry_until_matched(steps)
def test_institutions_record_updates_in_es_when_lit_rec_refers_to_it(
inspire_app, celery_app_with_context, celery_session_worker):
institution_1 = InstitutionsRecord.create(faker.record("ins"))
institution_1_control_number = institution_1["control_number"]
ref_1 = f"http://localhost:8000/api/institutions/{institution_1_control_number}"
db.session.commit()
expected_number_of_papers = 0
steps = [
{
"step": current_search.flush_and_refresh,
"args": ["records-institutions"]
},
{
"step": es_search,
"args": ["records-institutions"],
"expected_result": {
"expected_key": "hits.total.value",
"expected_result": 1,
},
},
{
"step": es_search,
"args": ["records-institutions"],
"expected_result": {
"expected_key": "hits.hits[0]._source.number_of_papers",
"expected_result": expected_number_of_papers,
},
},
]
retry_until_matched(steps)
data = {
"authors": [{
"full_name":
"John Doe",
"affiliations": [{
"value": "Institution",
"record": {
"$ref": ref_1
}
}],
}]
}
LiteratureRecord.create(faker.record("lit", data))
db.session.commit()
expected_number_of_papers = 1
steps = [
{
"step": current_search.flush_and_refresh,
"args": ["records-institutions"]
},
{
"step": es_search,
"args": ["records-institutions"],
"expected_result": {
"expected_key": "hits.hits[0]._source.number_of_papers",
"expected_result": expected_number_of_papers,
},
},
]
retry_until_matched(steps)
