def test_record_created_through_api_is_indexed(inspire_app, celery_app_with_context, celery_session_worker): data = faker.record("aut") token = AccessTokenFactory() db.session.commit() headers = {"Authorization": f"Bearer {token.access_token}"} content_type = "application/json" response = inspire_app.test_client().post("/api/authors", json=data, headers=headers, content_type=content_type) assert response.status_code == 201 current_search.flush_and_refresh("records-authors") steps = [ { "step": current_search.flush_and_refresh, "args": ["records-authors"] }, { "step": es_search, "args": ["records-authors"], "expected_result": { "expected_key": "hits.total.value", "expected_result": 1, }, }, ] retry_until_matched(steps)
def test_aut_record_update_when_changed(inspire_app, celery_app_with_context, celery_session_worker): data = faker.record("aut") rec = AuthorsRecord.create(data) db.session.commit() expected_death_date = "1900-01-01" data["death_date"] = expected_death_date data["control_number"] = rec["control_number"] rec.update(data) db.session.commit() steps = [ { "step": current_search.flush_and_refresh, "args": ["records-authors"] }, { "step": es_search, "args": ["records-authors"], "expected_result": { "expected_key": "hits.total.value", "expected_result": 1, }, }, { "step": es_search, "args": ["records-authors"], "expected_result": { "expected_key": "hits.hits[0]._source.death_date", "expected_result": expected_death_date, }, }, ] retry_until_matched(steps)["hits"]["hits"]
def test_aut_record_appear_in_es_when_created(inspire_app, celery_app_with_context, celery_session_worker): data = faker.record("aut") rec = AuthorsRecord.create(data) db.session.commit() expected_id = str(rec.id) steps = [ { "step": current_search.flush_and_refresh, "args": ["records-authors"] }, { "step": es_search, "args": ["records-authors"], "expected_result": { "expected_key": "hits.total.value", "expected_result": 1, }, }, { "step": es_search, "args": ["records-authors"], "expected_result": { "expected_key": "hits.hits[0]._id", "expected_result": expected_id, }, }, ] retry_until_matched(steps)
def test_retry_until_matched_raises_last_error_on_timeout(): def _test_func(): raise ConnectionError steps = [{"step": _test_func}] with pytest.raises(ConnectionError): retry_until_matched(steps, 1)
def test_experiment_record_updates_in_es_when_lit_rec_refers_to_it( inspire_app, celery_app_with_context, celery_session_worker): experiment_1 = ExperimentsRecord.create(faker.record("exp")) experiment_1_control_number = experiment_1["control_number"] ref_1 = f"http://localhost:8000/api/experiments/{experiment_1_control_number}" db.session.commit() expected_number_of_papers = 0 steps = [ { "step": current_search.flush_and_refresh, "args": ["records-experiments"] }, { "step": es_search, "args": ["records-experiments"], "expected_result": { "expected_key": "hits.total.value", "expected_result": 1, }, }, { "step": es_search, "args": ["records-experiments"], "expected_result": { "expected_key": "hits.hits[0]._source.number_of_papers", "expected_result": expected_number_of_papers, }, }, ] retry_until_matched(steps) data = { "accelerator_experiments": [{ "legacy_name": "LIGO", "record": { "$ref": ref_1 } }] } LiteratureRecord.create(faker.record("lit", data)) db.session.commit() expected_number_of_papers = 1 steps = [ { "step": current_search.flush_and_refresh, "args": ["records-experiments"] }, { "step": es_search, "args": ["records-experiments"], "expected_result": { "expected_key": "hits.hits[0]._source.number_of_papers", "expected_result": expected_number_of_papers, }, }, ] retry_until_matched(steps)
def test_lit_record_reindexes_references_when_earliest_date_changed( inspire_app, celery_app_with_context, celery_session_worker): data_cited_record = faker.record("lit") cited_record = LiteratureRecord.create(data_cited_record) db.session.commit() citations = [cited_record["control_number"]] data_citing_record = faker.record("lit", literature_citations=citations, data={"preprint_date": "2018-06-28"}) citing_record = LiteratureRecord.create(data_citing_record) db.session.commit() steps = [ { "step": current_search.flush_and_refresh, "args": ["records-hep"] }, { "step": LiteratureSearch.get_record_data_from_es, "args": [cited_record], "expected_result": { "expected_key": "citations_by_year", "expected_result": [{ "count": 1, "year": 2018 }], }, }, ] retry_until_matched(steps) data_citing_record["preprint_date"] = "2019-06-28" data_citing_record["control_number"] = citing_record["control_number"] citing_record.update(data_citing_record) db.session.commit() current_search.flush_and_refresh("records-hep") steps = [ { "step": current_search.flush_and_refresh, "args": ["records-hep"] }, { "step": LiteratureSearch.get_record_data_from_es, "args": [cited_record], "expected_result": { "expected_key": "citations_by_year", "expected_result": [{ "count": 1, "year": 2019 }], }, }, ] retry_until_matched(steps)
def test_literature_regression_changing_bai_in_record_reindex_records_which_are_citing_changed_one( inspire_app, celery_app_with_context, celery_session_worker, enable_self_citations): data = { "authors": [{ "full_name": "Jean-Luc Picard", "ids": [{ "schema": "INSPIRE BAI", "value": "Jean.L.Picard.1" }], }] } data = faker.record("lit", data=data) base_record = LiteratureRecord.create(data) citer_data = faker.record( "lit", literature_citations=[base_record["control_number"]]) citer = LiteratureRecord.create(citer_data) db.session.commit() steps = [ { "step": current_search.flush_and_refresh, "args": ["records-hep"] }, { "step": LiteratureSearch.get_record_data_from_es, "args": [citer], "expected_key": "referenced_authors_bais", "expected_result": ["Jean.L.Picard.1"], }, ] retry_until_matched(steps) data = dict(base_record) data["authors"][0]["ids"][0]["value"] = "Jean.L.Picard.2" base_record.update(data) db.session.commit() steps = [ { "step": current_search.flush_and_refresh, "args": ["records-hep"] }, { "step": LiteratureSearch.get_record_data_from_es, "args": [citer], "expected_key": "referenced_authors_bais", "expected_result": ["Jean.L.Picard.2"], }, ] retry_until_matched(steps)
def test_signature_linked_by_disambiguation_has_correct_facet_author_name( inspire_app, celery_app_with_context, celery_session_worker): data = faker.record("lit") data["authors"] = [{ "full_name": "Doe, John", "uuid": "94fc2b0a-dc17-42c2-bae3-ca0024079e51" }] record = LiteratureRecord.create(data) db.session.commit() clusters = [{ "signatures": [{ "publication_id": record["control_number"], "signature_uuid": "94fc2b0a-dc17-42c2-bae3-ca0024079e51", }], "authors": [], }] disambiguate_signatures(clusters) author_pids = PersistentIdentifier.query.filter_by(pid_type="aut").all() assert len(author_pids) == 1 pid_value = author_pids[0].pid_value author = AuthorsRecord.get_record_by_pid_value(pid_value) author_control_number = author.pop("control_number") expected_facet_author_name = [f"{author_control_number}_John Doe"] expected_record_ref = f"http://localhost:5000/api/authors/{pid_value}" steps = [ { "step": current_search.flush_and_refresh, "args": ["records-hep"] }, { "step": es_search, "args": ["records-hep"], "expected_result": { "expected_key": "hits.total.value", "expected_result": 1, }, }, { "expected_key": "hits.hits[0]._source.facet_author_name", "expected_result": expected_facet_author_name, }, { "expected_key": "hits.hits[0]._source.authors[0].record.$ref", "expected_result": expected_record_ref, }, ] retry_until_matched(steps)
def test_retry_until_matched_simple_step_with_expected_key(): steps = [{ "step": dict, "kwargs": { "a": 1, "b": [{ "z": 2 }], "c": 3 }, "expected_result": { "expected_key": "b[0].z", "expected_result": 2 }, }] retry_until_matched(steps)
def test_retry_until_matched_actually_retries_all_steps(): def _test_func(l, p): l.append(p) return l test_list = [] steps = [ { "step": _test_func, "args": [test_list, 1] }, { "expected_result": [1, 1, 1] }, ] retry_until_matched(steps)
def assert_citation_count(cited_record, expected_count): steps = [ { "step": current_search.flush_and_refresh, "args": ["records-hep"] }, { "step": LiteratureSearch.get_record_data_from_es, "args": [cited_record], "expected_result": { "expected_key": "citation_count", "expected_result": expected_count, }, }, ] retry_until_matched(steps)
def test_retry_until_matched_multi_step_wrong_value(): steps = [{"step": add, "args": [1, 2], "expected_result": 4}] with pytest.raises(AssertionError): retry_until_matched(steps, timeout=1) steps = [{ "step": dict, "kwargs": { "a": 1, "b": [{ "z": 3 }], "c": [3] }, "expected_key": "b[0].z", "expected_result": 2, }] with pytest.raises(AssertionError): retry_until_matched(steps, timeout=1)
def test_retry_until_matched_multi_step_with_expected_key_simpler_definition(): steps = [ { "step": dict, "kwargs": { "a": 1, "b": [{ "z": 2 }], "c": [3] }, "expected_key": "b[0].z", "expected_result": 2, }, { "expected_key": "c[0]", "expected_result": 3 }, ] retry_until_matched(steps)
def assert_es_hits_count(expected_hits_count, additional_steps=None): steps = [ { "step": current_search.flush_and_refresh, "args": ["records-hep"] }, { "step": es_search, "args": ["records-hep"], "expected_result": { "expected_key": "hits.total.value", "expected_result": expected_hits_count, }, }, ] if additional_steps: steps.extend(additional_steps) return retry_until_matched(steps)
def test_retry_until_matched_simple_step_without_expected_key(): steps = [{"step": add, "args": [1, 2], "expected_result": 3}] retry_until_matched(steps)
def test_retry_until_matched_raises_timeout_on_timeout_when_no_other_exceptions( ): steps = [{"step": sleep, "args": [2]}, {"step": sleep, "args": [2]}] with pytest.raises(TimeoutError): retry_until_matched(steps, 1)
def test_indexer_updates_authors_papers_when_name_changes( inspire_app, celery_app_with_context, celery_session_worker): author_data = faker.record("aut") author = AuthorsRecord.create(author_data) db.session.commit() current_search.flush_and_refresh("records-authors") author_cn = author["control_number"] lit_data = { "authors": [{ "record": { "$ref": f"https://labs.inspirehep.net/api/authors/{author_cn}" }, "full_name": author["name"]["value"], }] } lit_data = faker.record("lit", data=lit_data) lit_1 = LiteratureRecord.create(lit_data) db.session.commit() expected_hits = 1 expected_facet_author_name_count = 1 expected_facet_author_name = f"{author['control_number']}_{author['name']['value']}" steps = [ { "step": current_search.flush_and_refresh, "args": ["*"] }, { "step": es_search, "args": ["records-hep"], "expected_result": { "expected_key": "hits.total.value", "expected_result": expected_hits, }, }, { "expected_key": "hits.hits[0]._source.facet_author_name[0]", "expected_result": expected_facet_author_name, }, ] results = retry_until_matched(steps, timeout=45) assert (len(results["hits"]["hits"][0]["_source"]["facet_author_name"]) == expected_facet_author_name_count) data = dict(author) data["name"]["value"] = "Some other name" author.update(data) db.session.commit() expected_facet_author_name = f"{author['control_number']}_Some other name" steps = [ { "step": current_search.flush_and_refresh, "args": ["*"] }, { "step": es_search, "args": ["records-hep"], "expected_result": { "expected_key": "hits.total.value", "expected_result": expected_hits, }, }, { "expected_key": "hits.hits[0]._source.facet_author_name[0]", "expected_result": expected_facet_author_name, }, ] results = retry_until_matched(steps, timeout=45) assert (len(results["hits"]["hits"][0]["_source"]["facet_author_name"]) == expected_facet_author_name_count)
def test_conference_record_updates_in_es_when_lit_rec_reffers_to_it( inspire_app, celery_app_with_context, celery_session_worker): conference_1 = ConferencesRecord.create(faker.record("con")) conference_1_control_number = conference_1["control_number"] ref_1 = f"http://localhost:8000/api/conferences/{conference_1_control_number}" db.session.commit() expected_contributions_count = 0 steps = [ { "step": current_search.flush_and_refresh, "args": ["records-conferences"] }, { "step": es_search, "args": ["records-conferences"], "expected_result": { "expected_key": "hits.total.value", "expected_result": 1, }, }, { "step": es_search, "args": ["records-conferences"], "expected_result": { "expected_key": "hits.hits[0]._source.number_of_contributions", "expected_result": expected_contributions_count, }, }, ] retry_until_matched(steps) data = { "publication_info": [{ "conference_record": { "$ref": ref_1 } }], "document_type": ["conference paper"], } LiteratureRecord.create(faker.record("lit", data)) data = { "publication_info": [{ "conference_record": { "$ref": ref_1 }, "journal_title": "nice title" }], "document_type": ["proceedings"], } record2 = LiteratureRecord.create(faker.record("lit", data)) db.session.commit() expected_contributions_count = 1 steps = [ { "step": current_search.flush_and_refresh, "args": ["records-conferences"] }, { "step": es_search, "args": ["records-conferences"], "expected_result": { "expected_key": "hits.hits[0]._source.number_of_contributions", "expected_result": expected_contributions_count, }, }, ] retry_until_matched(steps) expected_proceedings = [ProceedingInfoItemSchemaV1().dump(record2).data] steps = [ { "step": current_search.flush_and_refresh, "args": ["records-conferences"] }, { "step": es_search, "args": ["records-conferences"], "expected_result": { "expected_key": "hits.hits[0]._source.proceedings", "expected_result": expected_proceedings, }, }, ] retry_until_matched(steps)
def test_institutions_record_updates_in_es_when_lit_rec_refers_to_it( inspire_app, celery_app_with_context, celery_session_worker): institution_1 = InstitutionsRecord.create(faker.record("ins")) institution_1_control_number = institution_1["control_number"] ref_1 = f"http://localhost:8000/api/institutions/{institution_1_control_number}" db.session.commit() expected_number_of_papers = 0 steps = [ { "step": current_search.flush_and_refresh, "args": ["records-institutions"] }, { "step": es_search, "args": ["records-institutions"], "expected_result": { "expected_key": "hits.total.value", "expected_result": 1, }, }, { "step": es_search, "args": ["records-institutions"], "expected_result": { "expected_key": "hits.hits[0]._source.number_of_papers", "expected_result": expected_number_of_papers, }, }, ] retry_until_matched(steps) data = { "authors": [{ "full_name": "John Doe", "affiliations": [{ "value": "Institution", "record": { "$ref": ref_1 } }], }] } LiteratureRecord.create(faker.record("lit", data)) db.session.commit() expected_number_of_papers = 1 steps = [ { "step": current_search.flush_and_refresh, "args": ["records-institutions"] }, { "step": es_search, "args": ["records-institutions"], "expected_result": { "expected_key": "hits.hits[0]._source.number_of_papers", "expected_result": expected_number_of_papers, }, }, ] retry_until_matched(steps)