def get(self, pid, record): page = request.values.get('page', 1, type=int) size = request.values.get('size', 10, type=int) if page < 1 or size < 1: abort(400) citing_records_results = get_citations_from_es(record, page, size) citing_records_count = citing_records_results.total citing_records = [ citation.to_dict() for citation in citing_records_results ] data = { 'citations': citing_records, 'citation_count': citing_records_count } return self.make_response(pid, data)
def _process_record(pid, app): if stop: return with app.app_context(): success = False deleted = False no_cits = False db_cits = None es_cits = None es_citation_count_field = None data = {} rec = get_db_record('lit', pid.pid_value) if rec.get('deleted'): success = True deleted = True if not deleted: try: es_cits = get_citations_from_es(rec).total search = LiteratureSearch().source(includes=['citation_count']) results = search.get_record(rec.id).execute() if not results.hits: es_citation_count_field = None else: es_citation_count_field = results.hits[0]['citation_count'] db_cits = rec.get_citations_count() except Exception as err: click.echo("Cannot prepare data for %s record. %s", pid.pid_value, err) if not deleted and es_cits is not None and es_cits == db_cits == es_citation_count_field: if es_cits == 0: no_cits = True success = True else: data = { 'pid_value': pid.pid_value, 'db_citations_count': db_cits, 'es_citations_count': es_cits, 'es_citations_field': es_citation_count_field } return (success, deleted, no_cits, data)
def test_regression_index_after_commit_retries_for_new_record_not_yet_in_db( mocked_indexing_task, mocked_permission_check, app, ): # this test doesn't use the isolated_app because it needs to commit to # the DB in order to create records versions. json_data = { '$schema': 'http://localhost:5000/schemas/records/hep.json', 'document_type': ['article'], 'titles': [{ 'title': 'This is the record being cited' }], 'control_number': 9999, '_collections': ['Literature'] } cited = InspireRecord.create(data=json_data, skip_files=True) cited.commit() db.session.commit() es.indices.refresh('records-hep') expected_args = ('lit', 9999, 2) mocked_indexing_task.assert_called_with(*expected_args) # execute mocked task index_modified_citations_from_record(*expected_args) es_rec = get_es_record('lit', 9999) assert es_rec['citation_count'] == 0 assert get_citations_from_es(es_rec).total == 0 citing_json = { '$schema': 'http://localhost:5000/schemas/records/hep.json', 'document_type': ['article'], 'titles': [{ 'title': 'Record citing the first one' }], '_collections': ['Literature'], 'control_number': 8888, 'references': [{ 'record': { '$ref': 'http://localhost:5000/api/literature/9999' }, 'reference': { 'authors': [{ 'full_name': 'Smith, J.' }], } }] } record = InspireRecord.create(data=citing_json, skip_files=True) record.commit() db.session.commit() es.indices.refresh('records-hep') expected_args = ('lit', record['control_number'], 2) mocked_indexing_task.assert_called_with(*expected_args) # execute mocked task pretending record is not committed yet to DB _delete_record('lit', record['control_number']) with pytest.raises(RecordGetterError): # XXX: celery in eager mode does not retry, so it raises the first time index_modified_citations_from_record(*expected_args) _delete_record('lit', cited['control_number'])
def test_index_after_commit_indexes_also_cites_two_records( mocked_indexing_task, mocked_permission_check, app, ): # this test doesn't use the isolated_app because it needs to commit to # the DB in order to create records versions. json1 = { '$schema': 'http://localhost:5000/schemas/records/hep.json', 'document_type': ['article'], 'titles': [{ 'title': 'This is the record being cited' }], 'control_number': 9999, '_collections': ['Literature'] } cited1 = InspireRecord.create(data=json1, skip_files=True) cited1.commit() db.session.commit() es.indices.refresh('records-hep') expected_args = ('lit', cited1['control_number'], 2) mocked_indexing_task.assert_called_with(*expected_args) # execute mocked task index_modified_citations_from_record(*expected_args) json2 = { '$schema': 'http://localhost:5000/schemas/records/hep.json', 'document_type': ['article'], 'titles': [{ 'title': 'This also is the record being cited' }], 'control_number': 9998, '_collections': ['Literature'] } cited2 = InspireRecord.create(data=json2, skip_files=True) cited2.commit() db.session.commit() es.indices.refresh('records-hep') expected_args = ('lit', cited2['control_number'], 2) mocked_indexing_task.assert_called_with(*expected_args) # execute mocked task index_modified_citations_from_record(*expected_args) es_rec1 = get_es_record('lit', 9999) es_rec2 = get_es_record('lit', 9998) assert es_rec1['citation_count'] == 0 assert es_rec2['citation_count'] == 0 assert get_citations_from_es(es_rec1).total == 0 assert get_citations_from_es(es_rec2).total == 0 citing_json = { '$schema': 'http://localhost:5000/schemas/records/hep.json', 'document_type': ['article'], 'titles': [{ 'title': 'Record citing the first one' }], '_collections': ['Literature'], 'control_number': 8888, 'references': [{ 'reference': { 'authors': [{ 'full_name': 'Smith, J.' }], } }] } record = InspireRecord.create(data=citing_json, skip_files=True) record.commit() db.session.commit() es.indices.refresh('records-hep') expected_args = ('lit', record['control_number'], 2) mocked_indexing_task.assert_called_with(*expected_args) # execute mocked task index_modified_citations_from_record(*expected_args) es_rec1 = get_es_record('lit', 9999) es_rec2 = get_es_record('lit', 9998) assert es_rec1['citation_count'] == 0 assert es_rec2['citation_count'] == 0 assert get_citations_from_es(es_rec1).total == 0 assert get_citations_from_es(es_rec2).total == 0 references = { 'references': [{ 'record': { '$ref': 'http://localhost:5000/api/literature/9998' }, }, { 'record': { '$ref': 'http://localhost:5000/api/literature/9999' }, }] } citing_json.update(references) record.clear() record.update(citing_json) record.commit() db.session.commit() es.indices.refresh('records-hep') expected_args = ('lit', record['control_number'], 3) mocked_indexing_task.assert_called_with(*expected_args) # execute mocked task index_modified_citations_from_record(*expected_args) es_rec1 = get_es_record('lit', 9999) es_rec2 = get_es_record('lit', 9998) assert es_rec1['citation_count'] == 1 assert es_rec2['citation_count'] == 1 assert get_citations_from_es(es_rec1).total == 1 assert get_citations_from_es(es_rec2).total == 1 _delete_record('lit', record['control_number']) _delete_record('lit', cited1['control_number']) _delete_record('lit', cited2['control_number'])
def test_index_after_commit_indexes_also_cites_record_when_new_citation_is_added( mocked_indexing_task, mocked_permission_check, app, ): # this test doesn't use the isolated_app because it needs to commit to # the DB in order to create records versions. json_data = { '$schema': 'http://localhost:5000/schemas/records/hep.json', 'document_type': ['article'], 'titles': [{ 'title': 'This is the record being cited' }], 'control_number': 9999, '_collections': ['Literature'] } cited = InspireRecord.create(data=json_data, skip_files=True) db.session.commit() es.indices.refresh('records-hep') expected_args = 'lit', cited['control_number'], 1 mocked_indexing_task.assert_called_with(*expected_args) # execute mocked task index_modified_citations_from_record(*expected_args) es_rec = get_es_record('lit', 9999) assert es_rec['citation_count'] == 0 assert get_citations_from_es(es_rec).total == 0 citing_json = { '$schema': 'http://localhost:5000/schemas/records/hep.json', 'document_type': ['article'], 'titles': [{ 'title': 'Record citing the first one' }], '_collections': ['Literature'], 'control_number': 8888, 'references': [{ "reference": { 'authors': [{ 'full_name': 'Smith, J.' }] } }] } record = InspireRecord.create(data=citing_json, skip_files=True) db.session.commit() es.indices.refresh('records-hep') expected_args = 'lit', record['control_number'], 1 mocked_indexing_task.assert_called_with(*expected_args) # execute mocked task index_modified_citations_from_record(*expected_args) es_rec = get_es_record('lit', 9999) assert es_rec['citation_count'] == 0 assert get_citations_from_es(es_rec).total == 0 references = { 'references': [{ "curated_relation": False, "record": { "$ref": "http://localhost:5000/api/literature/9999" }, "reference": { 'authors': [{ 'full_name': 'Smith, J.' }], } }] } citing_json.update(references) record.clear() record.update(citing_json) record.commit() db.session.commit() es.indices.refresh('records-hep') expected_args = 'lit', record['control_number'], 2 mocked_indexing_task.assert_called_with(*expected_args) # execute mocked task index_modified_citations_from_record(*expected_args) es_rec = get_es_record('lit', 9999) assert es_rec['citation_count'] == 1 assert get_citations_from_es(es_rec).total == 1 _delete_record('lit', 8888) _delete_record('lit', 9999)