def refextract(obj, eng): """Extract references from various sources and add them to the workflow. Runs ``refextract`` on both the PDF attached to the workflow and the references provided by the submitter, if any, then chooses the one that generated the most and attaches them to the workflow object. Args: obj: a workflow object. eng: a workflow engine. Returns: None """ if 'references' in obj.data: extracted_raw_references = dedupe_list( extract_references_from_raw_refs(obj.data['references'])) obj.log.info('Extracted %d references from raw refs.', len(extracted_raw_references)) obj.data['references'] = match_references(extracted_raw_references) return matched_pdf_references, matched_text_references = [], [] source = LiteratureReader(obj.data).source with get_document_in_workflow(obj) as tmp_document: if tmp_document: pdf_references = dedupe_list( extract_references_from_pdf(tmp_document, source)) matched_pdf_references = match_references(pdf_references) text = get_value(obj.extra_data, 'formdata.references') if text: text_references = dedupe_list( extract_references_from_text(text, source)) matched_text_references = match_references(text_references) if len(matched_pdf_references) == len(matched_text_references) == 0: obj.log.info('No references extracted.') elif len(matched_pdf_references) > len(matched_text_references): obj.log.info('Extracted %d references from PDF.', len(matched_pdf_references)) obj.data['references'] = matched_pdf_references elif len(matched_text_references) >= len(matched_pdf_references): obj.log.info('Extracted %d references from text.', len(matched_text_references)) obj.data['references'] = matched_text_references
def test_match_references_finds_match_when_repeated_record_with_different_scores( mocked_inspire_matcher_match, isolated_app ): references = [ { 'reference': { 'publication_info': { 'artid': '045', 'journal_title': 'JHEP', 'journal_volume': '06', 'page_start': '045', 'year': 2007 } } } ] schema = load_schema('hep') subschema = schema['properties']['references'] assert validate(references, subschema) is None references = match_references(references) assert len(references) == 1 assert references[0]['record']['$ref'] == 'http://localhost:5000/api/literature/1' assert validate(references, subschema) is None
def test_match_references_finds_match_when_repeated_record_with_different_scores( mocked_inspire_matcher_match, isolated_app ): references = [ { 'reference': { 'publication_info': { 'artid': '045', 'journal_title': 'JHEP', 'journal_volume': '06', 'page_start': '045', 'year': 2007 } } } ] schema = load_schema('hep') subschema = schema['properties']['references'] assert validate(references, subschema) is None references = match_references(references) assert len(references) == 1 assert references[0]['record']['$ref'] == 'http://localhost:5000/api/literature/1' assert validate(references, subschema) is None
def refextract_url(): """Run refextract on a URL.""" with local_refextract_kbs_path() as kbs_path: extracted_references = extract_references_from_url( request.json['url'], override_kbs_files=kbs_path, reference_format=u'{title},{volume},{page}') references = map_refextract_to_schema(extracted_references) references = match_references(references) return jsonify(references)
def refextract(obj, eng): """Extract references from various sources and add them to the workflow. Runs ``refextract`` on both the PDF attached to the workflow and the references provided by the submitter, if any, then chooses the one that generated the most and attaches them to the workflow object. Args: obj: a workflow object. eng: a workflow engine. Returns: None """ if 'references' in obj.data: extracted_raw_references = dedupe_list(extract_references_from_raw_refs(obj.data['references'])) obj.log.info('Extracted %d references from raw refs.', len(extracted_raw_references)) obj.data['references'] = match_references(extracted_raw_references) return matched_pdf_references, matched_text_references = [], [] source = LiteratureReader(obj.data).source with get_document_in_workflow(obj) as tmp_document: if tmp_document: pdf_references = dedupe_list(extract_references_from_pdf(tmp_document, source)) matched_pdf_references = match_references(pdf_references) text = get_value(obj.extra_data, 'formdata.references') if text: text_references = dedupe_list(extract_references_from_text(text, source)) matched_text_references = match_references(text_references) if len(matched_pdf_references) == len(matched_text_references) == 0: obj.log.info('No references extracted.') elif len(matched_pdf_references) > len(matched_text_references): obj.log.info('Extracted %d references from PDF.', len(matched_pdf_references)) obj.data['references'] = matched_pdf_references elif len(matched_text_references) >= len(matched_pdf_references): obj.log.info('Extracted %d references from text.', len(matched_text_references)) obj.data['references'] = matched_text_references
def refextract_text(): """Run refextract on a piece of text.""" with local_refextract_kbs_path() as kbs_path: extracted_references = extract_references_from_string( request.json['text'], override_kbs_files=kbs_path, reference_format=u'{title},{volume},{page}' ) references = map_refextract_to_schema(extracted_references) references = match_references(references) return jsonify(references)
def test_match_references_no_match_when_multiple_match_different_from_previous(isolated_app): """Test reference matcher for when inspire-matcher returns multiple matches where the matched record id is not the same as the previous matched record id""" original_cited_record_json = { '$schema': 'http://localhost:5000/schemas/records/hep.json', '_collections': ['Literature'], 'control_number': 1, 'document_type': ['article'], 'publication_info': [ { 'artid': '159', 'journal_title': 'JHEP', 'journal_volume': '03', 'page_start': '159', 'year': 2016 }, { 'artid': '074', 'journal_title': 'JHEP', 'journal_volume': '05', 'material': 'erratum', 'page_start': '074', 'year': 2017 } ] } errata_cited_record_json = { '$schema': 'http://localhost:5000/schemas/records/hep.json', '_collections': ['Literature'], 'control_number': 2, 'document_type': ['article'], 'publication_info': [ { 'artid': '074', 'journal_title': 'JHEP', 'journal_volume': '05', 'material': 'erratum', 'page_start': '074', 'year': 2017 } ] } TestRecordMetadata.create_from_kwargs( json=original_cited_record_json, index_name='records-hep') TestRecordMetadata.create_from_kwargs( json=errata_cited_record_json, index_name='records-hep') references = [ { 'reference': { 'publication_info': { 'artid': '074', 'journal_title': 'JHEP', 'journal_volume': '05', 'page_start': '074', 'year': 2017 } } } ] schema = load_schema('hep') subschema = schema['properties']['references'] assert validate(references, subschema) is None references = match_references(references) assert get_value(references[0], 'record') is None assert validate(references, subschema) is None
def test_match_references_no_match_when_multiple_match_different_from_previous( isolated_app): """Test reference matcher for when inspire-matcher returns multiple matches where the matched record id is not the same as the previous matched record id""" original_cited_record_json = { '$schema': 'http://localhost:5000/schemas/records/hep.json', '_collections': ['Literature'], 'control_number': 1, 'document_type': ['article'], 'publication_info': [{ 'artid': '159', 'journal_title': 'JHEP', 'journal_volume': '03', 'page_start': '159', 'year': 2016 }, { 'artid': '074', 'journal_title': 'JHEP', 'journal_volume': '05', 'material': 'erratum', 'page_start': '074', 'year': 2017 }] } errata_cited_record_json = { '$schema': 'http://localhost:5000/schemas/records/hep.json', '_collections': ['Literature'], 'control_number': 2, 'document_type': ['article'], 'publication_info': [{ 'artid': '074', 'journal_title': 'JHEP', 'journal_volume': '05', 'material': 'erratum', 'page_start': '074', 'year': 2017 }] } TestRecordMetadata.create_from_kwargs(json=original_cited_record_json, index_name='records-hep') TestRecordMetadata.create_from_kwargs(json=errata_cited_record_json, index_name='records-hep') references = [{ 'reference': { 'publication_info': { 'artid': '074', 'journal_title': 'JHEP', 'journal_volume': '05', 'page_start': '074', 'year': 2017 } } }] schema = load_schema('hep') subschema = schema['properties']['references'] assert validate(references, subschema) is None references = match_references(references) assert get_value(references[0], 'record') is None assert validate(references, subschema) is None
def get_linked_refs(): data = request.json matched_refs = match_references(data['references']) return jsonify({'references': matched_refs})
def get_linked_refs(): data = request.json matched_refs = match_references(data['references']) return jsonify({'references': matched_refs})
def match_references_based_on_flag(references): if current_app.config.get("FEATURE_FLAG_ENABLE_MATCH_REFERENCES_HEP"): return match_references_hep(references) return match_references(references)