def refextract(obj, eng):
"""Extract references from various sources and add them to the workflow.
Runs ``refextract`` on both the PDF attached to the workflow and the
references provided by the submitter, if any, then chooses the one
that generated the most and attaches them to the workflow object.
Args:
obj: a workflow object.
eng: a workflow engine.
Returns:
None
"""
if 'references' in obj.data:
extracted_raw_references = dedupe_list(
extract_references_from_raw_refs(obj.data['references']))
obj.log.info('Extracted %d references from raw refs.',
len(extracted_raw_references))
obj.data['references'] = match_references_based_on_flag(
extracted_raw_references)
return
matched_pdf_references, matched_text_references = [], []
source = LiteratureReader(obj.data).source
with get_document_in_workflow(obj) as tmp_document:
if tmp_document:
pdf_references = dedupe_list(
extract_references_from_pdf(tmp_document, source))
matched_pdf_references = match_references_based_on_flag(
pdf_references)
text = get_value(obj.extra_data, 'formdata.references')
if text:
text_references = dedupe_list(
extract_references_from_text(text, source))
matched_text_references = match_references_based_on_flag(
text_references)
if len(matched_pdf_references) == len(matched_text_references) == 0:
obj.log.info('No references extracted.')
elif len(matched_pdf_references) > len(matched_text_references):
obj.log.info('Extracted %d references from PDF.',
len(matched_pdf_references))
obj.data['references'] = matched_pdf_references
elif len(matched_text_references) >= len(matched_pdf_references):
obj.log.info('Extracted %d references from text.',
len(matched_text_references))
obj.data['references'] = matched_text_references
def test_get_document_in_workflow():
data = {
'documents': [
{
'key': 'fulltext.xml',
'fulltext': True,
},
],
}
files = MockFiles({})
files['fulltext.xml'] = None
obj = MockObj(data, {}, files=files)
with get_document_in_workflow(obj) as local_file:
assert local_file == files['fulltext.xml'].file.uri
def test_get_document_in_workflow():
data = {
'documents': [
{
'key': 'fulltext.xml',
'fulltext': True,
},
],
}
files = MockFiles({})
files['fulltext.xml'] = None
obj = MockObj(data, {}, files=files)
with get_document_in_workflow(obj) as local_file:
assert local_file == files['fulltext.xml'].file.uri
def refextract(obj, eng):
"""Extract references from various sources and add them to the workflow.
Runs ``refextract`` on both the PDF attached to the workflow and the
references provided by the submitter, if any, then chooses the one
that generated the most and attaches them to the workflow object.
Note:
We might want to compare the number of *matched* references instead.
Args:
obj: a workflow object.
eng: a workflow engine.
Returns:
None
"""
if 'references' in obj.data:
obj.log.info('Found references in metadata, extracting unextracted raw_refs')
obj.data['references'] = extract_references_from_raw_refs(obj.data['references'])
return
pdf_references, text_references = [], []
source = get_value(obj.data, 'acquisition_source.source')
with get_document_in_workflow(obj) as tmp_document:
if tmp_document:
try:
pdf_references = extract_references_from_pdf(tmp_document, source)
except TimeoutError:
obj.log.error('Timeout when extracting references from PDF.')
text = get_value(obj.extra_data, 'formdata.references')
if text:
try:
text_references = extract_references_from_text(text, source)
except TimeoutError:
obj.log.error('Timeout when extracting references from text.')
if len(pdf_references) == len(text_references) == 0:
obj.log.info('No references extracted.')
elif len(pdf_references) >= len(text_references):
obj.log.info('Extracted %d references from PDF.', len(pdf_references))
obj.data['references'] = pdf_references
elif len(text_references) > len(pdf_references):
obj.log.info('Extracted %d references from text.', len(text_references))
obj.data['references'] = text_references
def test_get_document_in_workflow_prefers_fulltext():
data = {
'documents': [
{
'key': 'table_of_contents.pdf',
},
{
'key': 'fulltext.xml',
'fulltext': True,
},
],
}
files = MockFiles({})
files['fulltext.xml'] = None
files['table_of_contents.pdf'] = None
obj = MockObj(data, {}, files=files)
with get_document_in_workflow(obj) as local_file:
assert local_file == files['fulltext.xml'].file.uri
def test_get_document_in_workflow_prefers_fulltext():
data = {
'documents': [
{
'key': 'table_of_contents.pdf',
},
{
'key': 'fulltext.xml',
'fulltext': True,
},
],
}
files = MockFiles({})
files['fulltext.xml'] = None
files['table_of_contents.pdf'] = None
obj = MockObj(data, {}, files=files)
with get_document_in_workflow(obj) as local_file:
assert local_file == files['fulltext.xml'].file.uri
def refextract(obj, eng):
"""Extract references from various sources and add them to the workflow.
Runs ``refextract`` on both the PDF attached to the workflow and the
references provided by the submitter, if any, then chooses the one
that generated the most and attaches them to the workflow object.
Args:
obj: a workflow object.
eng: a workflow engine.
Returns:
None
"""
if 'references' in obj.data:
extracted_raw_references = dedupe_list(extract_references_from_raw_refs(obj.data['references']))
obj.log.info('Extracted %d references from raw refs.', len(extracted_raw_references))
obj.data['references'] = match_references(extracted_raw_references)
return
matched_pdf_references, matched_text_references = [], []
source = LiteratureReader(obj.data).source
with get_document_in_workflow(obj) as tmp_document:
if tmp_document:
pdf_references = dedupe_list(extract_references_from_pdf(tmp_document, source))
matched_pdf_references = match_references(pdf_references)
text = get_value(obj.extra_data, 'formdata.references')
if text:
text_references = dedupe_list(extract_references_from_text(text, source))
matched_text_references = match_references(text_references)
if len(matched_pdf_references) == len(matched_text_references) == 0:
obj.log.info('No references extracted.')
elif len(matched_pdf_references) > len(matched_text_references):
obj.log.info('Extracted %d references from PDF.', len(matched_pdf_references))
obj.data['references'] = matched_pdf_references
elif len(matched_text_references) >= len(matched_pdf_references):
obj.log.info('Extracted %d references from text.', len(matched_text_references))
obj.data['references'] = matched_text_references
def test_get_document_in_workflow_takes_first_among_equals():
data = {
'documents': [
{
'key': 'table_of_contents.pdf',
},
{
'key': 'document.pdf',
},
],
}
files = MockFiles({})
files['document.pdf'] = None
files['table_of_contents.pdf'] = None
obj = MockObj(data, {}, files=files)
with get_document_in_workflow(obj) as local_file:
assert local_file == files['table_of_contents.pdf'].file.uri
assert 'More than one document in workflow, first one used' in obj.log._error.getvalue()
def test_get_document_in_workflow_takes_first_among_equals():
data = {
'documents': [
{
'key': 'table_of_contents.pdf',
},
{
'key': 'document.pdf',
},
],
}
files = MockFiles({})
files['document.pdf'] = None
files['table_of_contents.pdf'] = None
obj = MockObj(data, {}, files=files)
with get_document_in_workflow(obj) as local_file:
assert local_file == files['table_of_contents.pdf'].file.uri
assert 'More than one document in workflow, first one used' in obj.log._error.getvalue(
)
def test_get_document_in_workflow_returns_None_when_no_documents():
files = MockFiles({})
obj = MockObj({}, {}, files=files)
with get_document_in_workflow(obj) as local_file:
assert local_file is None
def test_get_document_in_workflow_returns_None_when_no_documents():
files = MockFiles({})
obj = MockObj({}, {}, files=files)
with get_document_in_workflow(obj) as local_file:
assert local_file is None