def submission_fulltext_download(obj, eng):
submission_pdf = obj.extra_data.get('submission_pdf')
if submission_pdf and is_pdf_link(submission_pdf):
filename = secure_filename('fulltext.pdf')
pdf = download_file_to_workflow(
workflow=obj,
name=filename,
url=submission_pdf,
)
if pdf:
obj.data['documents'] = [
document for document in obj.data.get('documents', ())
if document.get('key') != filename
]
lb = LiteratureBuilder(source=obj.data['acquisition_source']['source'], record=obj.data)
lb.add_document(
filename,
fulltext=True,
original_url=submission_pdf,
url='/api/files/{bucket}/{key}'.format(bucket=obj.files[filename].bucket_id, key=filename)
)
obj.data = lb.record
obj.log.info('PDF provided by user from %s', submission_pdf)
return obj.files[filename].file.uri
else:
obj.log.info('Cannot fetch PDF provided by user from %s', submission_pdf)
def populate_arxiv_document(obj, eng):
arxiv_id = LiteratureReader(obj.data).arxiv_id
for conf_name in ('ARXIV_PDF_URL', 'ARXIV_PDF_URL_ALTERNATIVE'):
url = current_app.config[conf_name].format(arxiv_id=arxiv_id)
is_valid_pdf_link = is_pdf_link(url)
if is_valid_pdf_link:
break
try:
if NO_PDF_ON_ARXIV in requests.get(url).content:
obj.log.info('No PDF is available for %s', arxiv_id)
return
except requests.exceptions.RequestException:
raise DownloadError("Error accessing url {url}".format(url=url))
if not is_valid_pdf_link:
raise DownloadError("{url} is not serving a PDF file.".format(url=url))
filename = secure_filename('{0}.pdf'.format(arxiv_id))
obj.data['documents'] = [
document for document in obj.data.get('documents', ())
if document.get('key') != filename
]
lb = LiteratureBuilder(source='arxiv', record=obj.data)
lb.add_document(
filename,
fulltext=True,
hidden=True,
material='preprint',
original_url=url,
url=url,
)
obj.data = lb.record
def populate_arxiv_document(obj, eng):
arxiv_id = get_arxiv_id(obj.data)
for conf_name in ('ARXIV_PDF_URL', 'ARXIV_PDF_URL_ALTERNATIVE'):
url = current_app.config[conf_name].format(arxiv_id=arxiv_id)
is_valid_pdf_link = is_pdf_link(url)
if is_valid_pdf_link:
break
if NO_PDF_ON_ARXIV in requests.get(url).content:
obj.log.info('No PDF is available for %s', arxiv_id)
return
if not is_valid_pdf_link:
raise DownloadError("{url} is not serving a PDF file.".format(url=url))
filename = secure_filename('{0}.pdf'.format(arxiv_id))
obj.data['documents'] = [
document for document in obj.data.get('documents', ())
if document.get('key') != filename
]
lb = LiteratureBuilder(source='arxiv', record=obj.data)
lb.add_document(
filename,
fulltext=True,
hidden=True,
material='preprint',
original_url=url,
url=url,
)
obj.data = lb.record
def populate_arxiv_document(obj, eng):
arxiv_id = get_arxiv_id(obj.data)
url = current_app.config['ARXIV_PDF_URL'].format(arxiv_id=arxiv_id)
if not is_pdf_link(url):
if NO_PDF_ON_ARXIV in requests.get(url).content:
obj.log.info('No PDF is available for %s', arxiv_id)
return
raise DownloadError("{url} is not serving a PDF file.".format(url=url))
filename = secure_filename('{0}.pdf'.format(arxiv_id))
obj.data['documents'] = [
document for document in obj.data.get('documents', ())
if document.get('key') != filename
]
lb = LiteratureBuilder(source='arxiv', record=obj.data)
lb.add_document(
filename,
fulltext=True,
hidden=True,
material='preprint',
original_url=url,
url=url,
)
obj.data = lb.record
def arxiv_fulltext_download(obj, eng):
"""Perform the fulltext download step for arXiv records.
:param obj: Workflow Object to process
:param eng: Workflow Engine processing the object
"""
arxiv_id = get_arxiv_id(obj.data)
filename = secure_filename('{0}.pdf'.format(arxiv_id))
url = current_app.config['ARXIV_PDF_URL'].format(arxiv_id=arxiv_id)
if not is_pdf_link(url):
if NO_PDF_ON_ARXIV in requests.get(url).content:
obj.log.info('No PDF is available for %s', arxiv_id)
return
raise DownloadError("{url} is not serving a PDF file.".format(url=url))
pdf = download_file_to_workflow(
workflow=obj,
name=filename,
url=url,
)
if pdf:
obj.log.info('PDF retrieved from arXiv for %s', arxiv_id)
else:
obj.log.error('Cannot retrieve PDF from arXiv for %s', arxiv_id)
def test_is_pdf_link_handles_empty_requests():
with requests_mock.Mocker() as requests_mocker:
requests_mocker.register_uri('GET',
'http://example.org/empty-pdf',
text='')
assert not is_pdf_link('http://example.org/empty-pdf')
def submission_fulltext_download(obj, eng):
submission_pdf = obj.extra_data.get('submission_pdf')
if submission_pdf and is_pdf_link(submission_pdf):
filename = secure_filename('fulltext.pdf')
pdf = download_file_to_workflow(
workflow=obj,
name=filename,
url=submission_pdf,
)
if pdf:
obj.log.info('PDF provided by user from %s', submission_pdf)
return obj.files[filename].file.uri
else:
obj.log.info('Cannot fetch PDF provided by user from %s', submission_pdf)
def populate_submission_document(obj, eng):
submission_pdf = obj.extra_data.get('submission_pdf')
if submission_pdf and is_pdf_link(submission_pdf):
filename = secure_filename('fulltext.pdf')
obj.data['documents'] = [
document for document in obj.data.get('documents', ())
if document.get('key') != filename
]
lb = LiteratureBuilder(source=obj.data['acquisition_source']['source'],
record=obj.data)
lb.add_document(
filename,
fulltext=True,
url=submission_pdf,
original_url=submission_pdf,
)
obj.data = lb.record
def populate_submission_document(obj, eng):
submission_pdf = obj.extra_data.get('submission_pdf')
if submission_pdf and is_pdf_link(submission_pdf):
filename = secure_filename('fulltext.pdf')
obj.data['documents'] = [
document for document in obj.data.get('documents', ())
if document.get('key') != filename
]
lb = LiteratureBuilder(
source=obj.data['acquisition_source']['source'], record=obj.data)
lb.add_document(
filename,
fulltext=True,
url=submission_pdf,
original_url=submission_pdf,
)
obj.data = lb.record
def arxiv_fulltext_download(obj, eng):
"""Perform the fulltext download step for arXiv records.
:param obj: Workflow Object to process
:param eng: Workflow Engine processing the object
"""
arxiv_id = get_arxiv_id(obj.data)
filename = secure_filename('{0}.pdf'.format(arxiv_id))
url = current_app.config['ARXIV_PDF_URL'].format(arxiv_id=arxiv_id)
if not is_pdf_link(url):
if NO_PDF_ON_ARXIV in requests.get(url).content:
obj.log.info('No PDF is available for %s', arxiv_id)
return
raise DownloadError("{url} is not serving a PDF file.".format(url=url))
pdf = download_file_to_workflow(
workflow=obj,
name=filename,
url=url,
)
if pdf:
obj.data['documents'] = [
document for document in obj.data.get('documents', ())
if document.get('key') != filename
]
lb = LiteratureBuilder(source='arxiv', record=obj.data)
lb.add_document(filename,
fulltext=True,
hidden=True,
material='preprint',
original_url=url,
url='/api/files/{bucket}/{key}'.format(
bucket=obj.files[filename].bucket_id,
key=filename))
obj.data = lb.record
obj.log.info('PDF retrieved from arXiv for %s', arxiv_id)
else:
obj.log.error('Cannot retrieve PDF from arXiv for %s', arxiv_id)
def test_is_pdf_link_handles_empty_requests():
with requests_mock.Mocker() as requests_mocker:
requests_mocker.register_uri('GET', 'http://example.org/empty-pdf', text='')
assert not is_pdf_link('http://example.org/empty-pdf')
def no_pdf_validator(form, field):
"""Validate that the field does not contain a link to a PDF."""
message = 'Please, use the field above to link to a PDF.'
if field.data and is_pdf_link(field.data):
raise StopValidation(message)
def pdf_validator(form, field):
"""Validate that the field contains a link to a PDF."""
message = 'Please, provide an accessible direct link to a PDF document.'
if field.data and not is_pdf_link(field.data):
raise StopValidation(message)
def test_is_pdf_link_handles_pdfs_starting_with_blank_lines():
assert is_pdf_link('https://arxiv.org/pdf/1803.01183.pdf')
def no_pdf_validator(form, field):
"""Validate that the field does not contain a link to a PDF."""
message = 'Please, use the field above to link to a PDF.'
if field.data and is_pdf_link(field.data):
raise StopValidation(message)
def pdf_validator(form, field):
"""Validate that the field contains a link to a PDF."""
message = 'Please, provide an accessible direct link to a PDF document.'
if field.data and not is_pdf_link(field.data):
raise StopValidation(message)
def test_is_pdf_handles_different_pdfs_correctly():
assert is_pdf_link('http://arxiv.org/pdf/physics/0404071')
