def test_arxiv_plot_extract_logs_when_images_are_invalid(mock_process_tarball):
mock_process_tarball.side_effect = DelegateError
schema = load_schema('hep')
subschema = schema['properties']['arxiv_eprints']
filename = pkg_resources.resource_filename(
__name__, os.path.join('fixtures', '1612.00624'))
data = {
'arxiv_eprints': [
{
'categories': [
'physics.ins-det',
],
'value': '1612.00624',
},
],
} # synthetic data
extra_data = {}
files = MockFiles({
'1612.00624.tar.gz':
AttrDict({'file': AttrDict({
'uri': filename,
})})
})
assert validate(data['arxiv_eprints'], subschema) is None
obj = MockObj(data, extra_data, files=files)
eng = MockEng()
assert arxiv_plot_extract(obj, eng) is None
assert '1612.00624' in obj.log._error.getvalue()
def test_populate_submission_document_without_pdf():
with requests_mock.Mocker() as requests_mocker:
requests_mocker.register_uri(
'GET', 'http://export.arxiv.org/pdf/1707.02785',
content=pkg_resources.resource_string(
__name__, os.path.join('fixtures', '1707.02785.html')),
)
schema = load_schema('hep')
subschema = schema['properties']['acquisition_source']
data = {
'acquisition_source': {
'datetime': '2017-11-30T16:38:43.352370',
'email': '*****@*****.**',
'internal_uid': 54252,
'method': 'submitter',
'orcid': '0000-0002-2174-4493',
'source': 'submitter',
'submission_number': '1'
}
}
assert validate(data['acquisition_source'], subschema) is None
extra_data = {
'submission_pdf': 'http://export.arxiv.org/pdf/1707.02785',
}
files = MockFiles({})
obj = MockObj(data, extra_data, files=files)
eng = MockEng()
assert populate_submission_document(obj, eng) is None
documents = obj.data.get('documents', [])
assert 0 == len(documents)
def test_arxiv_author_list_handles_auto_ignore_comment():
schema = load_schema('hep')
subschema = schema['properties']['arxiv_eprints']
filename = pkg_resources.resource_filename(
__name__, os.path.join('fixtures', '1703.09986.tar.gz'))
data = {
'arxiv_eprints': [
{
'categories': [
'hep-ex',
],
'value': '1703.09986',
},
],
} # record/1519995
extra_data = {}
files = MockFiles({
'1703.09986.tar.gz':
AttrDict({'file': AttrDict({
'uri': filename,
})})
})
assert validate(data['arxiv_eprints'], subschema) is None
obj = MockObj(data, extra_data, files=files)
eng = MockEng()
default_arxiv_author_list = arxiv_author_list()
assert default_arxiv_author_list(obj, eng) is None
def test_arxiv_author_list_with_missing_tarball():
schema = load_schema('hep')
eprints_subschema = schema['properties']['arxiv_eprints']
data = {
'arxiv_eprints': [
{
'categories': [
'hep-ex',
],
'value': '1703.09986',
},
],
} # record/1519995
validate(data['arxiv_eprints'], eprints_subschema)
extra_data = {}
files = MockFiles({
'jessica.jones.tar.gz':
AttrDict({'file': AttrDict({
'uri': 'alias.investigations',
})})
})
obj = MockObj(data, extra_data, files=files)
eng = MockEng()
default_arxiv_author_list = arxiv_author_list()
expected_message = \
'Skipping author list extraction, no tarball with name "1703.09986.tar.gz" found'
assert default_arxiv_author_list(obj, eng) is None
assert expected_message in obj.log._info.getvalue()
def test_arxiv_plot_extract_is_safe_to_rerun(mock_os):
schema = load_schema('hep')
subschema = schema['properties']['arxiv_eprints']
filename = pkg_resources.resource_filename(
__name__, os.path.join('fixtures', '0804.1873.tar.gz'))
data = {
'arxiv_eprints': [
{
'categories': [
'nucl-ex',
],
'value': '0804.1873',
},
],
} # literature/783246
extra_data = {}
files = MockFiles({
'0804.1873.tar.gz':
AttrDict({
'file': AttrDict({
'uri': filename,
}),
}),
})
assert validate(data['arxiv_eprints'], subschema) is None
obj = MockObj(data, extra_data, files=files)
eng = MockEng()
try:
temporary_dir = mkdtemp()
mock_os.path.abspath.return_value = temporary_dir
for _ in range(2):
assert arxiv_plot_extract(obj, eng) is None
expected_figures = [{
'url':
'/api/files/0b9dd5d1-feae-4ba5-809d-3a029b0bc110/figure1.png',
'source':
'arxiv',
'material':
'preprint',
'key':
'figure1.png',
'caption':
'Difference (in MeV) between the theoretical and experimental masses for the 2027 selected nuclei as a function of the mass number.'
}]
result = obj.data['figures']
assert expected_figures == result
expected_files = ['0804.1873.tar.gz', 'figure1.png']
assert expected_files == obj.files.keys
finally:
rmtree(temporary_dir)
def test_arxiv_package_download_logs_on_success():
httpretty.register_uri(httpretty.GET,
'http://export.arxiv.org/e-print/1605.03959',
body=pkg_resources.resource_string(
__name__,
os.path.join('fixtures', '1605.03959.tar.gz')))
schema = load_schema('hep')
subschema = schema['properties']['arxiv_eprints']
data = {
'arxiv_eprints': [
{
'categories': [
'hep-th',
'cond-mat.stat-mech',
'cond-mat.str-el',
],
'value': '1605.03959',
},
],
} # literature/1458968
extra_data = {}
files = MockFiles({})
assert validate(data['arxiv_eprints'], subschema) is None
obj = MockObj(data, extra_data, files=files)
eng = MockEng()
assert arxiv_package_download(obj, eng) is None
expected = 'Tarball retrieved from arXiv for 1605.03959'
result = obj.log._info.getvalue()
assert expected == result
def test_populate_arxiv_document_retries_on_connection_error(
mock_requests_get):
mock_requests_get.side_effect = side_effect_requests_get
schema = load_schema('hep')
subschema = schema['properties']['arxiv_eprints']
data = {
'arxiv_eprints': [
{
'categories': [
'hep-ex',
],
'value': '1605.03814',
},
],
} # literature/1458270
extra_data = {}
files = MockFiles({})
assert validate(data['arxiv_eprints'], subschema) is None
obj = MockObj(data, extra_data, files=files)
eng = MockEng()
with pytest.raises(DownloadError):
populate_arxiv_document(obj, eng)
assert mock_requests_get.call_count == 10
def test_prepare_files_ignores_keys_not_ending_with_pdf():
data = {}
extra_data = {}
files = MockFiles({
'foo.bar':
AttrDict({
'obj':
AttrDict({
'file': AttrDict({
'uri': '/data/foo.pdf',
}),
}),
}),
})
obj = MockObj(data, extra_data, files=files)
eng = MockEng()
assert prepare_files(obj, eng) is None
expected = {}
result = obj.data
assert expected == result
expected = ''
result = obj.log._info.getvalue()
assert expected == result
def test_download_documents():
with requests_mock.Mocker() as requests_mocker:
requests_mocker.register_uri(
'GET', 'http://export.arxiv.org/pdf/1605.03844',
content=pkg_resources.resource_string(
__name__, os.path.join('fixtures', '1605.03844.pdf')),
)
schema = load_schema('hep')
subschema = schema['properties']['documents']
data = {
'documents': [
{
'key': '1605.03844.pdf',
'url': 'http://export.arxiv.org/pdf/1605.03844'
},
],
} # literature/1458302
extra_data = {}
files = MockFiles({})
assert validate(data['documents'], subschema) is None
obj = MockObj(data, extra_data, files=files)
eng = MockEng()
assert download_documents(obj, eng) is None
documents = obj.data['documents']
expected_document_url = '/api/files/0b9dd5d1-feae-4ba5-809d-3a029b0bc110/1605.03844.pdf'
assert 1 == len(documents)
assert expected_document_url == documents[0]['url']
def test_arxiv_author_list_logs_on_error(mock_untar):
mock_untar.side_effect = InvalidTarball
schema = load_schema('hep')
subschema = schema['properties']['arxiv_eprints']
filename = pkg_resources.resource_filename(
__name__, os.path.join('fixtures', '1605.07707'))
data = {
'arxiv_eprints': [
{
'categories': [
'hep-th',
],
'value': '1605.07707',
},
],
} # synthethic data
extra_data = {}
files = MockFiles({
'1605.07707.tar.gz':
AttrDict({'file': AttrDict({
'uri': filename,
})})
})
assert validate(data['arxiv_eprints'], subschema) is None
obj = MockObj(data, extra_data, files=files)
eng = MockEng()
default_arxiv_author_list = arxiv_author_list()
assert default_arxiv_author_list(obj, eng) is None
assert '1605.07707' in obj.log._info.getvalue()
def test_prepare_files_annotates_files_from_arxiv():
schema = load_schema('hep')
_fft_schema = schema['properties']['_fft']
arxiv_eprints_schema = schema['properties']['arxiv_eprints']
data = {
'arxiv_eprints': [
{
'categories': ['hep-th'],
'value': 'hep-th/9711200',
},
],
}
extra_data = {}
files = MockFiles({
'foo.pdf':
AttrDict({
'obj':
AttrDict({
'file': AttrDict({
'uri': '/data/foo.pdf',
}),
}),
}),
})
assert validate(data['arxiv_eprints'], arxiv_eprints_schema) is None
obj = MockObj(data, extra_data, files=files)
eng = MockEng()
assert prepare_files(obj, eng) is None
expected_fft = [
{
'path': '/data/foo.pdf',
'type': 'arXiv',
'filename': 'arxiv:foo',
'format': '.pdf',
},
]
expected_arxiv_eprints = [
{
'categories': [
'hep-th',
],
'value': 'hep-th/9711200',
},
]
result = obj.data
assert validate(result['_fft'], _fft_schema) is None
assert expected_fft == result['_fft']
assert validate(result['arxiv_eprints'], arxiv_eprints_schema) is None
assert expected_arxiv_eprints == result['arxiv_eprints']
expected = 'Non-user PDF files added to FFT.'
result = obj.log._info.getvalue()
assert expected == result
def test_download_file_to_workflow_retries_on_protocol_error():
with requests_mock.Mocker() as requests_mocker:
filename = pkg_resources.resource_filename(
__name__, os.path.join('fixtures', '1605.03844.pdf'))
requests_mocker.register_uri(
'GET', 'http://export.arxiv.org/pdf/1605.03844', [
{
'exc': requests.packages.urllib3.exceptions.ProtocolError
},
{
'body': filename,
'status_code': 200
},
])
data = {}
extra_data = {}
files = MockFiles({})
obj = MockObj(data, extra_data, files=files)
expected = MockFileObject(key='1605.03844.pdf')
result = download_file_to_workflow(
obj, '1605.03844.pdf', 'http://export.arxiv.org/pdf/1605.03844')
assert expected == result
def test_arxiv_fulltext_download_logs_on_pdf_not_existing():
with requests_mock.Mocker() as requests_mocker:
requests_mocker.register_uri(
'GET',
'http://export.arxiv.org/pdf/1707.02785',
content=pkg_resources.resource_string(
__name__, os.path.join('fixtures', '1707.02785.html')),
)
schema = load_schema('hep')
subschema = schema['properties']['arxiv_eprints']
data = {
'arxiv_eprints': [
{
'categories': [
'cs.CV',
],
'value': '1707.02785',
},
],
} # literature/1458302
extra_data = {}
files = MockFiles({})
assert validate(data['arxiv_eprints'], subschema) is None
obj = MockObj(data, extra_data, files=files)
eng = MockEng()
assert arxiv_fulltext_download(obj, eng) is None
expected = 'No PDF is available for 1707.02785'
result = obj.log._info.getvalue()
assert expected == result
def test_arxiv_package_download_logs_on_error():
with requests_mock.Mocker() as requests_mocker:
requests_mocker.register_uri(
'GET',
'http://export.arxiv.org/e-print/1605.03951',
status_code=500,
)
schema = load_schema('hep')
subschema = schema['properties']['arxiv_eprints']
data = {
'arxiv_eprints': [
{
'categories': [
'astro-ph.HE',
],
'value': '1605.03951',
},
],
} # literature/1458254
extra_data = {}
files = MockFiles({})
assert validate(data['arxiv_eprints'], subschema) is None
obj = MockObj(data, extra_data, files=files)
eng = MockEng()
assert arxiv_package_download(obj, eng) is None
expected = 'Cannot retrieve tarball from arXiv for 1605.03951'
result = obj.log._error.getvalue()
assert expected == result
def test_arxiv_fulltext_download_logs_on_success():
with requests_mock.Mocker() as requests_mocker:
requests_mocker.register_uri(
'GET',
'http://export.arxiv.org/pdf/1605.03844',
content=pkg_resources.resource_string(
__name__, os.path.join('fixtures', '1605.03844.pdf')),
)
schema = load_schema('hep')
subschema = schema['properties']['arxiv_eprints']
data = {
'arxiv_eprints': [
{
'categories': [
'physics.ins-det',
],
'value': '1605.03844',
},
],
} # literature/1458302
extra_data = {}
files = MockFiles({})
assert validate(data['arxiv_eprints'], subschema) is None
obj = MockObj(data, extra_data, files=files)
eng = MockEng()
assert arxiv_fulltext_download(obj, eng) is None
expected = 'PDF retrieved from arXiv for 1605.03844'
result = obj.log._info.getvalue()
assert expected == result
def test_arxiv_author_list_handles_multiple_author_xml_files():
schema = load_schema('hep')
eprints_subschema = schema['properties']['arxiv_eprints']
filename = pkg_resources.resource_filename(
__name__, os.path.join('fixtures', '1703.09986.multiple_author_lists.tar.gz'))
data = {
'$schema': 'http://localhost:5000/hep.json',
'arxiv_eprints': [
{
'categories': [
'hep-ex',
],
'value': '1703.09986',
},
],
} # record/1519995
validate(data['arxiv_eprints'], eprints_subschema)
extra_data = {}
files = MockFiles({
'1703.09986.tar.gz': AttrDict({
'file': AttrDict({
'uri': filename,
})
})
})
obj = MockObj(data, extra_data, files=files)
eng = MockEng()
default_arxiv_author_list = arxiv_author_list()
default_arxiv_author_list(obj, eng)
authors_subschema = schema['properties']['authors']
expected_authors = [
{
'affiliations': [{'value': 'Yerevan Phys. Inst.'}],
'ids': [
{'value': 'INSPIRE-00312131', 'schema': 'INSPIRE ID'},
{'value': 'CERN-432142', 'schema': 'CERN'},
],
'full_name': 'Sirunyan, Albert M',
},
{
'affiliations': [{'value': 'Yerevan Phys. Inst.'}],
'ids': [
{'value': 'INSPIRE-00312132', 'schema': 'INSPIRE ID'},
{'value': 'CERN-432143', 'schema': 'CERN'},
],
'full_name': 'Weary, Jake',
}
]
validate(expected_authors, authors_subschema)
assert obj.data.get('authors') == expected_authors
def test_arxiv_plot_extract_populates_files_with_plots(mock_os, tmpdir):
schema = load_schema('hep')
subschema = schema['properties']['arxiv_eprints']
filename = pkg_resources.resource_filename(
__name__, os.path.join('fixtures', '0804.1873.tar.gz'))
data = {
'arxiv_eprints': [
{
'categories': [
'nucl-ex',
],
'value': '0804.1873',
},
],
} # literature/783246
extra_data = {}
files = MockFiles({
'0804.1873.tar.gz':
AttrDict({
'file': AttrDict({
'uri': filename,
}),
}),
})
assert validate(data['arxiv_eprints'], subschema) is None
obj = MockObj(data, extra_data, files=files)
eng = MockEng()
temporary_dir = tmpdir.mkdir('plots')
mock_os.path.abspath.return_value = str(temporary_dir)
assert arxiv_plot_extract(obj, eng) is None
expected = [{
'url':
'/api/files/0b9dd5d1-feae-4ba5-809d-3a029b0bc110/figure1.png',
'source':
'arxiv',
'material':
'preprint',
'key':
'figure1.png',
'caption':
'Difference (in MeV) between the theoretical and experimental masses for the 2027 selected nuclei as a function of the mass number.'
}]
result = obj.data['figures']
assert expected == result
expected = 'Added 1 plots.'
result = obj.log._info.getvalue()
assert expected == result
def test_submission_fulltext_download_does_not_duplicate_documents():
with requests_mock.Mocker() as requests_mocker:
requests_mocker.register_uri(
'GET',
'http://export.arxiv.org/pdf/1605.03844',
content=pkg_resources.resource_string(
__name__, os.path.join('fixtures', '1605.03844.pdf')),
)
schema = load_schema('hep')
subschema = schema['properties']['acquisition_source']
data = {
'acquisition_source': {
'datetime': '2017-11-30T16:38:43.352370',
'email': '*****@*****.**',
'internal_uid': 54252,
'method': 'submitter',
'orcid': '0000-0002-2174-4493',
'source': 'submitter',
'submission_number': '1'
}
}
assert validate(data['acquisition_source'], subschema) is None
extra_data = {
'submission_pdf': 'http://export.arxiv.org/pdf/1605.03844',
}
files = MockFiles({})
obj = MockObj(data, extra_data, files=files)
eng = MockEng()
assert submission_fulltext_download(obj, eng)
assert submission_fulltext_download(obj, eng)
expected_key = 'fulltext.pdf'
expected_documents = [{
'fulltext':
True,
'key':
expected_key,
'original_url':
'http://export.arxiv.org/pdf/1605.03844',
'source':
'submitter',
'url':
'/api/files/%s/%s' % (
obj.files[expected_key].bucket_id,
expected_key,
),
}]
result = obj.data['documents']
assert expected_documents == result
def test_populate_arxiv_document_retries_on_error():
response500 = {'content': '', 'status_code': 500}
response200 = {
'content': pkg_resources.resource_string(
__name__, os.path.join('fixtures', '1605.03814.pdf')),
'status_code': 200,
}
with requests_mock.Mocker() as requests_mocker:
requests_mocker.get(
'http://export.arxiv.org/pdf/1605.03814',
(response500, response200),
)
requests_mocker.get(
'http://arxiv.org/pdf/1605.03814',
(response500,)
)
schema = load_schema('hep')
subschema = schema['properties']['arxiv_eprints']
data = {
'arxiv_eprints': [
{
'categories': [
'hep-ex',
],
'value': '1605.03814',
},
],
} # literature/1458270
extra_data = {}
files = MockFiles({})
assert validate(data['arxiv_eprints'], subschema) is None
obj = MockObj(data, extra_data, files=files)
eng = MockEng()
assert populate_arxiv_document(obj, eng) is None
expected_url = 'http://export.arxiv.org/pdf/1605.03814'
expected_documents = [
{
'key': '1605.03814.pdf',
'fulltext': True,
'hidden': True,
'material': 'preprint',
'original_url': expected_url,
'url': expected_url,
'source': 'arxiv',
}
]
documents = obj.data['documents']
assert expected_documents == documents
def test_arxiv_plot_extract_populates_files_with_plots(mock_os):
schema = load_schema('hep')
subschema = schema['properties']['arxiv_eprints']
filename = pkg_resources.resource_filename(
__name__, os.path.join('fixtures', '0804.1873.tar.gz'))
data = {
'arxiv_eprints': [
{
'categories': [
'nucl-ex',
],
'value': '0804.1873',
},
],
} # literature/783246
extra_data = {}
files = MockFiles({
'0804.1873.tar.gz':
AttrDict({
'file': AttrDict({
'uri': filename,
}),
}),
})
assert validate(data['arxiv_eprints'], subschema) is None
obj = MockObj(data, extra_data, files=files)
eng = MockEng()
try:
temporary_dir = mkdtemp()
mock_os.path.abspath.return_value = temporary_dir
assert arxiv_plot_extract(obj, eng) is None
expected = obj.files['figure1']['description']
result = ('00000 Difference (in MeV) between the theoretical and '
'experimental masses for the 2027 selected nuclei as a '
'function of the mass number.')
assert expected == result
expected = 'Added 1 plots.'
result = obj.log._info.getvalue()
assert expected == result
finally:
rmtree(temporary_dir)
def test_get_document_in_workflow():
data = {
'documents': [
{
'key': 'fulltext.xml',
'fulltext': True,
},
],
}
files = MockFiles({})
files['fulltext.xml'] = None
obj = MockObj(data, {}, files=files)
with get_document_in_workflow(obj) as local_file:
assert local_file == files['fulltext.xml'].file.uri
def test_copy_file_to_workflow(mock_fsopen):
mock_fsopen.return_value = 'jessica jones'
data = {}
extra_data = {}
files = MockFiles({})
obj = MockObj(data, extra_data, files=files)
expected = MockFileObject(key='jessicajones.defenders;1')
result = copy_file_to_workflow(obj, 'jessicajones.defenders;1',
'file://jessicajones.defenders%3B1')
assert expected == result
mock_fsopen.assert_called_once_with('file://jessicajones.defenders;1',
mode='rb')
def test_populate_arxiv_document_does_not_duplicate_files_if_called_multiple_times(
):
with requests_mock.Mocker() as requests_mocker:
requests_mocker.register_uri(
'GET',
'http://export.arxiv.org/pdf/1605.03844',
content=pkg_resources.resource_string(
__name__, os.path.join('fixtures', '1605.03844.pdf')),
)
schema = load_schema('hep')
subschema = schema['properties']['arxiv_eprints']
data = {
'arxiv_eprints': [
{
'categories': [
'physics.ins-det',
],
'value': '1605.03844',
},
],
} # literature/1458302
extra_data = {}
files = MockFiles({})
assert validate(data['arxiv_eprints'], subschema) is None
obj = MockObj(data, extra_data, files=files)
eng = MockEng()
assert populate_arxiv_document(obj, eng) is None
assert populate_arxiv_document(obj, eng) is None
expected = [
{
'key': '1605.03844.pdf',
'fulltext': True,
'hidden': True,
'material': 'preprint',
'original_url': 'http://export.arxiv.org/pdf/1605.03844',
'url': 'http://export.arxiv.org/pdf/1605.03844',
'source': 'arxiv',
},
]
result = obj.data['documents']
assert expected == result
def test_arxiv_author_list_does_not_produce_latex():
schema = load_schema('hep')
filename = pkg_resources.resource_filename(
__name__, os.path.join('fixtures', '1802.03388.tar.gz'))
eprints_subschema = schema['properties']['arxiv_eprints']
data = {
'arxiv_eprints': [
{
'categories': [
'hep-ex',
],
'value': '1802.03388',
},
],
}
validate(data['arxiv_eprints'], eprints_subschema)
extra_data = {}
files = MockFiles({
'1802.03388.tar.gz': AttrDict({'file': AttrDict({'uri': filename})})
})
authors_subschema = schema['properties']['authors']
expected_authors = [
{
'affiliations': [{'value': 'Lund U.'}],
'ids': [
{
'value': 'INSPIRE-00061248',
'schema': 'INSPIRE ID'
}
],
'full_name': u'Åkesson, Torsten Paul Ake'
},
]
validate(expected_authors, authors_subschema)
obj = MockObj(data, extra_data, files=files)
eng = MockEng()
default_arxiv_author_list = arxiv_author_list()
assert default_arxiv_author_list(obj, eng) is None
assert obj.data.get('authors') == expected_authors
def test_arxiv_author_list_logs_on_error(mock_os, mock_untar):
mock_untar.side_effect = InvalidTarball
schema = load_schema('hep')
subschema = schema['properties']['arxiv_eprints']
data = {
'arxiv_eprints': [
{
'categories': [
'hep-th',
],
'value': '1605.07707',
},
],
} # synthethic data
extra_data = {}
files = MockFiles({
'1605.07707.tar.gz':
AttrDict({
'file':
AttrDict({
'uri': 'http://export.arxiv.org/e-print/1605.07707',
})
})
})
assert validate(data['arxiv_eprints'], subschema) is None
obj = MockObj(data, extra_data, files=files)
eng = MockEng()
default_arxiv_author_list = arxiv_author_list()
try:
temporary_dir = mkdtemp()
mock_os.path.abspath.return_value = temporary_dir
assert default_arxiv_author_list(obj, eng) is None
expected = 'Invalid tarball http://export.arxiv.org/e-print/1605.07707 for arxiv_id 1605.07707'
result = obj.log._error.getvalue()
assert expected == result
finally:
rmtree(temporary_dir)
def test_populate_arxiv_document_logs_on_pdf_not_existing():
response500 = {'content': '', 'status_code': 500}
response200 = {
'content':
pkg_resources.resource_string(
__name__, os.path.join('fixtures', '1707.02785.html')),
'status_code':
200,
}
with requests_mock.Mocker() as requests_mocker:
requests_mocker.get(
'http://export.arxiv.org/pdf/1707.02785',
(response200, ),
)
requests_mocker.get(
'http://arxiv.org/pdf/1707.02785',
(response500, ),
)
schema = load_schema('hep')
subschema = schema['properties']['arxiv_eprints']
data = {
'arxiv_eprints': [
{
'categories': [
'cs.CV',
],
'value': '1707.02785',
},
],
} # literature/1458302
extra_data = {}
files = MockFiles({})
assert validate(data['arxiv_eprints'], subschema) is None
obj = MockObj(data, extra_data, files=files)
eng = MockEng()
assert populate_arxiv_document(obj, eng) is None
expected = 'No PDF is available for 1707.02785'
result = obj.log._info.getvalue()
assert expected == result
def test_arxiv_fulltext_download_polulates_documents():
with requests_mock.Mocker() as requests_mocker:
requests_mocker.register_uri(
'GET',
'http://export.arxiv.org/pdf/1605.03844',
content=pkg_resources.resource_string(
__name__, os.path.join('fixtures', '1605.03844.pdf')),
)
schema = load_schema('hep')
subschema = schema['properties']['arxiv_eprints']
data = {
'arxiv_eprints': [
{
'categories': [
'physics.ins-det',
],
'value': '1605.03844',
},
],
} # literature/1458302
extra_data = {}
files = MockFiles({})
assert validate(data['arxiv_eprints'], subschema) is None
obj = MockObj(data, extra_data, files=files)
eng = MockEng()
assert arxiv_fulltext_download(obj, eng) is None
expected = [{
'fulltext': True,
'original_url': 'http://export.arxiv.org/pdf/1605.03844',
'url':
'/api/files/0b9dd5d1-feae-4ba5-809d-3a029b0bc110/1605.03844.pdf',
'material': 'preprint',
'source': 'arxiv',
'key': '1605.03844.pdf',
'hidden': True
}]
result = obj.data['documents']
assert expected == result
def test_get_document_in_workflow_prefers_fulltext():
data = {
'documents': [
{
'key': 'table_of_contents.pdf',
},
{
'key': 'fulltext.xml',
'fulltext': True,
},
],
}
files = MockFiles({})
files['fulltext.xml'] = None
files['table_of_contents.pdf'] = None
obj = MockObj(data, {}, files=files)
with get_document_in_workflow(obj) as local_file:
assert local_file == files['fulltext.xml'].file.uri
def test_prepare_files_does_nothing_when_obj_has_no_files():
data = {}
extra_data = {}
files = MockFiles({})
obj = MockObj(data, extra_data, files=files)
eng = MockEng()
assert prepare_files(obj, eng) is None
expected = {}
result = obj.data
assert expected == result
expected = ''
result = obj.log._info.getvalue()
assert expected == result
def test_arxiv_plot_extract_handles_duplicate_plot_names(mock_os):
schema = load_schema('hep')
subschema = schema['properties']['arxiv_eprints']
filename = pkg_resources.resource_filename(
__name__, os.path.join('fixtures', '1711.10662.tar.gz'))
data = {
'arxiv_eprints': [
{
'categories': [
'cs.CV',
],
'value': '1711.10662',
},
],
} # holdingpen/807096
extra_data = {}
files = MockFiles({
'1711.10662.tar.gz':
AttrDict({
'file': AttrDict({
'uri': filename,
}),
}),
})
assert validate(data['arxiv_eprints'], subschema) is None
obj = MockObj(data, extra_data, files=files)
eng = MockEng()
try:
temporary_dir = mkdtemp()
mock_os.path.abspath.return_value = temporary_dir
assert arxiv_plot_extract(obj, eng) is None
assert len(obj.data['figures']) == 66
assert len(obj.files.keys) == 67
finally:
rmtree(temporary_dir)