def test_get_solr_data(self):
"""
Tests query and bigquery requests from solr depending on the number of bibcodes
"""
# the mock is for solr call query, with 10 bibcodes
with mock.patch.object(self.current_app.client, 'get') as post_mock:
post_mock.return_value = mock_response = mock.Mock()
mock_response.json.return_value = solrdata.data_6
mock_response.status_code = 200
bibcodes = [
"2020AAS...23528705A", "2019EPSC...13.1911A",
"2019AAS...23338108A", "2019AAS...23320704A",
"2018EPJWC.18608001A", "2018AAS...23221409A",
"2018AAS...23136217A", "2018AAS...23130709A",
"2017ASPC..512...45A", "2015scop.confE...3A"
]
solr_data = get_solr_data(
bibcodes=bibcodes,
fields='bibcode,author,year,pub,bibstem',
sort=self.current_app.config['EXPORT_SERVICE_NO_SORT_SOLR'])
matched = 0
for i, doc in enumerate(solr_data['response']['docs']):
if doc['bibcode'] == bibcodes[i]:
matched += 1
self.assertEqual(matched, len(bibcodes))
# the mock is for solr call bigquery, with 22 bibcodes
with mock.patch.object(self.current_app.client, 'post') as post_mock:
post_mock.return_value = mock_response = mock.Mock()
mock_response.json.return_value = solrdata.data
mock_response.status_code = 200
bibcodes = [
"2018Wthr...73Q..35.", "2018TDM.....5a0201F",
"2018Spin....877001P", "2018SAAS...38.....D",
"2018PhRvL.120b9901P", "2017PhDT........14C",
"2017nova.pres.2388K", "2017CBET.4403....2G",
"2017ascl.soft06009C", "2017yCat.113380453S",
"2017AAVSN.429....1W", "2017sptz.prop13168Y",
"2017MsT..........2A", "2016emo6.rept.....R",
"2016iac..talk..872V", "2009bcet.book...65L",
"2007AAS...210.2104M", "2007RJPh....1...35.",
"1995ans..agar..390M", "1995anda.book.....N",
"1991hep.th....8028G", "1983aiaa.meetY....K"
]
solr_data = get_solr_data(bibcodes=bibcodes,
fields='bibcode,author,year,pub,bibstem',
sort='')
matched = 0
for i, doc in enumerate(solr_data['response']['docs']):
if doc['bibcode'] == bibcodes[i]:
matched += 1
self.assertEqual(matched, len(bibcodes))
def test_get_solr_data_when_error(self):
"""
Test when solr returns status_code 2xx vs when there is an error
:return:
"""
bibcodes = [
"2020AAS...23528705A", "2019EPSC...13.1911A",
"2019AAS...23338108A", "2019AAS...23320704A",
"2018EPJWC.18608001A", "2018AAS...23221409A",
"2018AAS...23136217A", "2018AAS...23130709A",
"2017ASPC..512...45A", "2015scop.confE...3A"
]
with mock.patch.object(self.current_app.client, 'get') as get_mock:
get_mock.return_value = mock_response = mock.Mock()
mock_response.raise_for_status = mock.Mock()
mock_response.raise_for_status.side_effect = exceptions.RequestException(
"Malformed request")
mock_response.status_code = 400
solr_data = get_solr_data(
bibcodes=bibcodes,
fields='bibcode,author,year,pub,bibstem',
sort=self.current_app.config['EXPORT_SERVICE_NO_SORT_SOLR'])
self.assertEqual(solr_data, None)
# with status code 200
with mock.patch.object(self.current_app.client, 'get') as get_mock:
get_mock.return_value = mock_response = mock.Mock()
mock_response.json.return_value = solrdata.data_6
mock_response.status_code = 200
solr_data = get_solr_data(
bibcodes=bibcodes,
fields='bibcode,author,year,pub,bibstem',
sort=self.current_app.config['EXPORT_SERVICE_NO_SORT_SOLR'])
self.assertEqual(len(solr_data['response']['docs']), len(bibcodes))
# response 203 is also acceptable, it means response is coming from another solr instance then
# where the service is running in
with mock.patch.object(self.current_app.client, 'get') as get_mock:
get_mock.return_value = mock_response = mock.Mock()
mock_response.status_code = 203
mock_response.json.return_value = solrdata.data_6
solr_data = get_solr_data(
bibcodes=bibcodes,
fields='bibcode,author,year,pub,bibstem',
sort=self.current_app.config['EXPORT_SERVICE_NO_SORT_SOLR'])
self.assertEqual(len(solr_data['response']['docs']), len(bibcodes))
def export_get(bibcode, style, format=-1):
"""
:param bibcode:
:param style:
:param format:
:return:
"""
if format == -1:
current_app.logger.debug(
'received request with bibcode={bibcode} to export in {style} style'
.format(bibcode=bibcode, style=style))
else:
current_app.logger.debug(
'received request with bibcode={bibcode} to export in {style} style with output format {format}'
.format(bibcode=bibcode, style=style, format=format))
sort = 'date desc, bibcode desc'
# if in the test mode, return test solr data
if current_app.config['EXPORT_SERVICE_TEST_BIBCODE_GET'] == bibcode:
return solrdata.data_2
return get_solr_data(bibcodes=[bibcode],
fields=default_solr_fields(),
sort=sort,
encode_style=adsFormatter().native_encoding(format))
def csl_soph_format_export_post():
"""
:return:
"""
try:
payload = request.get_json(force=True) # post data in json
except:
payload = dict(request.form) # post data in form encoding
if not payload:
return return_response({'error': 'no information received'}, 400)
if 'bibcode' not in payload:
return return_response(
{
'error':
'no bibcode found in payload (parameter name is `bibcode`)'
}, 400)
bibcodes = payload['bibcode']
csl_style = 'soph'
export_format = 2
current_app.logger.info(
'received request with bibcodes={bibcodes} to export in {csl_style} style with output format {export_format}'
.format(bibcodes=''.join(bibcodes),
csl_style=csl_style,
export_format=export_format))
solr_data = get_solr_data(bibcodes=bibcodes, fields=default_solr_fields())
return return_csl_format_export(solr_data, csl_style, export_format)
def custom_format_export():
try:
payload = request.get_json(force=True) # post data in json
except:
payload = dict(request.form) # post data in form encoding
if not payload:
return return_response({'error': 'no information received'}, 400)
if 'bibcode' not in payload:
return return_response(
{
'error':
'no bibcode found in payload (parameter name is `bibcode`)'
}, 400)
if 'format' not in payload:
return return_response(
{
'error':
'no custom format found in payload (parameter name is `format`)'
}, 400)
if 'sort' in payload:
if type(payload['sort']) is list:
sort = payload['sort'][0]
else:
sort = payload['sort']
else:
sort = 'date desc, bibcode desc'
bibcodes = payload['bibcode']
try:
if type(payload['format']) is list:
custom_format_str = payload['format'][0]
else:
custom_format_str = payload['format']
except Exception as e:
return return_response({'error': 'unable to read custom format'}, 400)
current_app.logger.info(
'received request with bibcodes={bibcodes} to export in a custom format: {custom_format_str} style using sort order={sort}'
.format(bibcodes=','.join(bibcodes),
custom_format_str=custom_format_str.encode('utf8'),
sort=sort))
if (len(bibcodes) == 0) or (len(custom_format_str) == 0):
return return_response(
{'error': 'not all the needed information received'}, 400)
# pass the user defined format to CustomFormat to parse and we would be able to get which fields
# in Solr we need to query on
custom_export = CustomFormat(custom_format=custom_format_str)
fields = custom_export.get_solr_fields()
# now get the required data from Solr and send it to customFormat for formatting
solr_data = get_solr_data(bibcodes=bibcodes, fields=fields, sort=sort)
if (solr_data is not None):
if ('error' in solr_data):
return return_response({'error': 'unable to query solr'}, 400)
custom_export.set_json_from_solr(solr_data)
return return_response(custom_export.get(), 200, 'POST')
return return_response({'error': 'no result from solr'}, 404)
def xml_refabs_format_export_post():
"""
:return:
"""
try:
payload = request.get_json(force=True) # post data in json
except:
payload = dict(request.form) # post data in form encoding
if not payload:
return return_response({'error': 'no information received'}, 400)
if 'bibcode' not in payload:
return return_response(
{
'error':
'no bibcode found in payload (parameter name is `bibcode`)'
}, 400)
bibcodes = payload['bibcode']
xml_style = 'ReferenceAbs'
current_app.logger.debug(
'received request with bibcodes={bibcodes} to export in {xml_style} style format'
.format(bibcodes=''.join(bibcodes), xml_style=xml_style))
solr_data = get_solr_data(bibcodes=bibcodes, fields=default_solr_fields())
return return_xml_format_export(solr_data, xml_style)
def rss_format_export_post():
"""
:return:
"""
try:
payload = request.get_json(force=True) # post data in json
except:
payload = dict(request.form) # post data in form encoding
if not payload:
return return_response({'error': 'no information received'}, 400)
if 'bibcode' not in payload:
return return_response(
{
'error':
'no bibcode found in payload (parameter name is `bibcode`)'
}, 400)
bibcodes = payload['bibcode']
if 'link' in payload:
link = payload['link']
else:
link = ''
current_app.logger.debug(
'received request with bibcodes={bibcodes} to export in RSS format'.
format(bibcodes=''.join(bibcodes)))
solr_data = get_solr_data(bibcodes=bibcodes, fields=default_solr_fields())
return return_rss_format_export(solr_data=solr_data, link=link)
def csl_format_export():
try:
payload = request.get_json(force=True) # post data in json
except:
payload = dict(request.form) # post data in form encoding
if not payload:
return return_response({'error': 'no information received'}, 400)
if 'bibcode' not in payload:
return return_response(
{
'error':
'no bibcode found in payload (parameter name is `bibcode`)'
}, 400)
if 'style' not in payload:
return return_response(
{'error': 'no style found in payload (parameter name is `style`)'},
400)
if 'format' not in payload:
return return_response(
{
'error':
'no output format found in payload (parameter name is `format`)'
}, 400)
bibcodes = payload['bibcode']
csl_style = payload['style']
export_format = payload['format']
if (len(bibcodes) == 0) or (len(csl_style) == 0) or (export_format
== None):
return return_response(
{'error': 'not all the needed information received'}, 400)
if (not adsCSLStyle().verify(csl_style)):
return return_response(
{
'error':
'unrecognizable style (supprted formats are: ' +
adsCSLStyle().get() + ')'
}, 400)
if (not adsFormatter().verify(export_format)):
return return_response(
{
'error':
'unrecognizable format (supprted formats are: unicode=1, html=2, latex=3)'
}, 400)
current_app.logger.debug(
'received request with bibcodes={bibcodes} to export in {csl_style} style with output format {export_format}'
.format(bibcodes=''.join(bibcodes),
csl_style=csl_style,
export_format=export_format))
solr_data = get_solr_data(bibcodes=bibcodes, fields=default_solr_fields())
return return_csl_format_export(solr_data, csl_style, export_format)
def votable_format_export_get(bibcode):
"""
:param bibcode:
:return:
"""
current_app.logger.debug(
'received request with bibcode={bibcode} to export in VOTable format'.
format(bibcode=bibcode))
solr_data = get_solr_data(bibcodes=[bibcode], fields=default_solr_fields())
return return_votable_format_export(solr_data=solr_data,
request_type='GET')
def xml_refabs_format_export_get(bibcode):
"""
:param bibcode:
:return:
"""
xml_style = 'ReferenceAbs'
current_app.logger.debug(
'received request with bibcode={bibcode} to export in {xml_style} style format'
.format(bibcode=bibcode, xml_style=xml_style))
solr_data = get_solr_data(bibcodes=[bibcode], fields=default_solr_fields())
return return_xml_format_export(solr_data, xml_style, request_type='GET')
def export_post(request, style, format=-1):
"""
:param request:
:param style:
:param format:
:return:
"""
try:
payload = request.get_json(force=True) # post data in json
except:
payload = dict(request.form) # post data in form encoding
if not payload:
return {'error': 'no information received'}, 400
if 'bibcode' not in payload:
return {
'error':
'no bibcode found in payload (parameter name is `bibcode`)'
}, 400
if 'sort' in payload:
if type(payload['sort']) is list:
sort = payload['sort'][0]
else:
sort = payload['sort']
else:
sort = 'date desc, bibcode desc'
bibcodes = payload['bibcode']
if format == -1:
current_app.logger.info(
'received request with bibcodes={bibcodes} to export in {style} style using sort order={sort}'
.format(bibcodes=','.join(bibcodes), style=style, sort=sort))
else:
current_app.logger.info(
'received request with bibcodes={bibcodes} to export in {style} style with output format {format} using sort order={sort}'
.format(bibcodes=','.join(bibcodes),
style=style,
format=format,
sort=sort))
# if in the test mode, return test solr data
if current_app.config['EXPORT_SERVICE_TEST_BIBCODE_GET'] == bibcodes:
return solrdata.data, 200
return get_solr_data(bibcodes=bibcodes,
fields=default_solr_fields(),
sort=sort), 200
def rss_format_export_get(bibcode, link):
"""
:param bibcode:
:param link:
:return:
"""
current_app.logger.debug(
'received request with bibcode={bibcode} to export in RSS format'.
format(bibcode=bibcode))
solr_data = get_solr_data(bibcodes=[bibcode], fields=default_solr_fields())
return return_rss_format_export(solr_data=solr_data,
link=link,
request_type='GET')
def fielded_medlars__format_export_get(bibcode):
"""
:param bibcode:
:return:
"""
fielded_style = 'MEDLARS'
current_app.logger.debug(
'received request with bibcode={bibcode} to export in {fielded_style} style format'
.format(bibcode=bibcode, fielded_style=fielded_style))
solr_data = get_solr_data(bibcodes=[bibcode], fields=default_solr_fields())
return return_fielded_format_export(solr_data,
fielded_style,
request_type='GET')
def bibTex_abs_format_export_get(bibcode):
"""
:param bibcode:
:return:
"""
bibTex_style = 'BibTex Abs'
current_app.logger.debug(
'received request with bibcode={bibcode} to export in {bibTex_style} style format'
.format(bibcode=bibcode, bibTex_style=bibTex_style))
solr_data = get_solr_data(bibcodes=[bibcode], fields=default_solr_fields())
return return_bibTex_format_export(solr_data=solr_data,
include_abs=True,
request_type='GET')
def custom_format_export():
try:
payload = request.get_json(force=True) # post data in json
except:
payload = dict(request.form) # post data in form encoding
if not payload:
return return_response({'error': 'no information received'}, 400)
if 'bibcode' not in payload:
return return_response(
{
'error':
'no bibcode found in payload (parameter name is `bibcode`)'
}, 400)
if 'format' not in payload:
return return_response(
{
'error':
'no custom format found in payload (parameter name is `format`)'
}, 400)
bibcodes = payload['bibcode']
custom_format_str = payload['format']
current_app.logger.debug(
'received request with bibcodes={bibcodes} to export in a custom format: {custom_format_str}'
.format(bibcodes=''.join(bibcodes),
custom_format_str=custom_format_str))
if (len(bibcodes) == 0) or (len(custom_format_str) == 0):
return return_response(
{'error': 'not all the needed information received'}, 400)
# pass the user defined format to CustomFormat to parse and we would be able to get which fields
# in Solr we need to query on
custom_export = CustomFormat(custom_format=custom_format_str)
fields = custom_export.get_solr_fields()
# now get the required data from Solr and send it to customFormat for formatting
solr_data = get_solr_data(bibcodes=bibcodes, fields=fields)
if (solr_data is not None):
if ('error' in solr_data):
return return_response({'error': 'unable to query solr'}, 400)
custom_export.set_json_from_solr(solr_data)
return return_response(custom_export.get(), 200)
return return_response({'error': 'no result from solr'}, 404)
def csl_soph_format_export_get(bibcode):
"""
:param bibcode:
:return:
"""
csl_style = 'soph'
export_format = 2
current_app.logger.info(
'received request with bibcode={bibcode} to export in {csl_style} style with output format {export_format}'
.format(bibcode=bibcode,
csl_style=csl_style,
export_format=export_format))
solr_data = get_solr_data(bibcodes=[bibcode], fields=default_solr_fields())
return return_csl_format_export(solr_data,
csl_style,
export_format,
request_type='GET')
