def test_setAttributes(self):
'''
Test setting attributes of a function with an explicit call to advertise()
and with @advertise. Assert that the function code executes as expected after
the operators.
'''
self.testfunction = advertise(thisattr='foo')(self.testfunction)
self.assertEqual(self.testfunction._advertised, [{'thisattr': 'foo'}])
self.testfunction = advertise(thisattr2='foo2')(self.testfunction)
self.assertListEqual(self.testfunction._advertised,
[{
'thisattr': 'foo'
}, {
'thisattr2': 'foo2'
}])
res = self.testfunction(1, 2, 3)
self.assertEqual(res, '1|2|3')
@advertise(decor1='foo', decor2='bar')
def testfunction(a, b, c):
return "{a}|{b}|{c}".format(a=a, b=b, c=c)
d = [{'decor1': 'foo'}, {'decor2': 'bar'}]
self.assertTrue((testfunction._advertised[0] == d[0]
and testfunction._advertised[1] == d[1])
or (testfunction._advertised[1] == d[0]
and testfunction._advertised[0] == d[1]))
res = testfunction(1, 2, 3)
self.assertEqual(res, '1|2|3')
def test_advertiseAttributes(self):
'''
Test advertising "empty" attributes
'''
self.testfunction = advertise('thisattr')(self.testfunction)
self.assertRaises(AttributeError, lambda: self.testfunction.thisattr)
self.assertEqual(self.testfunction._advertised, [{'thisattr': None}])
res = self.testfunction(1, 2, 3)
self.assertEqual(res, '1|2|3')
@advertise('decor1', 'decor2')
def testfunction(a, b, c):
return "{a}|{b}|{c}".format(a=a, b=b, c=c)
self.assertRaises(AttributeError, lambda: testfunction.decor1)
self.assertRaises(AttributeError, lambda: testfunction.decor2)
d = [{'decor1': None}, {'decor2': None}]
self.assertTrue((testfunction._advertised[0] == d[0]
and testfunction._advertised[1] == d[1])
or (testfunction._advertised[1] == d[0]
and testfunction._advertised[0] == d[1]))
res = testfunction(1, 2, 3)
self.assertEqual(res, '1|2|3')
class ScopedView(Resource):
'''Scoped route docstring'''
scopes = ['default']
decorators = [advertise('scopes')]
def get(self):
return "scoped route"
class PaperNetwork(Resource):
'''Returns paper network data for a solr query'''
decorators = [advertise('scopes', 'rate_limit')]
scopes = []
rate_limit = [500,60*60*24]
def post(self):
try:
required_fields = ['bibcode,title,first_author,year,citation_count,read_count,reference']
response = make_request(request, "PN", required_fields)
except QueryException as error:
return {'Error' : 'there was a problem with your request', 'Error Info': str(error)}, 403
if response.status_code == 200:
full_response = response.json()
else:
return {"Error": "There was a connection error. Please try again later", "Error Info": response.text}, response.status_code
#get_network_with_groups expects a list of normalized authors
data = full_response["response"]["docs"]
paper_network_json = paper_network.get_papernetwork(data, request.json.get("max_groups", current_app.config.get("VIS_SERVICE_PN_MAX_GROUPS")))
if paper_network_json:
return { "msg" : {
"numFound" : full_response["response"]["numFound"],
"start": full_response["response"].get("start", 0),
"rows": int(full_response["responseHeader"]["params"]["rows"])
},
"data" : paper_network_json}, 200
else:
return {"Error": "Empty network."}, 200
class AuthorNetwork(Resource):
'''Returns author network data for a solr query'''
decorators = [advertise('scopes', 'rate_limit')]
scopes = []
rate_limit = [500,60*60*24]
def post(self):
try:
required_fields = ['author_norm', 'title', 'citation_count', 'read_count','bibcode', 'pubdate']
response = make_request(request, "AN", required_fields)
except QueryException as error:
return {'Error' : 'there was a problem with your request', 'Error Info': str(error)}, 403
if response.status_code == 200:
full_response = response.json()
else:
return {"Error": "There was a connection error. Please try again later", "Error Info": response.text}, response.status_code
#get_network_with_groups expects a list of normalized authors
author_norm = [d.get("author_norm", []) for d in full_response["response"]["docs"]]
author_network_json = author_network.get_network_with_groups(author_norm, full_response["response"]["docs"])
if author_network_json:
return { "msg" :
{ "numFound" : full_response["response"]["numFound"],
"start": full_response["response"].get("start", 0),
"rows": int(full_response["responseHeader"]["params"]["rows"])
}, "data" : author_network_json}, 200
else:
return {"Error": "Empty network."}, 200
class PositionSearch(Resource):
"""Return publication information for a cone search"""
scopes = []
rate_limit = [1000, 60 * 60 * 24]
decorators = [advertise('scopes', 'rate_limit')]
def get(self, pstring):
# The following position strings are supported
# 1. 05 23 34.6 -69 45 22:0 6 (or 05h23m34.6s -69d45m22s:0m6s)
# 2. 05 23 34.6 -69 45 22:0.166666 (or 05h23m34.6s -69d45m22s:0.166666)
# 3. 80.89416667 -69.75611111:0.166666
stime = time.time()
# If we're given a string with qualifiers ('h', etc), convert to one without
current_app.logger.info('Attempting SIMBAD position search: %s' %
pstring)
try:
RA, DEC, radius = parse_position_string(pstring)
except Exception, err:
current_app.logger.error(
'Position string could not be parsed: %s' % pstring)
return {
'Error': 'Unable to get results!',
'Error Info': 'Invalid position string: %s' % pstring
}, 200
try:
result = do_position_query(RA, DEC, radius)
except timeout_decorator.timeout_decorator.TimeoutError:
current_app.logger.error('Position query %s timed out' % pstring)
return {
'Error': 'Unable to get results!',
'Error Info': 'Position query timed out'
}, 200
return result
class AllowedMirrors(BaseView):
"""
End point that returns the allowed list of mirror sites for either ADS
classic utilities or BEER/ADS2.0 utilities.
"""
decorators = [advertise('scopes', 'rate_limit')]
scopes = []
rate_limit = [1000, 60 * 60 * 24]
def get(self):
"""
HTTP GET request that returns the list of mirror sites that can be used
with this end point. Any end points not listed by this method cannot be
used for any of the other methods related to this service.
Return data (on success)
------------------------
list[<string>]
eg., list of mirrors, ['site1', 'site2', ...., 'siteN']
HTTP Responses:
--------------
Succeed authentication: 200
Any other responses will be default Flask errors
"""
return current_app.config.get('ADS_CLASSIC_MIRROR_LIST', [])
class Recommender(Resource):
"""Return recommender results for a given bibcode"""
scopes = []
rate_limit = [1000, 60 * 60 * 24]
decorators = [advertise('scopes', 'rate_limit')]
def get(self, bibcode):
stime = time.time()
try:
results = get_recommendations(bibcode)
except Exception, err:
current_app.logger.error('Recommender exception (%s): %s'%(bibcode, err))
# If the request for recommendations fails, we just want the UI to ignore and display nothing
return {'Error': 'Unable to get results!'}, 200
if 'Error' in results:
# In case of Solr connection errors, we want to capture more verbose information
if 'Reponse Code' in results:
error_code = results['Reponse Code']
error_msg = results['Error Info']
current_app.logger.error('Recommender failed (Solr request failed) for %s: Solr request returned %s (%s)'%(bibcode, error_code, error_msg))
else:
current_app.logger.error('Recommender failed (%s): %s'%(bibcode, results.get('Error')))
# If the request for recommendations fails, we just want the UI to ignore and display nothing
return {'Error': 'Unable to get results!'}, 200
else:
duration = time.time() - stime
current_app.logger.info('Recommendations for %s in %s user seconds'%(bibcode, duration))
return results
class ClassicView(HarbourView):
"""
Wrapper for importing libraries from ADS Classic
"""
decorators = [advertise('scopes', 'rate_limit')]
scopes = ['user']
rate_limit = [1000, 60 * 60 * 24]
service_url = 'BIBLIB_CLASSIC_SERVICE_URL'
class TwoPointOhView(HarbourView):
"""
Wrapper for importing libraries from ADS 2.0
"""
decorators = [advertise('scopes', 'rate_limit')]
scopes = ['user']
rate_limit = [1000, 60 * 60 * 24]
service_url = 'BIBLIB_TWOPOINTOH_SERVICE_URL'
class Graphics(Resource):
"""Return graphics information for a given bibcode"""
scopes = []
rate_limit = [1000, 60 * 60 * 24]
decorators = [advertise('scopes', 'rate_limit')]
def get(self, bibcode):
stime = time.time()
results = get_graphics(bibcode)
duration = time.time() - stime
current_app.logger.info('Graphics for %s in %s user seconds' %
(bibcode, duration))
return results, 200
class CitationHelper(Resource):
"""computes Citation Helper suggestions on the POST body"""
scopes = []
rate_limit = [1000, 60 * 60 * 24]
decorators = [advertise('scopes', 'rate_limit')]
def post(self):
stime = time.time()
if not request.json or 'bibcodes' not in request.json:
current_app.logger.error(
'No bibcodes were provided to Citation Helper')
return {
'Error': 'Unable to get results!',
'Error Info': 'No bibcodes found in POST body'
}, 200
bibcodes = map(str, request.json['bibcodes'])
if len(bibcodes) > \
current_app.config.get('CITATION_HELPER_MAX_SUBMITTED'):
current_app.logger.warning(
'Citation Helper called with %s bibcodes. Maximum is: %s!' %
(len(bibcodes),
current_app.config.get('CITATION_HELPER_MAX_SUBMITTED')))
return {
'Error': 'Unable to get results!',
'Error Info':
'Number of submitted bibcodes exceeds maximum number'
}, 200
results = get_suggestions(bibcodes=bibcodes)
if "Error" in results:
msg = 'Citation Helper request request blew up'
if 'Error Info' in results:
msg += ' (%s)' % str(results['Error Info'])
current_app.logger.error(msg)
return results, 200
if results:
duration = time.time() - stime
current_app.logger.info(
'Citation Helper request successfully completed in %s real seconds'
% duration)
return results
else:
current_app.logger.info(
'Citation Helper request returned empty result')
return {
'Error': 'Unable to get results!',
'Error Info': 'Could not find anything to suggest'
}, 200
class WordCloud(Resource):
'''Returns collated tf/idf data for a solr query'''
decorators = [advertise('scopes', 'rate_limit')]
scopes = []
rate_limit = [500,60*60*24]
def post(self):
solr_args = request.json
if not solr_args:
return {'Error' : 'there was a problem with your request', 'Error Info': 'no data provided with request'}, 403
if 'max_groups' in solr_args:
del solr_args['min_percent_word']
if 'min_occurrences_word' in solr_args:
del solr_args['min_occurrences_word']
elif 'query' in request.json:
try:
solr_args = json.loads(request.json["query"][0])
except Exception:
return {'Error' : 'there was a problem with your request', 'Error Info': 'couldn\'t decode query, it should be json-encoded before being sent (so double encoded)'}, 403
solr_args["rows"] = min(int(solr_args.get("rows", [current_app.config.get("VIS_SERVICE_WC_MAX_RECORDS")])[0]), current_app.config.get("VIS_SERVICE_WC_MAX_RECORDS"))
solr_args['fl'] ='abstract,title'
solr_args['wt'] = 'json'
headers = {'Authorization': request.headers.get('X-Forwarded-Authorization', request.headers.get('Authorization', ''))}
response = client().get(current_app.config.get("VIS_SERVICE_SOLR_PATH") , params = solr_args, headers=headers)
if response.status_code == 200:
data = response.json()
else:
return {"Error": "There was a connection error. Please try again later", "Error Info": response.text}, response.status_code
if data:
records = [str(". ".join(d.get('title', '')[:current_app.config.get("VIS_SERVICE_WC_MAX_TITLE_SIZE")]) + ". " + d.get('abstract', '')[:current_app.config.get("VIS_SERVICE_WC_MAX_ABSTRACT_SIZE")]) for d in data["response"]["docs"]]
word_cloud_json = word_cloud.generate_wordcloud(records, n_most_common=current_app.config.get("VIS_SERVICE_WC_MAX_WORDS"), accepted_pos=('NN', 'NNP', 'NNS', 'NNPS', 'JJ', 'RB', 'VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ'))
if word_cloud_json:
return word_cloud_json, 200
else:
return {"Error": "Empty word cloud. Try changing your minimum word parameters or expanding your query."}, 200
class ClassicUser(BaseView):
"""
End point to collect the user's ADS Classic information currently stored in
the database
"""
decorators = [advertise('scopes', 'rate_limit')]
scopes = ['user']
rate_limit = [1000, 60 * 60 * 24]
def get(self):
"""
HTTP GET request that returns the information currently stored about the
user's ADS Classic settings, currently stored in the service database.
Return data (on success)
------------------------
classic_email: <string> ADS Classic e-mail of the user
classic_mirror: <string> ADS Classic mirror this user belongs to
HTTP Responses:
--------------
Succeed authentication: 200
User unknown/wrong password/failed authentication: 404
Any other responses will be default Flask errors
"""
absolute_uid = self.helper_get_user_id()
try:
with current_app.session_scope() as session:
user = session.query(Users).filter(
Users.absolute_uid == absolute_uid).one()
return {
'classic_email': user.classic_email,
'classic_mirror': user.classic_mirror,
'twopointoh_email': user.twopointoh_email
}, 200
except NoResultFound:
return err(NO_CLASSIC_ACCOUNT)
class ClassicObjectSearch(Resource):
"""Return object NED refcodes for a given object list"""
scopes = []
rate_limit = [1000, 60 * 60 * 24]
decorators = [advertise('scopes', 'rate_limit')]
def post(self):
stime = time.time()
results = {}
# Get the supplied list of identifiers
if not request.json or 'objects' not in request.json:
current_app.logger.error(
'No objects were provided to Classic Object Search')
return {
'Error': 'Unable to get results!',
'Error Info': 'No object names found in POST body'
}, 200
results = get_NED_refcodes(request.json)
if "Error" in results:
current_app.logger.error(
'Classic Object Search request request blew up')
return results, 500
duration = time.time() - stime
current_app.logger.info(
'Classic Object Search request successfully completed in %s real seconds'
% duration)
# what output format?
try:
oformat = request.json['output_format']
except:
oformat = 'json'
# send the results back in the requested format
if oformat == 'json':
return results
else:
output = "\n".join(results['data'])
return Response(output, mimetype='text/plain; charset=us-ascii')
def init_app(self, app):
"""
Standard flask-extension initialisation
:param app: flask app
"""
if app is not None:
self.app = app
if 'watchman' in app.extensions:
raise RuntimeError('Flask application already initialised')
app.extensions['watchman'] = self
for view_name in self.allowed_endpoints.keys():
if view_name not in self.kwargs.keys():
continue
view = self.allowed_endpoints[view_name]['view']
route = self.allowed_endpoints[view_name]['route']
methods = self.allowed_endpoints[view_name]['methods']
# Does the user provide scopes?
if view_name in self.kwargs and self.kwargs[view_name].get(
'scopes', None) != None:
user_config = self.kwargs[view_name]
view.scopes = user_config.get('scopes', [])
view.decorators = user_config.get(
'decorators', ([advertise('scopes', 'rate_limit')]))
view.rate_limit = [1000, 60 * 60 * 24]
with app.app_context():
current_app.add_url_rule(route,
view_func=view.as_view(view_name),
methods=methods)
class ClassicMyADS(BaseView):
"""
End point to collect the user's ADS classic libraries with the external ADS
Classic end point
"""
decorators = [advertise('scopes', 'rate_limit')]
scopes = ['adsws:internal']
rate_limit = [1000, 60 * 60 * 24]
def get(self, uid):
"""
HTTP GET request that contacts the ADS Classic myADS end point to
obtain all the libraries relevant to that user.
:param uid: user ID for the API
:type uid: int
Return data (on success)
------------------------
HTTP Responses:
--------------
Succeed getting libraries: 200
User does not have a classic account: 400
ADS Classic give unknown messages: 500
ADS Classic times out: 504
Any other responses will be default Flask errors
"""
data = {}
with current_app.session_scope() as session:
try:
user = session.query(Users).filter(
Users.absolute_uid == uid).one()
if not user.classic_email:
raise NoResultFound
except NoResultFound:
current_app.logger.warning(
'User does not have an associated ADS Classic account')
return err(NO_CLASSIC_ACCOUNT)
url = current_app.config['ADS_CLASSIC_MYADS_URL'].format(
mirror='adsabs.harvard.edu', email=user.classic_email)
current_app.logger.debug('Obtaining libraries via: {}'.format(url))
try:
response = current_app.client.get(url)
except requests.exceptions.Timeout:
current_app.logger.warning(
'ADS Classic timed out before finishing: {}'.format(url))
return err(CLASSIC_TIMEOUT)
if response.status_code != 200:
current_app.logger.warning(
'ADS Classic returned an unkown status code: "{}" [code: {}]'
.format(response.text, response.status_code))
return err(CLASSIC_UNKNOWN_ERROR)
data = response.json()
return data, 200
class AuthenticateUserTwoPointOh(BaseView):
"""
End point to authenticate the user's ADS 2.0 credentials with the
external ADS Classic end point (2.0 and Classic accounts are on ADS 2.0,
interchangable)
"""
decorators = [advertise('scopes', 'rate_limit')]
scopes = ['user']
rate_limit = [1000, 60 * 60 * 24]
def post(self):
"""
HTTP POST request that receives the user's ADS 2.0 credentials, and
then contacts the Classic system to check that what the user provided
is indeed valid. If valid, the users ID is stored.
Post body:
----------
KEYWORD, VALUE
twopointoh_email: <string> ADS 2.0 e-mail of the user
twopointoh_password: <string> ADS 2.0 password of the user
Return data (on success):
-------------------------
twopointoh_authed: <boolean> were they authenticated
twopointoh_email: <string> e-mail that authenticated correctly
HTTP Responses:
--------------
Succeed authentication: 200
Bad/malformed data: 400
User unknown/wrong password/failed authentication: 404
ADS Classic give unknown messages: 500
ADS Classic times out: 504
Any other responses will be default Flask errors
"""
post_data = get_post_data(request)
# Collect the username, password from the request
try:
twopointoh_email = post_data['twopointoh_email']
twopointoh_password = post_data['twopointoh_password']
except KeyError:
current_app.logger.warning(
'User did not provide a required key: {}'.format(
traceback.print_exc()))
return err(CLASSIC_DATA_MALFORMED)
# Create the correct URL
url = current_app.config['ADS_CLASSIC_URL'].format(
mirror=current_app.config['ADS_TWO_POINT_OH_MIRROR'], )
params = {
'man_cmd': 'elogin',
'man_email': twopointoh_email,
'man_passwd': twopointoh_password
}
# Authenticate
current_app.logger.info(
'User "{email}" trying to authenticate"'.format(
email=twopointoh_email))
try:
response = current_app.client.post(url, params=params)
except requests.exceptions.Timeout:
current_app.logger.warning(
'ADS Classic end point timed out, returning to user')
return err(CLASSIC_TIMEOUT)
if response.status_code >= 500:
message, status_code = err(CLASSIC_UNKNOWN_ERROR)
message['ads_classic'] = {
'message': response.text,
'status_code': response.status_code
}
current_app.logger.warning(
'ADS Classic has responded with an unknown error: {}'.format(
response.text))
return message, status_code
# Sanity check the response
email = response.json()['email']
if email != twopointoh_email:
current_app.logger.warning(
'User email "{}" does not match ADS return email "{}"'.format(
twopointoh_email, email))
return err(CLASSIC_AUTH_FAILED)
# Respond to the user based on whether they were successful or not
if response.status_code == 200 \
and response.json()['message'] == 'LOGGED_IN' \
and int(response.json()['loggedin']):
current_app.logger.info(
'Authenticated successfully "{email}"'.format(
email=twopointoh_email))
absolute_uid = self.helper_get_user_id()
with current_app.session_scope() as session:
try:
user = session.query(Users).filter(
Users.absolute_uid == absolute_uid).one()
current_app.logger.info('User already exists in database')
user.twopointoh_email = twopointoh_email
except NoResultFound:
current_app.logger.info(
'Creating entry in database for user')
user = Users(absolute_uid=absolute_uid,
twopointoh_email=twopointoh_email)
session.add(user)
session.commit()
current_app.logger.info(
'Successfully saved content for "{}" to database'.format(
twopointoh_email))
return {
'twopointoh_email': email,
'twopointoh_authed': True
}, 200
return err(HARBOUR_SERVICE_FAIL)
else:
current_app.logger.warning(
'ADS 2.0 credentials for "{email}" did not succeed"'.format(
email=twopointoh_email))
return err(CLASSIC_AUTH_FAILED)
class AuthenticateUserClassic(BaseView):
"""
End point to authenticate the user's ADS classic credentials with the
external ADS Classic end point
"""
decorators = [advertise('scopes', 'rate_limit')]
scopes = ['user']
rate_limit = [1000, 60 * 60 * 24]
def post(self):
"""
HTTP POST request that receives the user's ADS Classic credentials, and
then contacts the Classic system to check that what the user provided is
indeed valid. If valid, the users ID is stored within the myADS service
store.
Post body:
----------
KEYWORD, VALUE
classic_email: <string> ADS Classic e-mail of the user
classic_password: <string> ADS Classic password of the user
classic_mirror: <string> ADS Classic mirror this user belongs to
Return data (on success):
-------------------------
classic_authed: <boolean> were they authenticated
classic_email: <string> e-mail that authenticated correctly
classic_mirror: <string> ADS Classic mirror this user selected
HTTP Responses:
--------------
Succeed authentication: 200
Bad/malformed data: 400
User unknown/wrong password/failed authentication: 404
ADS Classic give unknown messages: 500
ADS Classic times out: 504
Any other responses will be default Flask errors
"""
post_data = get_post_data(request)
with current_app.session_scope() as session:
# Collect the username, password from the request
try:
classic_email = post_data['classic_email']
classic_password = post_data['classic_password']
classic_mirror = post_data['classic_mirror']
except KeyError:
current_app.logger.warning(
'User did not provide a required key: {}'.format(
traceback.print_exc()))
return err(CLASSIC_DATA_MALFORMED)
# Check that the mirror exists and not man-in-the-middle
if classic_mirror not in current_app.config[
'ADS_CLASSIC_MIRROR_LIST']:
current_app.logger.warning(
'User "{}" tried to use a mirror that does not exist: "{}"'
.format(classic_email, classic_mirror))
return err(CLASSIC_BAD_MIRROR)
# Create the correct URL
url = current_app.config['ADS_CLASSIC_URL'].format(
mirror=classic_mirror, )
params = {
'man_cmd': 'elogin',
'man_email': classic_email,
'man_passwd': classic_password
}
# Authenticate
current_app.logger.info(
'User "{email}" trying to authenticate at mirror "{mirror}"'.
format(email=classic_email, mirror=classic_mirror))
try:
response = current_app.client.post(url, params=params)
except requests.exceptions.Timeout:
current_app.logger.warning(
'ADS Classic end point timed out, returning to user')
return err(CLASSIC_TIMEOUT)
if response.status_code >= 500:
message, status_code = err(CLASSIC_UNKNOWN_ERROR)
message['ads_classic'] = {
'message': response.text,
'status_code': response.status_code
}
current_app.logger.warning(
'ADS Classic has responded with an unknown error: {}'.
format(response.text))
return message, status_code
# Sanity check the response
email = response.json()['email']
if email != classic_email:
current_app.logger.warning(
'User email "{}" does not match ADS return email "{}"'.
format(classic_email, email))
return err(CLASSIC_AUTH_FAILED)
# Respond to the user based on whether they were successful or not
if response.status_code == 200 \
and response.json()['message'] == 'LOGGED_IN' \
and int(response.json()['loggedin']):
current_app.logger.info(
'Authenticated successfully "{email}" at mirror "{mirror}"'
.format(email=classic_email, mirror=classic_mirror))
# Save cookie in myADS
try:
cookie = response.json()['cookie']
except KeyError:
current_app.logger.warning(
'Classic returned no cookie, cannot continue: {}'.
format(response.json()))
return err(CLASSIC_NO_COOKIE)
absolute_uid = self.helper_get_user_id()
try:
user = session.query(Users).filter(
Users.absolute_uid == absolute_uid).one()
current_app.logger.info('User already exists in database')
user.classic_mirror = classic_mirror
user.classic_cookie = cookie
user.classic_email = classic_email
except NoResultFound:
current_app.logger.info(
'Creating entry in database for user')
user = Users(absolute_uid=absolute_uid,
classic_cookie=cookie,
classic_email=classic_email,
classic_mirror=classic_mirror)
session.add(user)
session.commit()
current_app.logger.info(
'Successfully saved content for "{}" to database: {{"cookie": "{}"}}'
.format(classic_email, '*' * len(user.classic_cookie)))
return {
'classic_email': email,
'classic_mirror': classic_mirror,
'classic_authed': True
}, 200
else:
current_app.logger.warning(
'Credentials for "{email}" did not succeed at mirror "{mirror}"'
.format(email=classic_email, mirror=classic_mirror))
return err(CLASSIC_AUTH_FAILED)
class ClassicLibraries(BaseView):
"""
End point to collect the user's ADS classic libraries with the external ADS
Classic end point
"""
decorators = [advertise('scopes', 'rate_limit')]
scopes = ['adsws:internal']
rate_limit = [1000, 60 * 60 * 24]
def get(self, uid):
"""
HTTP GET request that contacts the ADS Classic libraries end point to
obtain all the libraries relevant to that user.
:param uid: user ID for the API
:type uid: int
Return data (on success)
------------------------
libraries: <list<dict>> a list of dictionaries, that contains the
following for each library entry:
name: <string> name of the library
description: <string> description of the library
documents: <list<string>> list of documents
HTTP Responses:
--------------
Succeed getting libraries: 200
User does not have a classic account: 400
ADS Classic give unknown messages: 500
ADS Classic times out: 504
Any other responses will be default Flask errors
"""
with current_app.session_scope() as session:
try:
user = session.query(Users).filter(
Users.absolute_uid == uid).one()
if not user.classic_email:
raise NoResultFound
except NoResultFound:
current_app.logger.warning(
'User does not have an associated ADS Classic account')
return err(NO_CLASSIC_ACCOUNT)
url = current_app.config['ADS_CLASSIC_LIBRARIES_URL'].format(
mirror=user.classic_mirror, cookie=user.classic_cookie)
current_app.logger.debug('Obtaining libraries via: {}'.format(url))
try:
response = current_app.client.get(url)
except requests.exceptions.Timeout:
current_app.logger.warning(
'ADS Classic timed out before finishing: {}'.format(url))
return err(CLASSIC_TIMEOUT)
if response.status_code != 200:
current_app.logger.info(
'ADS Classic returned an unkown status code: "{}" [code: {}]'
.format(response.text, response.status_code))
return err(CLASSIC_UNKNOWN_ERROR)
data = response.json()
libraries = [
dict(name=i['name'],
description=i.get('desc', ''),
documents=[j['bibcode'] for j in i['entries']])
for i in data['libraries']
]
return {'libraries': libraries}, 200
class ExportTwoPointOhLibraries(BaseView):
"""
End point to return ADS 2.0 libraries in a format that users can use to
import them to other services. Currently, the following third-party
services are supported:
- Zotero (https://www.zotero.org/)
- Papers (http://www.papersapp.com/)
- Mendeley (https://www.mendeley.com/)
"""
decorators = [advertise('scopes', 'rate_limit')]
scopes = ['user']
rate_limit = [1000, 60 * 60 * 24]
def get(self, export):
"""
HTTP GET request that collects a users ADS 2.0 libraries from the flat
files on S3. It returns a file for download of the .bib format.
:param export: service to export to
:type export: str
Return data (on success)
------------------------
BibTeX file (.bib)
HTTP Responses:
--------------
Succeed getting libraries: 200
User does not have a classic/ADS 2.0 account: 400
User does not have any libraries in their ADS 2.0 account: 400
Unknown error: 500
Any other responses will be default Flask errors
"""
with current_app.session_scope() as session:
if export not in current_app.config['HARBOUR_EXPORT_TYPES']:
return err(TWOPOINTOH_WRONG_EXPORT_TYPE)
if not current_app.config['ADS_TWO_POINT_OH_LOADED_USERS']:
current_app.logger.error(
'Users from MongoDB have not been loaded into the app')
return err(TWOPOINTOH_AWS_PROBLEM)
absolute_uid = self.helper_get_user_id()
try:
user = session.query(Users).filter(
Users.absolute_uid == absolute_uid).one()
# Have they got an email for ADS 2.0?
if not user.twopointoh_email:
raise NoResultFound
except NoResultFound:
current_app.logger.warning(
'User does not have an associated ADS Classic/2.0 account')
return err(NO_TWOPOINTOH_ACCOUNT)
library_file_name = current_app.config[
'ADS_TWO_POINT_OH_USERS'].get(user.twopointoh_email, None)
if not library_file_name:
current_app.logger.warning(
'User does not have any libraries in ADS 2.0')
return err(NO_TWOPOINTOH_LIBRARIES)
try:
s3 = boto3.client('s3')
s3_presigned_url = s3.generate_presigned_url(
ClientMethod='get_object',
Params={
'Bucket':
current_app.config['ADS_TWO_POINT_OH_S3_MONGO_BUCKET'],
'Key':
library_file_name.replace('.json',
'.{}.zip'.format(export))
},
ExpiresIn=1800)
except Exception as error:
current_app.logger.error(
'Unknown error with AWS: {}'.format(error))
return err(TWOPOINTOH_AWS_PROBLEM)
return {'url': s3_presigned_url}, 200
class TwoPointOhLibraries(BaseView):
"""
End point to collect the user's ADS 2.0 libraries with the MongoDB dump
in a flat-file on S3 storage
Note: ADS 2.0 user accounts are tied to ADS Classic accounts. Therefore,
a user will need to regisiter their ADS Classic/2.0 account on this service
for them to also be able to access the ADS 2.0 libraries.
"""
decorators = [advertise('scopes', 'rate_limit')]
scopes = ['adsws:internal']
rate_limit = [1000, 60 * 60 * 24]
@staticmethod
def get_s3_library(library_file_name):
"""
Get the JSON MongoDB dump of the ADS 2.0 library of a specific user.
These files are stored on S3.
:param library_file_name: name of library file
:type library_file_name: str
:return: dict
"""
s3_resource = boto3.resource('s3')
bucket = s3_resource.Object(
current_app.config['ADS_TWO_POINT_OH_S3_MONGO_BUCKET'],
library_file_name)
body = bucket.get()['Body']
library_data = BytesIO()
for chunk in iter(lambda: body.read(1024), b''):
library_data.write(chunk)
library = json.loads(library_data.getvalue())
return library
def get(self, uid):
"""
HTTP GET request that finds the libraries within ADS 2.0 for that user.
:param uid: user ID for the API
:type uid: int
Return data (on success)
------------------------
libraries: <list<dict>> a list of dictionaries, that contains the
following for each library entry:
name: <string> name of the library
description: <string> description of the library
documents: <list<string>> list of documents
HTTP Responses:
--------------
Succeed getting libraries: 200
User does not have a classic/ADS 2.0 account: 400
User does not have any libraries in their ADS 2.0 account: 400
Unknown error: 500
Any other responses will be default Flask errors
"""
with current_app.session_scope() as session:
if not current_app.config['ADS_TWO_POINT_OH_LOADED_USERS']:
current_app.logger.error(
'Users from MongoDB have not been loaded into the app')
return err(TWOPOINTOH_AWS_PROBLEM)
try:
user = session.query(Users).filter(
Users.absolute_uid == uid).one()
# Have they got an email for ADS 2.0?
if not user.twopointoh_email:
raise NoResultFound
except NoResultFound:
current_app.logger.warning(
'User does not have an associated ADS 2.0 account')
return err(NO_TWOPOINTOH_ACCOUNT)
library_file_name = current_app.config[
'ADS_TWO_POINT_OH_USERS'].get(user.twopointoh_email, None)
if not library_file_name:
current_app.logger.warning(
'User does not have any libraries in ADS 2.0')
return err(NO_TWOPOINTOH_LIBRARIES)
try:
library = TwoPointOhLibraries.get_s3_library(library_file_name)
except Exception as error:
current_app.logger.error(
'Unknown error with AWS: {}'.format(error))
return err(TWOPOINTOH_AWS_PROBLEM)
return {'libraries': library}, 200
class UserView(BaseView):
"""
End point to create a library for a given user
"""
decorators = [advertise('scopes', 'rate_limit')]
scopes = ['user']
rate_limit = [1000, 60 * 60 * 24]
@staticmethod
def create_user(absolute_uid):
"""
Creates a user in the database with a UID from the API
:param absolute_uid: UID from the API
:return: no return
"""
try:
user = User(absolute_uid=absolute_uid)
db.session.add(user)
db.session.commit()
except IntegrityError as error:
current_app.logger.error('IntegrityError. User: {0:d} was not'
'added. Full traceback: {1}'.format(
absolute_uid, error))
raise
@staticmethod
def create_library(service_uid, library_data):
"""
Creates a library for a user
:param service_uid: the user ID within this microservice
:param library_data: content needed to create a library
:return: no return
"""
library_data = BaseView.helper_validate_library_data(
service_uid=service_uid, library_data=library_data)
_name = library_data.get('name')
_description = library_data.get('description')
_public = bool(library_data.get('public', False))
_bibcode = library_data.get('bibcode', False)
try:
# Make the library in the library table
library = Library(name=_name,
description=_description,
public=_public)
# If the user supplies bibcodes
if _bibcode and isinstance(_bibcode, list):
# Ensure unique content
_bibcode = uniquify(_bibcode)
current_app.logger.info(
'User supplied bibcodes: {0}'.format(_bibcode))
library.add_bibcodes(_bibcode)
elif _bibcode:
current_app.logger.error(
'Bibcode supplied not a list: {0}'.format(_bibcode))
raise TypeError('Bibcode should be a list.')
user = User.query.filter(User.id == service_uid).one()
# Make the permissions
permission = Permissions(owner=True, )
# Use the ORM to link the permissions to the library and user,
# so that no commit is required until the complete action is
# finished. This means any rollback will not leave a single
# library without permissions
library.permissions.append(permission)
user.permissions.append(permission)
db.session.add_all([library, permission, user])
db.session.commit()
current_app.logger.info(
'Library: "{0}" made, user_service: {1:d}'.format(
library.name, user.id))
return library
except IntegrityError as error:
# Roll back the changes
db.session.rollback()
current_app.logger.error('IntegitryError, database has been rolled'
'back. Caused by user_service: {0:d}.'
'Full error: {1}'.format(user.id, error))
# Log here
raise
except Exception:
db.session.rollback()
raise
@classmethod
def get_libraries(cls, service_uid, absolute_uid):
"""
Get all the libraries a user has
:param service_uid: microservice UID of the user
:param absolute_uid: unique UID of the user in the API
:return: list of libraries in json format
"""
# Get all the permissions for a user
# This can be improved into one database call rather than having
# one per each permission, but needs some checks in place.
result = db.session.query(Permissions, Library)\
.join(Permissions.library)\
.filter(Permissions.user_id == service_uid)\
.all()
output_libraries = []
for permission, library in result:
# For this library get all the people who have permissions
users = Permissions.query.filter(
Permissions.library_id == library.id).all()
num_documents = 0
if library.bibcode:
num_documents = len(library.bibcode)
if permission.owner:
main_permission = 'owner'
elif permission.admin:
main_permission = 'admin'
elif permission.write:
main_permission = 'write'
elif permission.read:
main_permission = 'read'
else:
main_permission = 'none'
if permission.owner or permission.admin and not library.public:
num_users = len(users)
elif library.public:
num_users = len(users)
else:
num_users = 0
service = '{api}/{uid}'.format(
api=current_app.config['BIBLIB_USER_EMAIL_ADSWS_API_URL'],
uid=absolute_uid)
current_app.logger.info(
'Obtaining email of user: {0} [API UID]'.format(absolute_uid))
response = client().get(service)
if response.status_code != 200:
current_app.logger.error('Could not find user in the API'
'database: {0}.'.format(service))
owner = 'Not available'
else:
owner = response.json()['email'].split('@')[0]
payload = dict(
name=library.name,
id='{0}'.format(cls.helper_uuid_to_slug(library.id)),
description=library.description,
num_documents=num_documents,
date_created=library.date_created.isoformat(),
date_last_modified=library.date_last_modified.isoformat(),
permission=main_permission,
public=library.public,
num_users=num_users,
owner=owner)
output_libraries.append(payload)
return output_libraries
# Methods
def get(self):
"""
HTTP GET request that returns all the libraries that belong to a given
user
:return: list of the users libraries with the relevant information
Header:
Must contain the API forwarded user ID of the user accessing the end
point
Post body:
----------
No post content accepted.
Return data:
-----------
name: <string> Name of the library
id: <string> ID of the library
description: <string> Description of the library
num_documents: <int> Number of documents in the library
date_created: <string> ISO date library was created
date_last_modified: <string> ISO date library was last modified
permission: <sting> Permission type, can be: 'read',
'write', 'admin', or 'owner'
public: <boolean> True means it is public
num_users: <int> Number of users with permissions to
this library
owner: <string> Identifier of the user who created
the library
Permissions:
-----------
The following type of user can read a library:
- user scope (authenticated via the API)
"""
# Check that they pass a user id
try:
user = self.helper_get_user_id()
except KeyError:
return err(MISSING_USERNAME_ERROR)
service_uid = \
self.helper_absolute_uid_to_service_uid(absolute_uid=user)
user_libraries = self.get_libraries(service_uid=service_uid,
absolute_uid=user)
return {'libraries': user_libraries}, 200
def post(self):
"""
HTTP POST request that creates a library for a given user
:return: the response for if the library was successfully created
Header:
-------
Must contain the API forwarded user ID of the user accessing the end
point
Post body:
----------
KEYWORD, VALUE
name: <string> name of the library (must be unique
for that user)
description: <string> description of the library
public: <boolean> is the library public to view
bibcode (OPTIONAL): <list> list of bibcodes to add
Return data:
-----------
name: <string> Name of the library
id: <string> ID of the library
description: <string> Description of the library
Permissions:
-----------
The following type of user can create a library
- must be logged in, i.e., scope = user
"""
# Check that they pass a user id
try:
user = self.helper_get_user_id()
except KeyError:
return err(MISSING_USERNAME_ERROR)
# Check if the user exists, if not, generate a user in the database
current_app.logger.info('Checking if the user exists')
if not self.helper_user_exists(absolute_uid=user):
current_app.logger.info(
'User: {0:d}, does not exist.'.format(user))
self.create_user(absolute_uid=user)
current_app.logger.info('User: {0:d}, created.'.format(user))
else:
current_app.logger.info('User already exists.')
# Switch to the service UID and not the API UID
service_uid = \
self.helper_absolute_uid_to_service_uid(absolute_uid=user)
current_app.logger.info(
'user_API: {0:d} is now user_service: {1:d}'.format(
user, service_uid))
# Create the library
try:
data = get_post_data(request,
types=dict(name=unicode,
description=unicode,
public=bool,
bibcode=list))
except TypeError as error:
current_app.logger.error(
'Wrong type passed for POST: {0} [{1}]'.format(
request.data, error))
return err(WRONG_TYPE_ERROR)
try:
library = \
self.create_library(service_uid=service_uid, library_data=data)
except BackendIntegrityError as error:
current_app.logger.error(error)
return err(DUPLICATE_LIBRARY_NAME_ERROR)
except TypeError as error:
current_app.logger.error(error)
return err(WRONG_TYPE_ERROR)
return_data = {
'name': library.name,
'id': '{0}'.format(self.helper_uuid_to_slug(library.id)),
'description': library.description
}
# If they added bibcodes include in the response
if hasattr(library, 'bibcode') and library.bibcode:
return_data['bibcode'] = library.get_bibcodes()
return return_data, 200
class OperationsView(BaseView):
"""
Endpoint to conduct operations on a given library or set of libraries. Supported operations are
union, intersection, difference, copy, and empty.
"""
decorators = [advertise('scopes', 'rate_limit')]
scopes = ['user']
rate_limit = [1000, 60 * 60 * 24]
@classmethod
def write_access(cls, service_uid, library_id):
"""
Defines which type of user has write permissions to a library.
:param service_uid: the user ID within this microservice
:param library_id: the unique ID of the library
:return: boolean, access (True), no access (False)
"""
read_allowed = ['write', 'admin', 'owner']
for access_type in read_allowed:
if cls.helper_access_allowed(service_uid=service_uid,
library_id=library_id,
access_type=access_type):
return True
return False
@classmethod
def setops_libraries(cls, library_id, document_data, operation='union'):
"""
Takes the union of two or more libraries
:param library_id: the primary library ID
:param document_data: dict containing the list 'libraries' that holds the secondary library IDs
:return: list of bibcodes in the union set
"""
current_app.logger.info(
'User requested to take the {0} of {1} with {2}'.format(
operation, library_id, document_data['libraries']))
with current_app.session_scope() as session:
# Find the specified library
primary_library = session.query(Library).filter_by(
id=library_id).one()
out_lib = set(primary_library.get_bibcodes())
for lib in document_data['libraries']:
if isinstance(lib, str):
lib = cls.helper_slug_to_uuid(lib)
secondary_library = session.query(Library).filter_by(
id=lib).one()
if operation == 'union':
out_lib = out_lib.union(
set(secondary_library.get_bibcodes()))
elif operation == 'intersection':
out_lib = out_lib.intersection(
set(secondary_library.get_bibcodes()))
elif operation == 'difference':
out_lib = out_lib.difference(
set(secondary_library.get_bibcodes()))
else:
current_app.logger.warning(
'Requested operation {0} is not allowed.'.format(
operation))
return
if len(out_lib) < 1:
current_app.logger.info(
'No records remain after taking the {0} of {1} and {2}'.format(
operation, library_id, document_data['libraries']))
return list(out_lib)
@classmethod
def copy_library(cls, library_id, document_data):
"""
Copies the contents of one library into another. Does not empty library first; call
empty_library on the target library first to do so
:param library_id: primary library ID, library to copy
:param document_data: dict containing the list 'libraries' which holds one secondary library ID; this is
the library to copy over
:return: dict containing the metadata of the copied-over library (the secondary library)
"""
current_app.logger.info(
'User requested to copy the contents of {0} into {1}'.format(
library_id, document_data['libraries']))
secondary_libid = document_data['libraries'][0]
if isinstance(secondary_libid, str):
secondary_libid = cls.helper_slug_to_uuid(secondary_libid)
metadata = {}
with current_app.session_scope() as session:
primary_library = session.query(Library).filter_by(
id=library_id).one()
good_bib = primary_library.get_bibcodes()
secondary_library = session.query(Library).filter_by(
id=secondary_libid).one()
secondary_library.add_bibcodes(good_bib)
metadata['name'] = secondary_library.name
metadata['description'] = secondary_library.description
metadata['public'] = secondary_library.public
session.add(secondary_library)
session.commit()
return metadata
@staticmethod
def empty_library(library_id):
"""
Empties the contents of one library
:param library_id: library to empty
:return: dict containing the metadata of the emptied library
"""
current_app.logger.info(
'User requested to empty the contents of {0}'.format(library_id))
metadata = {}
with current_app.session_scope() as session:
lib = session.query(Library).filter_by(id=library_id).one()
lib.remove_bibcodes(lib.get_bibcodes())
metadata['name'] = lib.name
metadata['description'] = lib.description
metadata['public'] = lib.public
session.add(lib)
session.commit()
return metadata
def post(self, library):
"""
HTTP POST request that conducts operations at the library level.
:param library: primary library ID
:return: response if operation was successful
Header:
-------
Must contain the API forwarded user ID of the user accessing the end
point
Post body:
----------
KEYWORD, VALUE
libraries: <list> List of secondary libraries to include in the action (optional, based on action)
action: <unicode> union, intersection, difference, copy, empty
Actions to perform on given libraries:
Union: requires one or more secondary libraries to be passed; takes the union of the
primary and secondary library sets; a new library is created
Intersection: requires one or more secondary libraries to be passed; takes the
intersection of the primary and secondary library sets; a new library is created
Difference: requires one or more secondary libraries to be passed; takes the difference
between the primary and secondary libraries; the primary library comes first in the
operation, so the secondary library is removed from the primary; a new library
is created
Copy: requires one and only one secondary library to be passed; the primary library
will be copied into the secondary library (so the secondary library will be
overwritten); no new library is created
Empty: secondary libraries are ignored; the primary library will be emptied of its
contents, though the library and metadata will remain; no new library is created
name: <string> (optional) name of the new library (must be unique for that user); used only for actions in
[union, intersection, difference]
description: <string> (optional) description of the new library; used only for actions in
[union, intersection, difference]
public: <boolean> (optional) is the new library public to view; used only for actions in
[union, intersection, difference]
-----------
Return data:
-----------
name: <string> Name of the library
id: <string> ID of the library
description: <string> Description of the library
Permissions:
-----------
The following type of user can conduct library operations:
- owner
- admin
- write
"""
# Get the user requesting this from the header
try:
user_editing = self.helper_get_user_id()
except KeyError:
return err(MISSING_USERNAME_ERROR)
# URL safe base64 string to UUID
try:
library_uuid = self.helper_slug_to_uuid(library)
except TypeError:
return err(BAD_LIBRARY_ID_ERROR)
user_editing_uid = \
self.helper_absolute_uid_to_service_uid(absolute_uid=user_editing)
# Check the permissions of the user
if not self.write_access(service_uid=user_editing_uid,
library_id=library_uuid):
return err(NO_PERMISSION_ERROR)
try:
data = get_post_data(request,
types=dict(libraries=list,
action=str,
name=str,
description=str,
public=bool))
except TypeError as error:
current_app.logger.error(
'Wrong type passed for POST: {0} [{1}]'.format(
request.data, error))
return err(WRONG_TYPE_ERROR)
if data['action'] in ['union', 'intersection', 'difference']:
if 'libraries' not in data:
return err(NO_LIBRARY_SPECIFIED_ERROR)
if 'name' not in data:
data['name'] = 'Untitled {0}.'.format(get_date().isoformat())
if 'public' not in data:
data['public'] = False
if data['action'] == 'copy':
if 'libraries' not in data:
return err(NO_LIBRARY_SPECIFIED_ERROR)
if len(data['libraries']) > 1:
return err(TOO_MANY_LIBRARIES_SPECIFIED_ERROR)
lib_names = []
with current_app.session_scope() as session:
primary = session.query(Library).filter_by(id=library_uuid).one()
lib_names.append(primary.name)
if 'libraries' in data:
for lib in data['libraries']:
try:
secondary_uuid = self.helper_slug_to_uuid(lib)
except TypeError:
return err(BAD_LIBRARY_ID_ERROR)
secondary = session.query(Library).filter_by(
id=secondary_uuid).one()
lib_names.append(secondary.name)
if data['action'] == 'union':
bib_union = self.setops_libraries(library_id=library_uuid,
document_data=data,
operation='union')
current_app.logger.info(
'Successfully took the union of the libraries {0} (IDs: {1}, {2})'
.format(', '.join(lib_names), library,
', '.join(data['libraries'])))
data['bibcode'] = bib_union
if 'description' not in data:
description = 'Union of libraries {0} (IDs: {1}, {2})' \
.format(', '.join(lib_names), library, ', '.join(data['libraries']))
# field length capped in model
if len(description) > 200:
description = 'Union of library {0} (ID: {1}) with {2} other libraries'\
.format(lib_names[0], library, len(lib_names[1:]))
data['description'] = description
try:
library_dict = self.create_library(
service_uid=user_editing_uid, library_data=data)
except BackendIntegrityError as error:
current_app.logger.error(error)
return err(DUPLICATE_LIBRARY_NAME_ERROR)
except TypeError as error:
current_app.logger.error(error)
return err(WRONG_TYPE_ERROR)
return library_dict, 200
elif data['action'] == 'intersection':
bib_intersect = self.setops_libraries(library_id=library_uuid,
document_data=data,
operation='intersection')
current_app.logger.info(
'Successfully took the intersection of the libraries {0} (IDs: {1}, {2})'
.format(', '.join(lib_names), library,
', '.join(data['libraries'])))
data['bibcode'] = bib_intersect
if 'description' not in data:
description = 'Intersection of {0} (IDs: {1}, {2})' \
.format(', '.join(lib_names), library, ', '.join(data['libraries']))
if len(description) > 200:
description = 'Intersection of {0} (ID: {1}) with {2} other libraries'\
.format(lib_names[0], library, len(lib_names[1:]))
data['description'] = description
try:
library_dict = self.create_library(
service_uid=user_editing_uid, library_data=data)
except BackendIntegrityError as error:
current_app.logger.error(error)
return err(DUPLICATE_LIBRARY_NAME_ERROR)
except TypeError as error:
current_app.logger.error(error)
return err(WRONG_TYPE_ERROR)
return library_dict, 200
elif data['action'] == 'difference':
bib_diff = self.setops_libraries(library_id=library_uuid,
document_data=data,
operation='difference')
current_app.logger.info(
'Successfully took the difference of {0} (ID {2}) - (minus) {1} (ID {3})'
.format(lib_names[0], ', '.join(lib_names[1:]), library,
', '.join(data['libraries'])))
data['bibcode'] = bib_diff
if 'description' not in data:
data['description'] = 'Records that are in {0} (ID {2}) but not in {1} (ID {3})' \
.format(lib_names[0], ', '.join(lib_names[1:]), library, ', '.join(data['libraries']))
try:
library_dict = self.create_library(
service_uid=user_editing_uid, library_data=data)
except BackendIntegrityError as error:
current_app.logger.error(error)
return err(DUPLICATE_LIBRARY_NAME_ERROR)
except TypeError as error:
current_app.logger.error(error)
return err(WRONG_TYPE_ERROR)
return library_dict, 200
elif data['action'] == 'copy':
library_dict = self.copy_library(library_id=library_uuid,
document_data=data)
current_app.logger.info(
'Successfully copied {0} (ID {2}) into {1} (ID {3})'.format(
lib_names[0], lib_names[1], library, data['libraries'][0]))
with current_app.session_scope() as session:
libid = self.helper_slug_to_uuid(data['libraries'][0])
library = session.query(Library).filter_by(id=libid).one()
bib = library.get_bibcodes()
library_dict['bibcode'] = bib
return library_dict, 200
elif data['action'] == 'empty':
library_dict = self.empty_library(library_id=library_uuid)
current_app.logger.info(
'Successfully emptied {0} (ID {1}) of all records'.format(
lib_names[0], library))
with current_app.session_scope() as session:
library = session.query(Library).filter_by(
id=library_uuid).one()
bib = library.get_bibcodes()
library_dict['bibcode'] = bib
return library_dict, 200
else:
current_app.logger.info('User requested a non-standard operation')
return {}, 400
class QuerySearch(Resource):
"""Given a Solr query with object names, return a Solr query with SIMBAD identifiers"""
scopes = []
rate_limit = [1000, 60 * 60 * 24]
decorators = [advertise('scopes', 'rate_limit')]
def post(self):
stime = time.time()
# Get the supplied list of identifiers
query = None
itype = None
name2id = {}
try:
query = request.json['query']
input_type = 'query'
except:
current_app.logger.error('No query was specified for the object search')
return {"Error": "Unable to get results!",
"Error Info": "No identifiers/objects found in POST body"}, 200
# If we get the request from BBB, the value of 'query' is actually an array
if isinstance(query, list):
solr_query = query[0]
else:
solr_query = query
translated_query = solr_query
current_app.logger.info('Received object query: %s'%solr_query)
# Check if an explicit target service was specified
try:
targets = [t.strip() for t in request.json['target'].lower().split(',')]
except:
targets = ['simbad', 'ned']
# Get the object names and individual object queries from the Solr query
try:
object_names, object_queries = parse_query_string(solr_query)
except Exception, err:
current_app.logger.error('Parsing the identifiers out of the query string blew up!')
return {"Error": "Unable to get results!",
"Error Info": "Parsing the identifiers out of the query string blew up! (%s)"%str(err)}, 200
# If no object names were found, return
if len(object_names) == 0:
return {"Error": "Unable to get results!",
"Error Info": "No identifiers/objects found in Solr object query"}, 200
# First we check if the object query was in fact a cone search. This would have been a queery of the form
# object:"80.89416667 -69.75611111:0.166666"
# resulting in 1 entry in the variable object_queries (namely: ['object:"80.89416667 -69.75611111:0.166666"'])
# and with 1 entry in object_names (namely: [u'80.89416667 -69.75611111:0.166666']).
is_cone_search = False
if len(object_queries) == 1:
# We received a cone search, which needs to be translated into a query in terms of 'simbid' and 'nedid'
try:
RA, DEC, radius = parse_position_string(object_names[0])
current_app.logger.info('Starting cone search at RA, DEC, radius: {0}, {1}, {2}'.format(RA, DEC, radius))
is_cone_search = True
except:
pass
# If this is a comne search, we go a different path
if is_cone_search:
result = {'simbad':[], 'ned':[]}
simbad_fail = False
ned_fail = False
result['simbad'] = simbad_position_query(RA, DEC, radius)
if 'Error' in result['simbad']:
simbad_fail = result['simbad']['Error Info']
result['ned'] = ned_position_query(RA, DEC, radius)
if 'Error' in result['ned']:
ned_fail = result['ned']['Error Info']
# If both SIMBAD and NED errored out, return an error
if simbad_fail and ned_fail:
return {"Error": "Unable to get results!",
"Error Info": "{0}, {1}".format(simbad_fail, ned_fail)}, 200
# Form the query in terms on simbid and nedid:
cone_components = []
if len(result['simbad']) > 0:
cone_components.append('simbid:({0})'.format(" OR ".join(result['simbad'])))
if len(result['ned']) > 0:
cone_components.append('nedid:({0})'.format(" OR ".join(result['ned'])))
oquery = "({0})".format(" OR ".join(cone_components))
translated_query = solr_query.replace(object_queries[0], oquery)
return {'query': translated_query}
# Create the translation map from the object names provided to identifiers indexed in Solr (simbid and nedid)
name2id = get_object_translations(object_names, targets)
# Now we have all necessary information to created the translated query
translated_query = translate_query(solr_query, object_queries, targets, object_names, name2id)
return {'query': translated_query}
class DocumentView(BaseView):
"""
End point to interact with a specific library, by adding documents and
removing documents. You also use this endpoint to delete the entire
library as this method should be scoped.
"""
# TODO: adding tags using PUT for RESTful endpoint?
decorators = [advertise('scopes', 'rate_limit')]
scopes = ['user']
rate_limit = [1000, 60 * 60 * 24]
@classmethod
def add_document_to_library(cls, library_id, document_data):
"""
Adds a document to a user's library
:param library_id: the library id to update
:param document_data: the meta data of the document
:return: number_added: number of documents successfully added
"""
current_app.logger.info(
'Adding a document: {0} to library_uuid: {1}'.format(
document_data, library_id))
with current_app.session_scope() as session:
# Find the specified library
library = session.query(Library).filter_by(id=library_id).one()
start_length = len(library.bibcode)
library.add_bibcodes(document_data['bibcode'])
session.add(library)
session.commit()
current_app.logger.info('Added: {0} is now {1}'.format(
document_data['bibcode'], library.bibcode))
end_length = len(library.bibcode)
return end_length - start_length
@classmethod
def remove_documents_from_library(cls, library_id, document_data):
"""
Remove a given document from a specific library
:param library_id: the unique ID of the library
:param document_data: the meta data of the document
:return: number_removed: number of documents successfully removed
"""
current_app.logger.info('Removing a document: {0} from library_uuid: '
'{1}'.format(document_data, library_id))
with current_app.session_scope() as session:
library = session.query(Library).filter_by(id=library_id).one()
start_length = len(library.bibcode)
library.remove_bibcodes(document_data['bibcode'])
session.add(library)
session.commit()
current_app.logger.info(
'Removed document successfully: {0}'.format(library.bibcode))
end_length = len(library.bibcode)
return start_length - end_length
@staticmethod
def update_library(library_id, library_data):
"""
Update the meta data of the library
:param library_id: the unique ID of the library
:param library_data: dictionary containing the updateable values
:return: values updated
"""
updateable = ['name', 'description', 'public']
updated = {}
with current_app.session_scope() as session:
library = session.query(Library).filter_by(id=library_id).one()
for key in library_data:
if key not in updateable:
continue
setattr(library, key, library_data[key])
updated[key] = library_data[key]
session.add(library)
session.commit()
return updated
@staticmethod
def delete_library(library_id):
"""
Delete the entire library from the database
:param library_id: the unique ID of the library
:return: no return
"""
with current_app.session_scope() as session:
library = session.query(Library).filter_by(id=library_id).one()
session.delete(library)
session.commit()
@classmethod
def update_access(cls, service_uid, library_id):
"""
Defines which type of user has delete permissions to a library.
:param service_uid: the user ID within this microservice
:param library_id: the unique ID of the library
:return: boolean, access (True), no access (False)
"""
update_allowed = ['admin', 'owner']
for access_type in update_allowed:
if cls.helper_access_allowed(service_uid=service_uid,
library_id=library_id,
access_type=access_type):
return True
return False
@classmethod
def delete_access(cls, service_uid, library_id):
"""
Defines which type of user has delete permissions to a library.
:param service_uid: the user ID within this microservice
:param library_id: the unique ID of the library
:return: boolean, access (True), no access (False)
"""
delete_allowed = cls.helper_access_allowed(service_uid=service_uid,
library_id=library_id,
access_type='owner')
return delete_allowed
@classmethod
def write_access(cls, service_uid, library_id):
"""
Defines which type of user has write permissions to a library.
:param service_uid: the user ID within this microservice
:param library_id: the unique ID of the library
:return: boolean, access (True), no access (False)
"""
read_allowed = ['write', 'admin', 'owner']
for access_type in read_allowed:
if cls.helper_access_allowed(service_uid=service_uid,
library_id=library_id,
access_type=access_type):
return True
return False
@staticmethod
def library_name_exists(service_uid, library_name):
"""
Checks to see if a library name already exists in the user's created
libraries
:param service_uid: the user ID within this microservice
:param library_name: name to check if it exists
:return: True (exists), False (does not exist)
"""
with current_app.session_scope() as session:
library_names = \
[i.library.name for i in
session.query(Permissions)\
.filter_by(user_id = service_uid)\
.filter(Permissions.permissions['owner'].astext.cast(Boolean).is_(True)).all()]
if library_name in library_names:
current_app.logger.error('Name supplied for the library already '
'exists: "{0}"'.format(library_name))
return True
else:
return False
def post(self, library):
"""
HTTP POST request that adds a document to a library for a given user
:param library: library ID
:return: the response for if the library was successfully created
Header:
-------
Must contain the API forwarded user ID of the user accessing the end
point
Post body:
----------
KEYWORD, VALUE
bibcode: <list> List of bibcodes to be added
action: add, remove add - adds a bibcode, remove - removes a
bibcode
Return data:
-----------
number_added: number of documents added (if 'add' is used)
number_removed: number of documents removed (if 'remove' is used)
Permissions:
-----------
The following type of user can add documents:
- owner
- admin
- write
"""
# Get the user requesting this from the header
try:
user_editing = self.helper_get_user_id()
except KeyError:
return err(MISSING_USERNAME_ERROR)
# URL safe base64 string to UUID
try:
library = self.helper_slug_to_uuid(library)
except TypeError:
return err(BAD_LIBRARY_ID_ERROR)
user_editing_uid = \
self.helper_absolute_uid_to_service_uid(absolute_uid=user_editing)
# Check the permissions of the user
if not self.write_access(service_uid=user_editing_uid,
library_id=library):
return err(NO_PERMISSION_ERROR)
try:
data = get_post_data(request, types=dict(bibcode=list, action=str))
except TypeError as error:
current_app.logger.error(
'Wrong type passed for POST: {0} [{1}]'.format(
request.data, error))
return err(WRONG_TYPE_ERROR)
if data['action'] == 'add':
current_app.logger.info('User requested to add a document')
number_added = self.add_document_to_library(library_id=library,
document_data=data)
current_app.logger.info(
'Successfully added {0} documents to {1} by {2}'.format(
number_added, library, user_editing_uid))
return {'number_added': number_added}, 200
elif data['action'] == 'remove':
current_app.logger.info('User requested to remove a document')
number_removed = self.remove_documents_from_library(
library_id=library, document_data=data)
current_app.logger.info(
'Successfully removed {0} documents to {1} by {2}'.format(
number_removed, library, user_editing_uid))
return {'number_removed': number_removed}, 200
else:
current_app.logger.info('User requested a non-standard action')
return {}, 400
def put(self, library):
"""
HTTP PUT request that updates the meta-data of the library
:param library: library ID
:return: the response for if the library was updated
Header:
-------
Must contain the API forwarded user ID of the user accessing the end
point
Post-body:
---------
name: name of the library
description: description of the library
public: boolean
Note: The above are optional, they can be empty if needed.
Return data:
-----------
returns the key/value that was requested to be updated
Permissions:
-----------
The following type of user can update the 'name', 'library', and
'public':
- owner
- admin
"""
try:
user = self.helper_get_user_id()
except KeyError:
return err(MISSING_USERNAME_ERROR)
# URL safe base64 string to UUID
try:
library = self.helper_slug_to_uuid(library)
except TypeError:
return err(BAD_LIBRARY_ID_ERROR)
if not self.helper_user_exists(user):
return err(NO_PERMISSION_ERROR)
if not self.helper_library_exists(library):
return err(MISSING_LIBRARY_ERROR)
user_updating_uid = \
self.helper_absolute_uid_to_service_uid(absolute_uid=user)
if not self.update_access(service_uid=user_updating_uid,
library_id=library):
return err(NO_PERMISSION_ERROR)
try:
library_data = get_post_data(request,
types=dict(name=str,
description=str,
public=bool))
except TypeError as error:
current_app.logger.error(
'Wrong type passed for POST: {0} [{1}]'.format(
request.data, error))
return err(WRONG_TYPE_ERROR)
# Remove content that is empty (note that the list() is necessary to create a copy, so pop will work)
for key in list(library_data.keys()):
if library_data[key] == ''.strip(' '):
current_app.logger.warning(
'Removing key: {0} as its empty.'.format(key))
library_data.pop(key)
# Check for duplicate namaes
if 'name' in library_data and \
self.library_name_exists(service_uid=user_updating_uid,
library_name=library_data['name']):
return err(DUPLICATE_LIBRARY_NAME_ERROR)
response = self.update_library(library_id=library,
library_data=library_data)
return response, 200
def delete(self, library):
"""
HTTP DELETE request that deletes a library defined by the number passed
:param library: library ID
:return: the response for it the library was deleted
Header:
-------
Must contain the API forwarded user ID of the user accessing the end
point
Post-body:
----------
No post content accepted.
Return data:
-----------
No data
Permissions:
-----------
The following type of user can update a library:
- owner
"""
try:
user = self.helper_get_user_id()
except KeyError:
return err(MISSING_USERNAME_ERROR)
# URL safe base64 string to UUID
try:
library = self.helper_slug_to_uuid(library)
except TypeError:
return err(BAD_LIBRARY_ID_ERROR)
if not self.helper_user_exists(user):
return err(NO_PERMISSION_ERROR)
if not self.helper_library_exists(library):
return err(MISSING_LIBRARY_ERROR)
user_deleting_uid = \
self.helper_absolute_uid_to_service_uid(absolute_uid=user)
try:
current_app.logger.info('user_API: {0:d} '
'requesting to delete library: {1}'.format(
user_deleting_uid, library))
if self.delete_access(service_uid=user_deleting_uid,
library_id=library):
self.delete_library(library_id=library)
current_app.logger.info(
'User: {0} deleted library: {1}.'.format(
user_deleting_uid, library))
else:
current_app.logger.error('User: {0} has incorrect permissions '
'to delete: {1}.'.format(
user_deleting_uid, library))
raise PermissionDeniedError('Incorrect permissions')
except NoResultFound as error:
current_app.logger.info('Failed to delete: {0}'.format(error))
return err(MISSING_LIBRARY_ERROR)
except PermissionDeniedError as error:
current_app.logger.info('Failed to delete: {0}'.format(error))
return err(NO_PERMISSION_ERROR)
return {}, 200
class QuerySearch(Resource):
"""Given a Solr query with object names, return a Solr query with SIMBAD identifiers"""
scopes = []
rate_limit = [1000, 60 * 60 * 24]
decorators = [advertise('scopes', 'rate_limit')]
def post(self):
stime = time.time()
# Get the supplied list of identifiers
identifiers = []
query = None
itype = None
name2id = {}
try:
query = request.json['query']
input_type = 'query'
except:
current_app.logger.error(
'No query was specified for the object search')
return {
"Error": "Unable to get results!",
"Error Info": "No identifiers/objects found in POST body"
}, 200
# If we get the request from BBB, the value of 'query' is actually an array
if isinstance(query, list):
solr_query = query[0]
else:
solr_query = query
current_app.logger.info('Received object query: %s' % solr_query)
# This query will be split up into two components: a SIMBAD and a NED object query
simbad_query = solr_query.replace('object:', 'simbid:')
ned_query = solr_query.replace('object:', 'nedid:')
# Check if an explicit target service was specified
try:
target = request.json['target']
except:
target = 'all'
# If we receive a (Solr) query string, we need to parse out the object names
try:
identifiers = get_objects_from_query_string(solr_query)
except Exception, err:
current_app.logger.error(
'Parsing the identifiers out of the query string blew up!')
return {
"Error":
"Unable to get results!",
"Error Info":
"Parsing the identifiers out of the query string blew up! (%s)"
% str(err)
}, 200
identifiers = [
iden for iden in identifiers
if iden.lower() not in ['object', ':']
]
# How many object names did we fid?
id_num = len(identifiers)
# Keep a list with the object names we found
identifiers_orig = identifiers
# If we did not find any object names, there is nothing to do!
if id_num == 0:
return {
"Error": "Unable to get results!",
"Error Info":
"No identifiers/objects found in Solr object query"
}, 200
# Get translations
simbad_query = ''
ned_query = ''
translated_query = ''
if target.lower() in ['simbad', 'all']:
name2simbid = {}
for ident in identifiers:
result = get_simbad_data([ident], 'objects')
if 'Error' in result or 'data' not in result:
# An error was returned!
current_app.logger.error(
'Failed to find data for SIMBAD object {0}!: {1}'.
format(ident, result.get('Error Info', 'NA')))
name2simbid[ident] = 0
continue
try:
SIMBADid = [
e.get('id', 0) for e in result['data'].values()
][0]
except:
SIMBADid = "0"
name2simbid[ident] = SIMBADid
simbad_query = translate_query(solr_query, identifiers,
name2simbid, 'simbid:')
if target.lower() in ['ned', 'all']:
name2nedid = {}
for ident in identifiers:
result = get_ned_data([ident], 'objects')
if 'Error' in result or 'data' not in result:
# An error was returned!
current_app.logger.error(
'Failed to find data for NED object {0}!: {1}'.format(
ident, result.get('Error Info', 'NA')))
name2nedid[ident] = 0
continue
try:
NEDid = [e.get('id', 0)
for e in result['data'].values()][0]
except:
NEDid = 0
name2nedid[ident] = str(NEDid)
ned_query = translate_query(solr_query, identifiers, name2nedid,
'nedid:')
if simbad_query and ned_query:
translated_query = '({0}) OR ({1})'.format(simbad_query, ned_query)
elif simbad_query:
translated_query = simbad_query
elif ned_query:
translated_query = ned_query
else:
translated_query = 'simbid:0'
return {'query': translated_query}
class LibraryView(BaseView):
"""
End point to interact with a specific library, only returns library content
if the user has the correct privileges.
The GET requests are separate from the POST, DELETE requests as this class
must be scopeless, whereas the others will have scope.
"""
decorators = [advertise('scopes', 'rate_limit')]
scopes = []
rate_limit = [1000, 60 * 60 * 24]
@classmethod
def get_documents_from_library(cls, library_id, service_uid):
"""
Retrieve all the documents that are within the library specified
:param library_id: the unique ID of the library
:param service_uid: the user ID within this microservice
:return: bibcodes
"""
with current_app.session_scope() as session:
# Get the library
library = session.query(Library).filter_by(id=library_id).one()
# Get the owner of the library
result = session.query(Permissions, User)\
.join(Permissions.user)\
.filter(Permissions.library_id == library_id) \
.filter(Permissions.permissions['owner'].astext.cast(Boolean).is_(True))\
.one()
owner_permissions, owner = result
service = '{api}/{uid}'.format(
api=current_app.config['BIBLIB_USER_EMAIL_ADSWS_API_URL'],
uid=owner.absolute_uid)
current_app.logger.info(
'Obtaining email of user: {0} [API UID]'.format(
owner.absolute_uid))
response = client().get(service)
# For this library get all the people who have permissions
users = session.query(Permissions).filter_by(
library_id=library.id).all()
if response.status_code != 200:
current_app.logger.error('Could not find user in the API'
'database: {0}.'.format(service))
owner = 'Not available'
else:
owner = response.json()['email'].split('@')[0]
# User requesting to see the content
if service_uid:
try:
permission = session.query(Permissions).filter(
Permissions.user_id == service_uid).filter(
Permissions.library_id == library_id).one()
if permission.permissions['owner']:
main_permission = 'owner'
elif permission.permissions['admin']:
main_permission = 'admin'
elif permission.permissions['write']:
main_permission = 'write'
elif permission.permissions['read']:
main_permission = 'read'
else:
main_permission = 'none'
except:
main_permission = 'none'
else:
main_permission = 'none'
if main_permission == 'owner' or main_permission == 'admin':
num_users = len(users)
elif library.public:
num_users = len(users)
else:
num_users = 0
metadata = dict(
name=library.name,
id='{0}'.format(cls.helper_uuid_to_slug(library.id)),
description=library.description,
num_documents=len(library.bibcode),
date_created=library.date_created.isoformat(),
date_last_modified=library.date_last_modified.isoformat(),
permission=main_permission,
public=library.public,
num_users=num_users,
owner=owner)
session.refresh(library)
session.expunge(library)
return library, metadata
@classmethod
def read_access(cls, service_uid, library_id):
"""
Defines which type of user has read permissions to a library.
:param service_uid: the user ID within this microservice
:param library_id: the unique ID of the library
:return: boolean, access (True), no access (False)
"""
read_allowed = ['read', 'write', 'admin', 'owner']
for access_type in read_allowed:
if cls.helper_access_allowed(service_uid=service_uid,
library_id=library_id,
access_type=access_type):
return True
return False
@staticmethod
def solr_big_query(bibcodes,
start=0,
rows=20,
sort='date desc',
fl='bibcode'):
"""
A thin wrapper for the solr bigquery service.
:param bibcodes: bibcodes
:type bibcodes: list
:param start: start index
:type start: int
:param rows: number of rows
:type rows: int
:param sort: how the response should be sorted
:type sort: str
:param fl: Solr fields to be returned
:type fl: str
:return: solr bigquery end point response
"""
bibcodes_string = 'bibcode\n' + '\n'.join(bibcodes)
# We need atleast bibcode and alternate bibcode for other methods
# to work properly
if fl == '':
fl = 'bibcode,alternate_bibcode'
else:
fl_split = fl.split(',')
for required_fl in ['bibcode', 'alternate_bibcode']:
if required_fl not in fl_split:
fl = '{},{}'.format(fl, required_fl)
params = {
'q': '*:*',
'wt': 'json',
'fl': fl,
'rows': rows,
'start': start,
'fq': '{!bitset}',
'sort': sort
}
headers = {
'Content-Type':
'big-query/csv',
'Authorization':
current_app.config.get(
'SERVICE_TOKEN',
request.headers.get('X-Forwarded-Authorization',
request.headers.get('Authorization', '')))
}
current_app.logger.info(
'Querying Solr bigquery microservice: {0}, {1}'.format(
params, bibcodes_string.replace('\n', ',')))
response = client().post(
url=current_app.config['BIBLIB_SOLR_BIG_QUERY_URL'],
params=params,
data=bibcodes_string,
headers=headers)
return response
@staticmethod
def solr_update_library(library_id, solr_docs):
"""
Updates the library based on the solr canonical bibcodes response
:param library: library_id of the library to update
:param solr_docs: solr docs from the bigquery response
:return: dictionary with details of files modified
num_updated: number of documents modified
duplicates_removed: number of files removed for duplication
update_list: list of changed bibcodes {'before': 'after'}
"""
# Definitions
update = False
canonical_bibcodes = []
alternate_bibcodes = {}
new_bibcode = {}
# Constants for the return dictionary
num_updated = 0
duplicates_removed = 0
update_list = []
# Extract the canonical bibcodes and create a hashmap for the
# alternate bibcodes
for doc in solr_docs:
canonical_bibcodes.append(doc['bibcode'])
if doc.get('alternate_bibcode'):
alternate_bibcodes.update(
{i: doc['bibcode']
for i in doc['alternate_bibcode']})
with current_app.session_scope() as session:
library = session.query(Library).filter(
Library.id == library_id).one()
for bibcode in library.bibcode:
# Skip if its already canonical
if bibcode in canonical_bibcodes:
new_bibcode[bibcode] = library.bibcode[bibcode]
continue
# Update if its an alternate
if bibcode in alternate_bibcodes.keys():
update = True
num_updated += 1
update_list.append({bibcode: alternate_bibcodes[bibcode]})
# Only add the bibcode if it is not there
if alternate_bibcodes[bibcode] not in new_bibcode:
new_bibcode[alternate_bibcodes[bibcode]] = \
library.bibcode[bibcode]
else:
duplicates_removed += 1
elif bibcode not in canonical_bibcodes and\
bibcode not in alternate_bibcodes.keys():
new_bibcode[bibcode] = library.bibcode[bibcode]
if update:
# Update the database
library.bibcode = new_bibcode
session.add(library)
session.commit()
updates = dict(num_updated=num_updated,
duplicates_removed=duplicates_removed,
update_list=update_list)
return updates
# Methods
def get(self, library):
"""
HTTP GET request that returns all the documents inside a given
user's library
:param library: library ID
:return: list of the users libraries with the relevant information
Header:
-------
Must contain the API forwarded user ID of the user accessing the end
point
Post body:
----------
No post content accepted.
Return data:
-----------
documents: <list> Currently, a list containing the bibcodes.
solr: <dict> The response from the solr bigquery end point
metadata: <dict> contains the following:
name: <string> Name of the library
id: <string> ID of the library
description: <string> Description of the library
num_documents: <int> Number of documents in the library
date_created: <string> ISO date library was created
date_last_modified: <string> ISO date library was last modified
permission: <sting> Permission type, can be: 'read',
'write', 'admin', or 'owner'
public: <boolean> True means it is public
num_users: <int> Number of users with permissions to
this library
owner: <string> Identifier of the user who created
the library
updates: <dict> contains the following
num_updated: <int> Number of documents modified based on
the response from solr
duplicates_removed: <int> Number of files removed because
they are duplications
update_list: <list>[<dict>]
List of dictionaries such that a
single element described the original
bibcode (key) and the updated bibcode
now being stored (item)
Permissions:
-----------
The following type of user can read a library:
- owner
- admin
- write
- read
Default Pagination Values:
-----------
- start: 0
- rows: 20 (max 100)
- sort: 'date desc'
- fl: 'bibcode'
"""
try:
user = int(request.headers[USER_ID_KEYWORD])
except KeyError:
current_app.logger.error('No username passed')
return err(MISSING_USERNAME_ERROR)
# Parameters to be forwarded to Solr: pagination, and fields
try:
start = int(request.args.get('start', 0))
max_rows = current_app.config.get('BIBLIB_MAX_ROWS', 100)
max_rows *= float(
request.headers.get('X-Adsws-Ratelimit-Level', 1.0))
max_rows = int(max_rows)
rows = min(int(request.args.get('rows', 20)), max_rows)
except ValueError:
start = 0
rows = 20
sort = request.args.get('sort', 'date desc')
fl = request.args.get('fl', 'bibcode')
current_app.logger.info('User gave pagination parameters:'
'start: {}, '
'rows: {}, '
'sort: "{}", '
'fl: "{}"'.format(start, rows, sort, fl))
try:
library = self.helper_slug_to_uuid(library)
except TypeError:
return err(BAD_LIBRARY_ID_ERROR)
current_app.logger.info('User: {0} requested library: {1}'.format(
user, library))
user_exists = self.helper_user_exists(absolute_uid=user)
if user_exists:
service_uid = \
self.helper_absolute_uid_to_service_uid(absolute_uid=user)
else:
service_uid = None
# If the library is public, allow access
try:
# Try to load the dictionary and obtain the solr content
library, metadata = self.get_documents_from_library(
library_id=library, service_uid=service_uid)
# pay attention to any functions that try to mutate the list
# this will alter expected returns later
try:
solr = self.solr_big_query(bibcodes=library.bibcode,
start=start,
rows=rows,
sort=sort,
fl=fl).json()
except Exception as error:
current_app.logger.warning(
'Could not parse solr data: {0}'.format(error))
solr = {'error': 'Could not parse solr data'}
# Now check if we can update the library database based on the
# returned canonical bibcodes
if solr.get('response'):
# Update bibcodes based on solrs response
updates = self.solr_update_library(
library_id=library.id, solr_docs=solr['response']['docs'])
documents = [i['bibcode'] for i in solr['response']['docs']]
else:
# Some problem occurred, we will just ignore it, but will
# definitely log it.
solr = SOLR_RESPONSE_MISMATCH_ERROR['body']
current_app.logger.warning(
'Problem with solr response: {0}'.format(solr))
updates = {}
documents = library.get_bibcodes()
documents.sort()
documents = documents[start:start + rows]
# Make the response dictionary
response = dict(documents=documents,
solr=solr,
metadata=metadata,
updates=updates)
except Exception as error:
current_app.logger.warning(
'Library missing or solr endpoint failed: {0}'.format(error))
return err(MISSING_LIBRARY_ERROR)
# Skip anymore logic if the library is public or the exception token is present
special_token = current_app.config.get('READONLY_ALL_LIBRARIES_TOKEN')
if library.public or (special_token and request.headers.get(
'Authorization', '').endswith(special_token)):
current_app.logger.info('Library: {0} is public'.format(
library.id))
return response, 200
else:
current_app.logger.warning('Library: {0} is private'.format(
library.id))
# If the user does not exist then there are no associated permissions
# If the user exists, they will have permissions
if self.helper_user_exists(absolute_uid=user):
service_uid = \
self.helper_absolute_uid_to_service_uid(absolute_uid=user)
else:
current_app.logger.error(
'User:{0} does not exist in the database.'
' Therefore will not have extra '
'privileges to view the library: {1}'.format(user, library.id))
return err(NO_PERMISSION_ERROR)
# If they do not have access, exit
if not self.read_access(service_uid=service_uid,
library_id=library.id):
current_app.logger.error(
'User: {0} does not have access to library: {1}. DENIED'.
format(service_uid, library.id))
return err(NO_PERMISSION_ERROR)
# If they have access, let them obtain the requested content
current_app.logger.info('User: {0} has access to library: {1}. '
'ALLOWED'.format(user, library.id))
return response, 200
class ObjectSearch(Resource):
"""Return object identifiers for a given object string"""
scopes = []
rate_limit = [1000, 60 * 60 * 24]
decorators = [advertise('scopes', 'rate_limit')]
def post(self):
stime = time.time()
# Get the supplied list of identifiers
identifiers = []
input_type = None
# determine whether a source for the data was specified
try:
source = request.json['source'].lower()
except:
source = 'simbad'
# We only deal with SIMBAD or NED as source
if source not in ['simbad', 'ned']:
current_app.logger.error(
'Unsupported source for object data specified: %s' % source)
return {
"Error":
"Unable to get results!",
"Error Info":
"Unsupported source for object data specified: %s" % source
}, 200
for itype in ['identifiers', 'objects']:
try:
identifiers = request.json[itype]
identifiers = map(str, identifiers)
input_type = itype
except:
pass
if not input_type:
current_app.logger.error(
'No identifiers and objects were specified for SIMBAD object query'
)
return {
"Error": "Unable to get results!",
"Error Info": "No identifiers/objects found in POST body"
}, 200
# We should either have a list of identifiers or a list of object names
if len(identifiers) == 0:
current_app.logger.error(
'No identifiers or objects were specified for SIMBAD object query'
)
return {
"Error": "Unable to get results!",
"Error Info": "No identifiers/objects found in POST body"
}, 200
# We have a known object data source and a list of identifiers. Let's start!
# We have identifiers
if source == 'simbad':
result = get_simbad_data(identifiers, input_type)
else:
if input_type == 'identifiers':
input_type = 'simple'
result = get_ned_data(identifiers, input_type)
if 'Error' in result:
# An error was returned!
err_msg = result['Error Info']
current_app.logger.error(
'Failed to find data for %s %s query (%s)!' %
(source.upper(), input_type, err_msg))
return result
else:
# We have results!
duration = time.time() - stime
current_app.logger.info(
'Found objects for %s %s in %s user seconds.' %
(source.upper(), input_type, duration))
# Now pick the entries in the results that correspond with the original object names
if input_type == 'objects':
# result['data'] = {k: result['data'].get(k.upper()) for k in identifiers}
result['data'] = {
k: result['data'].get(k) or result['data'].get(k.upper())
for k in identifiers
}
# Send back the results
return result.get('data', {})
class WordCloud(Resource):
'''Returns collated tf/idf data for a solr query'''
decorators = [advertise('scopes', 'rate_limit')]
scopes = []
rate_limit = [500, 60 * 60 * 24]
def post(self):
solr_args = request.json
if not solr_args:
return {
'Error': 'there was a problem with your request',
'Error Info': 'no data provided with request'
}, 403
if 'max_groups' in solr_args:
del solr_args['min_percent_word']
if 'min_occurrences_word' in solr_args:
del solr_args['min_occurrences_word']
elif 'query' in request.json:
try:
solr_args = json.loads(request.json["query"][0])
except Exception:
return {
'Error':
'there was a problem with your request',
'Error Info':
'couldn\'t decode query, it should be json-encoded before being sent (so double encoded)'
}, 403
solr_args["rows"] = min(
int(
solr_args.get(
"rows",
[current_app.config.get("VIS_SERVICE_WC_MAX_RECORDS")
])[0]),
current_app.config.get("VIS_SERVICE_WC_MAX_RECORDS"))
solr_args['fields'] = ['id']
solr_args['defType'] = 'aqp'
solr_args['tv'] = 'true'
solr_args['tv.tf_idf'] = 'true'
solr_args['tv.tf'] = 'true'
solr_args['tv.positions'] = 'false'
solr_args['tf.offsets'] = 'false'
solr_args['tv.fl'] = 'abstract,title'
solr_args['fl'] = 'id,abstract,title'
solr_args['wt'] = 'json'
headers = {
'X-Forwarded-Authorization': request.headers.get('Authorization')
}
response = client().get(
current_app.config.get("VIS_SERVICE_TVRH_PATH"),
params=solr_args,
headers=headers)
if response.status_code == 200:
data = response.json()
else:
return {
"Error":
"There was a connection error. Please try again later",
"Error Info": response.text
}, response.status_code
if data:
min_percent_word = request.args.get(
"min_percent_word",
current_app.config.get("VIS_SERVICE_WC_MIN_PERCENT_WORD"))
min_occurrences_word = request.args.get(
"min_occurrences_word",
current_app.config.get("VIS_SERVICE_WC_MIN_OCCURRENCES_WORD"))
word_cloud_json = word_cloud.generate_wordcloud(
data,
min_percent_word=min_percent_word,
min_occurrences_word=min_occurrences_word)
if word_cloud_json:
return word_cloud_json, 200
else:
return {
"Error":
"Empty word cloud. Try changing your minimum word parameters or expanding your query."
}, 200
