class Graphics(Resource):
"""Return graphics information for a given bibcode"""
scopes = []
rate_limit = [1000, 60 * 60 * 24]
decorators = [advertise('scopes', 'rate_limit')]
def get(self, bibcode):
stime = time.time()
try:
results = get_graphics(bibcode)
except Exception, err:
current_app.logger.error('Graphics exception (%s): %s' %
(bibcode, err))
return {'msg': 'Unable to get results! (%s)' % err}, 500
if results and results['query'] == 'OK':
duration = time.time() - stime
current_app.logger.info('Graphics for %s in %s user seconds' %
(bibcode, duration))
return results
else:
current_app.logger.error('Graphics failed (%s): %s' %
(bibcode, results.get('error', 'NA')))
return {
'Error': 'Unable to get results!',
'Error Info': results.get('error', 'NA')
}, 200
def init_app(self, app):
"""
Standard flask-extension initialisation
:param app: flask app
"""
if app is not None:
self.app = app
if "watchman" in app.extensions:
raise RuntimeError("Flask application already initialised")
app.extensions["watchman"] = self
for view_name in self.allowed_endpoints.keys():
if view_name not in self.kwargs.keys():
continue
view = self.allowed_endpoints[view_name]["view"]
route = self.allowed_endpoints[view_name]["route"]
methods = self.allowed_endpoints[view_name]["methods"]
# Does the user provide scopes?
if view_name in self.kwargs and self.kwargs[view_name].get("scopes", None) != None:
user_config = self.kwargs[view_name]
view.scopes = user_config.get("scopes", [])
view.decorators = user_config.get("decorators", ([advertise("scopes", "rate_limit")]))
view.rate_limit = [1000, 60 * 60 * 24]
with app.app_context():
current_app.add_url_rule(route, view_func=view.as_view(view_name), methods=methods)
class AuthorNetwork(Resource):
'''Returns author network data for a solr query'''
decorators = [advertise('scopes', 'rate_limit')]
scopes = []
rate_limit = [500, 60 * 60 * 24]
def get(self):
solr_args = dict(request.args)
solr_args["rows"] = min(
int(
solr_args.get(
"rows",
[current_app.config.get("VIS_SERVICE_AN_MAX_RECORDS")
])[0]),
current_app.config.get("VIS_SERVICE_AN_MAX_RECORDS"))
solr_args['fl'] = [
'author_norm', 'title', 'citation_count', 'read_count', 'bibcode',
'pubdate'
]
solr_args['wt'] = 'json'
headers = {
'X-Forwarded-Authorization': request.headers.get('Authorization')
}
response = client().get(
current_app.config.get("VIS_SERVICE_SOLR_PATH"),
params=solr_args,
headers=headers)
if response.status_code == 200:
full_response = response.json()
else:
return {
"Error":
"There was a connection error. Please try again later",
"Error Info": response.text
}, response.status_code
#get_network_with_groups expects a list of normalized authors
author_norm = [
d.get("author_norm", []) for d in full_response["response"]["docs"]
]
author_network_json = author_network.get_network_with_groups(
author_norm, full_response["response"]["docs"])
if author_network_json:
return {
"msg": {
"numFound": full_response["response"]["numFound"],
"start": full_response["response"].get("start", 0),
"rows":
int(full_response["responseHeader"]["params"]["rows"])
},
"data": author_network_json
}, 200
else:
return {"Error": "Empty network."}, 200
class PositionSearch(Resource):
"""Return publication information for a cone search"""
scopes = []
rate_limit = [1000, 60 * 60 * 24]
decorators = [advertise('scopes', 'rate_limit')]
def get(self, pstring):
# The following position strings are supported
# 1. 05 23 34.6 -69 45 22:0 6 (or 05h23m34.6s -69d45m22s:0m6s)
# 2. 05 23 34.6 -69 45 22:0.166666 (or 05h23m34.6s -69d45m22s:0.166666)
# 3. 80.89416667 -69.75611111:0.166666
stime = time.time()
# If we're given a string with qualifiers ('h', etc), convert to one without
current_app.logger.info('Attempting SIMBAD position search: %s' %
pstring)
try:
RA, DEC, radius = parse_position_string(pstring)
except Exception, err:
current_app.logger.error(
'Position string could not be parsed: %s' % pstring)
return {
'Error': 'Unable to get results!',
'Error Info': 'Invalid position string: %s' % pstring
}, 200
try:
result = do_position_query(RA, DEC, radius)
except timeout_decorator.timeout_decorator.TimeoutError:
current_app.logger.error('Position query %s timed out' % pstring)
return {
'Error': 'Unable to get results!',
'Error Info': 'Position query timed out'
}, 200
return result
class AllowedMirrors(BaseView):
"""
End point that returns the allowed list of mirror sites for either ADS
classic utilities or BEER/ADS2.0 utilities.
"""
decorators = [advertise('scopes', 'rate_limit')]
scopes = []
rate_limit = [1000, 60 * 60 * 24]
def get(self):
"""
HTTP GET request that returns the list of mirror sites that can be used
with this end point. Any end points not listed by this method cannot be
used for any of the other methods related to this service.
Return data (on success)
------------------------
list[<string>]
eg., list of mirrors, ['site1', 'site2', ...., 'siteN']
HTTP Responses:
--------------
Succeed authentication: 200
Any other responses will be default Flask errors
"""
return current_app.config.get('ADS_CLASSIC_MIRROR_LIST', [])
class BigQuery(SolrInterface):
"""Exposes the bigquery endpoint"""
scopes = ['api']
rate_limit = [100, 60 * 60 * 24]
decorators = [advertise('scopes', 'rate_limit')]
handler = 'SOLR_SERVICE_BIGQUERY_HANDLER'
def post(self):
payload = dict(request.form)
payload.update(request.args)
query, headers = self.cleanup_solr_request(payload)
if request.files and \
sum([len(i) for i in request.files.listvalues()]) > 1:
message = "You can only pass one content stream."
current_app.logger.error(message)
return json.dumps({'error': message}), 400
if 'fq' not in query:
query['fq'] = [u'{!bitset}']
elif len(filter(lambda x: '!bitset' in x, query['fq'])) == 0:
query['fq'].append(u'{!bitset}')
if 'big-query' not in headers.get('Content-Type', ''):
headers['Content-Type'] = 'big-query/csv'
if request.data:
current_app.logger.info(
"Dispatching 'POST' request to endpoint '{}'".format(
current_app.config[self.handler]))
r = current_app.client.post(
current_app.config[self.handler],
params=query,
data=request.data,
headers=headers,
cookies=SolrInterface.set_cookies(request),
)
current_app.logger.info(
"Received response from endpoint '{}' with status code '{}'".
format(current_app.config[self.handler], r.status_code))
elif request.files:
current_app.logger.info(
"Dispatching 'POST' request to endpoint '{}'".format(
current_app.config[self.handler]))
r = current_app.client.post(
current_app.config[self.handler],
params=query,
headers=headers,
files=request.files,
cookies=SolrInterface.set_cookies(request),
)
current_app.logger.info(
"Received response from endpoint '{}' with status code '{}'".
format(current_app.config[self.handler], r.status_code))
else:
message = "Malformed request"
current_app.logger.error(message)
return json.dumps({'error': message}), 400
return r.text, r.status_code, r.headers
class ClassicView(HarbourView):
"""
Wrapper for importing libraries from ADS Classic
"""
decorators = [advertise('scopes', 'rate_limit')]
scopes = ['user']
rate_limit = [1000, 60 * 60 * 24]
service_url = 'BIBLIB_CLASSIC_SERVICE_URL'
class TwoPointOhView(HarbourView):
"""
Wrapper for importing libraries from ADS 2.0
"""
decorators = [advertise('scopes', 'rate_limit')]
scopes = ['user']
rate_limit = [1000, 60 * 60 * 24]
service_url = 'BIBLIB_TWOPOINTOH_SERVICE_URL'
class StatusView(Resource):
"""Returns the status of this app"""
scopes = []
rate_limit = [1000, 60 * 60 * 24]
decorators = [advertise('scopes', 'rate_limit')]
def get(self):
return {'app': current_app.name, 'status': 'online'}, 200
def test_setAttributes(self):
"""
Test setting attributes of a function with an explicit call to advertise()
and with @advertise. Assert that the function code executes as expected after
the operators.
"""
self.testfunction = advertise(thisattr="foo")(self.testfunction)
self.assertEqual(self.testfunction._advertised, [{"thisattr": "foo"}])
self.testfunction = advertise(thisattr2="foo2")(self.testfunction)
self.assertItemsEqual(self.testfunction._advertised, [{"thisattr": "foo"}, {"thisattr2": "foo2"}])
res = self.testfunction(1, 2, 3)
self.assertEqual(res, "1|2|3")
@advertise(decor1="foo", decor2="bar")
def testfunction(a, b, c):
return "{a}|{b}|{c}".format(a=a, b=b, c=c)
self.assertItemsEqual(testfunction._advertised, [{"decor1": "foo"}, {"decor2": "bar"}])
res = testfunction(1, 2, 3)
self.assertEqual(res, "1|2|3")
def test_setAttributes(self):
'''
Test setting attributes of a function with an explicit call to advertise()
and with @advertise. Assert that the function code executes as expected after
the operators.
'''
self.testfunction = advertise(thisattr='foo')(self.testfunction)
self.assertEqual(self.testfunction._advertised,[{'thisattr':'foo'}])
self.testfunction = advertise(thisattr2='foo2')(self.testfunction)
self.assertItemsEqual(self.testfunction._advertised,[{'thisattr':'foo'},{'thisattr2':'foo2'}])
res = self.testfunction(1,2,3)
self.assertEqual(res,'1|2|3')
@advertise(decor1='foo',decor2='bar')
def testfunction(a,b,c):
return "{a}|{b}|{c}".format(a=a,b=b,c=c)
self.assertItemsEqual(testfunction._advertised,[{'decor1':'foo'},{'decor2':'bar'}])
res = testfunction(1,2,3)
self.assertEqual(res,'1|2|3')
class PrintArg(Resource):
"""
Returns the :arg in the route
"""
decorators = [advertise('scopes', 'rate_limit')]
scopes = ['scope1', 'scope2']
rate_limit = [1000, 60 * 60 * 24]
def get(self, arg):
"""
HTTP GET request that returns the string passed
:param arg: string to send as return
:return: argument given to the resource
"""
return {'arg': arg}, 200
class UnixTime(Resource):
"""
Returns the unix timestamp of the server
"""
decorators = [advertise('scopes', 'rate_limit')]
scopes = ['scope1', 'scope2']
rate_limit = [1000, 60 * 60 * 24]
def get(self):
"""
HTTP GET request
:return: the unix time now
"""
current_app.logger.info('Example of logging within the app.')
return {'now': time.time()}, 200
class BigQuery(Resource):
"""Exposes the bigquery endpoint"""
scopes = ['api']
rate_limit = [100, 60 * 60 * 24]
decorators = [advertise('scopes', 'rate_limit')]
handler = 'SOLR_SERVICE_BIGQUERY_HANDLER'
def post(self):
payload = dict(request.form)
payload.update(request.args)
headers = dict(request.headers)
query = SolrInterface.cleanup_solr_request(
payload, headers.get('X-Adsws-Uid', 'default'))
if request.files and \
sum([len(i) for i in request.files.listvalues()]) > 1:
return json.dumps(
{'error': 'You can only pass one content stream.'}), 400
if 'fq' not in query:
query['fq'] = [u'{!bitset}']
elif len(filter(lambda x: '!bitset' in x, query['fq'])) == 0:
query['fq'].append(u'{!bitset}')
if 'big-query' not in headers.get('Content-Type', ''):
headers['Content-Type'] = 'big-query/csv'
if request.data:
r = requests.post(
current_app.config[self.handler],
params=query,
data=request.data,
headers=headers,
cookies=SolrInterface.set_cookies(request),
)
elif request.files:
r = requests.post(
current_app.config[self.handler],
params=query,
headers=headers,
files=request.files,
cookies=SolrInterface.set_cookies(request),
)
else:
return json.dumps({'error': "malformed request"}), 400
return r.text, r.status_code, r.headers
class Export(Resource):
"""Returns export data for a list of bibcodes"""
decorators = [advertise('scopes', 'rate_limit')]
def get(self):
payload = dict(request.args)
return self.get_data_from_classic(payload)
def post(self):
try:
payload = request.get_json(force=True) # post data in json
except:
payload = dict(request.form) # post data in form encoding
return self.get_data_from_classic(payload)
def get_data_from_classic(self, payload):
if not payload:
return {'error': 'no information received'}, 400
elif 'bibcode' not in payload:
return {
'error':
'no bibcodes found in payload (parameter '
'name is "bibcode")'
}, 400
headers = {'User-Agent': 'ADS Script Request Agent'}
# assign data type based on endpoint
payload["data_type"] = self.data_type
# actual request
r = requests.post(
current_app.config.get("EXPORT_SERVICE_CLASSIC_EXPORT_URL"),
data=payload,
headers=headers)
r.raise_for_status()
hdr = re.match(
current_app.config['EXPORT_SERVICE_CLASSIC_SUCCESS_STRING'],
r.text)
if not hdr:
return {"error": "No records returned from ADS-Classic"}, 400
return {
"export": r.text.replace(hdr.group(), ''),
"msg": hdr.group().strip().split("\n")[::-1][0]
}
class BigQuery(SolrInterface):
"""Exposes the bigquery endpoint"""
scopes = ['api']
rate_limit = [100, 60 * 60 * 24]
decorators = [advertise('scopes', 'rate_limit')]
handler = 'SOLR_SERVICE_BIGQUERY_HANDLER'
def post(self):
payload = dict(request.form)
payload.update(request.args)
if request.is_json:
payload.update(request.json)
query, headers = self.cleanup_solr_request(payload)
files = self.check_for_embedded_bigquery(query, request, headers)
if files and len(files) > 0:
try:
current_user_id = current_user.get_id()
except:
# If solr service is not shipped with adsws, this will fail and it is ok
current_user_id = None
if current_user_id:
current_app.logger.info(
"Dispatching 'POST' request to endpoint '{}' for user '{}'"
.format(current_app.config[self.handler], current_user_id))
else:
current_app.logger.info(
"Dispatching 'POST' request to endpoint '{}'".format(
current_app.config[self.handler]))
r = requests.post(
current_app.config[self.handler],
params=query,
headers=headers,
files=files,
cookies=SolrInterface.set_cookies(request),
)
current_app.logger.info(
"Received response from endpoint '{}' with status code '{}'".
format(current_app.config[self.handler], r.status_code))
else:
message = "Malformed request"
current_app.logger.error(message)
return json.dumps({'error': message}), 400
return self.cleanup_solr_response_text(
r.text), r.status_code, r.headers
class ExampleApiUsage(Resource):
"""
This resource uses the client.session.get() method to access an api that
requires an oauth2 token, such as our own adsws
"""
decorators = [advertise('scopes', 'rate_limit')]
scopes = ['scope1']
rate_limit = [1000, 60 * 60 * 24]
def get(self):
"""
HTTP GET request using the apps client session defined in the config
:return: HTTP response from the API
"""
r = client().get(
current_app.config.get('SAMPLE_APPLICATION_ADSWS_API_URL'))
return r.json()
def test_advertiseAttributes(self):
"""
Test advertising "empty" attributes
"""
self.testfunction = advertise("thisattr")(self.testfunction)
self.assertRaises(AttributeError, lambda: self.testfunction.thisattr)
self.assertEqual(self.testfunction._advertised, [{"thisattr": None}])
res = self.testfunction(1, 2, 3)
self.assertEqual(res, "1|2|3")
@advertise("decor1", "decor2")
def testfunction(a, b, c):
return "{a}|{b}|{c}".format(a=a, b=b, c=c)
self.assertRaises(AttributeError, lambda: testfunction.decor1)
self.assertRaises(AttributeError, lambda: testfunction.decor2)
self.assertItemsEqual(testfunction._advertised, [{"decor1": None}, {"decor2": None}])
res = testfunction(1, 2, 3)
self.assertEqual(res, "1|2|3")
class PubMetrics(Resource):
"""Get metrics for a single publication (identified by its bibcode)"""
scopes = []
rate_limit = [1000, 60 * 60 * 24]
decorators = [advertise('scopes', 'rate_limit')]
def get(self, bibcode):
results = generate_metrics(bibcodes=[bibcode],
types=['basic', 'histograms'],
histograms=['reads', 'citations'])
# If the results contain an error message something went boink
if "Error" in results:
return results, 500
# otherwise we have real results or an empty dictionary
if results:
return results
else:
return {
'Error': 'Unable to get results!',
'Error Info': 'No data available to generate metrics'
}, 200
def test_advertiseAttributes(self):
'''
Test advertising "empty" attributes
'''
self.testfunction = advertise('thisattr')(self.testfunction)
self.assertRaises(AttributeError,lambda: self.testfunction.thisattr)
self.assertEqual(self.testfunction._advertised,[{'thisattr':None}])
res = self.testfunction(1,2,3)
self.assertEqual(res,'1|2|3')
@advertise('decor1','decor2')
def testfunction(a,b,c):
return "{a}|{b}|{c}".format(a=a,b=b,c=c)
self.assertRaises(AttributeError,lambda: testfunction.decor1)
self.assertRaises(AttributeError,lambda: testfunction.decor2)
self.assertItemsEqual(testfunction._advertised,[{'decor1':None},{'decor2':None}])
res = testfunction(1,2,3)
self.assertEqual(res,'1|2|3')
class Recommender(Resource):
"""Return recommender results for a given bibcode"""
scopes = []
rate_limit = [1000, 60 * 60 * 24]
decorators = [advertise('scopes', 'rate_limit')]
def get(self, bibcode):
stime = time.time()
try:
results = get_recommendations(bibcode)
except Exception, err:
current_app.logger.error('Recommender exception (%s): %s'%(bibcode, err))
return {'msg': 'Unable to get results! (%s)' % err}, 500
if 'Error' in results:
current_app.logger.error('Recommender failed (%s): %s'%(bibcode, results.get('Error')))
return results, results['Status Code']
else:
duration = time.time() - stime
current_app.logger.info('Recommendations for %s in %s user seconds'%(bibcode, duration))
return results
class ClassicUser(BaseView):
"""
End point to collect the user's ADS Classic information currently stored in
the database
"""
decorators = [advertise('scopes', 'rate_limit')]
scopes = ['user']
rate_limit = [1000, 60 * 60 * 24]
def get(self):
"""
HTTP GET request that returns the information currently stored about the
user's ADS Classic settings, currently stored in the service database.
Return data (on success)
------------------------
classic_email: <string> ADS Classic e-mail of the user
classic_mirror: <string> ADS Classic mirror this user belongs to
HTTP Responses:
--------------
Succeed authentication: 200
User unknown/wrong password/failed authentication: 404
Any other responses will be default Flask errors
"""
absolute_uid = self.helper_get_user_id()
try:
user = Users.query.filter(Users.absolute_uid == absolute_uid).one()
return {
'classic_email': user.classic_email,
'classic_mirror': user.classic_mirror,
'twopointoh_email': user.twopointoh_email
}, 200
except NoResultFound:
return err(NO_CLASSIC_ACCOUNT)
def init_app(self, app):
"""
Standard flask-extension initialisation
:param app: flask app
"""
if app is not None:
self.app = app
if 'watchman' in app.extensions:
raise RuntimeError('Flask application already initialised')
app.extensions['watchman'] = self
for view_name in self.allowed_endpoints.keys():
if view_name not in self.kwargs.keys():
continue
view = self.allowed_endpoints[view_name]['view']
route = self.allowed_endpoints[view_name]['route']
methods = self.allowed_endpoints[view_name]['methods']
# Does the user provide scopes?
if view_name in self.kwargs and self.kwargs[view_name].get('scopes', None) != None:
user_config = self.kwargs[view_name]
view.scopes = user_config.get('scopes', [])
view.decorators = user_config.get('decorators', ([advertise('scopes', 'rate_limit')]))
view.rate_limit = [1000, 60*60*24]
with app.app_context():
current_app.add_url_rule(
route,
view_func=view.as_view(view_name),
methods=methods
)
class Search(SolrInterface):
"""Exposes the solr select endpoint"""
scopes = []
rate_limit = [5000, 60 * 60 * 24]
decorators = [advertise('scopes', 'rate_limit')]
handler = {
'default': 'SOLR_SERVICE_SEARCH_HANDLER',
'default_embedded_bigquery': 'SOLR_SERVICE_BIGQUERY_HANDLER',
'bot': 'BOT_SOLR_SERVICE_SEARCH_HANDLER',
'bot_embedded_bigquery': 'BOT_SOLR_SERVICE_BIGQUERY_HANDLER'
}
def get_handler_class(self):
"""Identify bot requests based on their authentication token"""
forwarded_authorization = request.headers.get(
'X-Forwarded-Authorization', [])
if forwarded_authorization and len(forwarded_authorization) > 7:
request_token = forwarded_authorization[7:]
else:
request_token = request.headers.get('Authorization', [])[7:]
if request_token in current_app.config.get('BOT_TOKENS', []):
return "bot"
else:
return "default"
class ScopedView(Resource):
'''Scoped route docstring'''
scopes = ['default']
decorators = [advertise('scopes')]
def get(self):
return "scoped route"
class DocumentView(BaseView):
"""
End point to interact with a specific library, by adding documents and
removing documents. You also use this endpoint to delete the entire
library as this method should be scoped.
"""
# TODO: adding tags using PUT for RESTful endpoint?
decorators = [advertise('scopes', 'rate_limit')]
scopes = ['user']
rate_limit = [1000, 60 * 60 * 24]
@classmethod
def add_document_to_library(cls, library_id, document_data):
"""
Adds a document to a user's library
:param library_id: the library id to update
:param document_data: the meta data of the document
:return: number_added: number of documents successfully added
"""
current_app.logger.info(
'Adding a document: {0} to library_uuid: {1}'.format(
document_data, library_id))
# Find the specified library
library = Library.query.filter(Library.id == library_id).one()
start_length = len(library.bibcode)
library.add_bibcodes(document_data['bibcode'])
db.session.add(library)
db.session.commit()
current_app.logger.info('Added: {0} is now {1}'.format(
document_data['bibcode'], library.bibcode))
end_length = len(library.bibcode)
return end_length - start_length
@classmethod
def remove_documents_from_library(cls, library_id, document_data):
"""
Remove a given document from a specific library
:param library_id: the unique ID of the library
:param document_data: the meta data of the document
:return: number_removed: number of documents successfully removed
"""
current_app.logger.info('Removing a document: {0} from library_uuid: '
'{1}'.format(document_data, library_id))
library = Library.query.filter(Library.id == library_id).one()
start_length = len(library.bibcode)
library.remove_bibcodes(document_data['bibcode'])
db.session.add(library)
db.session.commit()
current_app.logger.info('Removed document successfully: {0}'.format(
library.bibcode))
end_length = len(library.bibcode)
return start_length - end_length
@staticmethod
def update_library(library_id, library_data):
"""
Update the meta data of the library
:param library_id: the unique ID of the library
:param library_data: dictionary containing the updateable values
:return: values updated
"""
updateable = ['name', 'description', 'public']
updated = {}
library = Library.query.filter(Library.id == library_id).one()
for key in library_data:
if key not in updateable:
continue
setattr(library, key, library_data[key])
updated[key] = library_data[key]
db.session.add(library)
db.session.commit()
return updated
@staticmethod
def delete_library(library_id):
"""
Delete the entire library from the database
:param library_id: the unique ID of the library
:return: no return
"""
library = Library.query.filter(Library.id == library_id).one()
db.session.delete(library)
db.session.commit()
@classmethod
def update_access(cls, service_uid, library_id):
"""
Defines which type of user has delete permissions to a library.
:param service_uid: the user ID within this microservice
:param library_id: the unique ID of the library
:return: boolean, access (True), no access (False)
"""
update_allowed = ['admin', 'owner']
for access_type in update_allowed:
if cls.helper_access_allowed(service_uid=service_uid,
library_id=library_id,
access_type=access_type):
return True
return False
@classmethod
def delete_access(cls, service_uid, library_id):
"""
Defines which type of user has delete permissions to a library.
:param service_uid: the user ID within this microservice
:param library_id: the unique ID of the library
:return: boolean, access (True), no access (False)
"""
delete_allowed = cls.helper_access_allowed(service_uid=service_uid,
library_id=library_id,
access_type='owner')
return delete_allowed
@classmethod
def write_access(cls, service_uid, library_id):
"""
Defines which type of user has write permissions to a library.
:param service_uid: the user ID within this microservice
:param library_id: the unique ID of the library
:return: boolean, access (True), no access (False)
"""
read_allowed = ['write', 'admin', 'owner']
for access_type in read_allowed:
if cls.helper_access_allowed(service_uid=service_uid,
library_id=library_id,
access_type=access_type):
return True
return False
@staticmethod
def library_name_exists(service_uid, library_name):
"""
Checks to see if a library name already exists in the user's created
libraries
:param service_uid: the user ID within this microservice
:param library_name: name to check if it exists
:return: True (exists), False (does not exist)
"""
library_names = \
[i.library.name for i in
Permissions.query.filter(Permissions.user_id == service_uid,
Permissions.owner == True).all()]
if library_name in library_names:
current_app.logger.error('Name supplied for the library already '
'exists: "{0}"'.format(library_name))
return True
else:
return False
def post(self, library):
"""
HTTP POST request that adds a document to a library for a given user
:param library: library ID
:return: the response for if the library was successfully created
Header:
-------
Must contain the API forwarded user ID of the user accessing the end
point
Post body:
----------
KEYWORD, VALUE
bibcode: <list> List of bibcodes to be added
action: add, remove add - adds a bibcode, remove - removes a
bibcode
Return data:
-----------
number_added: number of documents added (if 'add' is used)
number_removed: number of documents removed (if 'remove' is used)
Permissions:
-----------
The following type of user can add documents:
- owner
- admin
- write
"""
# Get the user requesting this from the header
try:
user_editing = self.helper_get_user_id()
except KeyError:
return err(MISSING_USERNAME_ERROR)
# URL safe base64 string to UUID
library = self.helper_slug_to_uuid(library)
user_editing_uid = \
self.helper_absolute_uid_to_service_uid(absolute_uid=user_editing)
# Check the permissions of the user
if not self.write_access(service_uid=user_editing_uid,
library_id=library):
return err(NO_PERMISSION_ERROR)
try:
data = get_post_data(request,
types=dict(bibcode=list, action=unicode))
except TypeError as error:
current_app.logger.error(
'Wrong type passed for POST: {0} [{1}]'.format(
request.data, error))
return err(WRONG_TYPE_ERROR)
if data['action'] == 'add':
current_app.logger.info('User requested to add a document')
number_added = self.add_document_to_library(library_id=library,
document_data=data)
current_app.logger.info(
'Successfully added {0} documents to {1} by {2}'.format(
number_added, library, user_editing_uid))
return {'number_added': number_added}, 200
elif data['action'] == 'remove':
current_app.logger.info('User requested to remove a document')
number_removed = self.remove_documents_from_library(
library_id=library, document_data=data)
current_app.logger.info(
'Successfully removed {0} documents to {1} by {2}'.format(
number_removed, library, user_editing_uid))
return {'number_removed': number_removed}, 200
else:
current_app.logger.info('User requested a non-standard action')
return {}, 400
def put(self, library):
"""
HTTP PUT request that updates the meta-data of the library
:param library: library ID
:return: the response for if the library was updated
Header:
-------
Must contain the API forwarded user ID of the user accessing the end
point
Post-body:
---------
name: name of the library
description: description of the library
public: boolean
Note: The above are optional, they can be empty if needed.
Return data:
-----------
returns the key/value that was requested to be updated
Permissions:
-----------
The following type of user can update the 'name', 'library', and
'public':
- owner
- admin
"""
try:
user = self.helper_get_user_id()
except KeyError:
return err(MISSING_USERNAME_ERROR)
# URL safe base64 string to UUID
library = self.helper_slug_to_uuid(library)
if not self.helper_user_exists(user):
return err(NO_PERMISSION_ERROR)
if not self.helper_library_exists(library):
return err(MISSING_LIBRARY_ERROR)
user_updating_uid = \
self.helper_absolute_uid_to_service_uid(absolute_uid=user)
if not self.update_access(service_uid=user_updating_uid,
library_id=library):
return err(NO_PERMISSION_ERROR)
try:
library_data = get_post_data(request,
types=dict(name=unicode,
description=unicode,
public=bool))
except TypeError as error:
current_app.logger.error(
'Wrong type passed for POST: {0} [{1}]'.format(
request.data, error))
return err(WRONG_TYPE_ERROR)
# Remove content that is empty
for key in library_data.keys():
if library_data[key] == ''.strip(' '):
current_app.logger.warning(
'Removing key: {0} as its empty.'.format(key))
library_data.pop(key)
# Check for duplicate namaes
if 'name' in library_data and \
self.library_name_exists(service_uid=user_updating_uid,
library_name=library_data['name']):
return err(DUPLICATE_LIBRARY_NAME_ERROR)
response = self.update_library(library_id=library,
library_data=library_data)
return response, 200
def delete(self, library):
"""
HTTP DELETE request that deletes a library defined by the number passed
:param library: library ID
:return: the response for it the library was deleted
Header:
-------
Must contain the API forwarded user ID of the user accessing the end
point
Post-body:
----------
No post content accepted.
Return data:
-----------
No data
Permissions:
-----------
The following type of user can update a library:
- owner
"""
try:
user = self.helper_get_user_id()
except KeyError:
return err(MISSING_USERNAME_ERROR)
# URL safe base64 string to UUID
library = self.helper_slug_to_uuid(library)
if not self.helper_user_exists(user):
return err(NO_PERMISSION_ERROR)
if not self.helper_library_exists(library):
return err(MISSING_LIBRARY_ERROR)
user_deleting_uid = \
self.helper_absolute_uid_to_service_uid(absolute_uid=user)
try:
current_app.logger.info('user_API: {0:d} '
'requesting to delete library: {1}'.format(
user_deleting_uid, library))
if self.delete_access(service_uid=user_deleting_uid,
library_id=library):
self.delete_library(library_id=library)
current_app.logger.info(
'User: {0} deleted library: {1}.'.format(
user_deleting_uid, library))
else:
current_app.logger.error('User: {0} has incorrect permissions '
'to delete: {1}.'.format(
user_deleting_uid, library))
raise PermissionDeniedError('Incorrect permissions')
except NoResultFound as error:
current_app.logger.info('Failed to delete: {0}'.format(error))
return err(MISSING_LIBRARY_ERROR)
except PermissionDeniedError as error:
current_app.logger.info('Failed to delete: {0}'.format(error))
return err(NO_PERMISSION_ERROR)
return {}, 200
class ObjectSearch(Resource):
"""Return object identifiers for a given object string"""
scopes = []
rate_limit = [1000, 60 * 60 * 24]
decorators = [advertise('scopes', 'rate_limit')]
def post(self):
stime = time.time()
# Get the supplied list of identifiers
identifiers = []
objects = []
facets = []
input_type = None
for itype in ['identifiers', 'objects', 'facets']:
try:
identifiers = request.json[itype]
identifiers = map(str, identifiers)
input_type = itype
except:
pass
if not input_type:
current_app.logger.error(
'No identifiers and objects were specified for SIMBAD object query'
)
return {
"Error": "Unable to get results!",
"Error Info": "No identifiers/objects found in POST body"
}, 200
# We should either have a list of identifiers, a list of object names or a list of facets
if len(identifiers) == 0 and len(objects) == 0 and len(facets) == 0:
current_app.logger.error(
'No identifiers, objects or facets were specified for SIMBAD object query'
)
return {
"Error": "Unable to get results!",
"Error Info": "No identifiers/objects found in POST body"
}, 200
# How many iden identifiers do we have?
id_num = len(identifiers)
if id_num == 0:
return {
"Error": "Unable to get results!",
"Error Info": "No identifiers/objects found in POST body"
}, 200
# Source to query
source = 'simbad'
# Now check if we have anything cached for them
cached = {
id: current_app.cache.get(id.upper())
for id in identifiers if current_app.cache.get(id.upper())
}
if source in ['simbad', 'all'] and len(identifiers) > 0:
# If we have cached values, filter those out from the initial list
if cached:
current_app.logger.debug(
'Received %s %s. Using %s entries from cache.' %
(id_num, input_type, len(cached)))
identifiers = [
id for id in identifiers
if not current_app.cache.get(id.upper())
]
if identifiers:
ident_upper = [i.upper() for i in identifiers]
# We have identifiers, not found in the cache
result = get_simbad_data(identifiers, input_type)
if 'Error' in result:
# An error was returned!
current_app.logger.error(
'Failed to find data for SIMBAD %s query!' %
input_type)
return result
else:
# We have results!
duration = time.time() - stime
current_app.logger.info(
'Found objects for SIMBAD %s in %s user seconds.' %
(input_type, duration))
# Before returning results, cache them
for ident, value in result['data'].items():
current_app.cache.set(ident.upper(),
value,
timeout=current_app.config.get(
'OBJECTS_CACHE_TIMEOUT'))
# Now pick the entries in the results that correspond with the original object names
if input_type == 'objects':
result['data'] = {
k: result['data'].get(k.upper())
for k in identifiers
}
# If we had results from cache, merge these in
if cached:
res = cached.copy()
res.update(result.get('data', {}))
return res
# Otherwise just send back the results
else:
return result.get('data', {})
elif cached:
# We only had cached results
return cached
else:
# This should never happen
current_app.logger.error(
'No data found, even though we had %s! Should never happen!'
% input_type)
result = {
"Error":
"Failed to find data for SIMBAD %s query!" % input_type,
"Error Info":
"No results found, where results were expected! Needs attention!"
}
return result
class Tvrh(SolrInterface):
"""Exposes the solr term-vector histogram endpoint"""
scopes = []
rate_limit = [500, 60 * 60 * 24]
decorators = [advertise('scopes', 'rate_limit')]
handler = 'SOLR_SERVICE_TVRH_HANDLER'
class HarbourView(BaseView):
"""
End point to import libraries from external systems
"""
decorators = [advertise('scopes', 'rate_limit')]
scopes = ['user']
rate_limit = [1000, 60 * 60 * 24]
service_url = 'default'
@staticmethod
def upsert_library(service_uid, library):
"""
Upsert a library into the database. This entails:
- Adding a library and bibcodes if there is no name conflict
- Not adding a library if name matches, but compare bibcodes
:param service_uid: microservice UID of the user
:param library: dictionary of the form:
{'name': str, 'description': str, 'documents': [str, ...., str]}
:return: boolean for success
"""
# Make the permissions
user = User.query.filter(User.id == service_uid).one()
try:
# we are passed the library from classic
# and need to find the corresponding library in bumblebee
# the corresponding library is the one with
# the same name and the same owner
# in raw sql, this is essentially
# q = "select library.id from library,permissions where library.name='{}' and permissions.library_id=library.id and permissions.user_id={} and permissions.owner=True"
# q = q.format(library['name'], user.id)
# but, this must be done via the orm api
q = db.session.query(Library).join(Permissions).filter(Library.id == Permissions.library_id)\
.filter(Permissions.user_id == user.id).filter(Permissions.owner == True).filter(Library.name == library['name'])
lib = q.all()
# Raise if there is not exactly one, it should be 1 or 0, but if
# multiple are returned, there is some problem
if len(lib) == 0:
raise NoResultFound
current_app.logger.info(
'User does not have a library with this name')
elif len(lib) > 1:
current_app.logger.warning(
'More than 1 library has the same name,'
' this should not happen: {}'.format(lib))
raise IntegrityError
# Get the single record returned, as names are considered unique in
# the workflow of creating libraries
lib = lib[0]
bibcode_before = len(lib.get_bibcodes())
lib.add_bibcodes(library['documents'])
bibcode_added = len(lib.get_bibcodes()) - bibcode_before
action = 'updated'
db.session.add(lib)
except NoResultFound:
current_app.logger.info(
'Creating library from scratch: {}'.format(library))
permission = Permissions(owner=True)
lib = Library(
name=library['name'][0:50],
description=library['description'][0:200],
)
lib.add_bibcodes(library['documents'])
lib.permissions.append(permission)
user.permissions.append(permission)
db.session.add_all([lib, permission, user])
bibcode_added = len(lib.get_bibcodes())
action = 'created'
db.session.commit()
return {
'library_id': BaseView.helper_uuid_to_slug(lib.id),
'name': lib.name,
'description': lib.description,
'num_added': bibcode_added,
'action': action
}
# Methods
def get(self):
"""
HTTP GET request that
:return:
Header:
Must contain the API forwarded user ID of the user accessing the end
point
Post body:
----------
No post content accepted.
Return data:
-----------
Permissions:
-----------
The following type of user can read a library:
- user scope (authenticated via the API)
"""
# Check that they pass a user id
try:
user = self.helper_get_user_id()
except KeyError:
return err(MISSING_USERNAME_ERROR)
service_uid = self.helper_absolute_uid_to_service_uid(
absolute_uid=user)
url = '{external_service}/{user_id}'.format(
external_service=current_app.config[self.service_url],
user_id=user)
current_app.logger.info(
'Collecting libraries for user {} from {}'.format(user, url))
response = client().get(url)
if response.status_code != 200:
return response.json(), response.status_code
resp = []
for library in response.json()['libraries']:
resp.append(
self.upsert_library(service_uid=service_uid, library=library))
return resp, 200
class Search(SolrInterface):
"""Exposes the solr select endpoint"""
scopes = []
rate_limit = [5000, 60 * 60 * 24]
decorators = [advertise('scopes', 'rate_limit')]
handler = 'SOLR_SERVICE_SEARCH_HANDLER'
class Qtree(SolrInterface):
"""Exposes the qtree endpoint"""
scopes = []
rate_limit = [500, 60 * 60 * 24]
decorators = [advertise('scopes', 'rate_limit')]
handler = 'SOLR_SERVICE_QTREE_HANDLER'
class QuerySearch(Resource):
"""Given a Solr query with object names, return a Solr query with SIMBAD identifiers"""
scopes = []
rate_limit = [1000, 60 * 60 * 24]
decorators = [advertise('scopes', 'rate_limit')]
def post(self):
stime = time.time()
# Get the supplied list of identifiers
identifiers = []
query = None
itype = None
name2id = {}
try:
query = request.json['query']
input_type = 'query'
except:
current_app.logger.error(
'No query was specified for SIMBAD object search')
return {
"Error": "Unable to get results!",
"Error Info": "No identifiers/objects found in POST body"
}, 200
# If we get the request from BBB, the value of 'query' is actually an array
if isinstance(query, list):
try:
solr_query = query[0]
except:
solr_query = ''
else:
solr_query = query
current_app.logger.info('Received SIMBAD object query: %s' %
solr_query)
new_query = solr_query.replace('object:', 'simbid:')
# If we receive a (Solr) query string, we need to parse out the object names
try:
identifiers = get_objects_from_query_string(solr_query)
except:
current_app.logger.error(
'Parsing the identifiers out of the query string blew up!')
return {
"Error": "Unable to get results!",
"Error Info": "No objects found in query string"
}, 200
identifiers = [
iden for iden in identifiers
if iden.lower() not in ['object', ':']
]
# How many object names did we fid?
id_num = len(identifiers)
# Keep a list with the object names we found
identifiers_orig = identifiers
# If we did not find any object names, there is nothing to do!
if id_num == 0:
return {
"Error": "Unable to get results!",
"Error Info": "No identifiers/objects found in POST body"
}, 200
# Source to query
source = 'simbad'
if source in ['simbad', 'all'] and len(identifiers) > 0:
if identifiers:
for ident in identifiers:
result = get_simbad_data([ident], 'objects')
if 'Error' in result:
# An error was returned!
current_app.logger.error(
'Failed to find data for SIMBAD %s query!' %
input_type)
return result
try:
SIMBADid = [
e.get('id', 0) for e in result['data'].values()
][0]
except:
SIMBADid = '0'
name2id[ident] = SIMBADid
for oname in identifiers:
try:
SIMBADid = name2id.get(oname)
except:
SIMBADid = '0'
new_query = new_query.replace(oname, SIMBADid)
return {"query": new_query}
else:
# This should never happen
current_app.logger.error(
'No data found, even though we had %s! Should never happen!'
% input_type)
result = {
"Error":
"Failed to find data for SIMBAD %s query!" % input_type,
"Error Info":
"No results found, where results were expected! Needs attention!"
}
return result
class UserView(BaseView):
"""
End point to create a library for a given user
"""
decorators = [advertise('scopes', 'rate_limit')]
scopes = ['user']
rate_limit = [1000, 60 * 60 * 24]
@staticmethod
def create_user(absolute_uid):
"""
Creates a user in the database with a UID from the API
:param absolute_uid: UID from the API
:return: no return
"""
try:
user = User(absolute_uid=absolute_uid)
db.session.add(user)
db.session.commit()
except IntegrityError as error:
current_app.logger.error('IntegrityError. User: {0:d} was not'
'added. Full traceback: {1}'.format(
absolute_uid, error))
raise
@staticmethod
def create_library(service_uid, library_data):
"""
Creates a library for a user
:param service_uid: the user ID within this microservice
:param library_data: content needed to create a library
:return: no return
"""
library_data = BaseView.helper_validate_library_data(
service_uid=service_uid, library_data=library_data)
_name = library_data.get('name')
_description = library_data.get('description')
_public = bool(library_data.get('public', False))
_bibcode = library_data.get('bibcode', False)
try:
# Make the library in the library table
library = Library(name=_name,
description=_description,
public=_public)
# If the user supplies bibcodes
if _bibcode and isinstance(_bibcode, list):
# Ensure unique content
_bibcode = uniquify(_bibcode)
current_app.logger.info(
'User supplied bibcodes: {0}'.format(_bibcode))
library.add_bibcodes(_bibcode)
elif _bibcode:
current_app.logger.error(
'Bibcode supplied not a list: {0}'.format(_bibcode))
raise TypeError('Bibcode should be a list.')
user = User.query.filter(User.id == service_uid).one()
# Make the permissions
permission = Permissions(owner=True, )
# Use the ORM to link the permissions to the library and user,
# so that no commit is required until the complete action is
# finished. This means any rollback will not leave a single
# library without permissions
library.permissions.append(permission)
user.permissions.append(permission)
db.session.add_all([library, permission, user])
db.session.commit()
current_app.logger.info(
'Library: "{0}" made, user_service: {1:d}'.format(
library.name, user.id))
return library
except IntegrityError as error:
# Roll back the changes
db.session.rollback()
current_app.logger.error('IntegitryError, database has been rolled'
'back. Caused by user_service: {0:d}.'
'Full error: {1}'.format(user.id, error))
# Log here
raise
except Exception:
db.session.rollback()
raise
@classmethod
def get_libraries(cls, service_uid, absolute_uid):
"""
Get all the libraries a user has
:param service_uid: microservice UID of the user
:param absolute_uid: unique UID of the user in the API
:return: list of libraries in json format
"""
# Get all the permissions for a user
# This can be improved into one database call rather than having
# one per each permission, but needs some checks in place.
result = db.session.query(Permissions, Library)\
.join(Permissions.library)\
.filter(Permissions.user_id == service_uid)\
.all()
output_libraries = []
for permission, library in result:
# For this library get all the people who have permissions
users = Permissions.query.filter(
Permissions.library_id == library.id).all()
num_documents = 0
if library.bibcode:
num_documents = len(library.bibcode)
if permission.owner:
main_permission = 'owner'
elif permission.admin:
main_permission = 'admin'
elif permission.write:
main_permission = 'write'
elif permission.read:
main_permission = 'read'
else:
main_permission = 'none'
if permission.owner or permission.admin and not library.public:
num_users = len(users)
elif library.public:
num_users = len(users)
else:
num_users = 0
service = '{api}/{uid}'.format(
api=current_app.config['BIBLIB_USER_EMAIL_ADSWS_API_URL'],
uid=absolute_uid)
current_app.logger.info(
'Obtaining email of user: {0} [API UID]'.format(absolute_uid))
response = client().get(service)
if response.status_code != 200:
current_app.logger.error('Could not find user in the API'
'database: {0}.'.format(service))
owner = 'Not available'
else:
owner = response.json()['email'].split('@')[0]
payload = dict(
name=library.name,
id='{0}'.format(cls.helper_uuid_to_slug(library.id)),
description=library.description,
num_documents=num_documents,
date_created=library.date_created.isoformat(),
date_last_modified=library.date_last_modified.isoformat(),
permission=main_permission,
public=library.public,
num_users=num_users,
owner=owner)
output_libraries.append(payload)
return output_libraries
# Methods
def get(self):
"""
HTTP GET request that returns all the libraries that belong to a given
user
:return: list of the users libraries with the relevant information
Header:
Must contain the API forwarded user ID of the user accessing the end
point
Post body:
----------
No post content accepted.
Return data:
-----------
name: <string> Name of the library
id: <string> ID of the library
description: <string> Description of the library
num_documents: <int> Number of documents in the library
date_created: <string> ISO date library was created
date_last_modified: <string> ISO date library was last modified
permission: <sting> Permission type, can be: 'read',
'write', 'admin', or 'owner'
public: <boolean> True means it is public
num_users: <int> Number of users with permissions to
this library
owner: <string> Identifier of the user who created
the library
Permissions:
-----------
The following type of user can read a library:
- user scope (authenticated via the API)
"""
# Check that they pass a user id
try:
user = self.helper_get_user_id()
except KeyError:
return err(MISSING_USERNAME_ERROR)
service_uid = \
self.helper_absolute_uid_to_service_uid(absolute_uid=user)
user_libraries = self.get_libraries(service_uid=service_uid,
absolute_uid=user)
return {'libraries': user_libraries}, 200
def post(self):
"""
HTTP POST request that creates a library for a given user
:return: the response for if the library was successfully created
Header:
-------
Must contain the API forwarded user ID of the user accessing the end
point
Post body:
----------
KEYWORD, VALUE
name: <string> name of the library (must be unique
for that user)
description: <string> description of the library
public: <boolean> is the library public to view
bibcode (OPTIONAL): <list> list of bibcodes to add
Return data:
-----------
name: <string> Name of the library
id: <string> ID of the library
description: <string> Description of the library
Permissions:
-----------
The following type of user can create a library
- must be logged in, i.e., scope = user
"""
# Check that they pass a user id
try:
user = self.helper_get_user_id()
except KeyError:
return err(MISSING_USERNAME_ERROR)
# Check if the user exists, if not, generate a user in the database
current_app.logger.info('Checking if the user exists')
if not self.helper_user_exists(absolute_uid=user):
current_app.logger.info(
'User: {0:d}, does not exist.'.format(user))
self.create_user(absolute_uid=user)
current_app.logger.info('User: {0:d}, created.'.format(user))
else:
current_app.logger.info('User already exists.')
# Switch to the service UID and not the API UID
service_uid = \
self.helper_absolute_uid_to_service_uid(absolute_uid=user)
current_app.logger.info(
'user_API: {0:d} is now user_service: {1:d}'.format(
user, service_uid))
# Create the library
try:
data = get_post_data(request,
types=dict(name=unicode,
description=unicode,
public=bool,
bibcode=list))
except TypeError as error:
current_app.logger.error(
'Wrong type passed for POST: {0} [{1}]'.format(
request.data, error))
return err(WRONG_TYPE_ERROR)
try:
library = \
self.create_library(service_uid=service_uid, library_data=data)
except BackendIntegrityError as error:
current_app.logger.error(error)
return err(DUPLICATE_LIBRARY_NAME_ERROR)
except TypeError as error:
current_app.logger.error(error)
return err(WRONG_TYPE_ERROR)
return_data = {
'name': library.name,
'id': '{0}'.format(self.helper_uuid_to_slug(library.id)),
'description': library.description
}
# If they added bibcodes include in the response
if hasattr(library, 'bibcode') and library.bibcode:
return_data['bibcode'] = library.get_bibcodes()
return return_data, 200
