class Graphics(Resource): """Return graphics information for a given bibcode""" scopes = [] rate_limit = [1000, 60 * 60 * 24] decorators = [advertise('scopes', 'rate_limit')] def get(self, bibcode): stime = time.time() try: results = get_graphics(bibcode) except Exception, err: current_app.logger.error('Graphics exception (%s): %s' % (bibcode, err)) return {'msg': 'Unable to get results! (%s)' % err}, 500 if results and results['query'] == 'OK': duration = time.time() - stime current_app.logger.info('Graphics for %s in %s user seconds' % (bibcode, duration)) return results else: current_app.logger.error('Graphics failed (%s): %s' % (bibcode, results.get('error', 'NA'))) return { 'Error': 'Unable to get results!', 'Error Info': results.get('error', 'NA') }, 200
def init_app(self, app): """ Standard flask-extension initialisation :param app: flask app """ if app is not None: self.app = app if "watchman" in app.extensions: raise RuntimeError("Flask application already initialised") app.extensions["watchman"] = self for view_name in self.allowed_endpoints.keys(): if view_name not in self.kwargs.keys(): continue view = self.allowed_endpoints[view_name]["view"] route = self.allowed_endpoints[view_name]["route"] methods = self.allowed_endpoints[view_name]["methods"] # Does the user provide scopes? if view_name in self.kwargs and self.kwargs[view_name].get("scopes", None) != None: user_config = self.kwargs[view_name] view.scopes = user_config.get("scopes", []) view.decorators = user_config.get("decorators", ([advertise("scopes", "rate_limit")])) view.rate_limit = [1000, 60 * 60 * 24] with app.app_context(): current_app.add_url_rule(route, view_func=view.as_view(view_name), methods=methods)
class AuthorNetwork(Resource): '''Returns author network data for a solr query''' decorators = [advertise('scopes', 'rate_limit')] scopes = [] rate_limit = [500, 60 * 60 * 24] def get(self): solr_args = dict(request.args) solr_args["rows"] = min( int( solr_args.get( "rows", [current_app.config.get("VIS_SERVICE_AN_MAX_RECORDS") ])[0]), current_app.config.get("VIS_SERVICE_AN_MAX_RECORDS")) solr_args['fl'] = [ 'author_norm', 'title', 'citation_count', 'read_count', 'bibcode', 'pubdate' ] solr_args['wt'] = 'json' headers = { 'X-Forwarded-Authorization': request.headers.get('Authorization') } response = client().get( current_app.config.get("VIS_SERVICE_SOLR_PATH"), params=solr_args, headers=headers) if response.status_code == 200: full_response = response.json() else: return { "Error": "There was a connection error. Please try again later", "Error Info": response.text }, response.status_code #get_network_with_groups expects a list of normalized authors author_norm = [ d.get("author_norm", []) for d in full_response["response"]["docs"] ] author_network_json = author_network.get_network_with_groups( author_norm, full_response["response"]["docs"]) if author_network_json: return { "msg": { "numFound": full_response["response"]["numFound"], "start": full_response["response"].get("start", 0), "rows": int(full_response["responseHeader"]["params"]["rows"]) }, "data": author_network_json }, 200 else: return {"Error": "Empty network."}, 200
class PositionSearch(Resource): """Return publication information for a cone search""" scopes = [] rate_limit = [1000, 60 * 60 * 24] decorators = [advertise('scopes', 'rate_limit')] def get(self, pstring): # The following position strings are supported # 1. 05 23 34.6 -69 45 22:0 6 (or 05h23m34.6s -69d45m22s:0m6s) # 2. 05 23 34.6 -69 45 22:0.166666 (or 05h23m34.6s -69d45m22s:0.166666) # 3. 80.89416667 -69.75611111:0.166666 stime = time.time() # If we're given a string with qualifiers ('h', etc), convert to one without current_app.logger.info('Attempting SIMBAD position search: %s' % pstring) try: RA, DEC, radius = parse_position_string(pstring) except Exception, err: current_app.logger.error( 'Position string could not be parsed: %s' % pstring) return { 'Error': 'Unable to get results!', 'Error Info': 'Invalid position string: %s' % pstring }, 200 try: result = do_position_query(RA, DEC, radius) except timeout_decorator.timeout_decorator.TimeoutError: current_app.logger.error('Position query %s timed out' % pstring) return { 'Error': 'Unable to get results!', 'Error Info': 'Position query timed out' }, 200 return result
class AllowedMirrors(BaseView): """ End point that returns the allowed list of mirror sites for either ADS classic utilities or BEER/ADS2.0 utilities. """ decorators = [advertise('scopes', 'rate_limit')] scopes = [] rate_limit = [1000, 60 * 60 * 24] def get(self): """ HTTP GET request that returns the list of mirror sites that can be used with this end point. Any end points not listed by this method cannot be used for any of the other methods related to this service. Return data (on success) ------------------------ list[<string>] eg., list of mirrors, ['site1', 'site2', ...., 'siteN'] HTTP Responses: -------------- Succeed authentication: 200 Any other responses will be default Flask errors """ return current_app.config.get('ADS_CLASSIC_MIRROR_LIST', [])
class BigQuery(SolrInterface): """Exposes the bigquery endpoint""" scopes = ['api'] rate_limit = [100, 60 * 60 * 24] decorators = [advertise('scopes', 'rate_limit')] handler = 'SOLR_SERVICE_BIGQUERY_HANDLER' def post(self): payload = dict(request.form) payload.update(request.args) query, headers = self.cleanup_solr_request(payload) if request.files and \ sum([len(i) for i in request.files.listvalues()]) > 1: message = "You can only pass one content stream." current_app.logger.error(message) return json.dumps({'error': message}), 400 if 'fq' not in query: query['fq'] = [u'{!bitset}'] elif len(filter(lambda x: '!bitset' in x, query['fq'])) == 0: query['fq'].append(u'{!bitset}') if 'big-query' not in headers.get('Content-Type', ''): headers['Content-Type'] = 'big-query/csv' if request.data: current_app.logger.info( "Dispatching 'POST' request to endpoint '{}'".format( current_app.config[self.handler])) r = current_app.client.post( current_app.config[self.handler], params=query, data=request.data, headers=headers, cookies=SolrInterface.set_cookies(request), ) current_app.logger.info( "Received response from endpoint '{}' with status code '{}'". format(current_app.config[self.handler], r.status_code)) elif request.files: current_app.logger.info( "Dispatching 'POST' request to endpoint '{}'".format( current_app.config[self.handler])) r = current_app.client.post( current_app.config[self.handler], params=query, headers=headers, files=request.files, cookies=SolrInterface.set_cookies(request), ) current_app.logger.info( "Received response from endpoint '{}' with status code '{}'". format(current_app.config[self.handler], r.status_code)) else: message = "Malformed request" current_app.logger.error(message) return json.dumps({'error': message}), 400 return r.text, r.status_code, r.headers
class ClassicView(HarbourView): """ Wrapper for importing libraries from ADS Classic """ decorators = [advertise('scopes', 'rate_limit')] scopes = ['user'] rate_limit = [1000, 60 * 60 * 24] service_url = 'BIBLIB_CLASSIC_SERVICE_URL'
class TwoPointOhView(HarbourView): """ Wrapper for importing libraries from ADS 2.0 """ decorators = [advertise('scopes', 'rate_limit')] scopes = ['user'] rate_limit = [1000, 60 * 60 * 24] service_url = 'BIBLIB_TWOPOINTOH_SERVICE_URL'
class StatusView(Resource): """Returns the status of this app""" scopes = [] rate_limit = [1000, 60 * 60 * 24] decorators = [advertise('scopes', 'rate_limit')] def get(self): return {'app': current_app.name, 'status': 'online'}, 200
def test_setAttributes(self): """ Test setting attributes of a function with an explicit call to advertise() and with @advertise. Assert that the function code executes as expected after the operators. """ self.testfunction = advertise(thisattr="foo")(self.testfunction) self.assertEqual(self.testfunction._advertised, [{"thisattr": "foo"}]) self.testfunction = advertise(thisattr2="foo2")(self.testfunction) self.assertItemsEqual(self.testfunction._advertised, [{"thisattr": "foo"}, {"thisattr2": "foo2"}]) res = self.testfunction(1, 2, 3) self.assertEqual(res, "1|2|3") @advertise(decor1="foo", decor2="bar") def testfunction(a, b, c): return "{a}|{b}|{c}".format(a=a, b=b, c=c) self.assertItemsEqual(testfunction._advertised, [{"decor1": "foo"}, {"decor2": "bar"}]) res = testfunction(1, 2, 3) self.assertEqual(res, "1|2|3")
def test_setAttributes(self): ''' Test setting attributes of a function with an explicit call to advertise() and with @advertise. Assert that the function code executes as expected after the operators. ''' self.testfunction = advertise(thisattr='foo')(self.testfunction) self.assertEqual(self.testfunction._advertised,[{'thisattr':'foo'}]) self.testfunction = advertise(thisattr2='foo2')(self.testfunction) self.assertItemsEqual(self.testfunction._advertised,[{'thisattr':'foo'},{'thisattr2':'foo2'}]) res = self.testfunction(1,2,3) self.assertEqual(res,'1|2|3') @advertise(decor1='foo',decor2='bar') def testfunction(a,b,c): return "{a}|{b}|{c}".format(a=a,b=b,c=c) self.assertItemsEqual(testfunction._advertised,[{'decor1':'foo'},{'decor2':'bar'}]) res = testfunction(1,2,3) self.assertEqual(res,'1|2|3')
class PrintArg(Resource): """ Returns the :arg in the route """ decorators = [advertise('scopes', 'rate_limit')] scopes = ['scope1', 'scope2'] rate_limit = [1000, 60 * 60 * 24] def get(self, arg): """ HTTP GET request that returns the string passed :param arg: string to send as return :return: argument given to the resource """ return {'arg': arg}, 200
class UnixTime(Resource): """ Returns the unix timestamp of the server """ decorators = [advertise('scopes', 'rate_limit')] scopes = ['scope1', 'scope2'] rate_limit = [1000, 60 * 60 * 24] def get(self): """ HTTP GET request :return: the unix time now """ current_app.logger.info('Example of logging within the app.') return {'now': time.time()}, 200
class BigQuery(Resource): """Exposes the bigquery endpoint""" scopes = ['api'] rate_limit = [100, 60 * 60 * 24] decorators = [advertise('scopes', 'rate_limit')] handler = 'SOLR_SERVICE_BIGQUERY_HANDLER' def post(self): payload = dict(request.form) payload.update(request.args) headers = dict(request.headers) query = SolrInterface.cleanup_solr_request( payload, headers.get('X-Adsws-Uid', 'default')) if request.files and \ sum([len(i) for i in request.files.listvalues()]) > 1: return json.dumps( {'error': 'You can only pass one content stream.'}), 400 if 'fq' not in query: query['fq'] = [u'{!bitset}'] elif len(filter(lambda x: '!bitset' in x, query['fq'])) == 0: query['fq'].append(u'{!bitset}') if 'big-query' not in headers.get('Content-Type', ''): headers['Content-Type'] = 'big-query/csv' if request.data: r = requests.post( current_app.config[self.handler], params=query, data=request.data, headers=headers, cookies=SolrInterface.set_cookies(request), ) elif request.files: r = requests.post( current_app.config[self.handler], params=query, headers=headers, files=request.files, cookies=SolrInterface.set_cookies(request), ) else: return json.dumps({'error': "malformed request"}), 400 return r.text, r.status_code, r.headers
class Export(Resource): """Returns export data for a list of bibcodes""" decorators = [advertise('scopes', 'rate_limit')] def get(self): payload = dict(request.args) return self.get_data_from_classic(payload) def post(self): try: payload = request.get_json(force=True) # post data in json except: payload = dict(request.form) # post data in form encoding return self.get_data_from_classic(payload) def get_data_from_classic(self, payload): if not payload: return {'error': 'no information received'}, 400 elif 'bibcode' not in payload: return { 'error': 'no bibcodes found in payload (parameter ' 'name is "bibcode")' }, 400 headers = {'User-Agent': 'ADS Script Request Agent'} # assign data type based on endpoint payload["data_type"] = self.data_type # actual request r = requests.post( current_app.config.get("EXPORT_SERVICE_CLASSIC_EXPORT_URL"), data=payload, headers=headers) r.raise_for_status() hdr = re.match( current_app.config['EXPORT_SERVICE_CLASSIC_SUCCESS_STRING'], r.text) if not hdr: return {"error": "No records returned from ADS-Classic"}, 400 return { "export": r.text.replace(hdr.group(), ''), "msg": hdr.group().strip().split("\n")[::-1][0] }
class BigQuery(SolrInterface): """Exposes the bigquery endpoint""" scopes = ['api'] rate_limit = [100, 60 * 60 * 24] decorators = [advertise('scopes', 'rate_limit')] handler = 'SOLR_SERVICE_BIGQUERY_HANDLER' def post(self): payload = dict(request.form) payload.update(request.args) if request.is_json: payload.update(request.json) query, headers = self.cleanup_solr_request(payload) files = self.check_for_embedded_bigquery(query, request, headers) if files and len(files) > 0: try: current_user_id = current_user.get_id() except: # If solr service is not shipped with adsws, this will fail and it is ok current_user_id = None if current_user_id: current_app.logger.info( "Dispatching 'POST' request to endpoint '{}' for user '{}'" .format(current_app.config[self.handler], current_user_id)) else: current_app.logger.info( "Dispatching 'POST' request to endpoint '{}'".format( current_app.config[self.handler])) r = requests.post( current_app.config[self.handler], params=query, headers=headers, files=files, cookies=SolrInterface.set_cookies(request), ) current_app.logger.info( "Received response from endpoint '{}' with status code '{}'". format(current_app.config[self.handler], r.status_code)) else: message = "Malformed request" current_app.logger.error(message) return json.dumps({'error': message}), 400 return self.cleanup_solr_response_text( r.text), r.status_code, r.headers
class ExampleApiUsage(Resource): """ This resource uses the client.session.get() method to access an api that requires an oauth2 token, such as our own adsws """ decorators = [advertise('scopes', 'rate_limit')] scopes = ['scope1'] rate_limit = [1000, 60 * 60 * 24] def get(self): """ HTTP GET request using the apps client session defined in the config :return: HTTP response from the API """ r = client().get( current_app.config.get('SAMPLE_APPLICATION_ADSWS_API_URL')) return r.json()
def test_advertiseAttributes(self): """ Test advertising "empty" attributes """ self.testfunction = advertise("thisattr")(self.testfunction) self.assertRaises(AttributeError, lambda: self.testfunction.thisattr) self.assertEqual(self.testfunction._advertised, [{"thisattr": None}]) res = self.testfunction(1, 2, 3) self.assertEqual(res, "1|2|3") @advertise("decor1", "decor2") def testfunction(a, b, c): return "{a}|{b}|{c}".format(a=a, b=b, c=c) self.assertRaises(AttributeError, lambda: testfunction.decor1) self.assertRaises(AttributeError, lambda: testfunction.decor2) self.assertItemsEqual(testfunction._advertised, [{"decor1": None}, {"decor2": None}]) res = testfunction(1, 2, 3) self.assertEqual(res, "1|2|3")
class PubMetrics(Resource): """Get metrics for a single publication (identified by its bibcode)""" scopes = [] rate_limit = [1000, 60 * 60 * 24] decorators = [advertise('scopes', 'rate_limit')] def get(self, bibcode): results = generate_metrics(bibcodes=[bibcode], types=['basic', 'histograms'], histograms=['reads', 'citations']) # If the results contain an error message something went boink if "Error" in results: return results, 500 # otherwise we have real results or an empty dictionary if results: return results else: return { 'Error': 'Unable to get results!', 'Error Info': 'No data available to generate metrics' }, 200
def test_advertiseAttributes(self): ''' Test advertising "empty" attributes ''' self.testfunction = advertise('thisattr')(self.testfunction) self.assertRaises(AttributeError,lambda: self.testfunction.thisattr) self.assertEqual(self.testfunction._advertised,[{'thisattr':None}]) res = self.testfunction(1,2,3) self.assertEqual(res,'1|2|3') @advertise('decor1','decor2') def testfunction(a,b,c): return "{a}|{b}|{c}".format(a=a,b=b,c=c) self.assertRaises(AttributeError,lambda: testfunction.decor1) self.assertRaises(AttributeError,lambda: testfunction.decor2) self.assertItemsEqual(testfunction._advertised,[{'decor1':None},{'decor2':None}]) res = testfunction(1,2,3) self.assertEqual(res,'1|2|3')
class Recommender(Resource): """Return recommender results for a given bibcode""" scopes = [] rate_limit = [1000, 60 * 60 * 24] decorators = [advertise('scopes', 'rate_limit')] def get(self, bibcode): stime = time.time() try: results = get_recommendations(bibcode) except Exception, err: current_app.logger.error('Recommender exception (%s): %s'%(bibcode, err)) return {'msg': 'Unable to get results! (%s)' % err}, 500 if 'Error' in results: current_app.logger.error('Recommender failed (%s): %s'%(bibcode, results.get('Error'))) return results, results['Status Code'] else: duration = time.time() - stime current_app.logger.info('Recommendations for %s in %s user seconds'%(bibcode, duration)) return results
class ClassicUser(BaseView): """ End point to collect the user's ADS Classic information currently stored in the database """ decorators = [advertise('scopes', 'rate_limit')] scopes = ['user'] rate_limit = [1000, 60 * 60 * 24] def get(self): """ HTTP GET request that returns the information currently stored about the user's ADS Classic settings, currently stored in the service database. Return data (on success) ------------------------ classic_email: <string> ADS Classic e-mail of the user classic_mirror: <string> ADS Classic mirror this user belongs to HTTP Responses: -------------- Succeed authentication: 200 User unknown/wrong password/failed authentication: 404 Any other responses will be default Flask errors """ absolute_uid = self.helper_get_user_id() try: user = Users.query.filter(Users.absolute_uid == absolute_uid).one() return { 'classic_email': user.classic_email, 'classic_mirror': user.classic_mirror, 'twopointoh_email': user.twopointoh_email }, 200 except NoResultFound: return err(NO_CLASSIC_ACCOUNT)
def init_app(self, app): """ Standard flask-extension initialisation :param app: flask app """ if app is not None: self.app = app if 'watchman' in app.extensions: raise RuntimeError('Flask application already initialised') app.extensions['watchman'] = self for view_name in self.allowed_endpoints.keys(): if view_name not in self.kwargs.keys(): continue view = self.allowed_endpoints[view_name]['view'] route = self.allowed_endpoints[view_name]['route'] methods = self.allowed_endpoints[view_name]['methods'] # Does the user provide scopes? if view_name in self.kwargs and self.kwargs[view_name].get('scopes', None) != None: user_config = self.kwargs[view_name] view.scopes = user_config.get('scopes', []) view.decorators = user_config.get('decorators', ([advertise('scopes', 'rate_limit')])) view.rate_limit = [1000, 60*60*24] with app.app_context(): current_app.add_url_rule( route, view_func=view.as_view(view_name), methods=methods )
class Search(SolrInterface): """Exposes the solr select endpoint""" scopes = [] rate_limit = [5000, 60 * 60 * 24] decorators = [advertise('scopes', 'rate_limit')] handler = { 'default': 'SOLR_SERVICE_SEARCH_HANDLER', 'default_embedded_bigquery': 'SOLR_SERVICE_BIGQUERY_HANDLER', 'bot': 'BOT_SOLR_SERVICE_SEARCH_HANDLER', 'bot_embedded_bigquery': 'BOT_SOLR_SERVICE_BIGQUERY_HANDLER' } def get_handler_class(self): """Identify bot requests based on their authentication token""" forwarded_authorization = request.headers.get( 'X-Forwarded-Authorization', []) if forwarded_authorization and len(forwarded_authorization) > 7: request_token = forwarded_authorization[7:] else: request_token = request.headers.get('Authorization', [])[7:] if request_token in current_app.config.get('BOT_TOKENS', []): return "bot" else: return "default"
class ScopedView(Resource): '''Scoped route docstring''' scopes = ['default'] decorators = [advertise('scopes')] def get(self): return "scoped route"
class DocumentView(BaseView): """ End point to interact with a specific library, by adding documents and removing documents. You also use this endpoint to delete the entire library as this method should be scoped. """ # TODO: adding tags using PUT for RESTful endpoint? decorators = [advertise('scopes', 'rate_limit')] scopes = ['user'] rate_limit = [1000, 60 * 60 * 24] @classmethod def add_document_to_library(cls, library_id, document_data): """ Adds a document to a user's library :param library_id: the library id to update :param document_data: the meta data of the document :return: number_added: number of documents successfully added """ current_app.logger.info( 'Adding a document: {0} to library_uuid: {1}'.format( document_data, library_id)) # Find the specified library library = Library.query.filter(Library.id == library_id).one() start_length = len(library.bibcode) library.add_bibcodes(document_data['bibcode']) db.session.add(library) db.session.commit() current_app.logger.info('Added: {0} is now {1}'.format( document_data['bibcode'], library.bibcode)) end_length = len(library.bibcode) return end_length - start_length @classmethod def remove_documents_from_library(cls, library_id, document_data): """ Remove a given document from a specific library :param library_id: the unique ID of the library :param document_data: the meta data of the document :return: number_removed: number of documents successfully removed """ current_app.logger.info('Removing a document: {0} from library_uuid: ' '{1}'.format(document_data, library_id)) library = Library.query.filter(Library.id == library_id).one() start_length = len(library.bibcode) library.remove_bibcodes(document_data['bibcode']) db.session.add(library) db.session.commit() current_app.logger.info('Removed document successfully: {0}'.format( library.bibcode)) end_length = len(library.bibcode) return start_length - end_length @staticmethod def update_library(library_id, library_data): """ Update the meta data of the library :param library_id: the unique ID of the library :param library_data: dictionary containing the updateable values :return: values updated """ updateable = ['name', 'description', 'public'] updated = {} library = Library.query.filter(Library.id == library_id).one() for key in library_data: if key not in updateable: continue setattr(library, key, library_data[key]) updated[key] = library_data[key] db.session.add(library) db.session.commit() return updated @staticmethod def delete_library(library_id): """ Delete the entire library from the database :param library_id: the unique ID of the library :return: no return """ library = Library.query.filter(Library.id == library_id).one() db.session.delete(library) db.session.commit() @classmethod def update_access(cls, service_uid, library_id): """ Defines which type of user has delete permissions to a library. :param service_uid: the user ID within this microservice :param library_id: the unique ID of the library :return: boolean, access (True), no access (False) """ update_allowed = ['admin', 'owner'] for access_type in update_allowed: if cls.helper_access_allowed(service_uid=service_uid, library_id=library_id, access_type=access_type): return True return False @classmethod def delete_access(cls, service_uid, library_id): """ Defines which type of user has delete permissions to a library. :param service_uid: the user ID within this microservice :param library_id: the unique ID of the library :return: boolean, access (True), no access (False) """ delete_allowed = cls.helper_access_allowed(service_uid=service_uid, library_id=library_id, access_type='owner') return delete_allowed @classmethod def write_access(cls, service_uid, library_id): """ Defines which type of user has write permissions to a library. :param service_uid: the user ID within this microservice :param library_id: the unique ID of the library :return: boolean, access (True), no access (False) """ read_allowed = ['write', 'admin', 'owner'] for access_type in read_allowed: if cls.helper_access_allowed(service_uid=service_uid, library_id=library_id, access_type=access_type): return True return False @staticmethod def library_name_exists(service_uid, library_name): """ Checks to see if a library name already exists in the user's created libraries :param service_uid: the user ID within this microservice :param library_name: name to check if it exists :return: True (exists), False (does not exist) """ library_names = \ [i.library.name for i in Permissions.query.filter(Permissions.user_id == service_uid, Permissions.owner == True).all()] if library_name in library_names: current_app.logger.error('Name supplied for the library already ' 'exists: "{0}"'.format(library_name)) return True else: return False def post(self, library): """ HTTP POST request that adds a document to a library for a given user :param library: library ID :return: the response for if the library was successfully created Header: ------- Must contain the API forwarded user ID of the user accessing the end point Post body: ---------- KEYWORD, VALUE bibcode: <list> List of bibcodes to be added action: add, remove add - adds a bibcode, remove - removes a bibcode Return data: ----------- number_added: number of documents added (if 'add' is used) number_removed: number of documents removed (if 'remove' is used) Permissions: ----------- The following type of user can add documents: - owner - admin - write """ # Get the user requesting this from the header try: user_editing = self.helper_get_user_id() except KeyError: return err(MISSING_USERNAME_ERROR) # URL safe base64 string to UUID library = self.helper_slug_to_uuid(library) user_editing_uid = \ self.helper_absolute_uid_to_service_uid(absolute_uid=user_editing) # Check the permissions of the user if not self.write_access(service_uid=user_editing_uid, library_id=library): return err(NO_PERMISSION_ERROR) try: data = get_post_data(request, types=dict(bibcode=list, action=unicode)) except TypeError as error: current_app.logger.error( 'Wrong type passed for POST: {0} [{1}]'.format( request.data, error)) return err(WRONG_TYPE_ERROR) if data['action'] == 'add': current_app.logger.info('User requested to add a document') number_added = self.add_document_to_library(library_id=library, document_data=data) current_app.logger.info( 'Successfully added {0} documents to {1} by {2}'.format( number_added, library, user_editing_uid)) return {'number_added': number_added}, 200 elif data['action'] == 'remove': current_app.logger.info('User requested to remove a document') number_removed = self.remove_documents_from_library( library_id=library, document_data=data) current_app.logger.info( 'Successfully removed {0} documents to {1} by {2}'.format( number_removed, library, user_editing_uid)) return {'number_removed': number_removed}, 200 else: current_app.logger.info('User requested a non-standard action') return {}, 400 def put(self, library): """ HTTP PUT request that updates the meta-data of the library :param library: library ID :return: the response for if the library was updated Header: ------- Must contain the API forwarded user ID of the user accessing the end point Post-body: --------- name: name of the library description: description of the library public: boolean Note: The above are optional, they can be empty if needed. Return data: ----------- returns the key/value that was requested to be updated Permissions: ----------- The following type of user can update the 'name', 'library', and 'public': - owner - admin """ try: user = self.helper_get_user_id() except KeyError: return err(MISSING_USERNAME_ERROR) # URL safe base64 string to UUID library = self.helper_slug_to_uuid(library) if not self.helper_user_exists(user): return err(NO_PERMISSION_ERROR) if not self.helper_library_exists(library): return err(MISSING_LIBRARY_ERROR) user_updating_uid = \ self.helper_absolute_uid_to_service_uid(absolute_uid=user) if not self.update_access(service_uid=user_updating_uid, library_id=library): return err(NO_PERMISSION_ERROR) try: library_data = get_post_data(request, types=dict(name=unicode, description=unicode, public=bool)) except TypeError as error: current_app.logger.error( 'Wrong type passed for POST: {0} [{1}]'.format( request.data, error)) return err(WRONG_TYPE_ERROR) # Remove content that is empty for key in library_data.keys(): if library_data[key] == ''.strip(' '): current_app.logger.warning( 'Removing key: {0} as its empty.'.format(key)) library_data.pop(key) # Check for duplicate namaes if 'name' in library_data and \ self.library_name_exists(service_uid=user_updating_uid, library_name=library_data['name']): return err(DUPLICATE_LIBRARY_NAME_ERROR) response = self.update_library(library_id=library, library_data=library_data) return response, 200 def delete(self, library): """ HTTP DELETE request that deletes a library defined by the number passed :param library: library ID :return: the response for it the library was deleted Header: ------- Must contain the API forwarded user ID of the user accessing the end point Post-body: ---------- No post content accepted. Return data: ----------- No data Permissions: ----------- The following type of user can update a library: - owner """ try: user = self.helper_get_user_id() except KeyError: return err(MISSING_USERNAME_ERROR) # URL safe base64 string to UUID library = self.helper_slug_to_uuid(library) if not self.helper_user_exists(user): return err(NO_PERMISSION_ERROR) if not self.helper_library_exists(library): return err(MISSING_LIBRARY_ERROR) user_deleting_uid = \ self.helper_absolute_uid_to_service_uid(absolute_uid=user) try: current_app.logger.info('user_API: {0:d} ' 'requesting to delete library: {1}'.format( user_deleting_uid, library)) if self.delete_access(service_uid=user_deleting_uid, library_id=library): self.delete_library(library_id=library) current_app.logger.info( 'User: {0} deleted library: {1}.'.format( user_deleting_uid, library)) else: current_app.logger.error('User: {0} has incorrect permissions ' 'to delete: {1}.'.format( user_deleting_uid, library)) raise PermissionDeniedError('Incorrect permissions') except NoResultFound as error: current_app.logger.info('Failed to delete: {0}'.format(error)) return err(MISSING_LIBRARY_ERROR) except PermissionDeniedError as error: current_app.logger.info('Failed to delete: {0}'.format(error)) return err(NO_PERMISSION_ERROR) return {}, 200
class ObjectSearch(Resource): """Return object identifiers for a given object string""" scopes = [] rate_limit = [1000, 60 * 60 * 24] decorators = [advertise('scopes', 'rate_limit')] def post(self): stime = time.time() # Get the supplied list of identifiers identifiers = [] objects = [] facets = [] input_type = None for itype in ['identifiers', 'objects', 'facets']: try: identifiers = request.json[itype] identifiers = map(str, identifiers) input_type = itype except: pass if not input_type: current_app.logger.error( 'No identifiers and objects were specified for SIMBAD object query' ) return { "Error": "Unable to get results!", "Error Info": "No identifiers/objects found in POST body" }, 200 # We should either have a list of identifiers, a list of object names or a list of facets if len(identifiers) == 0 and len(objects) == 0 and len(facets) == 0: current_app.logger.error( 'No identifiers, objects or facets were specified for SIMBAD object query' ) return { "Error": "Unable to get results!", "Error Info": "No identifiers/objects found in POST body" }, 200 # How many iden identifiers do we have? id_num = len(identifiers) if id_num == 0: return { "Error": "Unable to get results!", "Error Info": "No identifiers/objects found in POST body" }, 200 # Source to query source = 'simbad' # Now check if we have anything cached for them cached = { id: current_app.cache.get(id.upper()) for id in identifiers if current_app.cache.get(id.upper()) } if source in ['simbad', 'all'] and len(identifiers) > 0: # If we have cached values, filter those out from the initial list if cached: current_app.logger.debug( 'Received %s %s. Using %s entries from cache.' % (id_num, input_type, len(cached))) identifiers = [ id for id in identifiers if not current_app.cache.get(id.upper()) ] if identifiers: ident_upper = [i.upper() for i in identifiers] # We have identifiers, not found in the cache result = get_simbad_data(identifiers, input_type) if 'Error' in result: # An error was returned! current_app.logger.error( 'Failed to find data for SIMBAD %s query!' % input_type) return result else: # We have results! duration = time.time() - stime current_app.logger.info( 'Found objects for SIMBAD %s in %s user seconds.' % (input_type, duration)) # Before returning results, cache them for ident, value in result['data'].items(): current_app.cache.set(ident.upper(), value, timeout=current_app.config.get( 'OBJECTS_CACHE_TIMEOUT')) # Now pick the entries in the results that correspond with the original object names if input_type == 'objects': result['data'] = { k: result['data'].get(k.upper()) for k in identifiers } # If we had results from cache, merge these in if cached: res = cached.copy() res.update(result.get('data', {})) return res # Otherwise just send back the results else: return result.get('data', {}) elif cached: # We only had cached results return cached else: # This should never happen current_app.logger.error( 'No data found, even though we had %s! Should never happen!' % input_type) result = { "Error": "Failed to find data for SIMBAD %s query!" % input_type, "Error Info": "No results found, where results were expected! Needs attention!" } return result
class Tvrh(SolrInterface): """Exposes the solr term-vector histogram endpoint""" scopes = [] rate_limit = [500, 60 * 60 * 24] decorators = [advertise('scopes', 'rate_limit')] handler = 'SOLR_SERVICE_TVRH_HANDLER'
class HarbourView(BaseView): """ End point to import libraries from external systems """ decorators = [advertise('scopes', 'rate_limit')] scopes = ['user'] rate_limit = [1000, 60 * 60 * 24] service_url = 'default' @staticmethod def upsert_library(service_uid, library): """ Upsert a library into the database. This entails: - Adding a library and bibcodes if there is no name conflict - Not adding a library if name matches, but compare bibcodes :param service_uid: microservice UID of the user :param library: dictionary of the form: {'name': str, 'description': str, 'documents': [str, ...., str]} :return: boolean for success """ # Make the permissions user = User.query.filter(User.id == service_uid).one() try: # we are passed the library from classic # and need to find the corresponding library in bumblebee # the corresponding library is the one with # the same name and the same owner # in raw sql, this is essentially # q = "select library.id from library,permissions where library.name='{}' and permissions.library_id=library.id and permissions.user_id={} and permissions.owner=True" # q = q.format(library['name'], user.id) # but, this must be done via the orm api q = db.session.query(Library).join(Permissions).filter(Library.id == Permissions.library_id)\ .filter(Permissions.user_id == user.id).filter(Permissions.owner == True).filter(Library.name == library['name']) lib = q.all() # Raise if there is not exactly one, it should be 1 or 0, but if # multiple are returned, there is some problem if len(lib) == 0: raise NoResultFound current_app.logger.info( 'User does not have a library with this name') elif len(lib) > 1: current_app.logger.warning( 'More than 1 library has the same name,' ' this should not happen: {}'.format(lib)) raise IntegrityError # Get the single record returned, as names are considered unique in # the workflow of creating libraries lib = lib[0] bibcode_before = len(lib.get_bibcodes()) lib.add_bibcodes(library['documents']) bibcode_added = len(lib.get_bibcodes()) - bibcode_before action = 'updated' db.session.add(lib) except NoResultFound: current_app.logger.info( 'Creating library from scratch: {}'.format(library)) permission = Permissions(owner=True) lib = Library( name=library['name'][0:50], description=library['description'][0:200], ) lib.add_bibcodes(library['documents']) lib.permissions.append(permission) user.permissions.append(permission) db.session.add_all([lib, permission, user]) bibcode_added = len(lib.get_bibcodes()) action = 'created' db.session.commit() return { 'library_id': BaseView.helper_uuid_to_slug(lib.id), 'name': lib.name, 'description': lib.description, 'num_added': bibcode_added, 'action': action } # Methods def get(self): """ HTTP GET request that :return: Header: Must contain the API forwarded user ID of the user accessing the end point Post body: ---------- No post content accepted. Return data: ----------- Permissions: ----------- The following type of user can read a library: - user scope (authenticated via the API) """ # Check that they pass a user id try: user = self.helper_get_user_id() except KeyError: return err(MISSING_USERNAME_ERROR) service_uid = self.helper_absolute_uid_to_service_uid( absolute_uid=user) url = '{external_service}/{user_id}'.format( external_service=current_app.config[self.service_url], user_id=user) current_app.logger.info( 'Collecting libraries for user {} from {}'.format(user, url)) response = client().get(url) if response.status_code != 200: return response.json(), response.status_code resp = [] for library in response.json()['libraries']: resp.append( self.upsert_library(service_uid=service_uid, library=library)) return resp, 200
class Search(SolrInterface): """Exposes the solr select endpoint""" scopes = [] rate_limit = [5000, 60 * 60 * 24] decorators = [advertise('scopes', 'rate_limit')] handler = 'SOLR_SERVICE_SEARCH_HANDLER'
class Qtree(SolrInterface): """Exposes the qtree endpoint""" scopes = [] rate_limit = [500, 60 * 60 * 24] decorators = [advertise('scopes', 'rate_limit')] handler = 'SOLR_SERVICE_QTREE_HANDLER'
class QuerySearch(Resource): """Given a Solr query with object names, return a Solr query with SIMBAD identifiers""" scopes = [] rate_limit = [1000, 60 * 60 * 24] decorators = [advertise('scopes', 'rate_limit')] def post(self): stime = time.time() # Get the supplied list of identifiers identifiers = [] query = None itype = None name2id = {} try: query = request.json['query'] input_type = 'query' except: current_app.logger.error( 'No query was specified for SIMBAD object search') return { "Error": "Unable to get results!", "Error Info": "No identifiers/objects found in POST body" }, 200 # If we get the request from BBB, the value of 'query' is actually an array if isinstance(query, list): try: solr_query = query[0] except: solr_query = '' else: solr_query = query current_app.logger.info('Received SIMBAD object query: %s' % solr_query) new_query = solr_query.replace('object:', 'simbid:') # If we receive a (Solr) query string, we need to parse out the object names try: identifiers = get_objects_from_query_string(solr_query) except: current_app.logger.error( 'Parsing the identifiers out of the query string blew up!') return { "Error": "Unable to get results!", "Error Info": "No objects found in query string" }, 200 identifiers = [ iden for iden in identifiers if iden.lower() not in ['object', ':'] ] # How many object names did we fid? id_num = len(identifiers) # Keep a list with the object names we found identifiers_orig = identifiers # If we did not find any object names, there is nothing to do! if id_num == 0: return { "Error": "Unable to get results!", "Error Info": "No identifiers/objects found in POST body" }, 200 # Source to query source = 'simbad' if source in ['simbad', 'all'] and len(identifiers) > 0: if identifiers: for ident in identifiers: result = get_simbad_data([ident], 'objects') if 'Error' in result: # An error was returned! current_app.logger.error( 'Failed to find data for SIMBAD %s query!' % input_type) return result try: SIMBADid = [ e.get('id', 0) for e in result['data'].values() ][0] except: SIMBADid = '0' name2id[ident] = SIMBADid for oname in identifiers: try: SIMBADid = name2id.get(oname) except: SIMBADid = '0' new_query = new_query.replace(oname, SIMBADid) return {"query": new_query} else: # This should never happen current_app.logger.error( 'No data found, even though we had %s! Should never happen!' % input_type) result = { "Error": "Failed to find data for SIMBAD %s query!" % input_type, "Error Info": "No results found, where results were expected! Needs attention!" } return result
class UserView(BaseView): """ End point to create a library for a given user """ decorators = [advertise('scopes', 'rate_limit')] scopes = ['user'] rate_limit = [1000, 60 * 60 * 24] @staticmethod def create_user(absolute_uid): """ Creates a user in the database with a UID from the API :param absolute_uid: UID from the API :return: no return """ try: user = User(absolute_uid=absolute_uid) db.session.add(user) db.session.commit() except IntegrityError as error: current_app.logger.error('IntegrityError. User: {0:d} was not' 'added. Full traceback: {1}'.format( absolute_uid, error)) raise @staticmethod def create_library(service_uid, library_data): """ Creates a library for a user :param service_uid: the user ID within this microservice :param library_data: content needed to create a library :return: no return """ library_data = BaseView.helper_validate_library_data( service_uid=service_uid, library_data=library_data) _name = library_data.get('name') _description = library_data.get('description') _public = bool(library_data.get('public', False)) _bibcode = library_data.get('bibcode', False) try: # Make the library in the library table library = Library(name=_name, description=_description, public=_public) # If the user supplies bibcodes if _bibcode and isinstance(_bibcode, list): # Ensure unique content _bibcode = uniquify(_bibcode) current_app.logger.info( 'User supplied bibcodes: {0}'.format(_bibcode)) library.add_bibcodes(_bibcode) elif _bibcode: current_app.logger.error( 'Bibcode supplied not a list: {0}'.format(_bibcode)) raise TypeError('Bibcode should be a list.') user = User.query.filter(User.id == service_uid).one() # Make the permissions permission = Permissions(owner=True, ) # Use the ORM to link the permissions to the library and user, # so that no commit is required until the complete action is # finished. This means any rollback will not leave a single # library without permissions library.permissions.append(permission) user.permissions.append(permission) db.session.add_all([library, permission, user]) db.session.commit() current_app.logger.info( 'Library: "{0}" made, user_service: {1:d}'.format( library.name, user.id)) return library except IntegrityError as error: # Roll back the changes db.session.rollback() current_app.logger.error('IntegitryError, database has been rolled' 'back. Caused by user_service: {0:d}.' 'Full error: {1}'.format(user.id, error)) # Log here raise except Exception: db.session.rollback() raise @classmethod def get_libraries(cls, service_uid, absolute_uid): """ Get all the libraries a user has :param service_uid: microservice UID of the user :param absolute_uid: unique UID of the user in the API :return: list of libraries in json format """ # Get all the permissions for a user # This can be improved into one database call rather than having # one per each permission, but needs some checks in place. result = db.session.query(Permissions, Library)\ .join(Permissions.library)\ .filter(Permissions.user_id == service_uid)\ .all() output_libraries = [] for permission, library in result: # For this library get all the people who have permissions users = Permissions.query.filter( Permissions.library_id == library.id).all() num_documents = 0 if library.bibcode: num_documents = len(library.bibcode) if permission.owner: main_permission = 'owner' elif permission.admin: main_permission = 'admin' elif permission.write: main_permission = 'write' elif permission.read: main_permission = 'read' else: main_permission = 'none' if permission.owner or permission.admin and not library.public: num_users = len(users) elif library.public: num_users = len(users) else: num_users = 0 service = '{api}/{uid}'.format( api=current_app.config['BIBLIB_USER_EMAIL_ADSWS_API_URL'], uid=absolute_uid) current_app.logger.info( 'Obtaining email of user: {0} [API UID]'.format(absolute_uid)) response = client().get(service) if response.status_code != 200: current_app.logger.error('Could not find user in the API' 'database: {0}.'.format(service)) owner = 'Not available' else: owner = response.json()['email'].split('@')[0] payload = dict( name=library.name, id='{0}'.format(cls.helper_uuid_to_slug(library.id)), description=library.description, num_documents=num_documents, date_created=library.date_created.isoformat(), date_last_modified=library.date_last_modified.isoformat(), permission=main_permission, public=library.public, num_users=num_users, owner=owner) output_libraries.append(payload) return output_libraries # Methods def get(self): """ HTTP GET request that returns all the libraries that belong to a given user :return: list of the users libraries with the relevant information Header: Must contain the API forwarded user ID of the user accessing the end point Post body: ---------- No post content accepted. Return data: ----------- name: <string> Name of the library id: <string> ID of the library description: <string> Description of the library num_documents: <int> Number of documents in the library date_created: <string> ISO date library was created date_last_modified: <string> ISO date library was last modified permission: <sting> Permission type, can be: 'read', 'write', 'admin', or 'owner' public: <boolean> True means it is public num_users: <int> Number of users with permissions to this library owner: <string> Identifier of the user who created the library Permissions: ----------- The following type of user can read a library: - user scope (authenticated via the API) """ # Check that they pass a user id try: user = self.helper_get_user_id() except KeyError: return err(MISSING_USERNAME_ERROR) service_uid = \ self.helper_absolute_uid_to_service_uid(absolute_uid=user) user_libraries = self.get_libraries(service_uid=service_uid, absolute_uid=user) return {'libraries': user_libraries}, 200 def post(self): """ HTTP POST request that creates a library for a given user :return: the response for if the library was successfully created Header: ------- Must contain the API forwarded user ID of the user accessing the end point Post body: ---------- KEYWORD, VALUE name: <string> name of the library (must be unique for that user) description: <string> description of the library public: <boolean> is the library public to view bibcode (OPTIONAL): <list> list of bibcodes to add Return data: ----------- name: <string> Name of the library id: <string> ID of the library description: <string> Description of the library Permissions: ----------- The following type of user can create a library - must be logged in, i.e., scope = user """ # Check that they pass a user id try: user = self.helper_get_user_id() except KeyError: return err(MISSING_USERNAME_ERROR) # Check if the user exists, if not, generate a user in the database current_app.logger.info('Checking if the user exists') if not self.helper_user_exists(absolute_uid=user): current_app.logger.info( 'User: {0:d}, does not exist.'.format(user)) self.create_user(absolute_uid=user) current_app.logger.info('User: {0:d}, created.'.format(user)) else: current_app.logger.info('User already exists.') # Switch to the service UID and not the API UID service_uid = \ self.helper_absolute_uid_to_service_uid(absolute_uid=user) current_app.logger.info( 'user_API: {0:d} is now user_service: {1:d}'.format( user, service_uid)) # Create the library try: data = get_post_data(request, types=dict(name=unicode, description=unicode, public=bool, bibcode=list)) except TypeError as error: current_app.logger.error( 'Wrong type passed for POST: {0} [{1}]'.format( request.data, error)) return err(WRONG_TYPE_ERROR) try: library = \ self.create_library(service_uid=service_uid, library_data=data) except BackendIntegrityError as error: current_app.logger.error(error) return err(DUPLICATE_LIBRARY_NAME_ERROR) except TypeError as error: current_app.logger.error(error) return err(WRONG_TYPE_ERROR) return_data = { 'name': library.name, 'id': '{0}'.format(self.helper_uuid_to_slug(library.id)), 'description': library.description } # If they added bibcodes include in the response if hasattr(library, 'bibcode') and library.bibcode: return_data['bibcode'] = library.get_bibcodes() return return_data, 200