def test_setAttributes(self): ''' Test setting attributes of a function with an explicit call to advertise() and with @advertise. Assert that the function code executes as expected after the operators. ''' self.testfunction = advertise(thisattr='foo')(self.testfunction) self.assertEqual(self.testfunction._advertised, [{'thisattr': 'foo'}]) self.testfunction = advertise(thisattr2='foo2')(self.testfunction) self.assertListEqual(self.testfunction._advertised, [{ 'thisattr': 'foo' }, { 'thisattr2': 'foo2' }]) res = self.testfunction(1, 2, 3) self.assertEqual(res, '1|2|3') @advertise(decor1='foo', decor2='bar') def testfunction(a, b, c): return "{a}|{b}|{c}".format(a=a, b=b, c=c) d = [{'decor1': 'foo'}, {'decor2': 'bar'}] self.assertTrue((testfunction._advertised[0] == d[0] and testfunction._advertised[1] == d[1]) or (testfunction._advertised[1] == d[0] and testfunction._advertised[0] == d[1])) res = testfunction(1, 2, 3) self.assertEqual(res, '1|2|3')
def test_advertiseAttributes(self): ''' Test advertising "empty" attributes ''' self.testfunction = advertise('thisattr')(self.testfunction) self.assertRaises(AttributeError, lambda: self.testfunction.thisattr) self.assertEqual(self.testfunction._advertised, [{'thisattr': None}]) res = self.testfunction(1, 2, 3) self.assertEqual(res, '1|2|3') @advertise('decor1', 'decor2') def testfunction(a, b, c): return "{a}|{b}|{c}".format(a=a, b=b, c=c) self.assertRaises(AttributeError, lambda: testfunction.decor1) self.assertRaises(AttributeError, lambda: testfunction.decor2) d = [{'decor1': None}, {'decor2': None}] self.assertTrue((testfunction._advertised[0] == d[0] and testfunction._advertised[1] == d[1]) or (testfunction._advertised[1] == d[0] and testfunction._advertised[0] == d[1])) res = testfunction(1, 2, 3) self.assertEqual(res, '1|2|3')
class ScopedView(Resource): '''Scoped route docstring''' scopes = ['default'] decorators = [advertise('scopes')] def get(self): return "scoped route"
class PaperNetwork(Resource): '''Returns paper network data for a solr query''' decorators = [advertise('scopes', 'rate_limit')] scopes = [] rate_limit = [500,60*60*24] def post(self): try: required_fields = ['bibcode,title,first_author,year,citation_count,read_count,reference'] response = make_request(request, "PN", required_fields) except QueryException as error: return {'Error' : 'there was a problem with your request', 'Error Info': str(error)}, 403 if response.status_code == 200: full_response = response.json() else: return {"Error": "There was a connection error. Please try again later", "Error Info": response.text}, response.status_code #get_network_with_groups expects a list of normalized authors data = full_response["response"]["docs"] paper_network_json = paper_network.get_papernetwork(data, request.json.get("max_groups", current_app.config.get("VIS_SERVICE_PN_MAX_GROUPS"))) if paper_network_json: return { "msg" : { "numFound" : full_response["response"]["numFound"], "start": full_response["response"].get("start", 0), "rows": int(full_response["responseHeader"]["params"]["rows"]) }, "data" : paper_network_json}, 200 else: return {"Error": "Empty network."}, 200
class AuthorNetwork(Resource): '''Returns author network data for a solr query''' decorators = [advertise('scopes', 'rate_limit')] scopes = [] rate_limit = [500,60*60*24] def post(self): try: required_fields = ['author_norm', 'title', 'citation_count', 'read_count','bibcode', 'pubdate'] response = make_request(request, "AN", required_fields) except QueryException as error: return {'Error' : 'there was a problem with your request', 'Error Info': str(error)}, 403 if response.status_code == 200: full_response = response.json() else: return {"Error": "There was a connection error. Please try again later", "Error Info": response.text}, response.status_code #get_network_with_groups expects a list of normalized authors author_norm = [d.get("author_norm", []) for d in full_response["response"]["docs"]] author_network_json = author_network.get_network_with_groups(author_norm, full_response["response"]["docs"]) if author_network_json: return { "msg" : { "numFound" : full_response["response"]["numFound"], "start": full_response["response"].get("start", 0), "rows": int(full_response["responseHeader"]["params"]["rows"]) }, "data" : author_network_json}, 200 else: return {"Error": "Empty network."}, 200
class PositionSearch(Resource): """Return publication information for a cone search""" scopes = [] rate_limit = [1000, 60 * 60 * 24] decorators = [advertise('scopes', 'rate_limit')] def get(self, pstring): # The following position strings are supported # 1. 05 23 34.6 -69 45 22:0 6 (or 05h23m34.6s -69d45m22s:0m6s) # 2. 05 23 34.6 -69 45 22:0.166666 (or 05h23m34.6s -69d45m22s:0.166666) # 3. 80.89416667 -69.75611111:0.166666 stime = time.time() # If we're given a string with qualifiers ('h', etc), convert to one without current_app.logger.info('Attempting SIMBAD position search: %s' % pstring) try: RA, DEC, radius = parse_position_string(pstring) except Exception, err: current_app.logger.error( 'Position string could not be parsed: %s' % pstring) return { 'Error': 'Unable to get results!', 'Error Info': 'Invalid position string: %s' % pstring }, 200 try: result = do_position_query(RA, DEC, radius) except timeout_decorator.timeout_decorator.TimeoutError: current_app.logger.error('Position query %s timed out' % pstring) return { 'Error': 'Unable to get results!', 'Error Info': 'Position query timed out' }, 200 return result
class AllowedMirrors(BaseView): """ End point that returns the allowed list of mirror sites for either ADS classic utilities or BEER/ADS2.0 utilities. """ decorators = [advertise('scopes', 'rate_limit')] scopes = [] rate_limit = [1000, 60 * 60 * 24] def get(self): """ HTTP GET request that returns the list of mirror sites that can be used with this end point. Any end points not listed by this method cannot be used for any of the other methods related to this service. Return data (on success) ------------------------ list[<string>] eg., list of mirrors, ['site1', 'site2', ...., 'siteN'] HTTP Responses: -------------- Succeed authentication: 200 Any other responses will be default Flask errors """ return current_app.config.get('ADS_CLASSIC_MIRROR_LIST', [])
class Recommender(Resource): """Return recommender results for a given bibcode""" scopes = [] rate_limit = [1000, 60 * 60 * 24] decorators = [advertise('scopes', 'rate_limit')] def get(self, bibcode): stime = time.time() try: results = get_recommendations(bibcode) except Exception, err: current_app.logger.error('Recommender exception (%s): %s'%(bibcode, err)) # If the request for recommendations fails, we just want the UI to ignore and display nothing return {'Error': 'Unable to get results!'}, 200 if 'Error' in results: # In case of Solr connection errors, we want to capture more verbose information if 'Reponse Code' in results: error_code = results['Reponse Code'] error_msg = results['Error Info'] current_app.logger.error('Recommender failed (Solr request failed) for %s: Solr request returned %s (%s)'%(bibcode, error_code, error_msg)) else: current_app.logger.error('Recommender failed (%s): %s'%(bibcode, results.get('Error'))) # If the request for recommendations fails, we just want the UI to ignore and display nothing return {'Error': 'Unable to get results!'}, 200 else: duration = time.time() - stime current_app.logger.info('Recommendations for %s in %s user seconds'%(bibcode, duration)) return results
class ClassicView(HarbourView): """ Wrapper for importing libraries from ADS Classic """ decorators = [advertise('scopes', 'rate_limit')] scopes = ['user'] rate_limit = [1000, 60 * 60 * 24] service_url = 'BIBLIB_CLASSIC_SERVICE_URL'
class TwoPointOhView(HarbourView): """ Wrapper for importing libraries from ADS 2.0 """ decorators = [advertise('scopes', 'rate_limit')] scopes = ['user'] rate_limit = [1000, 60 * 60 * 24] service_url = 'BIBLIB_TWOPOINTOH_SERVICE_URL'
class Graphics(Resource): """Return graphics information for a given bibcode""" scopes = [] rate_limit = [1000, 60 * 60 * 24] decorators = [advertise('scopes', 'rate_limit')] def get(self, bibcode): stime = time.time() results = get_graphics(bibcode) duration = time.time() - stime current_app.logger.info('Graphics for %s in %s user seconds' % (bibcode, duration)) return results, 200
class CitationHelper(Resource): """computes Citation Helper suggestions on the POST body""" scopes = [] rate_limit = [1000, 60 * 60 * 24] decorators = [advertise('scopes', 'rate_limit')] def post(self): stime = time.time() if not request.json or 'bibcodes' not in request.json: current_app.logger.error( 'No bibcodes were provided to Citation Helper') return { 'Error': 'Unable to get results!', 'Error Info': 'No bibcodes found in POST body' }, 200 bibcodes = map(str, request.json['bibcodes']) if len(bibcodes) > \ current_app.config.get('CITATION_HELPER_MAX_SUBMITTED'): current_app.logger.warning( 'Citation Helper called with %s bibcodes. Maximum is: %s!' % (len(bibcodes), current_app.config.get('CITATION_HELPER_MAX_SUBMITTED'))) return { 'Error': 'Unable to get results!', 'Error Info': 'Number of submitted bibcodes exceeds maximum number' }, 200 results = get_suggestions(bibcodes=bibcodes) if "Error" in results: msg = 'Citation Helper request request blew up' if 'Error Info' in results: msg += ' (%s)' % str(results['Error Info']) current_app.logger.error(msg) return results, 200 if results: duration = time.time() - stime current_app.logger.info( 'Citation Helper request successfully completed in %s real seconds' % duration) return results else: current_app.logger.info( 'Citation Helper request returned empty result') return { 'Error': 'Unable to get results!', 'Error Info': 'Could not find anything to suggest' }, 200
class WordCloud(Resource): '''Returns collated tf/idf data for a solr query''' decorators = [advertise('scopes', 'rate_limit')] scopes = [] rate_limit = [500,60*60*24] def post(self): solr_args = request.json if not solr_args: return {'Error' : 'there was a problem with your request', 'Error Info': 'no data provided with request'}, 403 if 'max_groups' in solr_args: del solr_args['min_percent_word'] if 'min_occurrences_word' in solr_args: del solr_args['min_occurrences_word'] elif 'query' in request.json: try: solr_args = json.loads(request.json["query"][0]) except Exception: return {'Error' : 'there was a problem with your request', 'Error Info': 'couldn\'t decode query, it should be json-encoded before being sent (so double encoded)'}, 403 solr_args["rows"] = min(int(solr_args.get("rows", [current_app.config.get("VIS_SERVICE_WC_MAX_RECORDS")])[0]), current_app.config.get("VIS_SERVICE_WC_MAX_RECORDS")) solr_args['fl'] ='abstract,title' solr_args['wt'] = 'json' headers = {'Authorization': request.headers.get('X-Forwarded-Authorization', request.headers.get('Authorization', ''))} response = client().get(current_app.config.get("VIS_SERVICE_SOLR_PATH") , params = solr_args, headers=headers) if response.status_code == 200: data = response.json() else: return {"Error": "There was a connection error. Please try again later", "Error Info": response.text}, response.status_code if data: records = [str(". ".join(d.get('title', '')[:current_app.config.get("VIS_SERVICE_WC_MAX_TITLE_SIZE")]) + ". " + d.get('abstract', '')[:current_app.config.get("VIS_SERVICE_WC_MAX_ABSTRACT_SIZE")]) for d in data["response"]["docs"]] word_cloud_json = word_cloud.generate_wordcloud(records, n_most_common=current_app.config.get("VIS_SERVICE_WC_MAX_WORDS"), accepted_pos=('NN', 'NNP', 'NNS', 'NNPS', 'JJ', 'RB', 'VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ')) if word_cloud_json: return word_cloud_json, 200 else: return {"Error": "Empty word cloud. Try changing your minimum word parameters or expanding your query."}, 200
class ClassicUser(BaseView): """ End point to collect the user's ADS Classic information currently stored in the database """ decorators = [advertise('scopes', 'rate_limit')] scopes = ['user'] rate_limit = [1000, 60 * 60 * 24] def get(self): """ HTTP GET request that returns the information currently stored about the user's ADS Classic settings, currently stored in the service database. Return data (on success) ------------------------ classic_email: <string> ADS Classic e-mail of the user classic_mirror: <string> ADS Classic mirror this user belongs to HTTP Responses: -------------- Succeed authentication: 200 User unknown/wrong password/failed authentication: 404 Any other responses will be default Flask errors """ absolute_uid = self.helper_get_user_id() try: with current_app.session_scope() as session: user = session.query(Users).filter( Users.absolute_uid == absolute_uid).one() return { 'classic_email': user.classic_email, 'classic_mirror': user.classic_mirror, 'twopointoh_email': user.twopointoh_email }, 200 except NoResultFound: return err(NO_CLASSIC_ACCOUNT)
class ClassicObjectSearch(Resource): """Return object NED refcodes for a given object list""" scopes = [] rate_limit = [1000, 60 * 60 * 24] decorators = [advertise('scopes', 'rate_limit')] def post(self): stime = time.time() results = {} # Get the supplied list of identifiers if not request.json or 'objects' not in request.json: current_app.logger.error( 'No objects were provided to Classic Object Search') return { 'Error': 'Unable to get results!', 'Error Info': 'No object names found in POST body' }, 200 results = get_NED_refcodes(request.json) if "Error" in results: current_app.logger.error( 'Classic Object Search request request blew up') return results, 500 duration = time.time() - stime current_app.logger.info( 'Classic Object Search request successfully completed in %s real seconds' % duration) # what output format? try: oformat = request.json['output_format'] except: oformat = 'json' # send the results back in the requested format if oformat == 'json': return results else: output = "\n".join(results['data']) return Response(output, mimetype='text/plain; charset=us-ascii')
def init_app(self, app): """ Standard flask-extension initialisation :param app: flask app """ if app is not None: self.app = app if 'watchman' in app.extensions: raise RuntimeError('Flask application already initialised') app.extensions['watchman'] = self for view_name in self.allowed_endpoints.keys(): if view_name not in self.kwargs.keys(): continue view = self.allowed_endpoints[view_name]['view'] route = self.allowed_endpoints[view_name]['route'] methods = self.allowed_endpoints[view_name]['methods'] # Does the user provide scopes? if view_name in self.kwargs and self.kwargs[view_name].get( 'scopes', None) != None: user_config = self.kwargs[view_name] view.scopes = user_config.get('scopes', []) view.decorators = user_config.get( 'decorators', ([advertise('scopes', 'rate_limit')])) view.rate_limit = [1000, 60 * 60 * 24] with app.app_context(): current_app.add_url_rule(route, view_func=view.as_view(view_name), methods=methods)
class ClassicMyADS(BaseView): """ End point to collect the user's ADS classic libraries with the external ADS Classic end point """ decorators = [advertise('scopes', 'rate_limit')] scopes = ['adsws:internal'] rate_limit = [1000, 60 * 60 * 24] def get(self, uid): """ HTTP GET request that contacts the ADS Classic myADS end point to obtain all the libraries relevant to that user. :param uid: user ID for the API :type uid: int Return data (on success) ------------------------ HTTP Responses: -------------- Succeed getting libraries: 200 User does not have a classic account: 400 ADS Classic give unknown messages: 500 ADS Classic times out: 504 Any other responses will be default Flask errors """ data = {} with current_app.session_scope() as session: try: user = session.query(Users).filter( Users.absolute_uid == uid).one() if not user.classic_email: raise NoResultFound except NoResultFound: current_app.logger.warning( 'User does not have an associated ADS Classic account') return err(NO_CLASSIC_ACCOUNT) url = current_app.config['ADS_CLASSIC_MYADS_URL'].format( mirror='adsabs.harvard.edu', email=user.classic_email) current_app.logger.debug('Obtaining libraries via: {}'.format(url)) try: response = current_app.client.get(url) except requests.exceptions.Timeout: current_app.logger.warning( 'ADS Classic timed out before finishing: {}'.format(url)) return err(CLASSIC_TIMEOUT) if response.status_code != 200: current_app.logger.warning( 'ADS Classic returned an unkown status code: "{}" [code: {}]' .format(response.text, response.status_code)) return err(CLASSIC_UNKNOWN_ERROR) data = response.json() return data, 200
class AuthenticateUserTwoPointOh(BaseView): """ End point to authenticate the user's ADS 2.0 credentials with the external ADS Classic end point (2.0 and Classic accounts are on ADS 2.0, interchangable) """ decorators = [advertise('scopes', 'rate_limit')] scopes = ['user'] rate_limit = [1000, 60 * 60 * 24] def post(self): """ HTTP POST request that receives the user's ADS 2.0 credentials, and then contacts the Classic system to check that what the user provided is indeed valid. If valid, the users ID is stored. Post body: ---------- KEYWORD, VALUE twopointoh_email: <string> ADS 2.0 e-mail of the user twopointoh_password: <string> ADS 2.0 password of the user Return data (on success): ------------------------- twopointoh_authed: <boolean> were they authenticated twopointoh_email: <string> e-mail that authenticated correctly HTTP Responses: -------------- Succeed authentication: 200 Bad/malformed data: 400 User unknown/wrong password/failed authentication: 404 ADS Classic give unknown messages: 500 ADS Classic times out: 504 Any other responses will be default Flask errors """ post_data = get_post_data(request) # Collect the username, password from the request try: twopointoh_email = post_data['twopointoh_email'] twopointoh_password = post_data['twopointoh_password'] except KeyError: current_app.logger.warning( 'User did not provide a required key: {}'.format( traceback.print_exc())) return err(CLASSIC_DATA_MALFORMED) # Create the correct URL url = current_app.config['ADS_CLASSIC_URL'].format( mirror=current_app.config['ADS_TWO_POINT_OH_MIRROR'], ) params = { 'man_cmd': 'elogin', 'man_email': twopointoh_email, 'man_passwd': twopointoh_password } # Authenticate current_app.logger.info( 'User "{email}" trying to authenticate"'.format( email=twopointoh_email)) try: response = current_app.client.post(url, params=params) except requests.exceptions.Timeout: current_app.logger.warning( 'ADS Classic end point timed out, returning to user') return err(CLASSIC_TIMEOUT) if response.status_code >= 500: message, status_code = err(CLASSIC_UNKNOWN_ERROR) message['ads_classic'] = { 'message': response.text, 'status_code': response.status_code } current_app.logger.warning( 'ADS Classic has responded with an unknown error: {}'.format( response.text)) return message, status_code # Sanity check the response email = response.json()['email'] if email != twopointoh_email: current_app.logger.warning( 'User email "{}" does not match ADS return email "{}"'.format( twopointoh_email, email)) return err(CLASSIC_AUTH_FAILED) # Respond to the user based on whether they were successful or not if response.status_code == 200 \ and response.json()['message'] == 'LOGGED_IN' \ and int(response.json()['loggedin']): current_app.logger.info( 'Authenticated successfully "{email}"'.format( email=twopointoh_email)) absolute_uid = self.helper_get_user_id() with current_app.session_scope() as session: try: user = session.query(Users).filter( Users.absolute_uid == absolute_uid).one() current_app.logger.info('User already exists in database') user.twopointoh_email = twopointoh_email except NoResultFound: current_app.logger.info( 'Creating entry in database for user') user = Users(absolute_uid=absolute_uid, twopointoh_email=twopointoh_email) session.add(user) session.commit() current_app.logger.info( 'Successfully saved content for "{}" to database'.format( twopointoh_email)) return { 'twopointoh_email': email, 'twopointoh_authed': True }, 200 return err(HARBOUR_SERVICE_FAIL) else: current_app.logger.warning( 'ADS 2.0 credentials for "{email}" did not succeed"'.format( email=twopointoh_email)) return err(CLASSIC_AUTH_FAILED)
class AuthenticateUserClassic(BaseView): """ End point to authenticate the user's ADS classic credentials with the external ADS Classic end point """ decorators = [advertise('scopes', 'rate_limit')] scopes = ['user'] rate_limit = [1000, 60 * 60 * 24] def post(self): """ HTTP POST request that receives the user's ADS Classic credentials, and then contacts the Classic system to check that what the user provided is indeed valid. If valid, the users ID is stored within the myADS service store. Post body: ---------- KEYWORD, VALUE classic_email: <string> ADS Classic e-mail of the user classic_password: <string> ADS Classic password of the user classic_mirror: <string> ADS Classic mirror this user belongs to Return data (on success): ------------------------- classic_authed: <boolean> were they authenticated classic_email: <string> e-mail that authenticated correctly classic_mirror: <string> ADS Classic mirror this user selected HTTP Responses: -------------- Succeed authentication: 200 Bad/malformed data: 400 User unknown/wrong password/failed authentication: 404 ADS Classic give unknown messages: 500 ADS Classic times out: 504 Any other responses will be default Flask errors """ post_data = get_post_data(request) with current_app.session_scope() as session: # Collect the username, password from the request try: classic_email = post_data['classic_email'] classic_password = post_data['classic_password'] classic_mirror = post_data['classic_mirror'] except KeyError: current_app.logger.warning( 'User did not provide a required key: {}'.format( traceback.print_exc())) return err(CLASSIC_DATA_MALFORMED) # Check that the mirror exists and not man-in-the-middle if classic_mirror not in current_app.config[ 'ADS_CLASSIC_MIRROR_LIST']: current_app.logger.warning( 'User "{}" tried to use a mirror that does not exist: "{}"' .format(classic_email, classic_mirror)) return err(CLASSIC_BAD_MIRROR) # Create the correct URL url = current_app.config['ADS_CLASSIC_URL'].format( mirror=classic_mirror, ) params = { 'man_cmd': 'elogin', 'man_email': classic_email, 'man_passwd': classic_password } # Authenticate current_app.logger.info( 'User "{email}" trying to authenticate at mirror "{mirror}"'. format(email=classic_email, mirror=classic_mirror)) try: response = current_app.client.post(url, params=params) except requests.exceptions.Timeout: current_app.logger.warning( 'ADS Classic end point timed out, returning to user') return err(CLASSIC_TIMEOUT) if response.status_code >= 500: message, status_code = err(CLASSIC_UNKNOWN_ERROR) message['ads_classic'] = { 'message': response.text, 'status_code': response.status_code } current_app.logger.warning( 'ADS Classic has responded with an unknown error: {}'. format(response.text)) return message, status_code # Sanity check the response email = response.json()['email'] if email != classic_email: current_app.logger.warning( 'User email "{}" does not match ADS return email "{}"'. format(classic_email, email)) return err(CLASSIC_AUTH_FAILED) # Respond to the user based on whether they were successful or not if response.status_code == 200 \ and response.json()['message'] == 'LOGGED_IN' \ and int(response.json()['loggedin']): current_app.logger.info( 'Authenticated successfully "{email}" at mirror "{mirror}"' .format(email=classic_email, mirror=classic_mirror)) # Save cookie in myADS try: cookie = response.json()['cookie'] except KeyError: current_app.logger.warning( 'Classic returned no cookie, cannot continue: {}'. format(response.json())) return err(CLASSIC_NO_COOKIE) absolute_uid = self.helper_get_user_id() try: user = session.query(Users).filter( Users.absolute_uid == absolute_uid).one() current_app.logger.info('User already exists in database') user.classic_mirror = classic_mirror user.classic_cookie = cookie user.classic_email = classic_email except NoResultFound: current_app.logger.info( 'Creating entry in database for user') user = Users(absolute_uid=absolute_uid, classic_cookie=cookie, classic_email=classic_email, classic_mirror=classic_mirror) session.add(user) session.commit() current_app.logger.info( 'Successfully saved content for "{}" to database: {{"cookie": "{}"}}' .format(classic_email, '*' * len(user.classic_cookie))) return { 'classic_email': email, 'classic_mirror': classic_mirror, 'classic_authed': True }, 200 else: current_app.logger.warning( 'Credentials for "{email}" did not succeed at mirror "{mirror}"' .format(email=classic_email, mirror=classic_mirror)) return err(CLASSIC_AUTH_FAILED)
class ClassicLibraries(BaseView): """ End point to collect the user's ADS classic libraries with the external ADS Classic end point """ decorators = [advertise('scopes', 'rate_limit')] scopes = ['adsws:internal'] rate_limit = [1000, 60 * 60 * 24] def get(self, uid): """ HTTP GET request that contacts the ADS Classic libraries end point to obtain all the libraries relevant to that user. :param uid: user ID for the API :type uid: int Return data (on success) ------------------------ libraries: <list<dict>> a list of dictionaries, that contains the following for each library entry: name: <string> name of the library description: <string> description of the library documents: <list<string>> list of documents HTTP Responses: -------------- Succeed getting libraries: 200 User does not have a classic account: 400 ADS Classic give unknown messages: 500 ADS Classic times out: 504 Any other responses will be default Flask errors """ with current_app.session_scope() as session: try: user = session.query(Users).filter( Users.absolute_uid == uid).one() if not user.classic_email: raise NoResultFound except NoResultFound: current_app.logger.warning( 'User does not have an associated ADS Classic account') return err(NO_CLASSIC_ACCOUNT) url = current_app.config['ADS_CLASSIC_LIBRARIES_URL'].format( mirror=user.classic_mirror, cookie=user.classic_cookie) current_app.logger.debug('Obtaining libraries via: {}'.format(url)) try: response = current_app.client.get(url) except requests.exceptions.Timeout: current_app.logger.warning( 'ADS Classic timed out before finishing: {}'.format(url)) return err(CLASSIC_TIMEOUT) if response.status_code != 200: current_app.logger.info( 'ADS Classic returned an unkown status code: "{}" [code: {}]' .format(response.text, response.status_code)) return err(CLASSIC_UNKNOWN_ERROR) data = response.json() libraries = [ dict(name=i['name'], description=i.get('desc', ''), documents=[j['bibcode'] for j in i['entries']]) for i in data['libraries'] ] return {'libraries': libraries}, 200
class ExportTwoPointOhLibraries(BaseView): """ End point to return ADS 2.0 libraries in a format that users can use to import them to other services. Currently, the following third-party services are supported: - Zotero (https://www.zotero.org/) - Papers (http://www.papersapp.com/) - Mendeley (https://www.mendeley.com/) """ decorators = [advertise('scopes', 'rate_limit')] scopes = ['user'] rate_limit = [1000, 60 * 60 * 24] def get(self, export): """ HTTP GET request that collects a users ADS 2.0 libraries from the flat files on S3. It returns a file for download of the .bib format. :param export: service to export to :type export: str Return data (on success) ------------------------ BibTeX file (.bib) HTTP Responses: -------------- Succeed getting libraries: 200 User does not have a classic/ADS 2.0 account: 400 User does not have any libraries in their ADS 2.0 account: 400 Unknown error: 500 Any other responses will be default Flask errors """ with current_app.session_scope() as session: if export not in current_app.config['HARBOUR_EXPORT_TYPES']: return err(TWOPOINTOH_WRONG_EXPORT_TYPE) if not current_app.config['ADS_TWO_POINT_OH_LOADED_USERS']: current_app.logger.error( 'Users from MongoDB have not been loaded into the app') return err(TWOPOINTOH_AWS_PROBLEM) absolute_uid = self.helper_get_user_id() try: user = session.query(Users).filter( Users.absolute_uid == absolute_uid).one() # Have they got an email for ADS 2.0? if not user.twopointoh_email: raise NoResultFound except NoResultFound: current_app.logger.warning( 'User does not have an associated ADS Classic/2.0 account') return err(NO_TWOPOINTOH_ACCOUNT) library_file_name = current_app.config[ 'ADS_TWO_POINT_OH_USERS'].get(user.twopointoh_email, None) if not library_file_name: current_app.logger.warning( 'User does not have any libraries in ADS 2.0') return err(NO_TWOPOINTOH_LIBRARIES) try: s3 = boto3.client('s3') s3_presigned_url = s3.generate_presigned_url( ClientMethod='get_object', Params={ 'Bucket': current_app.config['ADS_TWO_POINT_OH_S3_MONGO_BUCKET'], 'Key': library_file_name.replace('.json', '.{}.zip'.format(export)) }, ExpiresIn=1800) except Exception as error: current_app.logger.error( 'Unknown error with AWS: {}'.format(error)) return err(TWOPOINTOH_AWS_PROBLEM) return {'url': s3_presigned_url}, 200
class TwoPointOhLibraries(BaseView): """ End point to collect the user's ADS 2.0 libraries with the MongoDB dump in a flat-file on S3 storage Note: ADS 2.0 user accounts are tied to ADS Classic accounts. Therefore, a user will need to regisiter their ADS Classic/2.0 account on this service for them to also be able to access the ADS 2.0 libraries. """ decorators = [advertise('scopes', 'rate_limit')] scopes = ['adsws:internal'] rate_limit = [1000, 60 * 60 * 24] @staticmethod def get_s3_library(library_file_name): """ Get the JSON MongoDB dump of the ADS 2.0 library of a specific user. These files are stored on S3. :param library_file_name: name of library file :type library_file_name: str :return: dict """ s3_resource = boto3.resource('s3') bucket = s3_resource.Object( current_app.config['ADS_TWO_POINT_OH_S3_MONGO_BUCKET'], library_file_name) body = bucket.get()['Body'] library_data = BytesIO() for chunk in iter(lambda: body.read(1024), b''): library_data.write(chunk) library = json.loads(library_data.getvalue()) return library def get(self, uid): """ HTTP GET request that finds the libraries within ADS 2.0 for that user. :param uid: user ID for the API :type uid: int Return data (on success) ------------------------ libraries: <list<dict>> a list of dictionaries, that contains the following for each library entry: name: <string> name of the library description: <string> description of the library documents: <list<string>> list of documents HTTP Responses: -------------- Succeed getting libraries: 200 User does not have a classic/ADS 2.0 account: 400 User does not have any libraries in their ADS 2.0 account: 400 Unknown error: 500 Any other responses will be default Flask errors """ with current_app.session_scope() as session: if not current_app.config['ADS_TWO_POINT_OH_LOADED_USERS']: current_app.logger.error( 'Users from MongoDB have not been loaded into the app') return err(TWOPOINTOH_AWS_PROBLEM) try: user = session.query(Users).filter( Users.absolute_uid == uid).one() # Have they got an email for ADS 2.0? if not user.twopointoh_email: raise NoResultFound except NoResultFound: current_app.logger.warning( 'User does not have an associated ADS 2.0 account') return err(NO_TWOPOINTOH_ACCOUNT) library_file_name = current_app.config[ 'ADS_TWO_POINT_OH_USERS'].get(user.twopointoh_email, None) if not library_file_name: current_app.logger.warning( 'User does not have any libraries in ADS 2.0') return err(NO_TWOPOINTOH_LIBRARIES) try: library = TwoPointOhLibraries.get_s3_library(library_file_name) except Exception as error: current_app.logger.error( 'Unknown error with AWS: {}'.format(error)) return err(TWOPOINTOH_AWS_PROBLEM) return {'libraries': library}, 200
class UserView(BaseView): """ End point to create a library for a given user """ decorators = [advertise('scopes', 'rate_limit')] scopes = ['user'] rate_limit = [1000, 60 * 60 * 24] @staticmethod def create_user(absolute_uid): """ Creates a user in the database with a UID from the API :param absolute_uid: UID from the API :return: no return """ try: user = User(absolute_uid=absolute_uid) db.session.add(user) db.session.commit() except IntegrityError as error: current_app.logger.error('IntegrityError. User: {0:d} was not' 'added. Full traceback: {1}'.format( absolute_uid, error)) raise @staticmethod def create_library(service_uid, library_data): """ Creates a library for a user :param service_uid: the user ID within this microservice :param library_data: content needed to create a library :return: no return """ library_data = BaseView.helper_validate_library_data( service_uid=service_uid, library_data=library_data) _name = library_data.get('name') _description = library_data.get('description') _public = bool(library_data.get('public', False)) _bibcode = library_data.get('bibcode', False) try: # Make the library in the library table library = Library(name=_name, description=_description, public=_public) # If the user supplies bibcodes if _bibcode and isinstance(_bibcode, list): # Ensure unique content _bibcode = uniquify(_bibcode) current_app.logger.info( 'User supplied bibcodes: {0}'.format(_bibcode)) library.add_bibcodes(_bibcode) elif _bibcode: current_app.logger.error( 'Bibcode supplied not a list: {0}'.format(_bibcode)) raise TypeError('Bibcode should be a list.') user = User.query.filter(User.id == service_uid).one() # Make the permissions permission = Permissions(owner=True, ) # Use the ORM to link the permissions to the library and user, # so that no commit is required until the complete action is # finished. This means any rollback will not leave a single # library without permissions library.permissions.append(permission) user.permissions.append(permission) db.session.add_all([library, permission, user]) db.session.commit() current_app.logger.info( 'Library: "{0}" made, user_service: {1:d}'.format( library.name, user.id)) return library except IntegrityError as error: # Roll back the changes db.session.rollback() current_app.logger.error('IntegitryError, database has been rolled' 'back. Caused by user_service: {0:d}.' 'Full error: {1}'.format(user.id, error)) # Log here raise except Exception: db.session.rollback() raise @classmethod def get_libraries(cls, service_uid, absolute_uid): """ Get all the libraries a user has :param service_uid: microservice UID of the user :param absolute_uid: unique UID of the user in the API :return: list of libraries in json format """ # Get all the permissions for a user # This can be improved into one database call rather than having # one per each permission, but needs some checks in place. result = db.session.query(Permissions, Library)\ .join(Permissions.library)\ .filter(Permissions.user_id == service_uid)\ .all() output_libraries = [] for permission, library in result: # For this library get all the people who have permissions users = Permissions.query.filter( Permissions.library_id == library.id).all() num_documents = 0 if library.bibcode: num_documents = len(library.bibcode) if permission.owner: main_permission = 'owner' elif permission.admin: main_permission = 'admin' elif permission.write: main_permission = 'write' elif permission.read: main_permission = 'read' else: main_permission = 'none' if permission.owner or permission.admin and not library.public: num_users = len(users) elif library.public: num_users = len(users) else: num_users = 0 service = '{api}/{uid}'.format( api=current_app.config['BIBLIB_USER_EMAIL_ADSWS_API_URL'], uid=absolute_uid) current_app.logger.info( 'Obtaining email of user: {0} [API UID]'.format(absolute_uid)) response = client().get(service) if response.status_code != 200: current_app.logger.error('Could not find user in the API' 'database: {0}.'.format(service)) owner = 'Not available' else: owner = response.json()['email'].split('@')[0] payload = dict( name=library.name, id='{0}'.format(cls.helper_uuid_to_slug(library.id)), description=library.description, num_documents=num_documents, date_created=library.date_created.isoformat(), date_last_modified=library.date_last_modified.isoformat(), permission=main_permission, public=library.public, num_users=num_users, owner=owner) output_libraries.append(payload) return output_libraries # Methods def get(self): """ HTTP GET request that returns all the libraries that belong to a given user :return: list of the users libraries with the relevant information Header: Must contain the API forwarded user ID of the user accessing the end point Post body: ---------- No post content accepted. Return data: ----------- name: <string> Name of the library id: <string> ID of the library description: <string> Description of the library num_documents: <int> Number of documents in the library date_created: <string> ISO date library was created date_last_modified: <string> ISO date library was last modified permission: <sting> Permission type, can be: 'read', 'write', 'admin', or 'owner' public: <boolean> True means it is public num_users: <int> Number of users with permissions to this library owner: <string> Identifier of the user who created the library Permissions: ----------- The following type of user can read a library: - user scope (authenticated via the API) """ # Check that they pass a user id try: user = self.helper_get_user_id() except KeyError: return err(MISSING_USERNAME_ERROR) service_uid = \ self.helper_absolute_uid_to_service_uid(absolute_uid=user) user_libraries = self.get_libraries(service_uid=service_uid, absolute_uid=user) return {'libraries': user_libraries}, 200 def post(self): """ HTTP POST request that creates a library for a given user :return: the response for if the library was successfully created Header: ------- Must contain the API forwarded user ID of the user accessing the end point Post body: ---------- KEYWORD, VALUE name: <string> name of the library (must be unique for that user) description: <string> description of the library public: <boolean> is the library public to view bibcode (OPTIONAL): <list> list of bibcodes to add Return data: ----------- name: <string> Name of the library id: <string> ID of the library description: <string> Description of the library Permissions: ----------- The following type of user can create a library - must be logged in, i.e., scope = user """ # Check that they pass a user id try: user = self.helper_get_user_id() except KeyError: return err(MISSING_USERNAME_ERROR) # Check if the user exists, if not, generate a user in the database current_app.logger.info('Checking if the user exists') if not self.helper_user_exists(absolute_uid=user): current_app.logger.info( 'User: {0:d}, does not exist.'.format(user)) self.create_user(absolute_uid=user) current_app.logger.info('User: {0:d}, created.'.format(user)) else: current_app.logger.info('User already exists.') # Switch to the service UID and not the API UID service_uid = \ self.helper_absolute_uid_to_service_uid(absolute_uid=user) current_app.logger.info( 'user_API: {0:d} is now user_service: {1:d}'.format( user, service_uid)) # Create the library try: data = get_post_data(request, types=dict(name=unicode, description=unicode, public=bool, bibcode=list)) except TypeError as error: current_app.logger.error( 'Wrong type passed for POST: {0} [{1}]'.format( request.data, error)) return err(WRONG_TYPE_ERROR) try: library = \ self.create_library(service_uid=service_uid, library_data=data) except BackendIntegrityError as error: current_app.logger.error(error) return err(DUPLICATE_LIBRARY_NAME_ERROR) except TypeError as error: current_app.logger.error(error) return err(WRONG_TYPE_ERROR) return_data = { 'name': library.name, 'id': '{0}'.format(self.helper_uuid_to_slug(library.id)), 'description': library.description } # If they added bibcodes include in the response if hasattr(library, 'bibcode') and library.bibcode: return_data['bibcode'] = library.get_bibcodes() return return_data, 200
class OperationsView(BaseView): """ Endpoint to conduct operations on a given library or set of libraries. Supported operations are union, intersection, difference, copy, and empty. """ decorators = [advertise('scopes', 'rate_limit')] scopes = ['user'] rate_limit = [1000, 60 * 60 * 24] @classmethod def write_access(cls, service_uid, library_id): """ Defines which type of user has write permissions to a library. :param service_uid: the user ID within this microservice :param library_id: the unique ID of the library :return: boolean, access (True), no access (False) """ read_allowed = ['write', 'admin', 'owner'] for access_type in read_allowed: if cls.helper_access_allowed(service_uid=service_uid, library_id=library_id, access_type=access_type): return True return False @classmethod def setops_libraries(cls, library_id, document_data, operation='union'): """ Takes the union of two or more libraries :param library_id: the primary library ID :param document_data: dict containing the list 'libraries' that holds the secondary library IDs :return: list of bibcodes in the union set """ current_app.logger.info( 'User requested to take the {0} of {1} with {2}'.format( operation, library_id, document_data['libraries'])) with current_app.session_scope() as session: # Find the specified library primary_library = session.query(Library).filter_by( id=library_id).one() out_lib = set(primary_library.get_bibcodes()) for lib in document_data['libraries']: if isinstance(lib, str): lib = cls.helper_slug_to_uuid(lib) secondary_library = session.query(Library).filter_by( id=lib).one() if operation == 'union': out_lib = out_lib.union( set(secondary_library.get_bibcodes())) elif operation == 'intersection': out_lib = out_lib.intersection( set(secondary_library.get_bibcodes())) elif operation == 'difference': out_lib = out_lib.difference( set(secondary_library.get_bibcodes())) else: current_app.logger.warning( 'Requested operation {0} is not allowed.'.format( operation)) return if len(out_lib) < 1: current_app.logger.info( 'No records remain after taking the {0} of {1} and {2}'.format( operation, library_id, document_data['libraries'])) return list(out_lib) @classmethod def copy_library(cls, library_id, document_data): """ Copies the contents of one library into another. Does not empty library first; call empty_library on the target library first to do so :param library_id: primary library ID, library to copy :param document_data: dict containing the list 'libraries' which holds one secondary library ID; this is the library to copy over :return: dict containing the metadata of the copied-over library (the secondary library) """ current_app.logger.info( 'User requested to copy the contents of {0} into {1}'.format( library_id, document_data['libraries'])) secondary_libid = document_data['libraries'][0] if isinstance(secondary_libid, str): secondary_libid = cls.helper_slug_to_uuid(secondary_libid) metadata = {} with current_app.session_scope() as session: primary_library = session.query(Library).filter_by( id=library_id).one() good_bib = primary_library.get_bibcodes() secondary_library = session.query(Library).filter_by( id=secondary_libid).one() secondary_library.add_bibcodes(good_bib) metadata['name'] = secondary_library.name metadata['description'] = secondary_library.description metadata['public'] = secondary_library.public session.add(secondary_library) session.commit() return metadata @staticmethod def empty_library(library_id): """ Empties the contents of one library :param library_id: library to empty :return: dict containing the metadata of the emptied library """ current_app.logger.info( 'User requested to empty the contents of {0}'.format(library_id)) metadata = {} with current_app.session_scope() as session: lib = session.query(Library).filter_by(id=library_id).one() lib.remove_bibcodes(lib.get_bibcodes()) metadata['name'] = lib.name metadata['description'] = lib.description metadata['public'] = lib.public session.add(lib) session.commit() return metadata def post(self, library): """ HTTP POST request that conducts operations at the library level. :param library: primary library ID :return: response if operation was successful Header: ------- Must contain the API forwarded user ID of the user accessing the end point Post body: ---------- KEYWORD, VALUE libraries: <list> List of secondary libraries to include in the action (optional, based on action) action: <unicode> union, intersection, difference, copy, empty Actions to perform on given libraries: Union: requires one or more secondary libraries to be passed; takes the union of the primary and secondary library sets; a new library is created Intersection: requires one or more secondary libraries to be passed; takes the intersection of the primary and secondary library sets; a new library is created Difference: requires one or more secondary libraries to be passed; takes the difference between the primary and secondary libraries; the primary library comes first in the operation, so the secondary library is removed from the primary; a new library is created Copy: requires one and only one secondary library to be passed; the primary library will be copied into the secondary library (so the secondary library will be overwritten); no new library is created Empty: secondary libraries are ignored; the primary library will be emptied of its contents, though the library and metadata will remain; no new library is created name: <string> (optional) name of the new library (must be unique for that user); used only for actions in [union, intersection, difference] description: <string> (optional) description of the new library; used only for actions in [union, intersection, difference] public: <boolean> (optional) is the new library public to view; used only for actions in [union, intersection, difference] ----------- Return data: ----------- name: <string> Name of the library id: <string> ID of the library description: <string> Description of the library Permissions: ----------- The following type of user can conduct library operations: - owner - admin - write """ # Get the user requesting this from the header try: user_editing = self.helper_get_user_id() except KeyError: return err(MISSING_USERNAME_ERROR) # URL safe base64 string to UUID try: library_uuid = self.helper_slug_to_uuid(library) except TypeError: return err(BAD_LIBRARY_ID_ERROR) user_editing_uid = \ self.helper_absolute_uid_to_service_uid(absolute_uid=user_editing) # Check the permissions of the user if not self.write_access(service_uid=user_editing_uid, library_id=library_uuid): return err(NO_PERMISSION_ERROR) try: data = get_post_data(request, types=dict(libraries=list, action=str, name=str, description=str, public=bool)) except TypeError as error: current_app.logger.error( 'Wrong type passed for POST: {0} [{1}]'.format( request.data, error)) return err(WRONG_TYPE_ERROR) if data['action'] in ['union', 'intersection', 'difference']: if 'libraries' not in data: return err(NO_LIBRARY_SPECIFIED_ERROR) if 'name' not in data: data['name'] = 'Untitled {0}.'.format(get_date().isoformat()) if 'public' not in data: data['public'] = False if data['action'] == 'copy': if 'libraries' not in data: return err(NO_LIBRARY_SPECIFIED_ERROR) if len(data['libraries']) > 1: return err(TOO_MANY_LIBRARIES_SPECIFIED_ERROR) lib_names = [] with current_app.session_scope() as session: primary = session.query(Library).filter_by(id=library_uuid).one() lib_names.append(primary.name) if 'libraries' in data: for lib in data['libraries']: try: secondary_uuid = self.helper_slug_to_uuid(lib) except TypeError: return err(BAD_LIBRARY_ID_ERROR) secondary = session.query(Library).filter_by( id=secondary_uuid).one() lib_names.append(secondary.name) if data['action'] == 'union': bib_union = self.setops_libraries(library_id=library_uuid, document_data=data, operation='union') current_app.logger.info( 'Successfully took the union of the libraries {0} (IDs: {1}, {2})' .format(', '.join(lib_names), library, ', '.join(data['libraries']))) data['bibcode'] = bib_union if 'description' not in data: description = 'Union of libraries {0} (IDs: {1}, {2})' \ .format(', '.join(lib_names), library, ', '.join(data['libraries'])) # field length capped in model if len(description) > 200: description = 'Union of library {0} (ID: {1}) with {2} other libraries'\ .format(lib_names[0], library, len(lib_names[1:])) data['description'] = description try: library_dict = self.create_library( service_uid=user_editing_uid, library_data=data) except BackendIntegrityError as error: current_app.logger.error(error) return err(DUPLICATE_LIBRARY_NAME_ERROR) except TypeError as error: current_app.logger.error(error) return err(WRONG_TYPE_ERROR) return library_dict, 200 elif data['action'] == 'intersection': bib_intersect = self.setops_libraries(library_id=library_uuid, document_data=data, operation='intersection') current_app.logger.info( 'Successfully took the intersection of the libraries {0} (IDs: {1}, {2})' .format(', '.join(lib_names), library, ', '.join(data['libraries']))) data['bibcode'] = bib_intersect if 'description' not in data: description = 'Intersection of {0} (IDs: {1}, {2})' \ .format(', '.join(lib_names), library, ', '.join(data['libraries'])) if len(description) > 200: description = 'Intersection of {0} (ID: {1}) with {2} other libraries'\ .format(lib_names[0], library, len(lib_names[1:])) data['description'] = description try: library_dict = self.create_library( service_uid=user_editing_uid, library_data=data) except BackendIntegrityError as error: current_app.logger.error(error) return err(DUPLICATE_LIBRARY_NAME_ERROR) except TypeError as error: current_app.logger.error(error) return err(WRONG_TYPE_ERROR) return library_dict, 200 elif data['action'] == 'difference': bib_diff = self.setops_libraries(library_id=library_uuid, document_data=data, operation='difference') current_app.logger.info( 'Successfully took the difference of {0} (ID {2}) - (minus) {1} (ID {3})' .format(lib_names[0], ', '.join(lib_names[1:]), library, ', '.join(data['libraries']))) data['bibcode'] = bib_diff if 'description' not in data: data['description'] = 'Records that are in {0} (ID {2}) but not in {1} (ID {3})' \ .format(lib_names[0], ', '.join(lib_names[1:]), library, ', '.join(data['libraries'])) try: library_dict = self.create_library( service_uid=user_editing_uid, library_data=data) except BackendIntegrityError as error: current_app.logger.error(error) return err(DUPLICATE_LIBRARY_NAME_ERROR) except TypeError as error: current_app.logger.error(error) return err(WRONG_TYPE_ERROR) return library_dict, 200 elif data['action'] == 'copy': library_dict = self.copy_library(library_id=library_uuid, document_data=data) current_app.logger.info( 'Successfully copied {0} (ID {2}) into {1} (ID {3})'.format( lib_names[0], lib_names[1], library, data['libraries'][0])) with current_app.session_scope() as session: libid = self.helper_slug_to_uuid(data['libraries'][0]) library = session.query(Library).filter_by(id=libid).one() bib = library.get_bibcodes() library_dict['bibcode'] = bib return library_dict, 200 elif data['action'] == 'empty': library_dict = self.empty_library(library_id=library_uuid) current_app.logger.info( 'Successfully emptied {0} (ID {1}) of all records'.format( lib_names[0], library)) with current_app.session_scope() as session: library = session.query(Library).filter_by( id=library_uuid).one() bib = library.get_bibcodes() library_dict['bibcode'] = bib return library_dict, 200 else: current_app.logger.info('User requested a non-standard operation') return {}, 400
class QuerySearch(Resource): """Given a Solr query with object names, return a Solr query with SIMBAD identifiers""" scopes = [] rate_limit = [1000, 60 * 60 * 24] decorators = [advertise('scopes', 'rate_limit')] def post(self): stime = time.time() # Get the supplied list of identifiers query = None itype = None name2id = {} try: query = request.json['query'] input_type = 'query' except: current_app.logger.error('No query was specified for the object search') return {"Error": "Unable to get results!", "Error Info": "No identifiers/objects found in POST body"}, 200 # If we get the request from BBB, the value of 'query' is actually an array if isinstance(query, list): solr_query = query[0] else: solr_query = query translated_query = solr_query current_app.logger.info('Received object query: %s'%solr_query) # Check if an explicit target service was specified try: targets = [t.strip() for t in request.json['target'].lower().split(',')] except: targets = ['simbad', 'ned'] # Get the object names and individual object queries from the Solr query try: object_names, object_queries = parse_query_string(solr_query) except Exception, err: current_app.logger.error('Parsing the identifiers out of the query string blew up!') return {"Error": "Unable to get results!", "Error Info": "Parsing the identifiers out of the query string blew up! (%s)"%str(err)}, 200 # If no object names were found, return if len(object_names) == 0: return {"Error": "Unable to get results!", "Error Info": "No identifiers/objects found in Solr object query"}, 200 # First we check if the object query was in fact a cone search. This would have been a queery of the form # object:"80.89416667 -69.75611111:0.166666" # resulting in 1 entry in the variable object_queries (namely: ['object:"80.89416667 -69.75611111:0.166666"']) # and with 1 entry in object_names (namely: [u'80.89416667 -69.75611111:0.166666']). is_cone_search = False if len(object_queries) == 1: # We received a cone search, which needs to be translated into a query in terms of 'simbid' and 'nedid' try: RA, DEC, radius = parse_position_string(object_names[0]) current_app.logger.info('Starting cone search at RA, DEC, radius: {0}, {1}, {2}'.format(RA, DEC, radius)) is_cone_search = True except: pass # If this is a comne search, we go a different path if is_cone_search: result = {'simbad':[], 'ned':[]} simbad_fail = False ned_fail = False result['simbad'] = simbad_position_query(RA, DEC, radius) if 'Error' in result['simbad']: simbad_fail = result['simbad']['Error Info'] result['ned'] = ned_position_query(RA, DEC, radius) if 'Error' in result['ned']: ned_fail = result['ned']['Error Info'] # If both SIMBAD and NED errored out, return an error if simbad_fail and ned_fail: return {"Error": "Unable to get results!", "Error Info": "{0}, {1}".format(simbad_fail, ned_fail)}, 200 # Form the query in terms on simbid and nedid: cone_components = [] if len(result['simbad']) > 0: cone_components.append('simbid:({0})'.format(" OR ".join(result['simbad']))) if len(result['ned']) > 0: cone_components.append('nedid:({0})'.format(" OR ".join(result['ned']))) oquery = "({0})".format(" OR ".join(cone_components)) translated_query = solr_query.replace(object_queries[0], oquery) return {'query': translated_query} # Create the translation map from the object names provided to identifiers indexed in Solr (simbid and nedid) name2id = get_object_translations(object_names, targets) # Now we have all necessary information to created the translated query translated_query = translate_query(solr_query, object_queries, targets, object_names, name2id) return {'query': translated_query}
class DocumentView(BaseView): """ End point to interact with a specific library, by adding documents and removing documents. You also use this endpoint to delete the entire library as this method should be scoped. """ # TODO: adding tags using PUT for RESTful endpoint? decorators = [advertise('scopes', 'rate_limit')] scopes = ['user'] rate_limit = [1000, 60 * 60 * 24] @classmethod def add_document_to_library(cls, library_id, document_data): """ Adds a document to a user's library :param library_id: the library id to update :param document_data: the meta data of the document :return: number_added: number of documents successfully added """ current_app.logger.info( 'Adding a document: {0} to library_uuid: {1}'.format( document_data, library_id)) with current_app.session_scope() as session: # Find the specified library library = session.query(Library).filter_by(id=library_id).one() start_length = len(library.bibcode) library.add_bibcodes(document_data['bibcode']) session.add(library) session.commit() current_app.logger.info('Added: {0} is now {1}'.format( document_data['bibcode'], library.bibcode)) end_length = len(library.bibcode) return end_length - start_length @classmethod def remove_documents_from_library(cls, library_id, document_data): """ Remove a given document from a specific library :param library_id: the unique ID of the library :param document_data: the meta data of the document :return: number_removed: number of documents successfully removed """ current_app.logger.info('Removing a document: {0} from library_uuid: ' '{1}'.format(document_data, library_id)) with current_app.session_scope() as session: library = session.query(Library).filter_by(id=library_id).one() start_length = len(library.bibcode) library.remove_bibcodes(document_data['bibcode']) session.add(library) session.commit() current_app.logger.info( 'Removed document successfully: {0}'.format(library.bibcode)) end_length = len(library.bibcode) return start_length - end_length @staticmethod def update_library(library_id, library_data): """ Update the meta data of the library :param library_id: the unique ID of the library :param library_data: dictionary containing the updateable values :return: values updated """ updateable = ['name', 'description', 'public'] updated = {} with current_app.session_scope() as session: library = session.query(Library).filter_by(id=library_id).one() for key in library_data: if key not in updateable: continue setattr(library, key, library_data[key]) updated[key] = library_data[key] session.add(library) session.commit() return updated @staticmethod def delete_library(library_id): """ Delete the entire library from the database :param library_id: the unique ID of the library :return: no return """ with current_app.session_scope() as session: library = session.query(Library).filter_by(id=library_id).one() session.delete(library) session.commit() @classmethod def update_access(cls, service_uid, library_id): """ Defines which type of user has delete permissions to a library. :param service_uid: the user ID within this microservice :param library_id: the unique ID of the library :return: boolean, access (True), no access (False) """ update_allowed = ['admin', 'owner'] for access_type in update_allowed: if cls.helper_access_allowed(service_uid=service_uid, library_id=library_id, access_type=access_type): return True return False @classmethod def delete_access(cls, service_uid, library_id): """ Defines which type of user has delete permissions to a library. :param service_uid: the user ID within this microservice :param library_id: the unique ID of the library :return: boolean, access (True), no access (False) """ delete_allowed = cls.helper_access_allowed(service_uid=service_uid, library_id=library_id, access_type='owner') return delete_allowed @classmethod def write_access(cls, service_uid, library_id): """ Defines which type of user has write permissions to a library. :param service_uid: the user ID within this microservice :param library_id: the unique ID of the library :return: boolean, access (True), no access (False) """ read_allowed = ['write', 'admin', 'owner'] for access_type in read_allowed: if cls.helper_access_allowed(service_uid=service_uid, library_id=library_id, access_type=access_type): return True return False @staticmethod def library_name_exists(service_uid, library_name): """ Checks to see if a library name already exists in the user's created libraries :param service_uid: the user ID within this microservice :param library_name: name to check if it exists :return: True (exists), False (does not exist) """ with current_app.session_scope() as session: library_names = \ [i.library.name for i in session.query(Permissions)\ .filter_by(user_id = service_uid)\ .filter(Permissions.permissions['owner'].astext.cast(Boolean).is_(True)).all()] if library_name in library_names: current_app.logger.error('Name supplied for the library already ' 'exists: "{0}"'.format(library_name)) return True else: return False def post(self, library): """ HTTP POST request that adds a document to a library for a given user :param library: library ID :return: the response for if the library was successfully created Header: ------- Must contain the API forwarded user ID of the user accessing the end point Post body: ---------- KEYWORD, VALUE bibcode: <list> List of bibcodes to be added action: add, remove add - adds a bibcode, remove - removes a bibcode Return data: ----------- number_added: number of documents added (if 'add' is used) number_removed: number of documents removed (if 'remove' is used) Permissions: ----------- The following type of user can add documents: - owner - admin - write """ # Get the user requesting this from the header try: user_editing = self.helper_get_user_id() except KeyError: return err(MISSING_USERNAME_ERROR) # URL safe base64 string to UUID try: library = self.helper_slug_to_uuid(library) except TypeError: return err(BAD_LIBRARY_ID_ERROR) user_editing_uid = \ self.helper_absolute_uid_to_service_uid(absolute_uid=user_editing) # Check the permissions of the user if not self.write_access(service_uid=user_editing_uid, library_id=library): return err(NO_PERMISSION_ERROR) try: data = get_post_data(request, types=dict(bibcode=list, action=str)) except TypeError as error: current_app.logger.error( 'Wrong type passed for POST: {0} [{1}]'.format( request.data, error)) return err(WRONG_TYPE_ERROR) if data['action'] == 'add': current_app.logger.info('User requested to add a document') number_added = self.add_document_to_library(library_id=library, document_data=data) current_app.logger.info( 'Successfully added {0} documents to {1} by {2}'.format( number_added, library, user_editing_uid)) return {'number_added': number_added}, 200 elif data['action'] == 'remove': current_app.logger.info('User requested to remove a document') number_removed = self.remove_documents_from_library( library_id=library, document_data=data) current_app.logger.info( 'Successfully removed {0} documents to {1} by {2}'.format( number_removed, library, user_editing_uid)) return {'number_removed': number_removed}, 200 else: current_app.logger.info('User requested a non-standard action') return {}, 400 def put(self, library): """ HTTP PUT request that updates the meta-data of the library :param library: library ID :return: the response for if the library was updated Header: ------- Must contain the API forwarded user ID of the user accessing the end point Post-body: --------- name: name of the library description: description of the library public: boolean Note: The above are optional, they can be empty if needed. Return data: ----------- returns the key/value that was requested to be updated Permissions: ----------- The following type of user can update the 'name', 'library', and 'public': - owner - admin """ try: user = self.helper_get_user_id() except KeyError: return err(MISSING_USERNAME_ERROR) # URL safe base64 string to UUID try: library = self.helper_slug_to_uuid(library) except TypeError: return err(BAD_LIBRARY_ID_ERROR) if not self.helper_user_exists(user): return err(NO_PERMISSION_ERROR) if not self.helper_library_exists(library): return err(MISSING_LIBRARY_ERROR) user_updating_uid = \ self.helper_absolute_uid_to_service_uid(absolute_uid=user) if not self.update_access(service_uid=user_updating_uid, library_id=library): return err(NO_PERMISSION_ERROR) try: library_data = get_post_data(request, types=dict(name=str, description=str, public=bool)) except TypeError as error: current_app.logger.error( 'Wrong type passed for POST: {0} [{1}]'.format( request.data, error)) return err(WRONG_TYPE_ERROR) # Remove content that is empty (note that the list() is necessary to create a copy, so pop will work) for key in list(library_data.keys()): if library_data[key] == ''.strip(' '): current_app.logger.warning( 'Removing key: {0} as its empty.'.format(key)) library_data.pop(key) # Check for duplicate namaes if 'name' in library_data and \ self.library_name_exists(service_uid=user_updating_uid, library_name=library_data['name']): return err(DUPLICATE_LIBRARY_NAME_ERROR) response = self.update_library(library_id=library, library_data=library_data) return response, 200 def delete(self, library): """ HTTP DELETE request that deletes a library defined by the number passed :param library: library ID :return: the response for it the library was deleted Header: ------- Must contain the API forwarded user ID of the user accessing the end point Post-body: ---------- No post content accepted. Return data: ----------- No data Permissions: ----------- The following type of user can update a library: - owner """ try: user = self.helper_get_user_id() except KeyError: return err(MISSING_USERNAME_ERROR) # URL safe base64 string to UUID try: library = self.helper_slug_to_uuid(library) except TypeError: return err(BAD_LIBRARY_ID_ERROR) if not self.helper_user_exists(user): return err(NO_PERMISSION_ERROR) if not self.helper_library_exists(library): return err(MISSING_LIBRARY_ERROR) user_deleting_uid = \ self.helper_absolute_uid_to_service_uid(absolute_uid=user) try: current_app.logger.info('user_API: {0:d} ' 'requesting to delete library: {1}'.format( user_deleting_uid, library)) if self.delete_access(service_uid=user_deleting_uid, library_id=library): self.delete_library(library_id=library) current_app.logger.info( 'User: {0} deleted library: {1}.'.format( user_deleting_uid, library)) else: current_app.logger.error('User: {0} has incorrect permissions ' 'to delete: {1}.'.format( user_deleting_uid, library)) raise PermissionDeniedError('Incorrect permissions') except NoResultFound as error: current_app.logger.info('Failed to delete: {0}'.format(error)) return err(MISSING_LIBRARY_ERROR) except PermissionDeniedError as error: current_app.logger.info('Failed to delete: {0}'.format(error)) return err(NO_PERMISSION_ERROR) return {}, 200
class QuerySearch(Resource): """Given a Solr query with object names, return a Solr query with SIMBAD identifiers""" scopes = [] rate_limit = [1000, 60 * 60 * 24] decorators = [advertise('scopes', 'rate_limit')] def post(self): stime = time.time() # Get the supplied list of identifiers identifiers = [] query = None itype = None name2id = {} try: query = request.json['query'] input_type = 'query' except: current_app.logger.error( 'No query was specified for the object search') return { "Error": "Unable to get results!", "Error Info": "No identifiers/objects found in POST body" }, 200 # If we get the request from BBB, the value of 'query' is actually an array if isinstance(query, list): solr_query = query[0] else: solr_query = query current_app.logger.info('Received object query: %s' % solr_query) # This query will be split up into two components: a SIMBAD and a NED object query simbad_query = solr_query.replace('object:', 'simbid:') ned_query = solr_query.replace('object:', 'nedid:') # Check if an explicit target service was specified try: target = request.json['target'] except: target = 'all' # If we receive a (Solr) query string, we need to parse out the object names try: identifiers = get_objects_from_query_string(solr_query) except Exception, err: current_app.logger.error( 'Parsing the identifiers out of the query string blew up!') return { "Error": "Unable to get results!", "Error Info": "Parsing the identifiers out of the query string blew up! (%s)" % str(err) }, 200 identifiers = [ iden for iden in identifiers if iden.lower() not in ['object', ':'] ] # How many object names did we fid? id_num = len(identifiers) # Keep a list with the object names we found identifiers_orig = identifiers # If we did not find any object names, there is nothing to do! if id_num == 0: return { "Error": "Unable to get results!", "Error Info": "No identifiers/objects found in Solr object query" }, 200 # Get translations simbad_query = '' ned_query = '' translated_query = '' if target.lower() in ['simbad', 'all']: name2simbid = {} for ident in identifiers: result = get_simbad_data([ident], 'objects') if 'Error' in result or 'data' not in result: # An error was returned! current_app.logger.error( 'Failed to find data for SIMBAD object {0}!: {1}'. format(ident, result.get('Error Info', 'NA'))) name2simbid[ident] = 0 continue try: SIMBADid = [ e.get('id', 0) for e in result['data'].values() ][0] except: SIMBADid = "0" name2simbid[ident] = SIMBADid simbad_query = translate_query(solr_query, identifiers, name2simbid, 'simbid:') if target.lower() in ['ned', 'all']: name2nedid = {} for ident in identifiers: result = get_ned_data([ident], 'objects') if 'Error' in result or 'data' not in result: # An error was returned! current_app.logger.error( 'Failed to find data for NED object {0}!: {1}'.format( ident, result.get('Error Info', 'NA'))) name2nedid[ident] = 0 continue try: NEDid = [e.get('id', 0) for e in result['data'].values()][0] except: NEDid = 0 name2nedid[ident] = str(NEDid) ned_query = translate_query(solr_query, identifiers, name2nedid, 'nedid:') if simbad_query and ned_query: translated_query = '({0}) OR ({1})'.format(simbad_query, ned_query) elif simbad_query: translated_query = simbad_query elif ned_query: translated_query = ned_query else: translated_query = 'simbid:0' return {'query': translated_query}
class LibraryView(BaseView): """ End point to interact with a specific library, only returns library content if the user has the correct privileges. The GET requests are separate from the POST, DELETE requests as this class must be scopeless, whereas the others will have scope. """ decorators = [advertise('scopes', 'rate_limit')] scopes = [] rate_limit = [1000, 60 * 60 * 24] @classmethod def get_documents_from_library(cls, library_id, service_uid): """ Retrieve all the documents that are within the library specified :param library_id: the unique ID of the library :param service_uid: the user ID within this microservice :return: bibcodes """ with current_app.session_scope() as session: # Get the library library = session.query(Library).filter_by(id=library_id).one() # Get the owner of the library result = session.query(Permissions, User)\ .join(Permissions.user)\ .filter(Permissions.library_id == library_id) \ .filter(Permissions.permissions['owner'].astext.cast(Boolean).is_(True))\ .one() owner_permissions, owner = result service = '{api}/{uid}'.format( api=current_app.config['BIBLIB_USER_EMAIL_ADSWS_API_URL'], uid=owner.absolute_uid) current_app.logger.info( 'Obtaining email of user: {0} [API UID]'.format( owner.absolute_uid)) response = client().get(service) # For this library get all the people who have permissions users = session.query(Permissions).filter_by( library_id=library.id).all() if response.status_code != 200: current_app.logger.error('Could not find user in the API' 'database: {0}.'.format(service)) owner = 'Not available' else: owner = response.json()['email'].split('@')[0] # User requesting to see the content if service_uid: try: permission = session.query(Permissions).filter( Permissions.user_id == service_uid).filter( Permissions.library_id == library_id).one() if permission.permissions['owner']: main_permission = 'owner' elif permission.permissions['admin']: main_permission = 'admin' elif permission.permissions['write']: main_permission = 'write' elif permission.permissions['read']: main_permission = 'read' else: main_permission = 'none' except: main_permission = 'none' else: main_permission = 'none' if main_permission == 'owner' or main_permission == 'admin': num_users = len(users) elif library.public: num_users = len(users) else: num_users = 0 metadata = dict( name=library.name, id='{0}'.format(cls.helper_uuid_to_slug(library.id)), description=library.description, num_documents=len(library.bibcode), date_created=library.date_created.isoformat(), date_last_modified=library.date_last_modified.isoformat(), permission=main_permission, public=library.public, num_users=num_users, owner=owner) session.refresh(library) session.expunge(library) return library, metadata @classmethod def read_access(cls, service_uid, library_id): """ Defines which type of user has read permissions to a library. :param service_uid: the user ID within this microservice :param library_id: the unique ID of the library :return: boolean, access (True), no access (False) """ read_allowed = ['read', 'write', 'admin', 'owner'] for access_type in read_allowed: if cls.helper_access_allowed(service_uid=service_uid, library_id=library_id, access_type=access_type): return True return False @staticmethod def solr_big_query(bibcodes, start=0, rows=20, sort='date desc', fl='bibcode'): """ A thin wrapper for the solr bigquery service. :param bibcodes: bibcodes :type bibcodes: list :param start: start index :type start: int :param rows: number of rows :type rows: int :param sort: how the response should be sorted :type sort: str :param fl: Solr fields to be returned :type fl: str :return: solr bigquery end point response """ bibcodes_string = 'bibcode\n' + '\n'.join(bibcodes) # We need atleast bibcode and alternate bibcode for other methods # to work properly if fl == '': fl = 'bibcode,alternate_bibcode' else: fl_split = fl.split(',') for required_fl in ['bibcode', 'alternate_bibcode']: if required_fl not in fl_split: fl = '{},{}'.format(fl, required_fl) params = { 'q': '*:*', 'wt': 'json', 'fl': fl, 'rows': rows, 'start': start, 'fq': '{!bitset}', 'sort': sort } headers = { 'Content-Type': 'big-query/csv', 'Authorization': current_app.config.get( 'SERVICE_TOKEN', request.headers.get('X-Forwarded-Authorization', request.headers.get('Authorization', ''))) } current_app.logger.info( 'Querying Solr bigquery microservice: {0}, {1}'.format( params, bibcodes_string.replace('\n', ','))) response = client().post( url=current_app.config['BIBLIB_SOLR_BIG_QUERY_URL'], params=params, data=bibcodes_string, headers=headers) return response @staticmethod def solr_update_library(library_id, solr_docs): """ Updates the library based on the solr canonical bibcodes response :param library: library_id of the library to update :param solr_docs: solr docs from the bigquery response :return: dictionary with details of files modified num_updated: number of documents modified duplicates_removed: number of files removed for duplication update_list: list of changed bibcodes {'before': 'after'} """ # Definitions update = False canonical_bibcodes = [] alternate_bibcodes = {} new_bibcode = {} # Constants for the return dictionary num_updated = 0 duplicates_removed = 0 update_list = [] # Extract the canonical bibcodes and create a hashmap for the # alternate bibcodes for doc in solr_docs: canonical_bibcodes.append(doc['bibcode']) if doc.get('alternate_bibcode'): alternate_bibcodes.update( {i: doc['bibcode'] for i in doc['alternate_bibcode']}) with current_app.session_scope() as session: library = session.query(Library).filter( Library.id == library_id).one() for bibcode in library.bibcode: # Skip if its already canonical if bibcode in canonical_bibcodes: new_bibcode[bibcode] = library.bibcode[bibcode] continue # Update if its an alternate if bibcode in alternate_bibcodes.keys(): update = True num_updated += 1 update_list.append({bibcode: alternate_bibcodes[bibcode]}) # Only add the bibcode if it is not there if alternate_bibcodes[bibcode] not in new_bibcode: new_bibcode[alternate_bibcodes[bibcode]] = \ library.bibcode[bibcode] else: duplicates_removed += 1 elif bibcode not in canonical_bibcodes and\ bibcode not in alternate_bibcodes.keys(): new_bibcode[bibcode] = library.bibcode[bibcode] if update: # Update the database library.bibcode = new_bibcode session.add(library) session.commit() updates = dict(num_updated=num_updated, duplicates_removed=duplicates_removed, update_list=update_list) return updates # Methods def get(self, library): """ HTTP GET request that returns all the documents inside a given user's library :param library: library ID :return: list of the users libraries with the relevant information Header: ------- Must contain the API forwarded user ID of the user accessing the end point Post body: ---------- No post content accepted. Return data: ----------- documents: <list> Currently, a list containing the bibcodes. solr: <dict> The response from the solr bigquery end point metadata: <dict> contains the following: name: <string> Name of the library id: <string> ID of the library description: <string> Description of the library num_documents: <int> Number of documents in the library date_created: <string> ISO date library was created date_last_modified: <string> ISO date library was last modified permission: <sting> Permission type, can be: 'read', 'write', 'admin', or 'owner' public: <boolean> True means it is public num_users: <int> Number of users with permissions to this library owner: <string> Identifier of the user who created the library updates: <dict> contains the following num_updated: <int> Number of documents modified based on the response from solr duplicates_removed: <int> Number of files removed because they are duplications update_list: <list>[<dict>] List of dictionaries such that a single element described the original bibcode (key) and the updated bibcode now being stored (item) Permissions: ----------- The following type of user can read a library: - owner - admin - write - read Default Pagination Values: ----------- - start: 0 - rows: 20 (max 100) - sort: 'date desc' - fl: 'bibcode' """ try: user = int(request.headers[USER_ID_KEYWORD]) except KeyError: current_app.logger.error('No username passed') return err(MISSING_USERNAME_ERROR) # Parameters to be forwarded to Solr: pagination, and fields try: start = int(request.args.get('start', 0)) max_rows = current_app.config.get('BIBLIB_MAX_ROWS', 100) max_rows *= float( request.headers.get('X-Adsws-Ratelimit-Level', 1.0)) max_rows = int(max_rows) rows = min(int(request.args.get('rows', 20)), max_rows) except ValueError: start = 0 rows = 20 sort = request.args.get('sort', 'date desc') fl = request.args.get('fl', 'bibcode') current_app.logger.info('User gave pagination parameters:' 'start: {}, ' 'rows: {}, ' 'sort: "{}", ' 'fl: "{}"'.format(start, rows, sort, fl)) try: library = self.helper_slug_to_uuid(library) except TypeError: return err(BAD_LIBRARY_ID_ERROR) current_app.logger.info('User: {0} requested library: {1}'.format( user, library)) user_exists = self.helper_user_exists(absolute_uid=user) if user_exists: service_uid = \ self.helper_absolute_uid_to_service_uid(absolute_uid=user) else: service_uid = None # If the library is public, allow access try: # Try to load the dictionary and obtain the solr content library, metadata = self.get_documents_from_library( library_id=library, service_uid=service_uid) # pay attention to any functions that try to mutate the list # this will alter expected returns later try: solr = self.solr_big_query(bibcodes=library.bibcode, start=start, rows=rows, sort=sort, fl=fl).json() except Exception as error: current_app.logger.warning( 'Could not parse solr data: {0}'.format(error)) solr = {'error': 'Could not parse solr data'} # Now check if we can update the library database based on the # returned canonical bibcodes if solr.get('response'): # Update bibcodes based on solrs response updates = self.solr_update_library( library_id=library.id, solr_docs=solr['response']['docs']) documents = [i['bibcode'] for i in solr['response']['docs']] else: # Some problem occurred, we will just ignore it, but will # definitely log it. solr = SOLR_RESPONSE_MISMATCH_ERROR['body'] current_app.logger.warning( 'Problem with solr response: {0}'.format(solr)) updates = {} documents = library.get_bibcodes() documents.sort() documents = documents[start:start + rows] # Make the response dictionary response = dict(documents=documents, solr=solr, metadata=metadata, updates=updates) except Exception as error: current_app.logger.warning( 'Library missing or solr endpoint failed: {0}'.format(error)) return err(MISSING_LIBRARY_ERROR) # Skip anymore logic if the library is public or the exception token is present special_token = current_app.config.get('READONLY_ALL_LIBRARIES_TOKEN') if library.public or (special_token and request.headers.get( 'Authorization', '').endswith(special_token)): current_app.logger.info('Library: {0} is public'.format( library.id)) return response, 200 else: current_app.logger.warning('Library: {0} is private'.format( library.id)) # If the user does not exist then there are no associated permissions # If the user exists, they will have permissions if self.helper_user_exists(absolute_uid=user): service_uid = \ self.helper_absolute_uid_to_service_uid(absolute_uid=user) else: current_app.logger.error( 'User:{0} does not exist in the database.' ' Therefore will not have extra ' 'privileges to view the library: {1}'.format(user, library.id)) return err(NO_PERMISSION_ERROR) # If they do not have access, exit if not self.read_access(service_uid=service_uid, library_id=library.id): current_app.logger.error( 'User: {0} does not have access to library: {1}. DENIED'. format(service_uid, library.id)) return err(NO_PERMISSION_ERROR) # If they have access, let them obtain the requested content current_app.logger.info('User: {0} has access to library: {1}. ' 'ALLOWED'.format(user, library.id)) return response, 200
class ObjectSearch(Resource): """Return object identifiers for a given object string""" scopes = [] rate_limit = [1000, 60 * 60 * 24] decorators = [advertise('scopes', 'rate_limit')] def post(self): stime = time.time() # Get the supplied list of identifiers identifiers = [] input_type = None # determine whether a source for the data was specified try: source = request.json['source'].lower() except: source = 'simbad' # We only deal with SIMBAD or NED as source if source not in ['simbad', 'ned']: current_app.logger.error( 'Unsupported source for object data specified: %s' % source) return { "Error": "Unable to get results!", "Error Info": "Unsupported source for object data specified: %s" % source }, 200 for itype in ['identifiers', 'objects']: try: identifiers = request.json[itype] identifiers = map(str, identifiers) input_type = itype except: pass if not input_type: current_app.logger.error( 'No identifiers and objects were specified for SIMBAD object query' ) return { "Error": "Unable to get results!", "Error Info": "No identifiers/objects found in POST body" }, 200 # We should either have a list of identifiers or a list of object names if len(identifiers) == 0: current_app.logger.error( 'No identifiers or objects were specified for SIMBAD object query' ) return { "Error": "Unable to get results!", "Error Info": "No identifiers/objects found in POST body" }, 200 # We have a known object data source and a list of identifiers. Let's start! # We have identifiers if source == 'simbad': result = get_simbad_data(identifiers, input_type) else: if input_type == 'identifiers': input_type = 'simple' result = get_ned_data(identifiers, input_type) if 'Error' in result: # An error was returned! err_msg = result['Error Info'] current_app.logger.error( 'Failed to find data for %s %s query (%s)!' % (source.upper(), input_type, err_msg)) return result else: # We have results! duration = time.time() - stime current_app.logger.info( 'Found objects for %s %s in %s user seconds.' % (source.upper(), input_type, duration)) # Now pick the entries in the results that correspond with the original object names if input_type == 'objects': # result['data'] = {k: result['data'].get(k.upper()) for k in identifiers} result['data'] = { k: result['data'].get(k) or result['data'].get(k.upper()) for k in identifiers } # Send back the results return result.get('data', {})
class WordCloud(Resource): '''Returns collated tf/idf data for a solr query''' decorators = [advertise('scopes', 'rate_limit')] scopes = [] rate_limit = [500, 60 * 60 * 24] def post(self): solr_args = request.json if not solr_args: return { 'Error': 'there was a problem with your request', 'Error Info': 'no data provided with request' }, 403 if 'max_groups' in solr_args: del solr_args['min_percent_word'] if 'min_occurrences_word' in solr_args: del solr_args['min_occurrences_word'] elif 'query' in request.json: try: solr_args = json.loads(request.json["query"][0]) except Exception: return { 'Error': 'there was a problem with your request', 'Error Info': 'couldn\'t decode query, it should be json-encoded before being sent (so double encoded)' }, 403 solr_args["rows"] = min( int( solr_args.get( "rows", [current_app.config.get("VIS_SERVICE_WC_MAX_RECORDS") ])[0]), current_app.config.get("VIS_SERVICE_WC_MAX_RECORDS")) solr_args['fields'] = ['id'] solr_args['defType'] = 'aqp' solr_args['tv'] = 'true' solr_args['tv.tf_idf'] = 'true' solr_args['tv.tf'] = 'true' solr_args['tv.positions'] = 'false' solr_args['tf.offsets'] = 'false' solr_args['tv.fl'] = 'abstract,title' solr_args['fl'] = 'id,abstract,title' solr_args['wt'] = 'json' headers = { 'X-Forwarded-Authorization': request.headers.get('Authorization') } response = client().get( current_app.config.get("VIS_SERVICE_TVRH_PATH"), params=solr_args, headers=headers) if response.status_code == 200: data = response.json() else: return { "Error": "There was a connection error. Please try again later", "Error Info": response.text }, response.status_code if data: min_percent_word = request.args.get( "min_percent_word", current_app.config.get("VIS_SERVICE_WC_MIN_PERCENT_WORD")) min_occurrences_word = request.args.get( "min_occurrences_word", current_app.config.get("VIS_SERVICE_WC_MIN_OCCURRENCES_WORD")) word_cloud_json = word_cloud.generate_wordcloud( data, min_percent_word=min_percent_word, min_occurrences_word=min_occurrences_word) if word_cloud_json: return word_cloud_json, 200 else: return { "Error": "Empty word cloud. Try changing your minimum word parameters or expanding your query." }, 200