def get_permissions(cls, library_id): """ Looks up and returns the permissions for all of the users that have any permissions. :param library_id: the library ID to look up permissions :return: list of dictionaries containing the user email and permission """ with current_app.session_scope() as session: # Find the permissions for the library result = session.query(Permissions, User)\ .join(Permissions.user)\ .filter(Permissions.library_id == library_id)\ .all() # Formulate the return content permission_list = [] for permission, user in result: # Convert the user id into user = cls.api_uid_email_lookup(user_info=user.absolute_uid) all_permissions = filter( lambda key: permission.__dict__[key], ['read', 'write', 'admin', 'owner'] ) permission_list.append( {user: all_permissions} ) return permission_list
def get(self): """ HTTP GET request that returns the information currently stored about the user's ADS Classic settings, currently stored in the service database. Return data (on success) ------------------------ classic_email: <string> ADS Classic e-mail of the user classic_mirror: <string> ADS Classic mirror this user belongs to HTTP Responses: -------------- Succeed authentication: 200 User unknown/wrong password/failed authentication: 404 Any other responses will be default Flask errors """ absolute_uid = self.helper_get_user_id() try: with current_app.session_scope() as session: user = session.query(Users).filter( Users.absolute_uid == absolute_uid).one() return { 'classic_email': user.classic_email, 'classic_mirror': user.classic_mirror, 'twopointoh_email': user.twopointoh_email }, 200 except NoResultFound: return err(NO_CLASSIC_ACCOUNT)
def copy_library(cls, library_id, document_data): """ Copies the contents of one library into another. Does not empty library first; call empty_library on the target library first to do so :param library_id: primary library ID, library to copy :param document_data: dict containing the list 'libraries' which holds one secondary library ID; this is the library to copy over :return: dict containing the metadata of the copied-over library (the secondary library) """ current_app.logger.info('User requested to copy the contents of {0} into {1}' .format(library_id, document_data['libraries'])) secondary_libid = document_data['libraries'][0] if isinstance(secondary_libid, basestring): secondary_libid = cls.helper_slug_to_uuid(secondary_libid) metadata = {} with current_app.session_scope() as session: primary_library = session.query(Library).filter_by(id=library_id).one() good_bib = primary_library.get_bibcodes() secondary_library = session.query(Library).filter_by(id=secondary_libid).one() secondary_library.add_bibcodes(good_bib) metadata['name'] = secondary_library.name metadata['description'] = secondary_library.description metadata['public'] = secondary_library.public session.add(secondary_library) session.commit() return metadata
def add_document_to_library(cls, library_id, document_data): """ Adds a document to a user's library :param library_id: the library id to update :param document_data: the meta data of the document :return: number_added: number of documents successfully added """ current_app.logger.info('Adding a document: {0} to library_uuid: {1}' .format(document_data, library_id)) with current_app.session_scope() as session: # Find the specified library library = session.query(Library).filter_by(id=library_id).one() start_length = len(library.bibcode) library.add_bibcodes(document_data['bibcode']) session.add(library) session.commit() current_app.logger.info('Added: {0} is now {1}'.format( document_data['bibcode'], library.bibcode) ) end_length = len(library.bibcode) return end_length - start_length
def configuration(key=None): '''Allows you to retrieve JSON data from VAULT_BUMBLEBEE_OPTIONS''' opts = current_app.config.get('VAULT_BUMBLEBEE_OPTIONS') or {} if not isinstance(opts, dict): return json.dumps({ 'msg': 'Server misconfiguration, VAULT_BUMBLEBEE_OPTIONS is of an invalid type' }), 500 if key: if key == 'link_servers': with current_app.session_scope() as session: res = session.query(Library).all() link_servers = [{ "name": l.libname, "link": l.libserver, "gif": l.iconurl } for l in res] link_servers = sorted(link_servers, key=itemgetter('name')) return json.dumps(link_servers), 200 elif key in opts: return json.dumps(opts[key]), 200 else: return '{}', 404 else: return json.dumps(opts), 200
def apply_protective_filters(self, payload, user_id, protected_fields): """ Adds filters to the query that should limit results to conditions that are associted with the user_id+protected_field. If a field is not found in the db of limits, it will not be returned to the user :param payload: raw request payload :param user_id: string, user id as known to ADS API :param protected_fields: list of strings, fields """ fl = payload.get('fl', 'id') fq = payload.get('fq', []) if not isinstance(fq, list): fq = [fq] payload['fq'] = fq with current_app.session_scope() as session: for f in session.query(Limits).filter( Limits.uid == user_id, or_(Limits.field == x for x in protected_fields)).all(): if f.filter: fl = u'{0},{1}'.format(fl, f.field) fq.append(unicode(f.filter)) payload['fl'] = fl session.commit()
def orcid_profile_local(orcid_id, type): '''Get /[orcid-id]/orcid-profile/<simple,full> - returns either bibcodes and statuses (/simple) or all records and saved metadata (/full) - all communication exclusively in JSON''' payload, headers = check_request(request) update = request.args.get('update', False) if type not in ['simple','full']: return json.dumps('Endpoint /orcid-profile/%s does not exist'.format(type)), 404 r = current_app.client.get(current_app.config['ORCID_API_ENDPOINT'] + '/' + orcid_id + '/record', headers=headers) if r.status_code == 200: update_profile_local(orcid_id, data=r.text, force=update) else: logging.warning('Failed fetching fresh profile from ORCID for %s'.format(orcid_id)) with current_app.session_scope() as session: profile = session.query(Profile).filter_by(orcid_id=orcid_id).first() if type == 'simple': bibcodes, statuses = profile.get_bibcodes() records = dict(zip(bibcodes, statuses)) elif type == 'full': records = profile.get_records() return json.dumps(records), 200
def update_stored_profile(orcid_id): '''Updates profile in orcid-service before processing''' with current_app.session_scope() as session: u = session.query(User).filter_by(orcid_id=orcid_id).first() if not u: return json.dumps({'error': 'We do not have a record for: %s' % orcid_id}), 404 if not u.access_token: return json.dumps({'error': 'We do not have access_token for: %s' % orcid_id}), 404 token = u.access_token headers = { 'Accept': 'application/json', 'Authorization': 'Bearer %s' % token, 'Content-Type': 'application/json' } try: r = current_app.client.get(current_app.config['ORCID_API_ENDPOINT'] + '/' + orcid_id + '/record', headers=headers, timeout=current_app.config.get('CONNECTION_TIMEOUT', 30)) except (ConnectionError, ConnectTimeout, ReadTimeout) as e: logging.error('For ORCID ID %s, there was a connection error with the ORCID API'.format(orcid_id)) return 'There was a connection error with the ORCID API', 502 if r.status_code == 200: update_profile_local(orcid_id, data=r.text, force=True) else: logging.warning('Failed fetching fresh profile from ORCID for %s'.format(orcid_id)) profile = session.query(Profile).filter_by(orcid_id=orcid_id).first() records = profile.get_records() return json.dumps(records), 200
def add_document_to_library(cls, library_id, document_data): """ Adds a document to a user's library :param library_id: the library id to update :param document_data: the meta data of the document :return: number_added: number of documents successfully added """ current_app.logger.info( 'Adding a document: {0} to library_uuid: {1}'.format( document_data, library_id)) with current_app.session_scope() as session: # Find the specified library library = session.query(Library).filter_by(id=library_id).one() start_length = len(library.bibcode) library.add_bibcodes(document_data['bibcode']) session.add(library) session.commit() current_app.logger.info('Added: {0} is now {1}'.format( document_data['bibcode'], library.bibcode)) end_length = len(library.bibcode) return end_length - start_length
def setops_libraries(cls, library_id, document_data, operation='union'): """ Takes the union of two or more libraries :param library_id: the primary library ID :param document_data: dict containing the list 'libraries' that holds the secondary library IDs :return: list of bibcodes in the union set """ current_app.logger.info('User requested to take the {0} of {1} with {2}' .format(operation, library_id, document_data['libraries'])) with current_app.session_scope() as session: # Find the specified library primary_library = session.query(Library).filter_by(id=library_id).one() out_lib = set(primary_library.get_bibcodes()) for lib in document_data['libraries']: if isinstance(lib, basestring): lib = cls.helper_slug_to_uuid(lib) secondary_library = session.query(Library).filter_by(id=lib).one() if operation == 'union': out_lib = out_lib.union(set(secondary_library.get_bibcodes())) elif operation == 'intersection': out_lib = out_lib.intersection(set(secondary_library.get_bibcodes())) elif operation == 'difference': out_lib = out_lib.difference(set(secondary_library.get_bibcodes())) else: current_app.logger.warning('Requested operation {0} is not allowed.'.format(operation)) return if len(out_lib) < 1: current_app.logger.info('No records remain after taking the {0} of {1} and {2}' .format(operation, library_id, document_data['libraries'])) return list(out_lib)
def library_name_exists(service_uid, library_name): """ Checks to see if a library name already exists in the user's created libraries :param service_uid: the user ID within this microservice :param library_name: name to check if it exists :return: True (exists), False (does not exist) """ with current_app.session_scope() as session: library_names = \ [i.library.name for i in session.query(Permissions)\ .filter_by(user_id = service_uid)\ .filter(Permissions.permissions['owner'].astext.cast(Boolean).is_(True)).all()] if library_name in library_names: current_app.logger.error('Name supplied for the library already ' 'exists: "{0}"'.format(library_name)) return True else: return False
def update_stored_profile(orcid_id): '''Updates profile in orcid-service before processing''' with current_app.session_scope() as session: u = session.query(User).filter_by(orcid_id=orcid_id).first() if not u: return json.dumps({'error': 'We do not have a record for: %s' % orcid_id}), 404 if not u.access_token: return json.dumps({'error': 'We do not have access_token for: %s' % orcid_id}), 404 token = u.access_token headers = { 'Accept': 'application/json', 'Authorization': 'Bearer %s' % token, 'Content-Type': 'application/json' } r = current_app.client.get(current_app.config['ORCID_API_ENDPOINT'] + '/' + orcid_id + '/record', headers=headers) if r.status_code == 200: update_profile_local(orcid_id, data=r.text, force=True) else: logging.warning('Failed fetching fresh profile from ORCID for %s'.format(orcid_id)) profile = session.query(Profile).filter_by(orcid_id=orcid_id).first() records = profile.get_records() return json.dumps(records), 200
def helper_access_allowed(service_uid, library_id, access_type): """ Determines if the given user has permissions to look at the content of a library. :param service_uid: the user ID within this microservice :param library_id: the unique ID of the library :param access_type: list of access types to check :return: boolean, access (True), no access (False) """ with current_app.session_scope() as session: try: permissions = session.query(Permissions).filter_by( library_id = library_id, user_id = service_uid ).one() return getattr(permissions, access_type) except NoResultFound as error: current_app.logger.error('No permissions for ' 'user: {0}, library: {1}, permission: {2}' ' [{3}]'.format(service_uid, library_id, access_type, error)) return False
def store_get(qid=None): if request.method == 'GET': with current_app.session_scope() as session: page = session.query(Pages).filter_by(qid=qid).first() if not page: return jsonify({'qid': qid, 'msg': 'Not found'}), 404 return current_app.wrap_response(page) elif request.method == 'HEAD': with current_app.session_scope() as session: # TODO: make it more efficient page = session.query(Pages).filter_by(qid=qid).options( load_only('id')).first() if page: return '', 200 else: return '', 404
def execute_myads_query(myads_id): """ Returns the constructed query for a single templated myADS notification, ready to execute :param myads_id: ID of a single notification :return: list of dicts; constructed query, dates are such that it's meant to be run today: [{q: query params, sort: sort string}] """ try: payload, headers = check_request(request) except Exception as e: return json.dumps({'msg': e.message or e.description}), 400 user_id = int(headers['X-Adsws-Uid']) if user_id == current_app.config['BOOTSTRAP_USER_ID']: return json.dumps( {'msg': 'Sorry, you can\'t use this service as an anonymous user'}), 400 with current_app.session_scope() as session: setup = session.query(MyADS).filter_by(user_id=user_id).filter_by( id=myads_id).first() if setup is None: return '{}', 404 query = _create_myads_query(setup.template, setup.frequency, setup.data, classes=setup.classes) return json.dumps(query)
def store_data(): '''Allows you to store/retrieve JSON data on the server side. It is always associated with the user id (which is communicated to us by API) - so there is no endpoint allowing you to access other users' data (should there be?) /user-data/<uid>?''' # get the query data try: payload, headers = check_request(request) except Exception as e: return json.dumps({'msg': e.message or e.description}), 400 user_id = int(headers['X-Adsws-Uid']) if user_id == 0: return json.dumps( {'msg': 'Sorry, you can\'t use this service as an anonymous user'}), 400 if request.method == 'GET': with current_app.session_scope() as session: q = session.query(User).filter_by(id=user_id).first() if not q: return '{}', 200 # or return 404? return q.user_data or '{}', 200 elif request.method == 'POST': d = json.dumps(payload) if len(d) > MAX_ALLOWED_JSON_SIZE: return json.dumps({ 'msg': 'You have exceeded the allowed storage limit, no data was saved' }), 400 u = User(id=user_id, user_data=d) with current_app.session_scope() as session: session.begin_nested() try: session.merge(u) session.commit() except exc.IntegrityError: session.rollback() return json.dumps({ 'msg': 'We have hit a db error! The world is crumbling all around... (eh, btw, your data was not saved)' }), 500 return d, 200
def get_myads(user_id): ''' Fetches a myADS profile for the pipeline for a given uid ''' # takes a given uid, grabs the user_data field, checks for the myADS key - if present, grabs the appropriate # queries/data from the queries/myADS table. Finally, returns the appropriate info along w/ the rest # of the setup from the dict (e.g. active, frequency, etc.) # TODO check rate limit # structure in vault: # "myADS": [{"name": user-supplied name, "qid": QID from query table (type="query") or ID from myads table # (type="template"), "type": "query" or "template", "active": true/false, "frequency": "daily" or "weekly", # "stateful": true/false}] output = [] with current_app.session_scope() as session: setups = session.query(MyADS).filter_by(user_id=user_id).filter_by( active=True).order_by(MyADS.id.asc()).all() if not setups: return '{}', 404 for s in setups: o = { 'id': s.id, 'name': s.name, 'type': s.type, 'active': s.active, 'stateful': s.stateful, 'frequency': s.frequency, 'template': s.template, 'classes': s.classes, 'data': s.data, 'created': s.created.isoformat(), 'updated': s.updated.isoformat() } if s.type == 'query': try: q = session.query(Query).filter_by(id=s.query_id).one() qid = q.qid query = None except ormexc.NoResultFound: qid = None query = None else: qid = None query = _create_myads_query(s.template, s.frequency, s.data, classes=s.classes) o['qid'] = qid o['query'] = query output.append(o) return json.dumps(output), 200
def get(self, uid): """ HTTP GET request that finds the libraries within ADS 2.0 for that user. :param uid: user ID for the API :type uid: int Return data (on success) ------------------------ libraries: <list<dict>> a list of dictionaries, that contains the following for each library entry: name: <string> name of the library description: <string> description of the library documents: <list<string>> list of documents HTTP Responses: -------------- Succeed getting libraries: 200 User does not have a classic/ADS 2.0 account: 400 User does not have any libraries in their ADS 2.0 account: 400 Unknown error: 500 Any other responses will be default Flask errors """ with current_app.session_scope() as session: if not current_app.config['ADS_TWO_POINT_OH_LOADED_USERS']: current_app.logger.error( 'Users from MongoDB have not been loaded into the app') return err(TWOPOINTOH_AWS_PROBLEM) try: user = session.query(Users).filter( Users.absolute_uid == uid).one() # Have they got an email for ADS 2.0? if not user.twopointoh_email: raise NoResultFound except NoResultFound: current_app.logger.warning( 'User does not have an associated ADS 2.0 account') return err(NO_TWOPOINTOH_ACCOUNT) library_file_name = current_app.config[ 'ADS_TWO_POINT_OH_USERS'].get(user.twopointoh_email, None) if not library_file_name: current_app.logger.warning( 'User does not have any libraries in ADS 2.0') return err(NO_TWOPOINTOH_LIBRARIES) try: library = TwoPointOhLibraries.get_s3_library(library_file_name) except Exception as error: current_app.logger.error( 'Unknown error with AWS: {}'.format(error)) return err(TWOPOINTOH_AWS_PROBLEM) return {'libraries': library}, 200
def preferences(orcid_id): '''Allows you to store/retrieve JSON data on the server side. It is always associated with the ORCID access token so there is no need for scope access ''' # get the query data try: payload, headers = check_request(request) except Exception as e: return json.dumps({'msg': e.message or e.description}), 400 access_token = headers['Authorization'][7:] # remove the 'Bearer:' thing if request.method == 'GET': with current_app.session_scope() as session: u = session.query(User).filter(and_(User.orcid_id==orcid_id, User.access_token==access_token)).options(load_only(User.orcid_id)).first() if not u: return '{}', 404 # not found return u.info or '{}', 200 elif request.method == 'POST': d = json.dumps(payload) if len(d) > current_app.config.get('MAX_ALLOWED_JSON_SIZE', 1000): return json.dumps({'msg': 'You have exceeded the allowed storage limit, no data was saved'}), 400 with current_app.session_scope() as session: u = session.query(User).filter(and_(User.orcid_id==orcid_id, User.access_token==access_token)).options(load_only(User.orcid_id)).first() if not u: return json.dumps({'error': 'We do not have a record for: %s' % orcid_id}), 404 u.info = d session.begin_nested() try: session.merge(u) session.commit() except exc.IntegrityError: session.rollback() return json.dumps({'msg': 'We have hit a db error! The world is crumbling all around... (eh, btw, your data was not saved)'}), 500 # per PEP-0249 a transaction is always in progress session.commit() return d, 200
def get_access_token(): '''Exchange 'code' for 'access_token' data''' payload = request.args.to_dict() if 'code' not in payload: raise Exception('Parameter code is missing') headers = {'Accept': 'application/json'} data = { 'client_id': current_app.config['ORCID_CLIENT_ID'], 'client_secret': current_app.config['ORCID_CLIENT_SECRET'], 'code': payload['code'], 'grant_type': 'authorization_code' } #print current_app.config['ORCID_OAUTH_ENDPOINT'], data, headers # do not use connection pool, always establish a new connection to the orcid remote server # we were having issue with dropped connectins mid-stream and this request is not idempotent # therefore we can't retry try: r = requests.post(current_app.config['ORCID_OAUTH_ENDPOINT'], data=data, headers=headers, timeout=current_app.config.get('CONNECTION_TIMEOUT', 30)) except (ConnectionError, ConnectTimeout, ReadTimeout) as e: logging.error('For ORCID code %s, there was a connection error with the ORCID API'.format(payload['code'])) return 'There was a connection error with the ORCID API', 502 if r.status_code != 200: logging.error('For ORCID code {}, there was an error getting the token from the ORCID API.'. format(payload['code'])) return r.text, r.status_code # update/create user account data = r.json() if 'orcid' in data: with current_app.session_scope() as session: u = session.query(User).filter_by(orcid_id=data['orcid']).options(load_only(User.orcid_id)).first() p = session.query(Profile).filter_by(orcid_id=data['orcid']).options(load_only(Profile.orcid_id)).first() if not u: u = User(orcid_id=data['orcid'], created=adsmutils.get_date()) if not p: p = Profile(orcid_id=data['orcid'], created=adsmutils.get_date()) u.updated = adsmutils.get_date() p. updated = adsmutils.get_date() u.access_token = data['access_token'] # save the user session.begin_nested() try: session.add(u) session.add(p) session.commit() except exc.IntegrityError as e: session.rollback() # per PEP-0249 a transaction is always in progress session.commit() return r.text, r.status_code
def query2svg(queryid): '''Returns the SVG form of the query - for better performance, will need to be cached/exported ''' with current_app.session_scope() as session: q = session.query(Query).filter_by(qid=queryid).first() if not q: return '<svg xmlns="http://www.w3.org/2000/svg"></svg>', 404, {'Content-Type': "image/svg+xml"} return SVG_TMPL % {'key': 'ADS query', 'value': q.numfound}, 200, {'Content-Type': "image/svg+xml"}
def delete_library(library_id): """ Delete the entire library from the database :param library_id: the unique ID of the library :return: no return """ with current_app.session_scope() as session: library = session.query(Library).filter_by(id=library_id).one() session.delete(library) session.commit()
def get(self, uid): """ HTTP GET request that contacts the ADS Classic myADS end point to obtain all the libraries relevant to that user. :param uid: user ID for the API :type uid: int Return data (on success) ------------------------ HTTP Responses: -------------- Succeed getting libraries: 200 User does not have a classic account: 400 ADS Classic give unknown messages: 500 ADS Classic times out: 504 Any other responses will be default Flask errors """ data = {} with current_app.session_scope() as session: try: user = session.query(Users).filter( Users.absolute_uid == uid).one() if not user.classic_email: raise NoResultFound except NoResultFound: current_app.logger.warning( 'User does not have an associated ADS Classic account') return err(NO_CLASSIC_ACCOUNT) url = current_app.config['ADS_CLASSIC_MYADS_URL'].format( mirror='adsabs.harvard.edu', email=user.classic_email) current_app.logger.debug('Obtaining libraries via: {}'.format(url)) try: response = current_app.client.get(url) except requests.exceptions.Timeout: current_app.logger.warning( 'ADS Classic timed out before finishing: {}'.format(url)) return err(CLASSIC_TIMEOUT) if response.status_code != 200: current_app.logger.warning( 'ADS Classic returned an unkown status code: "{}" [code: {}]' .format(response.text, response.status_code)) return err(CLASSIC_UNKNOWN_ERROR) data = response.json() return data, 200
def delete_library(library_id): """ Delete the entire library from the database :param library_id: the unique ID of the library :return: no return """ with current_app.session_scope() as session: library = session.query(Library).filter_by(id=library_id).one() session.delete(library) session.commit()
def update_status(orcid_id): """Gets/sets bibcode statuses for a given ORCID ID""" if request.method == 'GET': with current_app.session_scope() as session: profile = session.query(Profile).filter_by(orcid_id=orcid_id).first() recs = profile.get_records() statuses = profile.get_nested(recs,'status') records = dict(zip(recs, statuses)) return json.dumps(records), 200 if request.method == 'POST': payload = request.json if 'bibcodes' not in payload: raise Exception('Bibcodes are missing') if 'status' not in payload: raise Exception('Status is missing') with current_app.session_scope() as session: profile = session.query(Profile).filter_by(orcid_id=orcid_id).first() if type(payload['bibcodes']) != list: bibcodes = [payload['bibcodes']] else: bibcodes = payload['bibcodes'] profile.update_status(bibcodes,payload['status']) session.begin_nested() try: session.add(profile) session.commit() except exc.IntegrityError: session.rollback() return json.dumps({'msg': 'Updated status for bibcode {} not saved'.format(bibcodes)}), 500 # per PEP-0249 a transaction is always in progress session.commit() good_bibc, good_statuses = profile.get_status(bibcodes) records = dict(zip(good_bibc, good_statuses)) return json.dumps(records), 200
def update_profile_local(orcid_id, data=None, force=False): """Update local db with ORCID profile""" data = json.loads(data) with current_app.session_scope() as session: profile = session.query(Profile).filter_by(orcid_id=orcid_id).first() if not profile: logging.error('ORCID profile {} does not exist; creating'.format(orcid_id)) profile = Profile(orcid_id=orcid_id, created=adsmutils.get_date()) force = True # data assumed to come from ORCID API /works endpoint if data: # convert milliseconds since epoch to seconds since epoch last_modified = data['activities-summary']['last-modified-date']['value'] last_modified /= 1000. if force or (profile.updated < datetime.utcfromtimestamp(last_modified).replace(tzinfo=pytz.utc)): works = data['activities-summary']['works']['group'] new_recs = {} update_recs = {} orcid_recs = [] try: current_recs = profile.bibcode.keys() except: current_recs = [] for work in works: try: id0, rec = find_record(work) except: continue if id0 not in current_recs: new_recs.update(rec) else: # if bibcode already in the profile, keep its status rec[id0]['status'] = profile.bibcode[id0]['status'] update_recs.update(rec) orcid_recs.append(id0) profile.add_records(new_recs) profile.add_records(update_recs) # remove records from the profile that aren't in the ORCID set remove_recs = list(set(current_recs)-set(orcid_recs)) profile.remove_bibcodes(remove_recs) profile.updated = adsmutils.get_date() # save the user session.begin_nested() try: session.add(profile) session.commit() except exc.IntegrityError as e: session.rollback() logging.warning('ORCID profile database error - updated bibcodes for %s were not saved.'.format(orcid_id))
def store(qid=None): if request.method == 'POST': out = {} # there might be many objects in there... msgs = [] for _, fo in request.files.items(): if not hasattr(fo, 'read'): continue # not a file object # on error, we'll crash early; that's OK msg = TurboBeeMsg.loads('adsmsg.turbobee.TurboBeeMsg', fo.read()) msgs.append(msg) # also read data posted the normal way for k, v in request.form.items(): msg = TurboBeeMsg.loads('adsmsg.turbobee.TurboBeeMsg', base64.decodestring(v)) msgs.append(msg) if not len(msgs): return jsonify({'msg': 'Empty stream, no messages were received'}), 501 out = [] if len(msgs): out = current_app.set_pages(msgs) if 'errors' in out: return jsonify(out), 400 return jsonify(out), 200 elif request.method == 'DELETE': with current_app.session_scope() as session: pages = session.query(Pages).options( load_only('qid')).filter_by(qid=qid).first() qid = None if not pages: return jsonify({'qi': qid, 'msg': 'Not found'}), 404 else: qid = pages.qid session.delete(pages) try: session.commit() except exc.IntegrityError as e: session.rollback() return jsonify({'qid': qid, 'status': 'deleted'}), 200
def run(app=app): """ Carries out the deletion of the stale content """ with app.app_context(): with current_app.session_scope() as session: # Obtain the list of API users postgres_search_text = 'SELECT id FROM users;' result = session.execute(postgres_search_text).fetchall() list_of_api_users = [int(r[0]) for r in result] # Loop through every use in the service database removal_list = [] for service_user in session.query(User).all(): if service_user.absolute_uid not in list_of_api_users: try: # Obtain the libraries that should be deleted permissions = session.query(Permissions).filter( Permissions.user_id == service_user.id).all() libraries = [ session.query(Library).filter( Library.id == permission.library_id).one() for permission in permissions if permission.permissions['owner'] ] # Delete all the libraries found # By cascade this should delete all the permissions d = [ session.delete(library) for library in libraries ] p = [ session.delete(permission) for permission in permissions ] d = len(d) session.delete(service_user) session.commit() current_app.logger.info( 'Removed stale user: {} and {} libraries'. format(service_user, d)) removal_list.append(service_user) except Exception as error: current_app.logger.info( 'Problem with database, could not remove user {}: {}' .format(service_user, error)) session.rollback() current_app.logger.info('Deleted {} stale users: {}'.format( len(removal_list), removal_list))
def helper_validate_library_data(service_uid, library_data): """ Validates the library data to ensure the user does not give empty content for the title and description. :param service_uid: the user ID within this microservice :param library_data: content needed to create a library :return: validated name and description """ _name = library_data.get('name') or DEFAULT_LIBRARY_NAME_PREFIX _description = library_data.get('description') or \ DEFAULT_LIBRARY_DESCRIPTION current_app.logger.info('Creating library for user_service: {0:d}, ' 'with properties: {1}' .format(service_uid, library_data)) # We want to ensure that the users have unique library names. However, # it should be possible that they have access to other libraries from # other people, that have the same name with current_app.session_scope() as session: library_names = \ [i.library.name for i in session.query(Permissions).filter_by(user_id = service_uid, owner = True).all()] matches = [name for name in library_names if name == _name] if matches: current_app.logger.error('Name supplied for the library already ' 'exists: "{0}" ["{1}"]'.format(_name, matches)) raise BackendIntegrityError('Library name already exists.') if _name == DEFAULT_LIBRARY_NAME_PREFIX: default_names = [lib_name for lib_name in library_names if DEFAULT_LIBRARY_NAME_PREFIX in lib_name] _extension = len(default_names) + 1 _name = '{0} {1}'.format(_name, _extension) library_out = {} for key in library_data: library_out[key] = library_data[key] library_out['name'] = _name library_out['description'] = _description return library_out
def search(): keys = request.args.keys() # default is 50, max is 100 rows = min(current_app.config.get('MAX_RETURNED', 100), int(request.args.get('rows') or 50)) with current_app.session_scope() as session: if 'begin' in keys and 'end' in keys: begin = get_date(request.args['begin']) end = get_date(request.args['end']) query = session.query(Pages).filter( Pages.created.between(begin, end)) elif 'begin' in keys: # search for all records after begin begin = get_date(request.args['begin']) query = session.query(Pages).filter(Pages.created >= begin) elif 'end' in keys: # search for all records before end end = get_date(request.args['end']) query = session.query(Pages).filter(Pages.created <= end) elif 'at' in keys: # search for all records created at specific timestamp at = get_date(request.args['at']) query = session.query(Pages).filter(Pages.created == at) elif 'null' in keys: query = session.query(Pages).filter(Pages.created == None) else: return jsonify({'msg': 'Invalid parameters %s' % keys}), 505 if 'last_id' in keys: query = query.where(Pages.id > keys['last_id']) query = query.order_by(Pages.updated.asc()) \ .limit(rows) if 'fields' in keys: # load only some fields allowed_fields = [ 'qid', 'created', 'updated', 'expires', 'lifetime', 'content_type', 'content' ] fields = keys.get('fields', allowed_fields) fields_to_load = list(set(fields) & set(allowed_fields)) query = query.options(load_only(*fields_to_load)) try: pages = query.all() # it is possible that toJSON() will eagerly load all fields (defeating load_only() above) result = map(lambda page: page.toJSON(), pages) return jsonify(result) except Exception as e: current_app.logger.error('Failed request: %s (error=%s)', keys, e) return jsonify({'msg': e.message}), 500
def helper_library_exists(library_id): """ Helper function that checks if a library exists in the database or not by catching the raise and returning a True/False statement. :param library_id: the unique ID of the library :return: bool for exists (True) or does not (False) """ with current_app.session_scope() as session: try: session.query(Library).filter_by(id = library_id).one() return True except NoResultFound: return False
def add_records(datalinks_records_list): """ upserts records into db :param datalinks_records_list: :return: success boolean, plus a status text for retuning error message, if any, to the calling program """ rows = [] for i in range(len(datalinks_records_list.datalinks_records)): for j in range( len(datalinks_records_list.datalinks_records[i].data_links_rows )): rows.append([ datalinks_records_list.datalinks_records[i].bibcode, datalinks_records_list.datalinks_records[i].data_links_rows[j]. link_type, datalinks_records_list.datalinks_records[i]. data_links_rows[j].link_sub_type, datalinks_records_list. datalinks_records[i].data_links_rows[j].url, datalinks_records_list.datalinks_records[i].data_links_rows[j]. title, datalinks_records_list.datalinks_records[i]. data_links_rows[j].item_count ]) if len(rows) > 0: table = DataLinks.__table__ stmt = insert(table).values(rows) no_update_cols = [] update_cols = [ c.name for c in table.c if c not in list(table.primary_key.columns) and c.name not in no_update_cols ] on_conflict_stmt = stmt.on_conflict_do_update( index_elements=table.primary_key.columns, set_={k: getattr(stmt.excluded, k) for k in update_cols}) try: with current_app.session_scope() as session: session.execute(on_conflict_stmt) current_app.logger.info('updated db with new data successfully') return True, 'updated db with new data successfully' except (SQLAlchemyError, DBAPIError) as e: current_app.logger.error('SQLAlchemy: ' + str(e)) return False, 'SQLAlchemy: ' + str(e) return False, 'unable to extract data from protobuf structure'
def execute_query(queryid): '''Allows you to execute stored query. With this endpoint you can return parameters for the previously asigned and stored query in the database (such as current number of documents in the database. ''' with current_app.session_scope() as session: q = session.query(Query).filter_by(qid=queryid).first() if not q: return json.dumps({'msg': 'Query not found: ' + queryid}), 404 q_query = q.query.decode('utf8') try: payload, headers = check_request(request) except Exception as e: return json.dumps({'msg': e.message or e.description}), 400 dataq = json.loads(q_query) query = urlparse.parse_qs(dataq['query']) # override parameters using supplied params if len(payload) > 0: query.update(payload) # make sure the {!bitset} is there (when bigquery is used) if dataq['bigquery']: fq = query.get('fq') if not fq: fq = ['{!bitset}'] elif '!bitset' not in str(fq): if isinstance(fq, list): fq.append('{!bitset}') elif isinstance(fq, str): fq = [fq, '{!bitset}'] else: fq = ['{!bitset}'] query['fq'] = fq # always request json query['wt'] = 'json' r = make_solr_request(query=query, bigquery=dataq['bigquery'], headers=headers) return r.text, r.status_code
def query2svg(queryid): '''Returns the SVG form of the query - for better performance, will need to be cached/exported ''' with current_app.session_scope() as session: q = session.query(Query).filter_by(qid=queryid).first() if not q: return '<svg xmlns="http://www.w3.org/2000/svg"></svg>', 404, { 'Content-Type': "image/svg+xml" } return SVG_TMPL % { 'key': 'ADS query', 'value': q.numfound }, 200, { 'Content-Type': "image/svg+xml" }
def _delete_myads_notification(user_id=None, myads_id=None): """ Delete a single myADS notification setup :param myads_id: ID of a single notification :return: none """ with current_app.session_scope() as session: r = session.query(MyADS).filter_by(user_id=user_id).filter_by( id=myads_id).first() if not r: return '{}', 404 try: session.delete(r) session.commit() except exc.IntegrityError as e: session.rollback() return json.dumps({'msg': 'Query was not deleted'}), 500 return '{}', 204
def create_user(absolute_uid): """ Creates a user in the database with a UID from the API :param absolute_uid: UID from the API :return: no return """ try: user = User(absolute_uid=absolute_uid) with current_app.session_scope() as session: session.add(user) session.commit() except IntegrityError as error: current_app.logger.error('IntegrityError. User: {0:d} was not' 'added. Full traceback: {1}' .format(absolute_uid, error)) raise
def helper_user_exists(absolute_uid): """ Checks if a use exists before it would attempt to create one :param absolute_uid: UID from the API :return: boolean for if the user exists """ with current_app.session_scope() as session: user_count = session.query(User).filter_by(absolute_uid = absolute_uid).all() user_count = len(user_count) if user_count == 1: current_app.logger.info('User exists in database: {0} [API]' .format(absolute_uid)) return True elif user_count == 0: current_app.logger.warning('User does not exist in database: {0} ' '[API]'.format(absolute_uid)) return False
def helper_absolute_uid_to_service_uid(absolute_uid): """ Convert the API UID to the BibLib service ID. If the user does not exist in the database, first create a user. :param absolute_uid: API UID :return: BibLib service ID """ if not BaseView.helper_user_exists(absolute_uid=absolute_uid): user = BaseView.helper_create_user(absolute_uid=absolute_uid) with current_app.session_scope() as session: user = session.query(User).filter_by(absolute_uid = absolute_uid).one() current_app.logger.info('User found: {0} -> {1}' .format(absolute_uid, user.id)) return user.id
def del_records(bibcode_list): """ :param bibcode_list: :return: """ try: with current_app.session_scope() as session: count = 0 for bibcode in bibcode_list: count += session.query(DataLinks).filter( DataLinks.bibcode == bibcode).delete( synchronize_session=False) session.commit() except (SQLAlchemyError, DBAPIError) as e: current_app.logger.error('SQLAlchemy: ' + str(e)) return False, 'SQLAlchemy: ' + str(e) return True, count, 'removed ' + str(count) + ' records of ' + str( len(bibcode_list)) + ' bibcodes'
def setops_libraries(cls, library_id, document_data, operation='union'): """ Takes the union of two or more libraries :param library_id: the primary library ID :param document_data: dict containing the list 'libraries' that holds the secondary library IDs :return: list of bibcodes in the union set """ current_app.logger.info( 'User requested to take the {0} of {1} with {2}'.format( operation, library_id, document_data['libraries'])) with current_app.session_scope() as session: # Find the specified library primary_library = session.query(Library).filter_by( id=library_id).one() out_lib = set(primary_library.get_bibcodes()) for lib in document_data['libraries']: if isinstance(lib, str): lib = cls.helper_slug_to_uuid(lib) secondary_library = session.query(Library).filter_by( id=lib).one() if operation == 'union': out_lib = out_lib.union( set(secondary_library.get_bibcodes())) elif operation == 'intersection': out_lib = out_lib.intersection( set(secondary_library.get_bibcodes())) elif operation == 'difference': out_lib = out_lib.difference( set(secondary_library.get_bibcodes())) else: current_app.logger.warning( 'Requested operation {0} is not allowed.'.format( operation)) return if len(out_lib) < 1: current_app.logger.info( 'No records remain after taking the {0} of {1} and {2}'.format( operation, library_id, document_data['libraries'])) return list(out_lib)
def get_access_token(): '''Exchange 'code' for 'access_token' data''' payload = dict(request.args) if 'code' not in payload: raise Exception('Parameter code is missing') headers = {'Accept': 'application/json'} data = { 'client_id': current_app.config['ORCID_CLIENT_ID'], 'client_secret': current_app.config['ORCID_CLIENT_SECRET'], 'code': payload['code'][0], 'grant_type': 'authorization_code' } #print current_app.config['ORCID_OAUTH_ENDPOINT'], data, headers r = current_app.client.post(current_app.config['ORCID_OAUTH_ENDPOINT'], data=data, headers=headers) if r.status_code != 200: logging.error('For ORCID code {}, there was an error getting the token from the ORCID API.'. format(payload['code'][0])) # update/create user account data = r.json() if 'orcid' in data: with current_app.session_scope() as session: u = session.query(User).filter_by(orcid_id=data['orcid']).options(load_only(User.orcid_id)).first() p = session.query(Profile).filter_by(orcid_id=data['orcid']).options(load_only(Profile.orcid_id)).first() if not u: u = User(orcid_id=data['orcid'], created=adsmutils.get_date()) if not p: p = Profile(orcid_id=data['orcid'], created=adsmutils.get_date()) u.updated = adsmutils.get_date() p. updated = adsmutils.get_date() u.access_token = data['access_token'] # save the user session.begin_nested() try: session.add(u) session.add(p) session.commit() except exc.IntegrityError as e: session.rollback() # per PEP-0249 a transaction is always in progress session.commit() return r.text, r.status_code
def configuration(key=None): '''Allows you to retrieve JSON data from VAULT_BUMBLEBEE_OPTIONS''' opts = current_app.config.get('VAULT_BUMBLEBEE_OPTIONS') or {} if not isinstance(opts, dict): return json.dumps({'msg': 'Server misconfiguration, VAULT_BUMBLEBEE_OPTIONS is of an invalid type'}), 500 if key: if key == 'link_servers': with current_app.session_scope() as session: res = session.query(Library).all() link_servers = [{"name": l.libname, "link": l.libserver, "gif":l.iconurl} for l in res] link_servers = sorted(link_servers, key=itemgetter('name')) return json.dumps(link_servers), 200 elif key in opts: return json.dumps(opts[key]), 200 else: return '{}', 404 else: return json.dumps(opts)
def export(iso_datestring): ''' Get the latest changes (as recorded in users table) The optional argument latest_point is RFC3339, ie. '2008-09-03T20:56:35.450686Z' ''' # TODO check rate limit # TODO need to add a created/update field to users table # inspired by orcid-service endpoint of same endpoint, checks the users table for profiles w/ a myADS setup that # have been updated since a given date/time; returns these profiles to be added to the myADS processing latest = parser.parse(iso_datestring) # ISO 8601 output = [] with current_app.session_scope() as session: setups = session.query(MyADS).filter(MyADS.updated > latest).order_by( MyADS.updated.asc()).all() for s in setups: output.append(s.user_id) output = list(set(output)) return json.dumps({'users': output}), 200
def helper_create_user(absolute_uid): """ Creates a user in the database with a UID from the API :param absolute_uid: UID from the API :return: None """ with current_app.session_scope() as session: try: user = User(absolute_uid=absolute_uid) session.add(user) session.commit() current_app.logger.info('Successfully created user: {0} [API] as ' '{1} [Microservice]' .format(absolute_uid, user.id)) except IntegrityError as error: current_app.logger.error('IntegrityError. User: {0:d} was not' 'added. Full traceback: {1}' .format(absolute_uid, error)) raise
def empty_library(library_id): """ Empties the contents of one library :param library_id: library to empty :return: dict containing the metadata of the emptied library """ current_app.logger.info('User requested to empty the contents of {0}'.format(library_id)) metadata = {} with current_app.session_scope() as session: lib = session.query(Library).filter_by(id=library_id).one() lib.remove_bibcodes(lib.get_bibcodes()) metadata['name'] = lib.name metadata['description'] = lib.description metadata['public'] = lib.public session.add(lib) session.commit() return metadata
def execute_query(queryid): '''Allows you to execute stored query''' with current_app.session_scope() as session: q = session.query(Query).filter_by(qid=queryid).first() if not q: return json.dumps({msg: 'Query not found: ' + qid}), 404 q_query = q.query try: payload, headers = check_request(request) except Exception as e: return json.dumps({'msg': e.message or e.description}), 400 dataq = json.loads(q_query) query = urlparse.parse_qs(dataq['query']) # override parameters using supplied params if len(payload) > 0: query.update(payload) # make sure the {!bitset} is there (when bigquery is used) if dataq['bigquery']: fq = query.get('fq') if not fq: fq = ['{!bitset}'] elif '!bitset' not in str(fq): if isinstance(fq, list): fq.append('{!bitset}') else: fq = ['{!bitset}'] query['fq'] = fq # always request json query['wt'] = 'json' r = make_solr_request(query=query, bigquery=dataq['bigquery'], headers=headers) return r.text, r.status_code
def library_name_exists(service_uid, library_name): """ Checks to see if a library name already exists in the user's created libraries :param service_uid: the user ID within this microservice :param library_name: name to check if it exists :return: True (exists), False (does not exist) """ with current_app.session_scope() as session: library_names = \ [i.library.name for i in session.query(Permissions).filter_by(user_id = service_uid, owner = True).all()] if library_name in library_names: current_app.logger.error('Name supplied for the library already ' 'exists: "{0}"'.format(library_name)) return True else: return False
def remove_documents_from_library(cls, library_id, document_data): """ Remove a given document from a specific library :param library_id: the unique ID of the library :param document_data: the meta data of the document :return: number_removed: number of documents successfully removed """ current_app.logger.info('Removing a document: {0} from library_uuid: ' '{1}'.format(document_data, library_id)) with current_app.session_scope() as session: library = session.query(Library).filter_by(id=library_id).one() start_length = len(library.bibcode) library.remove_bibcodes(document_data['bibcode']) session.add(library) session.commit() current_app.logger.info('Removed document successfully: {0}' .format(library.bibcode)) end_length = len(library.bibcode) return start_length - end_length
def update_library(library_id, library_data): """ Update the meta data of the library :param library_id: the unique ID of the library :param library_data: dictionary containing the updateable values :return: values updated """ updateable = ['name', 'description', 'public'] updated = {} with current_app.session_scope() as session: library = session.query(Library).filter_by(id=library_id).one() for key in library_data: if key not in updateable: continue setattr(library, key, library_data[key]) updated[key] = library_data[key] session.add(library) session.commit() return updated
def execute_SQL_query(bibc): with current_app.session_scope() as session: resp = session.query(GraphicsModel).filter( GraphicsModel.bibcode == bibc).one() results = json.loads(json.dumps(resp, cls=AlchemyEncoder)) return results
def post(self, library): """ HTTP POST request that conducts operations at the library level. :param library: primary library ID :return: response if operation was successful Header: ------- Must contain the API forwarded user ID of the user accessing the end point Post body: ---------- KEYWORD, VALUE libraries: <list> List of secondary libraries to include in the action (optional, based on action) action: <unicode> union, intersection, difference, copy, empty Actions to perform on given libraries: Union: requires one or more secondary libraries to be passed; takes the union of the primary and secondary library sets; a new library is created Intersection: requires one or more secondary libraries to be passed; takes the intersection of the primary and secondary library sets; a new library is created Difference: requires one or more secondary libraries to be passed; takes the difference between the primary and secondary libraries; the primary library comes first in the operation, so the secondary library is removed from the primary; a new library is created Copy: requires one and only one secondary library to be passed; the primary library will be copied into the secondary library (so the secondary library will be overwritten); no new library is created Empty: secondary libraries are ignored; the primary library will be emptied of its contents, though the library and metadata will remain; no new library is created name: <string> (optional) name of the new library (must be unique for that user); used only for actions in [union, intersection, difference] description: <string> (optional) description of the new library; used only for actions in [union, intersection, difference] public: <boolean> (optional) is the new library public to view; used only for actions in [union, intersection, difference] ----------- Return data: ----------- name: <string> Name of the library id: <string> ID of the library description: <string> Description of the library Permissions: ----------- The following type of user can conduct library operations: - owner - admin - write """ # Get the user requesting this from the header try: user_editing = self.helper_get_user_id() except KeyError: return err(MISSING_USERNAME_ERROR) # URL safe base64 string to UUID library_uuid = self.helper_slug_to_uuid(library) user_editing_uid = \ self.helper_absolute_uid_to_service_uid(absolute_uid=user_editing) # Check the permissions of the user if not self.write_access(service_uid=user_editing_uid, library_id=library_uuid): return err(NO_PERMISSION_ERROR) try: data = get_post_data( request, types=dict(libraries=list, action=basestring, name=basestring, description=basestring, public=bool) ) except TypeError as error: current_app.logger.error('Wrong type passed for POST: {0} [{1}]' .format(request.data, error)) return err(WRONG_TYPE_ERROR) if data['action'] in ['union', 'intersection', 'difference']: if 'libraries' not in data: return err(NO_LIBRARY_SPECIFIED_ERROR) if 'name' not in data: data['name'] = 'Untitled {0}.'.format(get_date().isoformat()) if 'public' not in data: data['public'] = False if data['action'] == 'copy': if 'libraries' not in data: return err(NO_LIBRARY_SPECIFIED_ERROR) if len(data['libraries']) > 1: return err(TOO_MANY_LIBRARIES_SPECIFIED_ERROR) lib_names = [] with current_app.session_scope() as session: primary = session.query(Library).filter_by(id=library_uuid).one() lib_names.append(primary.name) if 'libraries' in data: for lib in data['libraries']: secondary_uuid = self.helper_slug_to_uuid(lib) secondary = session.query(Library).filter_by(id=secondary_uuid).one() lib_names.append(secondary.name) if data['action'] == 'union': bib_union = self.setops_libraries( library_id=library_uuid, document_data=data, operation='union' ) current_app.logger.info('Successfully took the union of the libraries {0} (IDs: {1}, {2})' .format(', '.join(lib_names), library, ', '.join(data['libraries']))) data['bibcode'] = bib_union if 'description' not in data: data['description'] = 'Union of libraries {0} (IDs: {1}, {2})' \ .format(', '.join(lib_names), library, ', '.join(data['libraries'])) try: library_dict = self.create_library(service_uid=user_editing_uid, library_data=data) except BackendIntegrityError as error: current_app.logger.error(error) return err(DUPLICATE_LIBRARY_NAME_ERROR) except TypeError as error: current_app.logger.error(error) return err(WRONG_TYPE_ERROR) return library_dict, 200 elif data['action'] == 'intersection': bib_intersect = self.setops_libraries( library_id=library_uuid, document_data=data, operation='intersection' ) current_app.logger.info('Successfully took the intersection of the libraries {0} (IDs: {1}, {2})' .format(', '.join(lib_names), library, ', '.join(data['libraries']))) data['bibcode'] = bib_intersect if 'description' not in data: data['description'] = 'Intersection of {0} (IDs: {1}, {2})' \ .format(', '.join(lib_names), library, ', '.join(data['libraries'])) try: library_dict = self.create_library(service_uid=user_editing_uid, library_data=data) except BackendIntegrityError as error: current_app.logger.error(error) return err(DUPLICATE_LIBRARY_NAME_ERROR) except TypeError as error: current_app.logger.error(error) return err(WRONG_TYPE_ERROR) return library_dict, 200 elif data['action'] == 'difference': bib_diff = self.setops_libraries( library_id=library_uuid, document_data=data, operation='difference' ) current_app.logger.info('Successfully took the difference of {0} (ID {2}) - (minus) {1} (ID {3})' .format(lib_names[0], ', '.join(lib_names[1:]), library, ', '.join(data['libraries']))) data['bibcode'] = bib_diff if 'description' not in data: data['description'] = 'Records that are in {0} (ID {2}) but not in {1} (ID {3})' \ .format(lib_names[0], ', '.join(lib_names[1:]), library, ', '.join(data['libraries'])) try: library_dict = self.create_library(service_uid=user_editing_uid, library_data=data) except BackendIntegrityError as error: current_app.logger.error(error) return err(DUPLICATE_LIBRARY_NAME_ERROR) except TypeError as error: current_app.logger.error(error) return err(WRONG_TYPE_ERROR) return library_dict, 200 elif data['action'] == 'copy': library_dict = self.copy_library( library_id=library_uuid, document_data=data ) current_app.logger.info('Successfully copied {0} (ID {2}) into {1} (ID {3})' .format(lib_names[0], lib_names[1], library, data['libraries'][0])) with current_app.session_scope() as session: libid = self.helper_slug_to_uuid(data['libraries'][0]) library = session.query(Library).filter_by(id=libid).one() bib = library.get_bibcodes() library_dict['bibcode'] = bib return library_dict, 200 elif data['action'] == 'empty': library_dict = self.empty_library( library_id=library_uuid ) current_app.logger.info('Successfully emptied {0} (ID {1}) of all records' .format(lib_names[0], library)) with current_app.session_scope() as session: library = session.query(Library).filter_by(id=library_uuid).one() bib = library.get_bibcodes() library_dict['bibcode'] = bib return library_dict, 200 else: current_app.logger.info('User requested a non-standard operation') return {}, 400
def execute_SQL_query(query): with current_app.session_scope() as session: results = session.execute(query).fetchall() return results
def has_permission(service_uid_editor, service_uid_modify, library_id): """ Check if the user wanting to change the library has the correct permissions to do so, and the user to be changed is not the owner. :param service_uid_editor: the user ID of the editor :param service_uid_modify: the user ID of the user to be edited :param library_id: the library id :return: boolean """ if service_uid_editor == service_uid_modify: current_app.logger.error('Editing user: {0} and user to edit: {1}' ' are the same. This is not allowed.' .format(service_uid_modify, service_uid_editor)) return False current_app.logger.info('Checking if user: {0}, can edit the ' 'permissions of user: {1}' .format( service_uid_editor, service_uid_modify )) # Check if the editor has permissions with current_app.session_scope() as session: try: editor_permissions = session.query(Permissions).filter( Permissions.user_id == service_uid_editor, Permissions.library_id == library_id ).one() except NoResultFound as error: current_app.logger.error( 'User: {0} has no permissions for this library: {1}' .format(service_uid_editor, error) ) return False if editor_permissions.owner: current_app.logger.info('User: {0} is owner, so is allowed to ' 'change permissions' .format(service_uid_editor)) return True # Check if the user to be modified has permissions try: modify_permissions = session.query(Permissions).filter( Permissions.user_id == service_uid_modify, Permissions.library_id == library_id ).one() except NoResultFound: modify_permissions = False # if the editor is admin, and the modifier has no permissions if editor_permissions.admin: # user to be modified has no permissions if not modify_permissions: return True # user to be modified is not owner if not modify_permissions.owner: return True else: return False
def get_libraries(cls, service_uid, absolute_uid): """ Get all the libraries a user has :param service_uid: microservice UID of the user :param absolute_uid: unique UID of the user in the API :return: list of libraries in json format """ # Get all the permissions for a user # This can be improved into one database call rather than having # one per each permission, but needs some checks in place. with current_app.session_scope() as session: result = session.query(Permissions, Library)\ .join(Permissions.library)\ .filter(Permissions.user_id == service_uid)\ .all() output_libraries = [] for permission, library in result: # For this library get all the people who have permissions users = session.query(Permissions).filter_by( library_id = library.id ).all() num_documents = 0 if library.bibcode: num_documents = len(library.bibcode) if permission.owner: main_permission = 'owner' elif permission.admin: main_permission = 'admin' elif permission.write: main_permission = 'write' elif permission.read: main_permission = 'read' else: main_permission = 'none' if permission.owner or permission.admin and not library.public: num_users = len(users) elif library.public: num_users = len(users) else: num_users = 0 service = '{api}/{uid}'.format( api=current_app.config['BIBLIB_USER_EMAIL_ADSWS_API_URL'], uid=absolute_uid ) current_app.logger.info('Obtaining email of user: {0} [API UID]' .format(absolute_uid)) response = client().get( service ) if response.status_code != 200: current_app.logger.error('Could not find user in the API' 'database: {0}.'.format(service)) owner = 'Not available' else: owner = response.json()['email'].split('@')[0] payload = dict( name=library.name, id='{0}'.format(cls.helper_uuid_to_slug(library.id)), description=library.description, num_documents=num_documents, date_created=library.date_created.isoformat(), date_last_modified=library.date_last_modified.isoformat(), permission=main_permission, public=library.public, num_users=num_users, owner=owner ) output_libraries.append(payload) return output_libraries
def post(self): """ HTTP POST request that creates a library for a given user :return: the response for if the library was successfully created Header: ------- Must contain the API forwarded user ID of the user accessing the end point Post body: ---------- KEYWORD, VALUE name: <string> name of the library (must be unique for that user) description: <string> description of the library public: <boolean> is the library public to view bibcode (OPTIONAL): <list> list of bibcodes to add Return data: ----------- name: <string> Name of the library id: <string> ID of the library description: <string> Description of the library Permissions: ----------- The following type of user can create a library - must be logged in, i.e., scope = user """ # Check that they pass a user id try: user = self.helper_get_user_id() except KeyError: return err(MISSING_USERNAME_ERROR) # Check if the user exists, if not, generate a user in the database current_app.logger.info('Checking if the user exists') if not self.helper_user_exists(absolute_uid=user): current_app.logger.info('User: {0:d}, does not exist.' .format(user)) self.create_user(absolute_uid=user) current_app.logger.info('User: {0:d}, created.'.format(user)) else: current_app.logger.info('User already exists.') # Switch to the service UID and not the API UID service_uid = \ self.helper_absolute_uid_to_service_uid(absolute_uid=user) current_app.logger.info('user_API: {0:d} is now user_service: {1:d}' .format(user, service_uid)) # Create the library try: data = get_post_data( request, types=dict( name=unicode, description=unicode, public=bool, bibcode=list ) ) except TypeError as error: current_app.logger.error('Wrong type passed for POST: {0} [{1}]' .format(request.data, error)) return err(WRONG_TYPE_ERROR) with current_app.session_scope() as session: try: library_dict = \ self.create_library(service_uid=service_uid, library_data=data) except BackendIntegrityError as error: current_app.logger.error(error) return err(DUPLICATE_LIBRARY_NAME_ERROR) except TypeError as error: current_app.logger.error(error) return err(WRONG_TYPE_ERROR) return library_dict, 200
def store_data(): '''Allows you to store/retrieve JSON data on the server side. It is always associated with the user id (which is communicated to us by API) - so there is no endpoint allowing you to access other users' data (should there be?) /user-data/<uid>?''' # get the query data try: payload, headers = check_request(request) except Exception as e: return json.dumps({'msg': e.message or e.description}), 400 user_id = int(headers['X-Adsws-Uid']) if user_id == 0: return json.dumps({'msg': 'Sorry, you can\'t use this service as an anonymous user'}), 400 if request.method == 'GET': with current_app.session_scope() as session: q = session.query(User).filter_by(id=user_id).first() if not q: return '{}', 200 # or return 404? return json.dumps(q.user_data) or '{}', 200 elif request.method == 'POST': # limit both number of keys and length of value to keep db clean if len(max(payload.values(), key=len)) > current_app.config['MAX_ALLOWED_JSON_SIZE']: return json.dumps({'msg': 'You have exceeded the allowed storage limit (length of values), no data was saved'}), 400 if len(payload.keys()) > current_app.config['MAX_ALLOWED_JSON_KEYS']: return json.dumps({'msg': 'You have exceeded the allowed storage limit (number of keys), no data was saved'}), 400 with current_app.session_scope() as session: try: q = session.query(User).filter_by(id=user_id).with_for_update(of=User).one() try: data = q.user_data except TypeError: data = {} except ormexc.NoResultFound: data = payload u = User(id=user_id, user_data=data) try: session.add(u) session.commit() return json.dumps(data), 200 except exc.IntegrityError: q = session.query(User).filter_by(id=user_id).with_for_update(of=User).one() try: data = q.user_data except TypeError: data = {} data.update(payload) q.user_data = data session.begin_nested() try: session.commit() except exc.IntegrityError: session.rollback() return json.dumps({'msg': 'We have hit a db error! The world is crumbling all around... (eh, btw, your data was not saved)'}), 500 return json.dumps(data), 200
def query(queryid=None): '''Stores/retrieves the montysolr query; it can receive data in urlencoded format or as application/json encoded data. In the second case, you can pass 'bigquery' together with the 'query' like so: { query: {foo: bar}, bigquery: 'data\ndata\n....' } ''' if request.method == 'GET' and queryid: with current_app.session_scope() as session: q = session.query(Query).filter_by(qid=queryid).first() if not q: return json.dumps({'msg': 'Query not found: ' + queryid}), 404 return json.dumps({ 'qid': q.qid, 'query': q.query, 'numfound': q.numfound }), 200 # get the query data try: payload, headers = check_request(request) except Exception as e: return json.dumps({'msg': e.message or e.description}), 400 if len(payload.keys()) == 0: raise Exception('Query cannot be empty') payload = cleanup_payload(payload) # check we don't have this query already query = json.dumps(payload) qid = md5.new(headers['X-Adsws-Uid'] + query).hexdigest() with current_app.session_scope() as session: q = session.query(Query).filter_by(qid=qid).first() if q: return json.dumps({'qid': qid, 'numFound': q.numfound}), 200 # check the query is valid solrq = payload['query'] + '&wt=json' r = make_solr_request(query=solrq, bigquery=payload['bigquery'], headers=headers) if r.status_code != 200: return json.dumps({'msg': 'Could not verify the query.', 'query': payload, 'reason': r.text}), 404 # extract number of docs found num_found = 0 try: num_found = int(r.json()['response']['numFound']) except: pass # save the query q = Query(qid=qid, query=query, numfound=num_found) with current_app.session_scope() as session: session.begin_nested() try: session.add(q) session.commit() except exc.IntegrityError as e: session.rollback() return json.dumps({'msg': e.message or e.description}), 400 # TODO: update #q = session.merge(q) # force re-sync from database #q.updated = datetime.datetime.utcnow() #session.commit() return json.dumps({'qid': qid, 'numFound': num_found}), 200
def solr_update_library(library_id, solr_docs): """ Updates the library based on the solr canonical bibcodes response :param library: library_id of the library to update :param solr_docs: solr docs from the bigquery response :return: dictionary with details of files modified num_updated: number of documents modified duplicates_removed: number of files removed for duplication update_list: list of changed bibcodes {'before': 'after'} """ # Definitions update = False canonical_bibcodes = [] alternate_bibcodes = {} new_bibcode = {} # Constants for the return dictionary num_updated = 0 duplicates_removed = 0 update_list = [] # Extract the canonical bibcodes and create a hashmap for the # alternate bibcodes for doc in solr_docs: canonical_bibcodes.append(doc['bibcode']) if doc.get('alternate_bibcode'): alternate_bibcodes.update( {i: doc['bibcode'] for i in doc['alternate_bibcode']} ) with current_app.session_scope() as session: library = session.query(Library).filter(Library.id == library_id).one() for bibcode in library.bibcode: # Skip if its already canonical if bibcode in canonical_bibcodes: new_bibcode[bibcode] = library.bibcode[bibcode] continue # Update if its an alternate if bibcode in alternate_bibcodes.keys(): update = True num_updated += 1 update_list.append({bibcode: alternate_bibcodes[bibcode]}) # Only add the bibcode if it is not there if alternate_bibcodes[bibcode] not in new_bibcode: new_bibcode[alternate_bibcodes[bibcode]] = \ library.bibcode[bibcode] else: duplicates_removed += 1 elif bibcode not in canonical_bibcodes and\ bibcode not in alternate_bibcodes.keys(): new_bibcode[bibcode] = library.bibcode[bibcode] if update: # Update the database library.bibcode = new_bibcode session.add(library) session.commit() updates = dict( num_updated=num_updated, duplicates_removed=duplicates_removed, update_list=update_list ) return updates
def get_documents_from_library(cls, library_id, service_uid): """ Retrieve all the documents that are within the library specified :param library_id: the unique ID of the library :param service_uid: the user ID within this microservice :return: bibcodes """ with current_app.session_scope() as session: # Get the library library = session.query(Library).filter_by(id=library_id).one() # Get the owner of the library result = session.query(Permissions, User)\ .join(Permissions.user)\ .filter(Permissions.library_id == library_id)\ .filter(Permissions.owner == True)\ .one() owner_permissions, owner = result service = '{api}/{uid}'.format( api=current_app.config['BIBLIB_USER_EMAIL_ADSWS_API_URL'], uid=owner.absolute_uid ) current_app.logger.info('Obtaining email of user: {0} [API UID]' .format(owner.absolute_uid)) response = client().get( service ) # For this library get all the people who have permissions users = session.query(Permissions).filter_by( library_id = library.id ).all() if response.status_code != 200: current_app.logger.error('Could not find user in the API' 'database: {0}.'.format(service)) owner = 'Not available' else: owner = response.json()['email'].split('@')[0] # User requesting to see the content if service_uid: try: permission = session.query(Permissions).filter( Permissions.user_id == service_uid ).filter( Permissions.library_id == library_id ).one() if permission.owner: main_permission = 'owner' elif permission.admin: main_permission = 'admin' elif permission.write: main_permission = 'write' elif permission.read: main_permission = 'read' else: main_permission = 'none' except: main_permission = 'none' else: main_permission = 'none' if main_permission == 'owner' or main_permission == 'admin': num_users = len(users) elif library.public: num_users = len(users) else: num_users = 0 metadata = dict( name=library.name, id='{0}'.format(cls.helper_uuid_to_slug(library.id)), description=library.description, num_documents=len(library.bibcode), date_created=library.date_created.isoformat(), date_last_modified=library.date_last_modified.isoformat(), permission=main_permission, public=library.public, num_users=num_users, owner=owner ) session.refresh(library) session.expunge(library) return library, metadata
def create_library(cls, service_uid, library_data): """ Creates a library for a user :param service_uid: the user ID within this microservice :param library_data: content needed to create a library :return: library dict """ library_data = BaseView.helper_validate_library_data( service_uid=service_uid, library_data=library_data ) _name = library_data.get('name') _description = library_data.get('description') _public = bool(library_data.get('public', False)) _bibcode = library_data.get('bibcode', False) with current_app.session_scope() as session: try: # Make the library in the library table library = Library(name=_name, description=_description, public=_public) # If the user supplies bibcodes if _bibcode and isinstance(_bibcode, list): # Ensure unique content _bibcode = uniquify(_bibcode) current_app.logger.info('User supplied bibcodes: {0}' .format(_bibcode)) library.add_bibcodes(_bibcode) elif _bibcode: current_app.logger.error('Bibcode supplied not a list: {0}' .format(_bibcode)) raise TypeError('Bibcode should be a list.') user = session.query(User).filter_by(id=service_uid).one() # Make the permissions permission = Permissions( owner=True, ) # Use the ORM to link the permissions to the library and user, # so that no commit is required until the complete action is # finished. This means any rollback will not leave a single # library without permissions library.permissions.append(permission) user.permissions.append(permission) session.add_all([library, permission, user]) session.commit() current_app.logger.info('Library: "{0}" made, user_service: {1:d}' .format(library.name, user.id)) library_dict = dict( name=library.name, id='{0}'.format(cls.helper_uuid_to_slug(library.id)), description=library.description, ) # If they added bibcodes include in the response if hasattr(library, 'bibcode') and library.bibcode: library_dict['bibcode'] = library.get_bibcodes() return library_dict except IntegrityError as error: # Roll back the changes session.rollback() current_app.logger.error('IntegitryError, database has been rolled' 'back. Caused by user_service: {0:d}.' 'Full error: {1}' .format(user.id, error)) # Log here raise except Exception: session.rollback() raise