def get_permissions(cls, library_id):
"""
Looks up and returns the permissions for all of the users that have
any permissions.
:param library_id: the library ID to look up permissions
:return: list of dictionaries containing the user email and permission
"""
with current_app.session_scope() as session:
# Find the permissions for the library
result = session.query(Permissions, User)\
.join(Permissions.user)\
.filter(Permissions.library_id == library_id)\
.all()
# Formulate the return content
permission_list = []
for permission, user in result:
# Convert the user id into
user = cls.api_uid_email_lookup(user_info=user.absolute_uid)
all_permissions = filter(
lambda key: permission.__dict__[key],
['read', 'write', 'admin', 'owner']
)
permission_list.append(
{user: all_permissions}
)
return permission_list
def get(self):
"""
HTTP GET request that returns the information currently stored about the
user's ADS Classic settings, currently stored in the service database.
Return data (on success)
------------------------
classic_email: <string> ADS Classic e-mail of the user
classic_mirror: <string> ADS Classic mirror this user belongs to
HTTP Responses:
--------------
Succeed authentication: 200
User unknown/wrong password/failed authentication: 404
Any other responses will be default Flask errors
"""
absolute_uid = self.helper_get_user_id()
try:
with current_app.session_scope() as session:
user = session.query(Users).filter(
Users.absolute_uid == absolute_uid).one()
return {
'classic_email': user.classic_email,
'classic_mirror': user.classic_mirror,
'twopointoh_email': user.twopointoh_email
}, 200
except NoResultFound:
return err(NO_CLASSIC_ACCOUNT)
def copy_library(cls, library_id, document_data):
"""
Copies the contents of one library into another. Does not empty library first; call
empty_library on the target library first to do so
:param library_id: primary library ID, library to copy
:param document_data: dict containing the list 'libraries' which holds one secondary library ID; this is
the library to copy over
:return: dict containing the metadata of the copied-over library (the secondary library)
"""
current_app.logger.info('User requested to copy the contents of {0} into {1}'
.format(library_id, document_data['libraries']))
secondary_libid = document_data['libraries'][0]
if isinstance(secondary_libid, basestring):
secondary_libid = cls.helper_slug_to_uuid(secondary_libid)
metadata = {}
with current_app.session_scope() as session:
primary_library = session.query(Library).filter_by(id=library_id).one()
good_bib = primary_library.get_bibcodes()
secondary_library = session.query(Library).filter_by(id=secondary_libid).one()
secondary_library.add_bibcodes(good_bib)
metadata['name'] = secondary_library.name
metadata['description'] = secondary_library.description
metadata['public'] = secondary_library.public
session.add(secondary_library)
session.commit()
return metadata
def add_document_to_library(cls, library_id, document_data):
"""
Adds a document to a user's library
:param library_id: the library id to update
:param document_data: the meta data of the document
:return: number_added: number of documents successfully added
"""
current_app.logger.info('Adding a document: {0} to library_uuid: {1}'
.format(document_data, library_id))
with current_app.session_scope() as session:
# Find the specified library
library = session.query(Library).filter_by(id=library_id).one()
start_length = len(library.bibcode)
library.add_bibcodes(document_data['bibcode'])
session.add(library)
session.commit()
current_app.logger.info('Added: {0} is now {1}'.format(
document_data['bibcode'],
library.bibcode)
)
end_length = len(library.bibcode)
return end_length - start_length
def configuration(key=None):
'''Allows you to retrieve JSON data from VAULT_BUMBLEBEE_OPTIONS'''
opts = current_app.config.get('VAULT_BUMBLEBEE_OPTIONS') or {}
if not isinstance(opts, dict):
return json.dumps({
'msg':
'Server misconfiguration, VAULT_BUMBLEBEE_OPTIONS is of an invalid type'
}), 500
if key:
if key == 'link_servers':
with current_app.session_scope() as session:
res = session.query(Library).all()
link_servers = [{
"name": l.libname,
"link": l.libserver,
"gif": l.iconurl
} for l in res]
link_servers = sorted(link_servers, key=itemgetter('name'))
return json.dumps(link_servers), 200
elif key in opts:
return json.dumps(opts[key]), 200
else:
return '{}', 404
else:
return json.dumps(opts), 200
def apply_protective_filters(self, payload, user_id, protected_fields):
"""
Adds filters to the query that should limit results to conditions
that are associted with the user_id+protected_field. If a field is
not found in the db of limits, it will not be returned to the user
:param payload: raw request payload
:param user_id: string, user id as known to ADS API
:param protected_fields: list of strings, fields
"""
fl = payload.get('fl', 'id')
fq = payload.get('fq', [])
if not isinstance(fq, list):
fq = [fq]
payload['fq'] = fq
with current_app.session_scope() as session:
for f in session.query(Limits).filter(
Limits.uid == user_id,
or_(Limits.field == x for x in protected_fields)).all():
if f.filter:
fl = u'{0},{1}'.format(fl, f.field)
fq.append(unicode(f.filter))
payload['fl'] = fl
session.commit()
def orcid_profile_local(orcid_id, type):
'''Get /[orcid-id]/orcid-profile/<simple,full> - returns either bibcodes and statuses (/simple) or all
records and saved metadata (/full) - all communication exclusively in JSON'''
payload, headers = check_request(request)
update = request.args.get('update', False)
if type not in ['simple','full']:
return json.dumps('Endpoint /orcid-profile/%s does not exist'.format(type)), 404
r = current_app.client.get(current_app.config['ORCID_API_ENDPOINT'] + '/' + orcid_id + '/record',
headers=headers)
if r.status_code == 200:
update_profile_local(orcid_id, data=r.text, force=update)
else:
logging.warning('Failed fetching fresh profile from ORCID for %s'.format(orcid_id))
with current_app.session_scope() as session:
profile = session.query(Profile).filter_by(orcid_id=orcid_id).first()
if type == 'simple':
bibcodes, statuses = profile.get_bibcodes()
records = dict(zip(bibcodes, statuses))
elif type == 'full':
records = profile.get_records()
return json.dumps(records), 200
def update_stored_profile(orcid_id):
'''Updates profile in orcid-service before processing'''
with current_app.session_scope() as session:
u = session.query(User).filter_by(orcid_id=orcid_id).first()
if not u:
return json.dumps({'error': 'We do not have a record for: %s' % orcid_id}), 404
if not u.access_token:
return json.dumps({'error': 'We do not have access_token for: %s' % orcid_id}), 404
token = u.access_token
headers = {
'Accept': 'application/json',
'Authorization': 'Bearer %s' % token,
'Content-Type': 'application/json'
}
try:
r = current_app.client.get(current_app.config['ORCID_API_ENDPOINT'] + '/' + orcid_id + '/record',
headers=headers, timeout=current_app.config.get('CONNECTION_TIMEOUT', 30))
except (ConnectionError, ConnectTimeout, ReadTimeout) as e:
logging.error('For ORCID ID %s, there was a connection error with the ORCID API'.format(orcid_id))
return 'There was a connection error with the ORCID API', 502
if r.status_code == 200:
update_profile_local(orcid_id, data=r.text, force=True)
else:
logging.warning('Failed fetching fresh profile from ORCID for %s'.format(orcid_id))
profile = session.query(Profile).filter_by(orcid_id=orcid_id).first()
records = profile.get_records()
return json.dumps(records), 200
def add_document_to_library(cls, library_id, document_data):
"""
Adds a document to a user's library
:param library_id: the library id to update
:param document_data: the meta data of the document
:return: number_added: number of documents successfully added
"""
current_app.logger.info(
'Adding a document: {0} to library_uuid: {1}'.format(
document_data, library_id))
with current_app.session_scope() as session:
# Find the specified library
library = session.query(Library).filter_by(id=library_id).one()
start_length = len(library.bibcode)
library.add_bibcodes(document_data['bibcode'])
session.add(library)
session.commit()
current_app.logger.info('Added: {0} is now {1}'.format(
document_data['bibcode'], library.bibcode))
end_length = len(library.bibcode)
return end_length - start_length
def setops_libraries(cls, library_id, document_data, operation='union'):
"""
Takes the union of two or more libraries
:param library_id: the primary library ID
:param document_data: dict containing the list 'libraries' that holds the secondary library IDs
:return: list of bibcodes in the union set
"""
current_app.logger.info('User requested to take the {0} of {1} with {2}'
.format(operation, library_id, document_data['libraries']))
with current_app.session_scope() as session:
# Find the specified library
primary_library = session.query(Library).filter_by(id=library_id).one()
out_lib = set(primary_library.get_bibcodes())
for lib in document_data['libraries']:
if isinstance(lib, basestring):
lib = cls.helper_slug_to_uuid(lib)
secondary_library = session.query(Library).filter_by(id=lib).one()
if operation == 'union':
out_lib = out_lib.union(set(secondary_library.get_bibcodes()))
elif operation == 'intersection':
out_lib = out_lib.intersection(set(secondary_library.get_bibcodes()))
elif operation == 'difference':
out_lib = out_lib.difference(set(secondary_library.get_bibcodes()))
else:
current_app.logger.warning('Requested operation {0} is not allowed.'.format(operation))
return
if len(out_lib) < 1:
current_app.logger.info('No records remain after taking the {0} of {1} and {2}'
.format(operation, library_id, document_data['libraries']))
return list(out_lib)
def library_name_exists(service_uid, library_name):
"""
Checks to see if a library name already exists in the user's created
libraries
:param service_uid: the user ID within this microservice
:param library_name: name to check if it exists
:return: True (exists), False (does not exist)
"""
with current_app.session_scope() as session:
library_names = \
[i.library.name for i in
session.query(Permissions)\
.filter_by(user_id = service_uid)\
.filter(Permissions.permissions['owner'].astext.cast(Boolean).is_(True)).all()]
if library_name in library_names:
current_app.logger.error('Name supplied for the library already '
'exists: "{0}"'.format(library_name))
return True
else:
return False
def update_stored_profile(orcid_id):
'''Updates profile in orcid-service before processing'''
with current_app.session_scope() as session:
u = session.query(User).filter_by(orcid_id=orcid_id).first()
if not u:
return json.dumps({'error': 'We do not have a record for: %s' % orcid_id}), 404
if not u.access_token:
return json.dumps({'error': 'We do not have access_token for: %s' % orcid_id}), 404
token = u.access_token
headers = {
'Accept': 'application/json',
'Authorization': 'Bearer %s' % token,
'Content-Type': 'application/json'
}
r = current_app.client.get(current_app.config['ORCID_API_ENDPOINT'] + '/' + orcid_id + '/record',
headers=headers)
if r.status_code == 200:
update_profile_local(orcid_id, data=r.text, force=True)
else:
logging.warning('Failed fetching fresh profile from ORCID for %s'.format(orcid_id))
profile = session.query(Profile).filter_by(orcid_id=orcid_id).first()
records = profile.get_records()
return json.dumps(records), 200
def helper_access_allowed(service_uid, library_id, access_type):
"""
Determines if the given user has permissions to look at the content
of a library.
:param service_uid: the user ID within this microservice
:param library_id: the unique ID of the library
:param access_type: list of access types to check
:return: boolean, access (True), no access (False)
"""
with current_app.session_scope() as session:
try:
permissions = session.query(Permissions).filter_by(
library_id = library_id,
user_id = service_uid
).one()
return getattr(permissions, access_type)
except NoResultFound as error:
current_app.logger.error('No permissions for '
'user: {0}, library: {1}, permission: {2}'
' [{3}]'.format(service_uid, library_id,
access_type, error))
return False
def store_get(qid=None):
if request.method == 'GET':
with current_app.session_scope() as session:
page = session.query(Pages).filter_by(qid=qid).first()
if not page:
return jsonify({'qid': qid, 'msg': 'Not found'}), 404
return current_app.wrap_response(page)
elif request.method == 'HEAD':
with current_app.session_scope() as session:
# TODO: make it more efficient
page = session.query(Pages).filter_by(qid=qid).options(
load_only('id')).first()
if page:
return '', 200
else:
return '', 404
def execute_myads_query(myads_id):
"""
Returns the constructed query for a single templated myADS notification, ready to execute
:param myads_id: ID of a single notification
:return: list of dicts; constructed query, dates are such that it's meant to be run today:
[{q: query params,
sort: sort string}]
"""
try:
payload, headers = check_request(request)
except Exception as e:
return json.dumps({'msg': e.message or e.description}), 400
user_id = int(headers['X-Adsws-Uid'])
if user_id == current_app.config['BOOTSTRAP_USER_ID']:
return json.dumps(
{'msg':
'Sorry, you can\'t use this service as an anonymous user'}), 400
with current_app.session_scope() as session:
setup = session.query(MyADS).filter_by(user_id=user_id).filter_by(
id=myads_id).first()
if setup is None:
return '{}', 404
query = _create_myads_query(setup.template,
setup.frequency,
setup.data,
classes=setup.classes)
return json.dumps(query)
def store_data():
'''Allows you to store/retrieve JSON data on the server side.
It is always associated with the user id (which is communicated
to us by API) - so there is no endpoint allowing you to access
other users' data (should there be?) /user-data/<uid>?'''
# get the query data
try:
payload, headers = check_request(request)
except Exception as e:
return json.dumps({'msg': e.message or e.description}), 400
user_id = int(headers['X-Adsws-Uid'])
if user_id == 0:
return json.dumps(
{'msg':
'Sorry, you can\'t use this service as an anonymous user'}), 400
if request.method == 'GET':
with current_app.session_scope() as session:
q = session.query(User).filter_by(id=user_id).first()
if not q:
return '{}', 200 # or return 404?
return q.user_data or '{}', 200
elif request.method == 'POST':
d = json.dumps(payload)
if len(d) > MAX_ALLOWED_JSON_SIZE:
return json.dumps({
'msg':
'You have exceeded the allowed storage limit, no data was saved'
}), 400
u = User(id=user_id, user_data=d)
with current_app.session_scope() as session:
session.begin_nested()
try:
session.merge(u)
session.commit()
except exc.IntegrityError:
session.rollback()
return json.dumps({
'msg':
'We have hit a db error! The world is crumbling all around... (eh, btw, your data was not saved)'
}), 500
return d, 200
def get_myads(user_id):
'''
Fetches a myADS profile for the pipeline for a given uid
'''
# takes a given uid, grabs the user_data field, checks for the myADS key - if present, grabs the appropriate
# queries/data from the queries/myADS table. Finally, returns the appropriate info along w/ the rest
# of the setup from the dict (e.g. active, frequency, etc.)
# TODO check rate limit
# structure in vault:
# "myADS": [{"name": user-supplied name, "qid": QID from query table (type="query") or ID from myads table
# (type="template"), "type": "query" or "template", "active": true/false, "frequency": "daily" or "weekly",
# "stateful": true/false}]
output = []
with current_app.session_scope() as session:
setups = session.query(MyADS).filter_by(user_id=user_id).filter_by(
active=True).order_by(MyADS.id.asc()).all()
if not setups:
return '{}', 404
for s in setups:
o = {
'id': s.id,
'name': s.name,
'type': s.type,
'active': s.active,
'stateful': s.stateful,
'frequency': s.frequency,
'template': s.template,
'classes': s.classes,
'data': s.data,
'created': s.created.isoformat(),
'updated': s.updated.isoformat()
}
if s.type == 'query':
try:
q = session.query(Query).filter_by(id=s.query_id).one()
qid = q.qid
query = None
except ormexc.NoResultFound:
qid = None
query = None
else:
qid = None
query = _create_myads_query(s.template,
s.frequency,
s.data,
classes=s.classes)
o['qid'] = qid
o['query'] = query
output.append(o)
return json.dumps(output), 200
def get(self, uid):
"""
HTTP GET request that finds the libraries within ADS 2.0 for that user.
:param uid: user ID for the API
:type uid: int
Return data (on success)
------------------------
libraries: <list<dict>> a list of dictionaries, that contains the
following for each library entry:
name: <string> name of the library
description: <string> description of the library
documents: <list<string>> list of documents
HTTP Responses:
--------------
Succeed getting libraries: 200
User does not have a classic/ADS 2.0 account: 400
User does not have any libraries in their ADS 2.0 account: 400
Unknown error: 500
Any other responses will be default Flask errors
"""
with current_app.session_scope() as session:
if not current_app.config['ADS_TWO_POINT_OH_LOADED_USERS']:
current_app.logger.error(
'Users from MongoDB have not been loaded into the app')
return err(TWOPOINTOH_AWS_PROBLEM)
try:
user = session.query(Users).filter(
Users.absolute_uid == uid).one()
# Have they got an email for ADS 2.0?
if not user.twopointoh_email:
raise NoResultFound
except NoResultFound:
current_app.logger.warning(
'User does not have an associated ADS 2.0 account')
return err(NO_TWOPOINTOH_ACCOUNT)
library_file_name = current_app.config[
'ADS_TWO_POINT_OH_USERS'].get(user.twopointoh_email, None)
if not library_file_name:
current_app.logger.warning(
'User does not have any libraries in ADS 2.0')
return err(NO_TWOPOINTOH_LIBRARIES)
try:
library = TwoPointOhLibraries.get_s3_library(library_file_name)
except Exception as error:
current_app.logger.error(
'Unknown error with AWS: {}'.format(error))
return err(TWOPOINTOH_AWS_PROBLEM)
return {'libraries': library}, 200
def preferences(orcid_id):
'''Allows you to store/retrieve JSON data on the server side.
It is always associated with the ORCID access token so there
is no need for scope access
'''
# get the query data
try:
payload, headers = check_request(request)
except Exception as e:
return json.dumps({'msg': e.message or e.description}), 400
access_token = headers['Authorization'][7:] # remove the 'Bearer:' thing
if request.method == 'GET':
with current_app.session_scope() as session:
u = session.query(User).filter(and_(User.orcid_id==orcid_id, User.access_token==access_token)).options(load_only(User.orcid_id)).first()
if not u:
return '{}', 404 # not found
return u.info or '{}', 200
elif request.method == 'POST':
d = json.dumps(payload)
if len(d) > current_app.config.get('MAX_ALLOWED_JSON_SIZE', 1000):
return json.dumps({'msg': 'You have exceeded the allowed storage limit, no data was saved'}), 400
with current_app.session_scope() as session:
u = session.query(User).filter(and_(User.orcid_id==orcid_id, User.access_token==access_token)).options(load_only(User.orcid_id)).first()
if not u:
return json.dumps({'error': 'We do not have a record for: %s' % orcid_id}), 404
u.info = d
session.begin_nested()
try:
session.merge(u)
session.commit()
except exc.IntegrityError:
session.rollback()
return json.dumps({'msg': 'We have hit a db error! The world is crumbling all around... (eh, btw, your data was not saved)'}), 500
# per PEP-0249 a transaction is always in progress
session.commit()
return d, 200
def get_access_token():
'''Exchange 'code' for 'access_token' data'''
payload = request.args.to_dict()
if 'code' not in payload:
raise Exception('Parameter code is missing')
headers = {'Accept': 'application/json'}
data = {
'client_id': current_app.config['ORCID_CLIENT_ID'],
'client_secret': current_app.config['ORCID_CLIENT_SECRET'],
'code': payload['code'],
'grant_type': 'authorization_code'
}
#print current_app.config['ORCID_OAUTH_ENDPOINT'], data, headers
# do not use connection pool, always establish a new connection to the orcid remote server
# we were having issue with dropped connectins mid-stream and this request is not idempotent
# therefore we can't retry
try:
r = requests.post(current_app.config['ORCID_OAUTH_ENDPOINT'], data=data, headers=headers,
timeout=current_app.config.get('CONNECTION_TIMEOUT', 30))
except (ConnectionError, ConnectTimeout, ReadTimeout) as e:
logging.error('For ORCID code %s, there was a connection error with the ORCID API'.format(payload['code']))
return 'There was a connection error with the ORCID API', 502
if r.status_code != 200:
logging.error('For ORCID code {}, there was an error getting the token from the ORCID API.'.
format(payload['code']))
return r.text, r.status_code
# update/create user account
data = r.json()
if 'orcid' in data:
with current_app.session_scope() as session:
u = session.query(User).filter_by(orcid_id=data['orcid']).options(load_only(User.orcid_id)).first()
p = session.query(Profile).filter_by(orcid_id=data['orcid']).options(load_only(Profile.orcid_id)).first()
if not u:
u = User(orcid_id=data['orcid'], created=adsmutils.get_date())
if not p:
p = Profile(orcid_id=data['orcid'], created=adsmutils.get_date())
u.updated = adsmutils.get_date()
p. updated = adsmutils.get_date()
u.access_token = data['access_token']
# save the user
session.begin_nested()
try:
session.add(u)
session.add(p)
session.commit()
except exc.IntegrityError as e:
session.rollback()
# per PEP-0249 a transaction is always in progress
session.commit()
return r.text, r.status_code
def query2svg(queryid):
'''Returns the SVG form of the query - for better performance, will need
to be cached/exported
'''
with current_app.session_scope() as session:
q = session.query(Query).filter_by(qid=queryid).first()
if not q:
return '<svg xmlns="http://www.w3.org/2000/svg"></svg>', 404, {'Content-Type': "image/svg+xml"}
return SVG_TMPL % {'key': 'ADS query', 'value': q.numfound}, 200, {'Content-Type': "image/svg+xml"}
def delete_library(library_id):
"""
Delete the entire library from the database
:param library_id: the unique ID of the library
:return: no return
"""
with current_app.session_scope() as session:
library = session.query(Library).filter_by(id=library_id).one()
session.delete(library)
session.commit()
def get(self, uid):
"""
HTTP GET request that contacts the ADS Classic myADS end point to
obtain all the libraries relevant to that user.
:param uid: user ID for the API
:type uid: int
Return data (on success)
------------------------
HTTP Responses:
--------------
Succeed getting libraries: 200
User does not have a classic account: 400
ADS Classic give unknown messages: 500
ADS Classic times out: 504
Any other responses will be default Flask errors
"""
data = {}
with current_app.session_scope() as session:
try:
user = session.query(Users).filter(
Users.absolute_uid == uid).one()
if not user.classic_email:
raise NoResultFound
except NoResultFound:
current_app.logger.warning(
'User does not have an associated ADS Classic account')
return err(NO_CLASSIC_ACCOUNT)
url = current_app.config['ADS_CLASSIC_MYADS_URL'].format(
mirror='adsabs.harvard.edu', email=user.classic_email)
current_app.logger.debug('Obtaining libraries via: {}'.format(url))
try:
response = current_app.client.get(url)
except requests.exceptions.Timeout:
current_app.logger.warning(
'ADS Classic timed out before finishing: {}'.format(url))
return err(CLASSIC_TIMEOUT)
if response.status_code != 200:
current_app.logger.warning(
'ADS Classic returned an unkown status code: "{}" [code: {}]'
.format(response.text, response.status_code))
return err(CLASSIC_UNKNOWN_ERROR)
data = response.json()
return data, 200
def delete_library(library_id):
"""
Delete the entire library from the database
:param library_id: the unique ID of the library
:return: no return
"""
with current_app.session_scope() as session:
library = session.query(Library).filter_by(id=library_id).one()
session.delete(library)
session.commit()
def update_status(orcid_id):
"""Gets/sets bibcode statuses for a given ORCID ID"""
if request.method == 'GET':
with current_app.session_scope() as session:
profile = session.query(Profile).filter_by(orcid_id=orcid_id).first()
recs = profile.get_records()
statuses = profile.get_nested(recs,'status')
records = dict(zip(recs, statuses))
return json.dumps(records), 200
if request.method == 'POST':
payload = request.json
if 'bibcodes' not in payload:
raise Exception('Bibcodes are missing')
if 'status' not in payload:
raise Exception('Status is missing')
with current_app.session_scope() as session:
profile = session.query(Profile).filter_by(orcid_id=orcid_id).first()
if type(payload['bibcodes']) != list:
bibcodes = [payload['bibcodes']]
else:
bibcodes = payload['bibcodes']
profile.update_status(bibcodes,payload['status'])
session.begin_nested()
try:
session.add(profile)
session.commit()
except exc.IntegrityError:
session.rollback()
return json.dumps({'msg': 'Updated status for bibcode {} not saved'.format(bibcodes)}), 500
# per PEP-0249 a transaction is always in progress
session.commit()
good_bibc, good_statuses = profile.get_status(bibcodes)
records = dict(zip(good_bibc, good_statuses))
return json.dumps(records), 200
def update_profile_local(orcid_id, data=None, force=False):
"""Update local db with ORCID profile"""
data = json.loads(data)
with current_app.session_scope() as session:
profile = session.query(Profile).filter_by(orcid_id=orcid_id).first()
if not profile:
logging.error('ORCID profile {} does not exist; creating'.format(orcid_id))
profile = Profile(orcid_id=orcid_id, created=adsmutils.get_date())
force = True
# data assumed to come from ORCID API /works endpoint
if data:
# convert milliseconds since epoch to seconds since epoch
last_modified = data['activities-summary']['last-modified-date']['value']
last_modified /= 1000.
if force or (profile.updated < datetime.utcfromtimestamp(last_modified).replace(tzinfo=pytz.utc)):
works = data['activities-summary']['works']['group']
new_recs = {}
update_recs = {}
orcid_recs = []
try:
current_recs = profile.bibcode.keys()
except:
current_recs = []
for work in works:
try:
id0, rec = find_record(work)
except:
continue
if id0 not in current_recs:
new_recs.update(rec)
else:
# if bibcode already in the profile, keep its status
rec[id0]['status'] = profile.bibcode[id0]['status']
update_recs.update(rec)
orcid_recs.append(id0)
profile.add_records(new_recs)
profile.add_records(update_recs)
# remove records from the profile that aren't in the ORCID set
remove_recs = list(set(current_recs)-set(orcid_recs))
profile.remove_bibcodes(remove_recs)
profile.updated = adsmutils.get_date()
# save the user
session.begin_nested()
try:
session.add(profile)
session.commit()
except exc.IntegrityError as e:
session.rollback()
logging.warning('ORCID profile database error - updated bibcodes for %s were not saved.'.format(orcid_id))
def store(qid=None):
if request.method == 'POST':
out = {}
# there might be many objects in there...
msgs = []
for _, fo in request.files.items():
if not hasattr(fo, 'read'):
continue # not a file object
# on error, we'll crash early; that's OK
msg = TurboBeeMsg.loads('adsmsg.turbobee.TurboBeeMsg', fo.read())
msgs.append(msg)
# also read data posted the normal way
for k, v in request.form.items():
msg = TurboBeeMsg.loads('adsmsg.turbobee.TurboBeeMsg',
base64.decodestring(v))
msgs.append(msg)
if not len(msgs):
return jsonify({'msg':
'Empty stream, no messages were received'}), 501
out = []
if len(msgs):
out = current_app.set_pages(msgs)
if 'errors' in out:
return jsonify(out), 400
return jsonify(out), 200
elif request.method == 'DELETE':
with current_app.session_scope() as session:
pages = session.query(Pages).options(
load_only('qid')).filter_by(qid=qid).first()
qid = None
if not pages:
return jsonify({'qi': qid, 'msg': 'Not found'}), 404
else:
qid = pages.qid
session.delete(pages)
try:
session.commit()
except exc.IntegrityError as e:
session.rollback()
return jsonify({'qid': qid, 'status': 'deleted'}), 200
def run(app=app):
"""
Carries out the deletion of the stale content
"""
with app.app_context():
with current_app.session_scope() as session:
# Obtain the list of API users
postgres_search_text = 'SELECT id FROM users;'
result = session.execute(postgres_search_text).fetchall()
list_of_api_users = [int(r[0]) for r in result]
# Loop through every use in the service database
removal_list = []
for service_user in session.query(User).all():
if service_user.absolute_uid not in list_of_api_users:
try:
# Obtain the libraries that should be deleted
permissions = session.query(Permissions).filter(
Permissions.user_id == service_user.id).all()
libraries = [
session.query(Library).filter(
Library.id == permission.library_id).one()
for permission in permissions
if permission.permissions['owner']
]
# Delete all the libraries found
# By cascade this should delete all the permissions
d = [
session.delete(library)
for library in libraries
]
p = [
session.delete(permission)
for permission in permissions
]
d = len(d)
session.delete(service_user)
session.commit()
current_app.logger.info(
'Removed stale user: {} and {} libraries'.
format(service_user, d))
removal_list.append(service_user)
except Exception as error:
current_app.logger.info(
'Problem with database, could not remove user {}: {}'
.format(service_user, error))
session.rollback()
current_app.logger.info('Deleted {} stale users: {}'.format(
len(removal_list), removal_list))
def helper_validate_library_data(service_uid, library_data):
"""
Validates the library data to ensure the user does not give empty
content for the title and description.
:param service_uid: the user ID within this microservice
:param library_data: content needed to create a library
:return: validated name and description
"""
_name = library_data.get('name') or DEFAULT_LIBRARY_NAME_PREFIX
_description = library_data.get('description') or \
DEFAULT_LIBRARY_DESCRIPTION
current_app.logger.info('Creating library for user_service: {0:d}, '
'with properties: {1}'
.format(service_uid, library_data))
# We want to ensure that the users have unique library names. However,
# it should be possible that they have access to other libraries from
# other people, that have the same name
with current_app.session_scope() as session:
library_names = \
[i.library.name for i in
session.query(Permissions).filter_by(user_id = service_uid,
owner = True).all()]
matches = [name for name in library_names if name == _name]
if matches:
current_app.logger.error('Name supplied for the library already '
'exists: "{0}" ["{1}"]'.format(_name,
matches))
raise BackendIntegrityError('Library name already exists.')
if _name == DEFAULT_LIBRARY_NAME_PREFIX:
default_names = [lib_name for lib_name
in library_names
if DEFAULT_LIBRARY_NAME_PREFIX
in lib_name]
_extension = len(default_names) + 1
_name = '{0} {1}'.format(_name,
_extension)
library_out = {}
for key in library_data:
library_out[key] = library_data[key]
library_out['name'] = _name
library_out['description'] = _description
return library_out
def search():
keys = request.args.keys()
# default is 50, max is 100
rows = min(current_app.config.get('MAX_RETURNED', 100),
int(request.args.get('rows') or 50))
with current_app.session_scope() as session:
if 'begin' in keys and 'end' in keys:
begin = get_date(request.args['begin'])
end = get_date(request.args['end'])
query = session.query(Pages).filter(
Pages.created.between(begin, end))
elif 'begin' in keys: # search for all records after begin
begin = get_date(request.args['begin'])
query = session.query(Pages).filter(Pages.created >= begin)
elif 'end' in keys: # search for all records before end
end = get_date(request.args['end'])
query = session.query(Pages).filter(Pages.created <= end)
elif 'at' in keys: # search for all records created at specific timestamp
at = get_date(request.args['at'])
query = session.query(Pages).filter(Pages.created == at)
elif 'null' in keys:
query = session.query(Pages).filter(Pages.created == None)
else:
return jsonify({'msg': 'Invalid parameters %s' % keys}), 505
if 'last_id' in keys:
query = query.where(Pages.id > keys['last_id'])
query = query.order_by(Pages.updated.asc()) \
.limit(rows)
if 'fields' in keys: # load only some fields
allowed_fields = [
'qid', 'created', 'updated', 'expires', 'lifetime',
'content_type', 'content'
]
fields = keys.get('fields', allowed_fields)
fields_to_load = list(set(fields) & set(allowed_fields))
query = query.options(load_only(*fields_to_load))
try:
pages = query.all()
# it is possible that toJSON() will eagerly load all fields (defeating load_only() above)
result = map(lambda page: page.toJSON(), pages)
return jsonify(result)
except Exception as e:
current_app.logger.error('Failed request: %s (error=%s)', keys, e)
return jsonify({'msg': e.message}), 500
def helper_library_exists(library_id):
"""
Helper function that checks if a library exists in the database or not
by catching the raise and returning a True/False statement.
:param library_id: the unique ID of the library
:return: bool for exists (True) or does not (False)
"""
with current_app.session_scope() as session:
try:
session.query(Library).filter_by(id = library_id).one()
return True
except NoResultFound:
return False
def add_records(datalinks_records_list):
"""
upserts records into db
:param datalinks_records_list:
:return: success boolean, plus a status text for retuning error message, if any, to the calling program
"""
rows = []
for i in range(len(datalinks_records_list.datalinks_records)):
for j in range(
len(datalinks_records_list.datalinks_records[i].data_links_rows
)):
rows.append([
datalinks_records_list.datalinks_records[i].bibcode,
datalinks_records_list.datalinks_records[i].data_links_rows[j].
link_type, datalinks_records_list.datalinks_records[i].
data_links_rows[j].link_sub_type, datalinks_records_list.
datalinks_records[i].data_links_rows[j].url,
datalinks_records_list.datalinks_records[i].data_links_rows[j].
title, datalinks_records_list.datalinks_records[i].
data_links_rows[j].item_count
])
if len(rows) > 0:
table = DataLinks.__table__
stmt = insert(table).values(rows)
no_update_cols = []
update_cols = [
c.name for c in table.c if c not in list(table.primary_key.columns)
and c.name not in no_update_cols
]
on_conflict_stmt = stmt.on_conflict_do_update(
index_elements=table.primary_key.columns,
set_={k: getattr(stmt.excluded, k)
for k in update_cols})
try:
with current_app.session_scope() as session:
session.execute(on_conflict_stmt)
current_app.logger.info('updated db with new data successfully')
return True, 'updated db with new data successfully'
except (SQLAlchemyError, DBAPIError) as e:
current_app.logger.error('SQLAlchemy: ' + str(e))
return False, 'SQLAlchemy: ' + str(e)
return False, 'unable to extract data from protobuf structure'
def execute_query(queryid):
'''Allows you to execute stored query. With this endpoint you can return parameters for the
previously asigned and stored query in the database (such as current number of documents in
the database.
'''
with current_app.session_scope() as session:
q = session.query(Query).filter_by(qid=queryid).first()
if not q:
return json.dumps({'msg': 'Query not found: ' + queryid}), 404
q_query = q.query.decode('utf8')
try:
payload, headers = check_request(request)
except Exception as e:
return json.dumps({'msg': e.message or e.description}), 400
dataq = json.loads(q_query)
query = urlparse.parse_qs(dataq['query'])
# override parameters using supplied params
if len(payload) > 0:
query.update(payload)
# make sure the {!bitset} is there (when bigquery is used)
if dataq['bigquery']:
fq = query.get('fq')
if not fq:
fq = ['{!bitset}']
elif '!bitset' not in str(fq):
if isinstance(fq, list):
fq.append('{!bitset}')
elif isinstance(fq, str):
fq = [fq, '{!bitset}']
else:
fq = ['{!bitset}']
query['fq'] = fq
# always request json
query['wt'] = 'json'
r = make_solr_request(query=query,
bigquery=dataq['bigquery'],
headers=headers)
return r.text, r.status_code
def query2svg(queryid):
'''Returns the SVG form of the query - for better performance, will need
to be cached/exported
'''
with current_app.session_scope() as session:
q = session.query(Query).filter_by(qid=queryid).first()
if not q:
return '<svg xmlns="http://www.w3.org/2000/svg"></svg>', 404, {
'Content-Type': "image/svg+xml"
}
return SVG_TMPL % {
'key': 'ADS query',
'value': q.numfound
}, 200, {
'Content-Type': "image/svg+xml"
}
def _delete_myads_notification(user_id=None, myads_id=None):
"""
Delete a single myADS notification setup
:param myads_id: ID of a single notification
:return: none
"""
with current_app.session_scope() as session:
r = session.query(MyADS).filter_by(user_id=user_id).filter_by(
id=myads_id).first()
if not r:
return '{}', 404
try:
session.delete(r)
session.commit()
except exc.IntegrityError as e:
session.rollback()
return json.dumps({'msg': 'Query was not deleted'}), 500
return '{}', 204
def create_user(absolute_uid):
"""
Creates a user in the database with a UID from the API
:param absolute_uid: UID from the API
:return: no return
"""
try:
user = User(absolute_uid=absolute_uid)
with current_app.session_scope() as session:
session.add(user)
session.commit()
except IntegrityError as error:
current_app.logger.error('IntegrityError. User: {0:d} was not'
'added. Full traceback: {1}'
.format(absolute_uid, error))
raise
def helper_user_exists(absolute_uid):
"""
Checks if a use exists before it would attempt to create one
:param absolute_uid: UID from the API
:return: boolean for if the user exists
"""
with current_app.session_scope() as session:
user_count = session.query(User).filter_by(absolute_uid = absolute_uid).all()
user_count = len(user_count)
if user_count == 1:
current_app.logger.info('User exists in database: {0} [API]'
.format(absolute_uid))
return True
elif user_count == 0:
current_app.logger.warning('User does not exist in database: {0} '
'[API]'.format(absolute_uid))
return False
def helper_absolute_uid_to_service_uid(absolute_uid):
"""
Convert the API UID to the BibLib service ID.
If the user does not exist in the database, first create a user.
:param absolute_uid: API UID
:return: BibLib service ID
"""
if not BaseView.helper_user_exists(absolute_uid=absolute_uid):
user = BaseView.helper_create_user(absolute_uid=absolute_uid)
with current_app.session_scope() as session:
user = session.query(User).filter_by(absolute_uid = absolute_uid).one()
current_app.logger.info('User found: {0} -> {1}'
.format(absolute_uid, user.id))
return user.id
def del_records(bibcode_list):
"""
:param bibcode_list:
:return:
"""
try:
with current_app.session_scope() as session:
count = 0
for bibcode in bibcode_list:
count += session.query(DataLinks).filter(
DataLinks.bibcode == bibcode).delete(
synchronize_session=False)
session.commit()
except (SQLAlchemyError, DBAPIError) as e:
current_app.logger.error('SQLAlchemy: ' + str(e))
return False, 'SQLAlchemy: ' + str(e)
return True, count, 'removed ' + str(count) + ' records of ' + str(
len(bibcode_list)) + ' bibcodes'
def setops_libraries(cls, library_id, document_data, operation='union'):
"""
Takes the union of two or more libraries
:param library_id: the primary library ID
:param document_data: dict containing the list 'libraries' that holds the secondary library IDs
:return: list of bibcodes in the union set
"""
current_app.logger.info(
'User requested to take the {0} of {1} with {2}'.format(
operation, library_id, document_data['libraries']))
with current_app.session_scope() as session:
# Find the specified library
primary_library = session.query(Library).filter_by(
id=library_id).one()
out_lib = set(primary_library.get_bibcodes())
for lib in document_data['libraries']:
if isinstance(lib, str):
lib = cls.helper_slug_to_uuid(lib)
secondary_library = session.query(Library).filter_by(
id=lib).one()
if operation == 'union':
out_lib = out_lib.union(
set(secondary_library.get_bibcodes()))
elif operation == 'intersection':
out_lib = out_lib.intersection(
set(secondary_library.get_bibcodes()))
elif operation == 'difference':
out_lib = out_lib.difference(
set(secondary_library.get_bibcodes()))
else:
current_app.logger.warning(
'Requested operation {0} is not allowed.'.format(
operation))
return
if len(out_lib) < 1:
current_app.logger.info(
'No records remain after taking the {0} of {1} and {2}'.format(
operation, library_id, document_data['libraries']))
return list(out_lib)
def get_access_token():
'''Exchange 'code' for 'access_token' data'''
payload = dict(request.args)
if 'code' not in payload:
raise Exception('Parameter code is missing')
headers = {'Accept': 'application/json'}
data = {
'client_id': current_app.config['ORCID_CLIENT_ID'],
'client_secret': current_app.config['ORCID_CLIENT_SECRET'],
'code': payload['code'][0],
'grant_type': 'authorization_code'
}
#print current_app.config['ORCID_OAUTH_ENDPOINT'], data, headers
r = current_app.client.post(current_app.config['ORCID_OAUTH_ENDPOINT'], data=data, headers=headers)
if r.status_code != 200:
logging.error('For ORCID code {}, there was an error getting the token from the ORCID API.'.
format(payload['code'][0]))
# update/create user account
data = r.json()
if 'orcid' in data:
with current_app.session_scope() as session:
u = session.query(User).filter_by(orcid_id=data['orcid']).options(load_only(User.orcid_id)).first()
p = session.query(Profile).filter_by(orcid_id=data['orcid']).options(load_only(Profile.orcid_id)).first()
if not u:
u = User(orcid_id=data['orcid'], created=adsmutils.get_date())
if not p:
p = Profile(orcid_id=data['orcid'], created=adsmutils.get_date())
u.updated = adsmutils.get_date()
p. updated = adsmutils.get_date()
u.access_token = data['access_token']
# save the user
session.begin_nested()
try:
session.add(u)
session.add(p)
session.commit()
except exc.IntegrityError as e:
session.rollback()
# per PEP-0249 a transaction is always in progress
session.commit()
return r.text, r.status_code
def configuration(key=None):
'''Allows you to retrieve JSON data from VAULT_BUMBLEBEE_OPTIONS'''
opts = current_app.config.get('VAULT_BUMBLEBEE_OPTIONS') or {}
if not isinstance(opts, dict):
return json.dumps({'msg': 'Server misconfiguration, VAULT_BUMBLEBEE_OPTIONS is of an invalid type'}), 500
if key:
if key == 'link_servers':
with current_app.session_scope() as session:
res = session.query(Library).all()
link_servers = [{"name": l.libname, "link": l.libserver, "gif":l.iconurl} for l in res]
link_servers = sorted(link_servers, key=itemgetter('name'))
return json.dumps(link_servers), 200
elif key in opts:
return json.dumps(opts[key]), 200
else:
return '{}', 404
else:
return json.dumps(opts)
def export(iso_datestring):
'''
Get the latest changes (as recorded in users table)
The optional argument latest_point is RFC3339, ie. '2008-09-03T20:56:35.450686Z'
'''
# TODO check rate limit
# TODO need to add a created/update field to users table
# inspired by orcid-service endpoint of same endpoint, checks the users table for profiles w/ a myADS setup that
# have been updated since a given date/time; returns these profiles to be added to the myADS processing
latest = parser.parse(iso_datestring) # ISO 8601
output = []
with current_app.session_scope() as session:
setups = session.query(MyADS).filter(MyADS.updated > latest).order_by(
MyADS.updated.asc()).all()
for s in setups:
output.append(s.user_id)
output = list(set(output))
return json.dumps({'users': output}), 200
def helper_create_user(absolute_uid):
"""
Creates a user in the database with a UID from the API
:param absolute_uid: UID from the API
:return: None
"""
with current_app.session_scope() as session:
try:
user = User(absolute_uid=absolute_uid)
session.add(user)
session.commit()
current_app.logger.info('Successfully created user: {0} [API] as '
'{1} [Microservice]'
.format(absolute_uid, user.id))
except IntegrityError as error:
current_app.logger.error('IntegrityError. User: {0:d} was not'
'added. Full traceback: {1}'
.format(absolute_uid, error))
raise
def empty_library(library_id):
"""
Empties the contents of one library
:param library_id: library to empty
:return: dict containing the metadata of the emptied library
"""
current_app.logger.info('User requested to empty the contents of {0}'.format(library_id))
metadata = {}
with current_app.session_scope() as session:
lib = session.query(Library).filter_by(id=library_id).one()
lib.remove_bibcodes(lib.get_bibcodes())
metadata['name'] = lib.name
metadata['description'] = lib.description
metadata['public'] = lib.public
session.add(lib)
session.commit()
return metadata
def execute_query(queryid):
'''Allows you to execute stored query'''
with current_app.session_scope() as session:
q = session.query(Query).filter_by(qid=queryid).first()
if not q:
return json.dumps({msg: 'Query not found: ' + qid}), 404
q_query = q.query
try:
payload, headers = check_request(request)
except Exception as e:
return json.dumps({'msg': e.message or e.description}), 400
dataq = json.loads(q_query)
query = urlparse.parse_qs(dataq['query'])
# override parameters using supplied params
if len(payload) > 0:
query.update(payload)
# make sure the {!bitset} is there (when bigquery is used)
if dataq['bigquery']:
fq = query.get('fq')
if not fq:
fq = ['{!bitset}']
elif '!bitset' not in str(fq):
if isinstance(fq, list):
fq.append('{!bitset}')
else:
fq = ['{!bitset}']
query['fq'] = fq
# always request json
query['wt'] = 'json'
r = make_solr_request(query=query, bigquery=dataq['bigquery'], headers=headers)
return r.text, r.status_code
def library_name_exists(service_uid, library_name):
"""
Checks to see if a library name already exists in the user's created
libraries
:param service_uid: the user ID within this microservice
:param library_name: name to check if it exists
:return: True (exists), False (does not exist)
"""
with current_app.session_scope() as session:
library_names = \
[i.library.name for i in
session.query(Permissions).filter_by(user_id = service_uid,
owner = True).all()]
if library_name in library_names:
current_app.logger.error('Name supplied for the library already '
'exists: "{0}"'.format(library_name))
return True
else:
return False
def remove_documents_from_library(cls, library_id, document_data):
"""
Remove a given document from a specific library
:param library_id: the unique ID of the library
:param document_data: the meta data of the document
:return: number_removed: number of documents successfully removed
"""
current_app.logger.info('Removing a document: {0} from library_uuid: '
'{1}'.format(document_data, library_id))
with current_app.session_scope() as session:
library = session.query(Library).filter_by(id=library_id).one()
start_length = len(library.bibcode)
library.remove_bibcodes(document_data['bibcode'])
session.add(library)
session.commit()
current_app.logger.info('Removed document successfully: {0}'
.format(library.bibcode))
end_length = len(library.bibcode)
return start_length - end_length
def update_library(library_id, library_data):
"""
Update the meta data of the library
:param library_id: the unique ID of the library
:param library_data: dictionary containing the updateable values
:return: values updated
"""
updateable = ['name', 'description', 'public']
updated = {}
with current_app.session_scope() as session:
library = session.query(Library).filter_by(id=library_id).one()
for key in library_data:
if key not in updateable:
continue
setattr(library, key, library_data[key])
updated[key] = library_data[key]
session.add(library)
session.commit()
return updated
def execute_SQL_query(bibc):
with current_app.session_scope() as session:
resp = session.query(GraphicsModel).filter(
GraphicsModel.bibcode == bibc).one()
results = json.loads(json.dumps(resp, cls=AlchemyEncoder))
return results
def post(self, library):
"""
HTTP POST request that conducts operations at the library level.
:param library: primary library ID
:return: response if operation was successful
Header:
-------
Must contain the API forwarded user ID of the user accessing the end
point
Post body:
----------
KEYWORD, VALUE
libraries: <list> List of secondary libraries to include in the action (optional, based on action)
action: <unicode> union, intersection, difference, copy, empty
Actions to perform on given libraries:
Union: requires one or more secondary libraries to be passed; takes the union of the
primary and secondary library sets; a new library is created
Intersection: requires one or more secondary libraries to be passed; takes the
intersection of the primary and secondary library sets; a new library is created
Difference: requires one or more secondary libraries to be passed; takes the difference
between the primary and secondary libraries; the primary library comes first in the
operation, so the secondary library is removed from the primary; a new library
is created
Copy: requires one and only one secondary library to be passed; the primary library
will be copied into the secondary library (so the secondary library will be
overwritten); no new library is created
Empty: secondary libraries are ignored; the primary library will be emptied of its
contents, though the library and metadata will remain; no new library is created
name: <string> (optional) name of the new library (must be unique for that user); used only for actions in
[union, intersection, difference]
description: <string> (optional) description of the new library; used only for actions in
[union, intersection, difference]
public: <boolean> (optional) is the new library public to view; used only for actions in
[union, intersection, difference]
-----------
Return data:
-----------
name: <string> Name of the library
id: <string> ID of the library
description: <string> Description of the library
Permissions:
-----------
The following type of user can conduct library operations:
- owner
- admin
- write
"""
# Get the user requesting this from the header
try:
user_editing = self.helper_get_user_id()
except KeyError:
return err(MISSING_USERNAME_ERROR)
# URL safe base64 string to UUID
library_uuid = self.helper_slug_to_uuid(library)
user_editing_uid = \
self.helper_absolute_uid_to_service_uid(absolute_uid=user_editing)
# Check the permissions of the user
if not self.write_access(service_uid=user_editing_uid,
library_id=library_uuid):
return err(NO_PERMISSION_ERROR)
try:
data = get_post_data(
request,
types=dict(libraries=list, action=basestring, name=basestring, description=basestring, public=bool)
)
except TypeError as error:
current_app.logger.error('Wrong type passed for POST: {0} [{1}]'
.format(request.data, error))
return err(WRONG_TYPE_ERROR)
if data['action'] in ['union', 'intersection', 'difference']:
if 'libraries' not in data:
return err(NO_LIBRARY_SPECIFIED_ERROR)
if 'name' not in data:
data['name'] = 'Untitled {0}.'.format(get_date().isoformat())
if 'public' not in data:
data['public'] = False
if data['action'] == 'copy':
if 'libraries' not in data:
return err(NO_LIBRARY_SPECIFIED_ERROR)
if len(data['libraries']) > 1:
return err(TOO_MANY_LIBRARIES_SPECIFIED_ERROR)
lib_names = []
with current_app.session_scope() as session:
primary = session.query(Library).filter_by(id=library_uuid).one()
lib_names.append(primary.name)
if 'libraries' in data:
for lib in data['libraries']:
secondary_uuid = self.helper_slug_to_uuid(lib)
secondary = session.query(Library).filter_by(id=secondary_uuid).one()
lib_names.append(secondary.name)
if data['action'] == 'union':
bib_union = self.setops_libraries(
library_id=library_uuid,
document_data=data,
operation='union'
)
current_app.logger.info('Successfully took the union of the libraries {0} (IDs: {1}, {2})'
.format(', '.join(lib_names), library, ', '.join(data['libraries'])))
data['bibcode'] = bib_union
if 'description' not in data:
data['description'] = 'Union of libraries {0} (IDs: {1}, {2})' \
.format(', '.join(lib_names), library, ', '.join(data['libraries']))
try:
library_dict = self.create_library(service_uid=user_editing_uid, library_data=data)
except BackendIntegrityError as error:
current_app.logger.error(error)
return err(DUPLICATE_LIBRARY_NAME_ERROR)
except TypeError as error:
current_app.logger.error(error)
return err(WRONG_TYPE_ERROR)
return library_dict, 200
elif data['action'] == 'intersection':
bib_intersect = self.setops_libraries(
library_id=library_uuid,
document_data=data,
operation='intersection'
)
current_app.logger.info('Successfully took the intersection of the libraries {0} (IDs: {1}, {2})'
.format(', '.join(lib_names), library, ', '.join(data['libraries'])))
data['bibcode'] = bib_intersect
if 'description' not in data:
data['description'] = 'Intersection of {0} (IDs: {1}, {2})' \
.format(', '.join(lib_names), library, ', '.join(data['libraries']))
try:
library_dict = self.create_library(service_uid=user_editing_uid, library_data=data)
except BackendIntegrityError as error:
current_app.logger.error(error)
return err(DUPLICATE_LIBRARY_NAME_ERROR)
except TypeError as error:
current_app.logger.error(error)
return err(WRONG_TYPE_ERROR)
return library_dict, 200
elif data['action'] == 'difference':
bib_diff = self.setops_libraries(
library_id=library_uuid,
document_data=data,
operation='difference'
)
current_app.logger.info('Successfully took the difference of {0} (ID {2}) - (minus) {1} (ID {3})'
.format(lib_names[0], ', '.join(lib_names[1:]), library, ', '.join(data['libraries'])))
data['bibcode'] = bib_diff
if 'description' not in data:
data['description'] = 'Records that are in {0} (ID {2}) but not in {1} (ID {3})' \
.format(lib_names[0], ', '.join(lib_names[1:]), library, ', '.join(data['libraries']))
try:
library_dict = self.create_library(service_uid=user_editing_uid, library_data=data)
except BackendIntegrityError as error:
current_app.logger.error(error)
return err(DUPLICATE_LIBRARY_NAME_ERROR)
except TypeError as error:
current_app.logger.error(error)
return err(WRONG_TYPE_ERROR)
return library_dict, 200
elif data['action'] == 'copy':
library_dict = self.copy_library(
library_id=library_uuid,
document_data=data
)
current_app.logger.info('Successfully copied {0} (ID {2}) into {1} (ID {3})'
.format(lib_names[0], lib_names[1], library, data['libraries'][0]))
with current_app.session_scope() as session:
libid = self.helper_slug_to_uuid(data['libraries'][0])
library = session.query(Library).filter_by(id=libid).one()
bib = library.get_bibcodes()
library_dict['bibcode'] = bib
return library_dict, 200
elif data['action'] == 'empty':
library_dict = self.empty_library(
library_id=library_uuid
)
current_app.logger.info('Successfully emptied {0} (ID {1}) of all records'
.format(lib_names[0], library))
with current_app.session_scope() as session:
library = session.query(Library).filter_by(id=library_uuid).one()
bib = library.get_bibcodes()
library_dict['bibcode'] = bib
return library_dict, 200
else:
current_app.logger.info('User requested a non-standard operation')
return {}, 400
def execute_SQL_query(query):
with current_app.session_scope() as session:
results = session.execute(query).fetchall()
return results
def has_permission(service_uid_editor,
service_uid_modify,
library_id):
"""
Check if the user wanting to change the library has the correct
permissions to do so, and the user to be changed is not the owner.
:param service_uid_editor: the user ID of the editor
:param service_uid_modify: the user ID of the user to be edited
:param library_id: the library id
:return: boolean
"""
if service_uid_editor == service_uid_modify:
current_app.logger.error('Editing user: {0} and user to edit: {1}'
' are the same. This is not allowed.'
.format(service_uid_modify,
service_uid_editor))
return False
current_app.logger.info('Checking if user: {0}, can edit the '
'permissions of user: {1}'
.format(
service_uid_editor,
service_uid_modify
))
# Check if the editor has permissions
with current_app.session_scope() as session:
try:
editor_permissions = session.query(Permissions).filter(
Permissions.user_id == service_uid_editor,
Permissions.library_id == library_id
).one()
except NoResultFound as error:
current_app.logger.error(
'User: {0} has no permissions for this library: {1}'
.format(service_uid_editor, error)
)
return False
if editor_permissions.owner:
current_app.logger.info('User: {0} is owner, so is allowed to '
'change permissions'
.format(service_uid_editor))
return True
# Check if the user to be modified has permissions
try:
modify_permissions = session.query(Permissions).filter(
Permissions.user_id == service_uid_modify,
Permissions.library_id == library_id
).one()
except NoResultFound:
modify_permissions = False
# if the editor is admin, and the modifier has no permissions
if editor_permissions.admin:
# user to be modified has no permissions
if not modify_permissions:
return True
# user to be modified is not owner
if not modify_permissions.owner:
return True
else:
return False
def get_libraries(cls, service_uid, absolute_uid):
"""
Get all the libraries a user has
:param service_uid: microservice UID of the user
:param absolute_uid: unique UID of the user in the API
:return: list of libraries in json format
"""
# Get all the permissions for a user
# This can be improved into one database call rather than having
# one per each permission, but needs some checks in place.
with current_app.session_scope() as session:
result = session.query(Permissions, Library)\
.join(Permissions.library)\
.filter(Permissions.user_id == service_uid)\
.all()
output_libraries = []
for permission, library in result:
# For this library get all the people who have permissions
users = session.query(Permissions).filter_by(
library_id = library.id
).all()
num_documents = 0
if library.bibcode:
num_documents = len(library.bibcode)
if permission.owner:
main_permission = 'owner'
elif permission.admin:
main_permission = 'admin'
elif permission.write:
main_permission = 'write'
elif permission.read:
main_permission = 'read'
else:
main_permission = 'none'
if permission.owner or permission.admin and not library.public:
num_users = len(users)
elif library.public:
num_users = len(users)
else:
num_users = 0
service = '{api}/{uid}'.format(
api=current_app.config['BIBLIB_USER_EMAIL_ADSWS_API_URL'],
uid=absolute_uid
)
current_app.logger.info('Obtaining email of user: {0} [API UID]'
.format(absolute_uid))
response = client().get(
service
)
if response.status_code != 200:
current_app.logger.error('Could not find user in the API'
'database: {0}.'.format(service))
owner = 'Not available'
else:
owner = response.json()['email'].split('@')[0]
payload = dict(
name=library.name,
id='{0}'.format(cls.helper_uuid_to_slug(library.id)),
description=library.description,
num_documents=num_documents,
date_created=library.date_created.isoformat(),
date_last_modified=library.date_last_modified.isoformat(),
permission=main_permission,
public=library.public,
num_users=num_users,
owner=owner
)
output_libraries.append(payload)
return output_libraries
def post(self):
"""
HTTP POST request that creates a library for a given user
:return: the response for if the library was successfully created
Header:
-------
Must contain the API forwarded user ID of the user accessing the end
point
Post body:
----------
KEYWORD, VALUE
name: <string> name of the library (must be unique
for that user)
description: <string> description of the library
public: <boolean> is the library public to view
bibcode (OPTIONAL): <list> list of bibcodes to add
Return data:
-----------
name: <string> Name of the library
id: <string> ID of the library
description: <string> Description of the library
Permissions:
-----------
The following type of user can create a library
- must be logged in, i.e., scope = user
"""
# Check that they pass a user id
try:
user = self.helper_get_user_id()
except KeyError:
return err(MISSING_USERNAME_ERROR)
# Check if the user exists, if not, generate a user in the database
current_app.logger.info('Checking if the user exists')
if not self.helper_user_exists(absolute_uid=user):
current_app.logger.info('User: {0:d}, does not exist.'
.format(user))
self.create_user(absolute_uid=user)
current_app.logger.info('User: {0:d}, created.'.format(user))
else:
current_app.logger.info('User already exists.')
# Switch to the service UID and not the API UID
service_uid = \
self.helper_absolute_uid_to_service_uid(absolute_uid=user)
current_app.logger.info('user_API: {0:d} is now user_service: {1:d}'
.format(user, service_uid))
# Create the library
try:
data = get_post_data(
request,
types=dict(
name=unicode,
description=unicode,
public=bool,
bibcode=list
)
)
except TypeError as error:
current_app.logger.error('Wrong type passed for POST: {0} [{1}]'
.format(request.data, error))
return err(WRONG_TYPE_ERROR)
with current_app.session_scope() as session:
try:
library_dict = \
self.create_library(service_uid=service_uid, library_data=data)
except BackendIntegrityError as error:
current_app.logger.error(error)
return err(DUPLICATE_LIBRARY_NAME_ERROR)
except TypeError as error:
current_app.logger.error(error)
return err(WRONG_TYPE_ERROR)
return library_dict, 200
def store_data():
'''Allows you to store/retrieve JSON data on the server side.
It is always associated with the user id (which is communicated
to us by API) - so there is no endpoint allowing you to access
other users' data (should there be?) /user-data/<uid>?'''
# get the query data
try:
payload, headers = check_request(request)
except Exception as e:
return json.dumps({'msg': e.message or e.description}), 400
user_id = int(headers['X-Adsws-Uid'])
if user_id == 0:
return json.dumps({'msg': 'Sorry, you can\'t use this service as an anonymous user'}), 400
if request.method == 'GET':
with current_app.session_scope() as session:
q = session.query(User).filter_by(id=user_id).first()
if not q:
return '{}', 200 # or return 404?
return json.dumps(q.user_data) or '{}', 200
elif request.method == 'POST':
# limit both number of keys and length of value to keep db clean
if len(max(payload.values(), key=len)) > current_app.config['MAX_ALLOWED_JSON_SIZE']:
return json.dumps({'msg': 'You have exceeded the allowed storage limit (length of values), no data was saved'}), 400
if len(payload.keys()) > current_app.config['MAX_ALLOWED_JSON_KEYS']:
return json.dumps({'msg': 'You have exceeded the allowed storage limit (number of keys), no data was saved'}), 400
with current_app.session_scope() as session:
try:
q = session.query(User).filter_by(id=user_id).with_for_update(of=User).one()
try:
data = q.user_data
except TypeError:
data = {}
except ormexc.NoResultFound:
data = payload
u = User(id=user_id, user_data=data)
try:
session.add(u)
session.commit()
return json.dumps(data), 200
except exc.IntegrityError:
q = session.query(User).filter_by(id=user_id).with_for_update(of=User).one()
try:
data = q.user_data
except TypeError:
data = {}
data.update(payload)
q.user_data = data
session.begin_nested()
try:
session.commit()
except exc.IntegrityError:
session.rollback()
return json.dumps({'msg': 'We have hit a db error! The world is crumbling all around... (eh, btw, your data was not saved)'}), 500
return json.dumps(data), 200
def query(queryid=None):
'''Stores/retrieves the montysolr query; it can receive data in urlencoded
format or as application/json encoded data. In the second case, you can
pass 'bigquery' together with the 'query' like so:
{
query: {foo: bar},
bigquery: 'data\ndata\n....'
}
'''
if request.method == 'GET' and queryid:
with current_app.session_scope() as session:
q = session.query(Query).filter_by(qid=queryid).first()
if not q:
return json.dumps({'msg': 'Query not found: ' + queryid}), 404
return json.dumps({
'qid': q.qid,
'query': q.query,
'numfound': q.numfound }), 200
# get the query data
try:
payload, headers = check_request(request)
except Exception as e:
return json.dumps({'msg': e.message or e.description}), 400
if len(payload.keys()) == 0:
raise Exception('Query cannot be empty')
payload = cleanup_payload(payload)
# check we don't have this query already
query = json.dumps(payload)
qid = md5.new(headers['X-Adsws-Uid'] + query).hexdigest()
with current_app.session_scope() as session:
q = session.query(Query).filter_by(qid=qid).first()
if q:
return json.dumps({'qid': qid, 'numFound': q.numfound}), 200
# check the query is valid
solrq = payload['query'] + '&wt=json'
r = make_solr_request(query=solrq, bigquery=payload['bigquery'], headers=headers)
if r.status_code != 200:
return json.dumps({'msg': 'Could not verify the query.', 'query': payload, 'reason': r.text}), 404
# extract number of docs found
num_found = 0
try:
num_found = int(r.json()['response']['numFound'])
except:
pass
# save the query
q = Query(qid=qid, query=query, numfound=num_found)
with current_app.session_scope() as session:
session.begin_nested()
try:
session.add(q)
session.commit()
except exc.IntegrityError as e:
session.rollback()
return json.dumps({'msg': e.message or e.description}), 400
# TODO: update
#q = session.merge(q) # force re-sync from database
#q.updated = datetime.datetime.utcnow()
#session.commit()
return json.dumps({'qid': qid, 'numFound': num_found}), 200
def solr_update_library(library_id, solr_docs):
"""
Updates the library based on the solr canonical bibcodes response
:param library: library_id of the library to update
:param solr_docs: solr docs from the bigquery response
:return: dictionary with details of files modified
num_updated: number of documents modified
duplicates_removed: number of files removed for duplication
update_list: list of changed bibcodes {'before': 'after'}
"""
# Definitions
update = False
canonical_bibcodes = []
alternate_bibcodes = {}
new_bibcode = {}
# Constants for the return dictionary
num_updated = 0
duplicates_removed = 0
update_list = []
# Extract the canonical bibcodes and create a hashmap for the
# alternate bibcodes
for doc in solr_docs:
canonical_bibcodes.append(doc['bibcode'])
if doc.get('alternate_bibcode'):
alternate_bibcodes.update(
{i: doc['bibcode'] for i in doc['alternate_bibcode']}
)
with current_app.session_scope() as session:
library = session.query(Library).filter(Library.id == library_id).one()
for bibcode in library.bibcode:
# Skip if its already canonical
if bibcode in canonical_bibcodes:
new_bibcode[bibcode] = library.bibcode[bibcode]
continue
# Update if its an alternate
if bibcode in alternate_bibcodes.keys():
update = True
num_updated += 1
update_list.append({bibcode: alternate_bibcodes[bibcode]})
# Only add the bibcode if it is not there
if alternate_bibcodes[bibcode] not in new_bibcode:
new_bibcode[alternate_bibcodes[bibcode]] = \
library.bibcode[bibcode]
else:
duplicates_removed += 1
elif bibcode not in canonical_bibcodes and\
bibcode not in alternate_bibcodes.keys():
new_bibcode[bibcode] = library.bibcode[bibcode]
if update:
# Update the database
library.bibcode = new_bibcode
session.add(library)
session.commit()
updates = dict(
num_updated=num_updated,
duplicates_removed=duplicates_removed,
update_list=update_list
)
return updates
def get_documents_from_library(cls, library_id, service_uid):
"""
Retrieve all the documents that are within the library specified
:param library_id: the unique ID of the library
:param service_uid: the user ID within this microservice
:return: bibcodes
"""
with current_app.session_scope() as session:
# Get the library
library = session.query(Library).filter_by(id=library_id).one()
# Get the owner of the library
result = session.query(Permissions, User)\
.join(Permissions.user)\
.filter(Permissions.library_id == library_id)\
.filter(Permissions.owner == True)\
.one()
owner_permissions, owner = result
service = '{api}/{uid}'.format(
api=current_app.config['BIBLIB_USER_EMAIL_ADSWS_API_URL'],
uid=owner.absolute_uid
)
current_app.logger.info('Obtaining email of user: {0} [API UID]'
.format(owner.absolute_uid))
response = client().get(
service
)
# For this library get all the people who have permissions
users = session.query(Permissions).filter_by(
library_id = library.id
).all()
if response.status_code != 200:
current_app.logger.error('Could not find user in the API'
'database: {0}.'.format(service))
owner = 'Not available'
else:
owner = response.json()['email'].split('@')[0]
# User requesting to see the content
if service_uid:
try:
permission = session.query(Permissions).filter(
Permissions.user_id == service_uid
).filter(
Permissions.library_id == library_id
).one()
if permission.owner:
main_permission = 'owner'
elif permission.admin:
main_permission = 'admin'
elif permission.write:
main_permission = 'write'
elif permission.read:
main_permission = 'read'
else:
main_permission = 'none'
except:
main_permission = 'none'
else:
main_permission = 'none'
if main_permission == 'owner' or main_permission == 'admin':
num_users = len(users)
elif library.public:
num_users = len(users)
else:
num_users = 0
metadata = dict(
name=library.name,
id='{0}'.format(cls.helper_uuid_to_slug(library.id)),
description=library.description,
num_documents=len(library.bibcode),
date_created=library.date_created.isoformat(),
date_last_modified=library.date_last_modified.isoformat(),
permission=main_permission,
public=library.public,
num_users=num_users,
owner=owner
)
session.refresh(library)
session.expunge(library)
return library, metadata
def create_library(cls, service_uid, library_data):
"""
Creates a library for a user
:param service_uid: the user ID within this microservice
:param library_data: content needed to create a library
:return: library dict
"""
library_data = BaseView.helper_validate_library_data(
service_uid=service_uid,
library_data=library_data
)
_name = library_data.get('name')
_description = library_data.get('description')
_public = bool(library_data.get('public', False))
_bibcode = library_data.get('bibcode', False)
with current_app.session_scope() as session:
try:
# Make the library in the library table
library = Library(name=_name,
description=_description,
public=_public)
# If the user supplies bibcodes
if _bibcode and isinstance(_bibcode, list):
# Ensure unique content
_bibcode = uniquify(_bibcode)
current_app.logger.info('User supplied bibcodes: {0}'
.format(_bibcode))
library.add_bibcodes(_bibcode)
elif _bibcode:
current_app.logger.error('Bibcode supplied not a list: {0}'
.format(_bibcode))
raise TypeError('Bibcode should be a list.')
user = session.query(User).filter_by(id=service_uid).one()
# Make the permissions
permission = Permissions(
owner=True,
)
# Use the ORM to link the permissions to the library and user,
# so that no commit is required until the complete action is
# finished. This means any rollback will not leave a single
# library without permissions
library.permissions.append(permission)
user.permissions.append(permission)
session.add_all([library, permission, user])
session.commit()
current_app.logger.info('Library: "{0}" made, user_service: {1:d}'
.format(library.name, user.id))
library_dict = dict(
name=library.name,
id='{0}'.format(cls.helper_uuid_to_slug(library.id)),
description=library.description,
)
# If they added bibcodes include in the response
if hasattr(library, 'bibcode') and library.bibcode:
library_dict['bibcode'] = library.get_bibcodes()
return library_dict
except IntegrityError as error:
# Roll back the changes
session.rollback()
current_app.logger.error('IntegitryError, database has been rolled'
'back. Caused by user_service: {0:d}.'
'Full error: {1}'
.format(user.id, error))
# Log here
raise
except Exception:
session.rollback()
raise
