def POST(self, name):
"""Create an account"""
logger.debug(web.data())
try:
data = json.loads(web.data().decode('utf-8'))
except json.decoder.JSONDecodeError:
raise Error(BADPARAMS, msg="Could not decode JSON.")
account_id = data.get('account_id')
email = data.get('email')
passwd = data.get('password')
name = data.get('name')
surname = data.get('surname')
authority = data.get('authority')
if not passwd or not name or not surname:
raise Error(BADPARAMS)
account = AccountController.create_acccount(email, passwd, account_id,
name, surname, authority)
account = account.__dict__
del account['password']
del account['hash']
return [account]
def get_string_field(name, data):
value = data.get(name)
if value is None:
raise Error(name + ' required', 400)
if not isinstance(value, (str, unicode)):
raise Error('invalid ' + name, 400)
return value
def POST(self, name):
"""Login - obtain a token"""
logger.debug(web.data())
try:
data = json.loads(web.data().decode('utf-8'))
except json.decoder.JSONDecodeError:
raise Error(BADPARAMS, msg="Could not decode JSON.")
email = data.get('email')
passwd = data.get('password')
try:
assert email and is_valid_email(email)
except AssertionError:
raise Error(BADPARAMS, msg="Invalid email provided.")
try:
assert passwd
account = Account(email, passwd)
assert account.is_valid()
except AssertionError:
raise Error(BADAUTH)
account.reload()
result = account.__dict__
result['token'] = account.issue_token()
del result['password']
del result['hash']
return [result]
def choose_best_candidate(self, results):
"""Attempt to return a single result from the array provided.
We assume the first result will always be the fittest candidate, then
we iterate the results and compare the rest with the fittest to see
if we cannot determine a unique result, this happens when either:
(a) more than one work_id share the lowest score;
(b) there is no canonical among multiple URIs of the fittest result.
"""
best = result_to_identifier(results.pop(0))
for e in results:
candidate = result_to_identifier(e)
same_score = candidate.score == best.score
same_work = candidate.work == best.work
if not same_score:
# we already have the lowest score (results are in ASC order)
break
elif same_score and not same_work:
raise Error(AMBIGUOUS, data=results_to_identifiers(results))
elif same_score and same_work and not best.is_canonical():
raise Error(NONCANONICAL, data=results_to_identifiers(results))
elif same_work and best.is_canonical():
# we continue to see wether following results correspond to a
# different work with the same score
continue
return best
def decode_token(intoken):
try:
return jwt.decode(intoken, SECRET_KEY)
except jwt.exceptions.DecodeError:
raise Error(FORBIDDEN)
except jwt.ExpiredSignatureError:
raise Error(UNAUTHORIZED, msg="Signature expired.")
except jwt.InvalidTokenError:
raise Error(UNAUTHORIZED, msg="Invalid token.")
def validate_username(username):
if not username:
raise Error('username can not be empty')
if len(username) > conf.max_username_length:
raise Error('username too long, should be less than {}'.format(
conf.max_username_length))
if not re.match('[-0-9a-z]+', username):
raise Error('username can only contain ' +
'lowercase letters, digits and dash')
def do_login(username, password):
validate_username(username)
validate_password(password)
user = dbutil.get_user(username)
if not user:
raise Error('not found', 404)
if hash_password(password, user['salt']) != user['hashed_password']:
raise Error('wrong password')
return token_response(dbutil.get_user_for_token(username))
def GET(self, name):
"""Get Events with various filtering options"""
logger.debug("Query: %s" % (web.input()))
event_id = web.input().get('event_id')
if event_id:
results = Event.get_from_event_id(event_id)
data = results_to_events(results)
else:
filters = web.input().get('filter')
clause, params = build_params(filters)
start_date = web.input().get('start_date')
end_date = web.input().get('end_date')
if start_date:
dclause, dparams = build_date_clause(start_date, end_date)
clause += dclause
params.update(dparams)
agg = web.input().get('aggregation') or ''
if agg == '':
results = Event.get_all(clause, params)
data = results_to_events(results)
elif agg == 'measure_uri':
results = Event.aggregate_by_measure(clause, params)
data = results_to_measure_aggregation(results)
elif agg == 'measure_uri,country_uri':
results = Event.aggregate_by_measure_country(clause, params)
data = results_to_measure_country_aggregation(results)
elif agg == 'country_uri,measure_uri':
results = Event.aggregate_by_country_measure(clause, params)
data = results_to_country_measure_aggregation(results)
elif agg == 'measure_uri,year':
results = Event.aggregate_by_measure_year(clause, params)
data = results_to_measure_year_aggregation(results)
elif agg == 'year,measure_uri':
results = Event.aggregate_by_year_measure(clause, params)
data = results_to_year_measure_aggregation(results)
elif agg == 'measure_uri,month':
results = Event.aggregate_by_measure_month(clause, params)
data = results_to_measure_month_aggregation(results)
elif agg == 'month,measure_uri':
results = Event.aggregate_by_month_measure(clause, params)
data = results_to_month_measure_aggregation(results)
else:
m = "Aggregation must be one of the following: 'measure_uri', "
"'measure_uri,country_uri', 'country_uri,measure_uri', "
"'measure_uri,year', 'year,measure_uri', 'measure_uri,month', "
"'month,measure_uri'"
raise Error(BADPARAMS, msg=m)
if not data:
raise Error(NORESULT)
return data
def create_account(email, password, account_id, name,
surname, authority='user'):
if not email or not is_valid_email(email):
raise Error(BADPARAMS, msg="Invalid email provided.")
if not account_id or not is_uri(account_id):
raise Error(BADPARAMS,
msg="Invalid account_id provided. Not a URI")
acct = Account(email, password, account_id, name, surname, authority)
acct.save()
return acct
def validate(self):
try:
payload = jwt.decode(self.token, SECRET_KEY)
self.update_from_payload(payload)
if not is_uri(self.sub):
raise AssertionError
except jwt.exceptions.DecodeError:
raise Error(FORBIDDEN)
except jwt.ExpiredSignatureError:
raise Error(UNAUTHORIZED, msg="Signature expired.")
except (AssertionError, jwt.InvalidTokenError):
raise Error(UNAUTHORIZED, msg="Invalid token.")
def response(self, *args, **kw):
if not JWT_DISABLED:
intoken = get_token_from_header()
try:
jwt.decode(intoken, SECRET_KEY)
except jwt.exceptions.DecodeError:
raise Error(FORBIDDEN)
except jwt.ExpiredSignatureError:
raise Error(UNAUTHORIZED, msg="Signature expired.")
except jwt.InvalidTokenError:
raise Error(UNAUTHORIZED, msg="Invalid token.")
return fn(self, *args, **kw)
def insert(self, params):
tablename, attnames, attvalues = params
# tablename = tablename.lower()
# attnames = [attname.lower() for attname in attnames]
if not len(attnames) == len(attvalues):
raise Error("number of columns does not match number of values")
mapping = dict(zip(attnames, attvalues))
try:
idvalue = mapping["id"]
except KeyError:
raise Error("you must specify the 'id' column and value")
self.conn.sadd("%s_id" % tablename, idvalue)
self.conn.hmset("%s:%s" % (tablename, idvalue), mapping)
def put_user_info(username):
visitor = util.get_visitor()
if not visitor or visitor['username'] != username:
raise Error('you are not {}'.format(username))
user = User.get_user(username)
if user is None:
raise NotFound(username)
user_dict_patch = request.json
if not user_dict_patch:
raise Error('empty update')
if user.update(user_dict_patch):
return util.success_response()
else:
raise Error('update failed')
def Solve(self, x):
for c in range(self.layers[0].channels):
for h in range(self.layers[0].height):
for w in range(self.layers[0].width):
self.layers[0].values[c][h][w] = x[c][h][w]
for i in range(1, len(self.layers)):
for c in range(self.layers[i].channels):
for h in range(self.layers[i].height):
for w in range(self.layers[i].width):
self.layers[i].values[c][h][w] = Error.GetError(
self.layers[i].func,
Layers.ApplyConv(self.layers[i].weights[c][h][w],
self.layers[i].address[c][h][w],
self.layers[i - 1].values) +
self.layers[i].biases[c][h][w])
ans = [
self.layers[-1].values[i][0][0]
for i in range(self.layers[-1].channels)
]
if self.use_softmax:
summ = sum([math.exp(ans[i]) for i in range(len(ans))])
ans = [math.exp(ans[i]) / summ for i in range(len(ans))]
return ans
def save_event(data, from_nameko=False):
"""Store a new event. Ignore token if the event comes from Nameko."""
try:
work_uri = data.get('work_uri')
measure_uri = data.get('measure_uri')
timestamp = parser.parse(data.get('timestamp'))
value = data.get('value')
event_uri = data.get('event_uri') or None
country_uri = data.get('country_uri') or None
if from_nameko:
uploader_uri = data.get('uploader_uri')
else:
uploader_uri = get_uploader_from_token()
if not all([work_uri, measure_uri, timestamp, value, uploader_uri]):
raise AssertionError
except BaseException:
raise Error(BADPARAMS)
event_id = str(uuid.uuid4())
event = Event(event_id, work_uri, measure_uri, timestamp, value, event_uri,
country_uri, uploader_uri)
event.save()
return [event.__dict__]
def _check_values(self):
if not (self.user_id and self.to_url and self.data and self.type):
Error("Please provide all the values", 400)
self.correct_data = False
self.message = "Please provide all the values"
return False
return True
def response(self, *args, **kw):
data = fn(self, *args, **kw)
count = len(data)
if count > 0:
return {'status': 'ok', 'code': 200, 'count': count, 'data': data}
else:
raise Error(NORESULT)
def aggregate_by_measure_country(clause, params):
try:
q = '''SELECT
measure_uri,
namespace,
source,
type,
version,
country_uri,
country_code,
country_name,
continent_code,
SUM(value) as value
FROM event
INNER JOIN measure USING(measure_uri)
LEFT JOIN country USING(country_uri)
WHERE 1=1 ''' + clause + '''
GROUP BY measure_uri, namespace, source, type, version,
country_uri, country_code, country_name, continent_code
ORDER BY measure_uri, country_uri;'''
result = db.query(q, params)
return result
except (Exception, psycopg2.DatabaseError) as error:
logger.error(error)
raise Error(FATAL)
def GET(self, name):
"""List a work if UUID provided otherwise list all works"""
work_id = web.input().get('uuid') or web.input().get('UUID')
if work_id:
results = Work.get_from_work_id(work_id)
sort = ""
else:
filters = web.input().get('filter')
sort = web.input().get('sort')
order = web.input().get('order', 'asc')
clause, params = build_parms(filters)
if sort:
validate_sorting_or_fail(["title"], sort, order)
results = Work.get_all(clause, params)
if not results:
raise Error(NORESULT)
include_relatives = work_id is not None
data = results_to_works(results, include_relatives)
if sort:
# we sort by each work's (first) title
return sort_alphabetically(data, sort, order)
return data
def post_note():
note = request.json
if not note:
raise Error('note required')
note = Note.post(note)
if note:
return note.note
def get_username_and_password(data=None):
if data is None:
data = request.json
if not data:
raise Error('username and password required')
username = get_string_field('username', data)
password = get_string_field('password', data)
return username, password
def save_if_not_exists(self):
try:
option = dict(title=self.title)
q = '''INSERT INTO title VALUES ($title) ON CONFLICT DO NOTHING'''
return db.query(q, option)
except (Exception, psycopg2.DatabaseError) as error:
logger.error(error)
raise Error(FATAL)
def get_from_id(account_id):
params = {'account_id': account_id}
q = '''SELECT * FROM account WHERE account_id = $account_id;'''
try:
return db.query(q, params)
except (Exception, psycopg2.DatabaseError) as error:
logger.error(error)
raise Error(FATAL)
def response(self, *args, **kw):
logger.debug("Data: %s" % (get_input()))
data = fn(self, *args, **kw)
count = len(data)
if count > 0:
return {'status': 'ok', 'code': 200, 'count': count, 'data': data}
else:
raise Error(NORESULT)
def GET(self, name):
"""Get Measures with descriptions"""
results = Measure.get_all()
data = results_to_measures(results, True)
if not data:
raise Error(NORESULT)
return data
def get_all(clause, params):
try:
q = '''SELECT * FROM event WHERE 1=1 ''' + clause + '''
ORDER BY timestamp;'''
return db.query(q, params)
except (Exception, psycopg2.DatabaseError) as error:
logger.error(error)
raise Error(FATAL)
def save(self):
from .title import Title
from .identifier import Identifier
try:
with db.transaction():
q = '''INSERT INTO work (work_id, work_type)
VALUES ($work_id, $work_type) ON CONFLICT DO NOTHING'''
db.query(q, dict(work_id=self.UUID, work_type=self.type))
if not self.exists():
logger.error('Could not save record.')
raise Error(FATAL)
for title in self.title:
t = Title(title)
t.save_if_not_exists()
q = '''INSERT INTO work_title (work_id, title)
VALUES ($work_id, $title) ON CONFLICT DO NOTHING'''
db.query(q, dict(work_id=self.UUID, title=title))
for i in self.URI:
uri = i.get('URI') or i.get('uri')
is_canonical = i['canonical']
scheme, value = Identifier.split_uri(uri)
Identifier.insert_if_not_exist(scheme, value)
q = '''INSERT INTO work_uri (work_id, uri_scheme, uri_value,
canonical) VALUES
($work_id, $uri_scheme, $uri_value, $canonical)
ON CONFLICT DO NOTHING'''
db.query(q, dict(work_id=self.UUID, uri_scheme=scheme,
uri_value=value, canonical=is_canonical))
try:
for c in self.child:
db.insert('work_relation', parent_work_id=self.UUID,
child_work_id=c)
except AttributeError:
pass
try:
for p in self.parent:
db.insert('work_relation', parent_work_id=p,
child_work_id=self.UUID)
except AttributeError:
pass
except (Exception, psycopg2.DatabaseError) as error:
logger.debug(error)
raise Error(FATAL)
def raise_exception(data):
import re
from errors import Error
errre = re.compile('Code: (\d+), e.displayText\(\) = DB::Exception: (.+?), e.what\(\) = (.+)')
m = errre.search(data)
if m:
raise Error(*m.groups())
else:
raise Exception('unexpected answer: {}'.format(data))
def GET(self, name):
""" Get a matching ID in the desired format for a given object ID.
1. get uri or title from request
- if uri: get scheme and path lowecased
- if isbn: remove hyphens
2. get parameters from filters
3. query by URI or title or both
4. iterate through results and check them
- if single: output
- if multiple && strict: attempt to choose fittest result
- if multiple && !strict: output all
"""
uri = web.input().get('uri') or web.input().get('URI')
title = web.input().get('title')
filters = web.input().get('filter')
strict = web.input().get('strict') in ("true", "True")
try:
if uri:
scheme, value = Identifier.split_uri(uri)
require_params_or_fail([scheme, value], 'a valid URI')
if title:
title = urllib.parse.unquote(title.strip())
require_params_or_fail([title], 'a valid title')
if not uri and not title:
raise Error
except BaseException:
raise Error(BADPARAMS, msg="Invalid URI or title provided")
clause, params = build_parms(filters)
if uri and not title:
results = Identifier.get_from_uri(scheme, value, clause, params)
elif title and not uri:
results = Identifier.get_from_title(title, clause, params)
else:
results = Identifier.get_from_title(title, clause, params,
scheme, value)
if not results:
raise Error(NORESULT)
return self.process_results(list(results), strict)
def build_params(filters):
if not filters:
return "", {}
# split by ',' except those preceeded by a top level domain, which will
# be a tag URI scheme (e.g. tag:openbookpublishers.com,2009)
params = re.split(r"(?<!\.[a-z]{3}),", filters)
options = {}
uris = []
measures = []
countries = []
uploaders = []
clause = ""
for p in params:
try:
field, val = p.split(':', 1)
if field == "work_uri":
uris.append(val)
elif field == "measure_uri":
measures.append(val)
elif field == "country_uri":
countries.append(val)
elif field == "uploader_uri":
uploaders.append(val)
else:
raise Error(BADFILTERS)
except BaseException:
raise Error(BADFILTERS, msg="Unknown filter '%s'" % (p))
process = {
"work_uri": uris,
"measure_uri": measures,
"country_uri": countries,
"uploader_uri": uploaders
}
for key, values in process.items():
if len(values) > 0:
try:
andclause, ops = build_clause(key, values)
options.update(ops)
clause = clause + andclause
except BaseException:
raise Error(BADFILTERS)
return clause, options
