def normalize(uristr):
"""Translate the given URI into a normalized form."""
uristr = uristr.encode('utf-8')
# Strip proxy prefix for proxied URLs
for scheme in URL_SCHEMES:
if uristr.startswith(VIA_PREFIX + scheme + ':'):
uristr = uristr[len(VIA_PREFIX):]
break
# Try to extract the scheme
uri = urlparse.urlsplit(uristr)
# If this isn't a URL, we don't perform any normalization
if uri.scheme.lower() not in URL_SCHEMES:
return text_type(uristr, 'utf-8')
# Don't perform normalization on URLs with no hostname.
if uri.hostname is None:
return text_type(uristr, 'utf-8')
scheme = _normalize_scheme(uri)
netloc = _normalize_netloc(uri)
path = _normalize_path(uri)
query = _normalize_query(uri)
fragment = None
uri = urlparse.SplitResult(scheme, netloc, path, query, fragment)
return text_type(uri.geturl(), 'utf-8')
def test_filtered_windowing(self, db_session, windowsize, expected):
"""Check that windowing respects the where clause."""
testdata = []
enabled = ASCII_LOWERCASE[:13]
disabled = ASCII_LOWERCASE[13:]
testdata.extend([{"name": text_type(l), "enabled": True} for l in enabled])
testdata.extend([{"name": text_type(l), "enabled": False} for l in disabled])
db_session.execute(test_cw.insert().values(testdata))
filter_ = test_cw.c.enabled
windows = column_windows(
db_session, test_cw.c.name, windowsize=windowsize, where=filter_
)
assert window_query_results(db_session, windows, filter_) == expected
def auth_token(request):
"""
Fetch the token (if any) associated with a request.
:param request: the request object
:type request: pyramid.request.Request
:returns: the auth token carried by the request, or None
:rtype: h.models.Token or None
"""
try:
header = request.headers['Authorization']
except KeyError:
return None
if not header.startswith('Bearer '):
return None
token = text_type(header[len('Bearer '):]).strip()
# If the token is empty at this point, it is clearly invalid and we
# should reject it.
if not token:
return None
token_model = (request.db.query(models.Token)
.filter_by(value=token)
.one_or_none())
if token_model is not None:
return Token(token_model)
# If we've got this far it's possible the token is a legacy client JWT.
return _maybe_jwt(token, request)
def auth_token(request):
"""
Fetch the token (if any) associated with a request.
:param request: the request object
:type request: pyramid.request.Request
:returns: the auth token carried by the request, or None
:rtype: h.models.Token or None
"""
try:
header = request.headers["Authorization"]
except KeyError:
return None
if not header.startswith("Bearer "):
return None
token = text_type(header[len("Bearer ") :]).strip()
# If the token is empty at this point, it is clearly invalid and we
# should reject it.
if not token:
return None
return token
def app(pyramid_app, db_engine):
from h import db
_clean_database(db_engine)
db.init(db_engine, authority=text_type(TEST_SETTINGS['h.authority']))
return TestApp(pyramid_app)
def unauthenticated_userid(self, request):
"""
Return the userid implied by the token in the passed request, if any.
This function inspects the passed request for bearer tokens, and
attempts to interpret any found tokens as either API tokens or JWTs,
in that order.
:param request: a request object
:type request: pyramid.request.Request
:returns: the userid authenticated for the passed request or None
:rtype: unicode or None
"""
try:
header = request.headers['Authorization']
except KeyError:
return None
if not header.startswith('Bearer '):
return None
token = text_type(header[len('Bearer '):]).strip()
# If the token is empty at this point, it is clearly invalid and we
# should reject it.
if not token:
return None
return (tokens.userid_from_api_token(token) or
tokens.userid_from_jwt(token, request))
def check_password(self, secret):
"""Check the passed password for this user."""
if not self.password:
return False
# Old-style separate salt.
#
# TODO: remove this deprecated code path when a suitable proportion of
# users have updated their password by logging-in. (Check how many
# users still have a non-null salt in the database.)
if self.salt is not None:
verified = password_context.verify(secret + self.salt,
self.password)
# If the password is correct, take this opportunity to upgrade the
# password and remove the salt.
if verified:
self.password = secret
return verified
verified, new_hash = password_context.verify_and_update(secret,
self.password)
if not verified:
return False
if new_hash is not None:
self._password = text_type(new_hash)
return verified
def default_authority(request):
"""
Return the value of the h.authority config settings.
Falls back on returning request.domain if h.authority isn't set.
"""
return text_type(request.registry.settings.get("h.authority", request.domain))
def title(self):
"""Return a title for this annotation.
Return the annotated document's title or if the document has no title
then return its filename (if it's a file:// URI) or its URI for
non-file URIs.
The title is escaped and safe to be rendered.
If it contains escaped characters then the title will be a
Markup object, so that it won't be double-escaped.
"""
document_ = self.annotation.document
if document_:
try:
title = document_["title"]
except (KeyError, TypeError):
# Sometimes document_ has no "title" key or isn't a dict at
# all.
title = ""
if title:
# Convert non-string titles into strings.
# We're assuming that title cannot be a byte string.
title = text_type(title)
return jinja2.escape(title)
if self.filename:
return jinja2.escape(urllib2.unquote(self.filename))
else:
return jinja2.escape(urllib2.unquote(self.uri))
def auth_domain(request):
"""
Return the value of the h.auth_domain config settings.
Falls back on returning request.domain if h.auth_domain isn't set.
"""
return text_type(request.registry.settings.get('h.auth_domain', request.domain))
def check_password(self, user, password):
"""Check the password for this user, and upgrade it if necessary."""
if not user.password:
return False
# Old-style separate salt.
#
# TODO: remove this deprecated code path when a suitable proportion of
# users have updated their password by logging-in. (Check how many
# users still have a non-null salt in the database.)
if user.salt is not None:
verified = self.hasher.verify(password + user.salt, user.password)
# If the password is correct, take this opportunity to upgrade the
# password and remove the salt.
if verified:
self.update_password(user, password)
return verified
verified, new_hash = self.hasher.verify_and_update(password,
user.password)
if not verified:
return False
if new_hash is not None:
user.password = text_type(new_hash)
return verified
def update_password(self, user, new_password):
"""Update the user's password."""
# Remove any existing explicit salt (the password context salts the
# password automatically).
user.salt = None
user.password = text_type(self.hasher.hash(new_password))
user.password_updated = datetime.datetime.utcnow()
def _generate_random_string(length=12):
"""Generate a random ascii string of the requested length."""
msg = hashlib.sha256()
word = ''
for _ in range(length):
word += random.choice(string.ascii_letters)
msg.update(word.encode('ascii'))
return text_type(msg.hexdigest()[:length])
def test_basic_windowing(self, db_session, windowsize, expected):
"""Check that windowing returns the correct batches of rows."""
testdata = [{"name": text_type(l), "enabled": True} for l in ASCII_LOWERCASE]
db_session.execute(test_cw.insert().values(testdata))
windows = column_windows(db_session, test_cw.c.name, windowsize=windowsize)
assert window_query_results(db_session, windows) == expected
def pyramid_request(db_session, fake_feature, pyramid_settings):
"""Dummy Pyramid request object."""
request = testing.DummyRequest(db=db_session, feature=fake_feature)
request.auth_domain = text_type(request.domain)
request.create_form = mock.Mock()
request.matched_route = mock.Mock()
request.registry.settings = pyramid_settings
request.is_xhr = False
return request
def password(self, secret):
if len(secret) < PASSWORD_MIN_LENGTH:
raise ValueError('password must be more than {min} characters '
'long'.format(min=PASSWORD_MIN_LENGTH))
# Remove any existing explicit salt (the password context salts the
# password automatically).
self.salt = None
self._password = text_type(password_context.encrypt(secret))
self.password_updated = datetime.datetime.utcnow()
def _init_db(settings):
engine = db.make_engine(settings)
# If the alembic_version table is present, then the database is managed by
# alembic, and we shouldn't call `db.init`.
try:
engine.execute('select 1 from alembic_version')
except sqlalchemy.exc.ProgrammingError:
log.info("initializing database")
db.init(engine, should_create=True, authority=text_type(settings['h.authority']))
else:
log.info("detected alembic_version table, skipping db initialization")
def pyramid_request(db_session, fake_feature, pyramid_settings):
"""Dummy Pyramid request object."""
request = testing.DummyRequest(db=db_session, feature=fake_feature)
request.authority = text_type(TEST_AUTHORITY)
request.create_form = mock.Mock()
request.matched_route = mock.Mock()
request.registry.settings = pyramid_settings
request.is_xhr = False
request.params = MultiDict()
request.GET = request.params
request.POST = request.params
return request
def find_by_uris(cls, session, uris):
"""Find documents by a list of uris."""
query_uris = [text_type(uri.normalize(u), 'utf-8') for u in uris]
matching_claims = (
session.query(DocumentURI)
.filter(DocumentURI.uri_normalized.in_(query_uris))
.distinct(DocumentURI.document_id)
.subquery()
)
return session.query(Document).join(matching_claims)
def uni_fold(text):
# Convert bytes to text
if isinstance(text, bytes):
text = text_type(text, "utf-8")
# Do not touch other types
if not isinstance(text, text_type):
return text
text = text.lower()
text = unicodedata.normalize('NFKD', text)
return u"".join([c for c in text if not unicodedata.combining(c)])
def bearer_token(request):
"""
Return the bearer token from the request's Authorization header.
The "Bearer " prefix will be stripped from the token.
If the request has no Authorization header or the Authorization header
doesn't contain a bearer token, returns ''.
:rtype: unicode
"""
if request.headers.get('Authorization', '').startswith('Bearer '):
return text_type(request.headers['Authorization'][len('Bearer '):])
else:
return u''
def uri(self):
"""Return this annotation's URI or an empty string.
The uri is escaped and safe to be rendered.
The uri is a Markup object so it won't be double-escaped.
"""
uri_ = self.get("uri")
if uri_:
# Convert non-string URIs into strings.
# If the URI is already a unicode string this will do nothing.
# We're assuming that URI cannot be a byte string.
return text_type(uri_)
else:
return ""
def uni_fold(text):
"""
Return a case-folded and Unicode-normalized copy of ``text``.
This is used to ensure matching of filters against annotations ignores
differences in case or different ways of representing the same characters.
"""
# Convert bytes to text
if isinstance(text, bytes):
text = text_type(text, "utf-8")
# Do not touch other types
if not isinstance(text, text_type):
return text
text = text.lower()
text = unicodedata.normalize("NFKD", text)
return "".join([c for c in text if not unicodedata.combining(c)])
def _init_db(settings):
engine = db.make_engine(settings)
# If the alembic_version table is present, then the database is managed by
# alembic, and we shouldn't call `db.init`.
try:
engine.execute("select 1 from alembic_version")
except sqlalchemy.exc.ProgrammingError:
log.info("initializing database")
db.init(
engine, should_create=True, authority=text_type(settings["h.authority"])
)
# Stamp the database with the current schema version so that future
# migrations start from the correct point.
alembic_cfg = alembic.config.Config("conf/alembic.ini")
alembic.command.stamp(alembic_cfg, "head")
else:
log.info("detected alembic_version table, skipping db initialization")
def handle_annotation_event(message, sockets, settings, session):
id_ = message['annotation_id']
annotation = storage.fetch_annotation(session, id_)
if annotation is None:
log.warn('received annotation event for missing annotation: %s', id_)
return
nipsa_service = NipsaService(session)
user_nipsad = nipsa_service.is_flagged(annotation.userid)
authority = text_type(settings.get('h.authority', 'localhost'))
group_service = GroupfinderService(session, authority)
for socket in sockets:
reply = _generate_annotation_event(message, socket, annotation, user_nipsad, group_service)
if reply is None:
continue
socket.send_json(reply)
def handle_annotation_event(message, sockets, settings, session):
id_ = message["annotation_id"]
annotation = storage.fetch_annotation(session, id_)
if annotation is None:
log.warning("received annotation event for missing annotation: %s", id_)
return
nipsa_service = NipsaService(session)
user_nipsad = nipsa_service.is_flagged(annotation.userid)
authority = text_type(settings.get("h.authority", "localhost"))
group_service = GroupfinderService(session, authority)
user_service = UserService(authority, session)
formatters = [AnnotationUserInfoFormatter(session, user_service)]
for socket in sockets:
reply = _generate_annotation_event(
message, socket, annotation, user_nipsad, group_service, formatters
)
if reply is None:
continue
socket.send_json(reply)
def authenticated_userid(request):
"""
Return the token-authenticated userid for the passed request.
This function inspects the passed request for bearer tokens, and attempts
to interpret any found tokens as either API tokens or JWTs, in that order.
:param request: a request object
:type request: pyramid.request.Request
:returns: the userid authenticated for the passed request or None
:rtype: unicode or None
"""
token = None
if request.headers.get('Authorization', '').startswith('Bearer '):
token = text_type(request.headers['Authorization'][len('Bearer '):])
# If token is None or an empty string, it is clearly invalid and we should
# reject it.
if not token:
return None
return (userid_from_api_token(token) or
userid_from_jwt(token, request))
def title(self):
"""
Return a title for this document.
Return the document's title or if the document has no title then return
its filename (if it's a file:// URI) or its URI for non-file URIs.
The title is escaped and safe to be rendered.
If it contains escaped characters then the title will be a
Markup object, so that it won't be double-escaped.
"""
title = self.document.title
if title:
# Convert non-string titles into strings.
# We're assuming that title cannot be a byte string.
title = text_type(title)
return jinja2.escape(title)
if self.filename:
return jinja2.escape(url_unquote(self.filename))
else:
return jinja2.escape(url_unquote(self.uri))
def test_validate_returns_legacy_client_token(self, svc):
token = text_type(jwt.encode({'exp': self.time(1)}, key='secret'))
result = svc.validate(token)
assert isinstance(result, LegacyClientJWT)
def claimant(self, value):
self._claimant = value
self._claimant_normalized = text_type(uri.normalize(value), 'utf-8')
def test_validate_returns_legacy_client_token(self, svc):
token = text_type(jwt.encode({'exp': self.time(1)}, key='secret'))
result = svc.validate(token)
assert isinstance(result, LegacyClientJWT)
def target_uri(self, value):
self._target_uri = value
self._target_uri_normalized = text_type(uri.normalize(value), 'utf-8')
def init_db(db_engine):
from h import db
authority = text_type(TEST_SETTINGS['h.authority'])
db.init(db_engine, should_drop=True, should_create=True, authority=authority)
def groups(self, factories):
return {
'climate': text_type(factories.Group(name='Climate').pubid),
'politics': text_type(factories.Group(name='Politics').pubid)
}
def claimant_normalized(self):
claimant = self.claimant
if claimant:
return text_type(uri.normalize(claimant), 'utf-8')
def test_returns_false_for_missing_client(self, svc):
id_ = text_type(uuid.uuid1())
assert svc.validate_response_type(id_, 'code', None) is False
def user_service_factory(context, request):
"""Return a UserService instance for the passed context and request."""
return UserService(default_authority=text_type(request.auth_domain),
session=request.db)
def test_returns_none_when_client_missing(self, svc):
id_ = text_type(uuid.uuid1())
assert svc.get_default_redirect_uri(id_, None) is None
def uri_normalized(self):
uri_ = self.uri
if uri_:
return text_type(uri.normalize(uri_), 'utf-8')
parser_initdb = subparsers.add_parser('initdb', help=initdb.__doc__)
_add_common_args(parser_initdb)
def admin(args):
"""Make a user an admin."""
request = bootstrap(args)
accounts.make_admin(args.username)
request.tm.commit()
parser_admin = subparsers.add_parser('admin', help=admin.__doc__)
_add_common_args(parser_admin)
parser_admin.add_argument(
'username',
type=lambda s: text_type(s, sys.getfilesystemencoding())
if PY2 else text_type,
help="the name of the user to make into an admin, e.g. 'fred'")
def annotool(args):
"""
Perform operations on the annotation database.
This command provides a way of running named commands across the database
of annotations and can be used to run data migrations, analytics, etc.
**NB**: This tool cannot currently be used for making changes to the
annotation "document" field.
"""
request = bootstrap(args)
def _hash_password(self, password):
if not self.salt:
self.salt = _generate_random_string(24)
return text_type(CRYPT.encode(password + self.salt))
def test_validate_returns_none_for_invalid_legacy_client_token(self, svc):
token = text_type(jwt.encode({'exp': self.time(-1)}, key='secret'))
result = svc.validate(token)
assert result is None
def test_returns_false_when_client_missing(self, svc, oauth_request):
assert svc.authenticate_client_id(text_type(uuid.uuid1()), oauth_request) is False
def test_it_raises_for_missing_client(self, svc, code, oauth_request):
id_ = text_type(uuid.uuid1())
with pytest.raises(InvalidClientIdError):
svc.save_authorization_code(id_, code, oauth_request)
def test_validate_returns_none_for_invalid_legacy_client_token(self, svc):
token = text_type(jwt.encode({'exp': self.time(-1)}, key='secret'))
result = svc.validate(token)
assert result is None
def test_returns_none_when_not_found(self, svc, client):
id_ = text_type(uuid.uuid1())
assert svc.find_client(id_) is None
def jwt_token(self, claims, secret, algorithm='HS256'):
return text_type(jwt.encode(claims, secret, algorithm=algorithm))