def test_get_response_doesnt_reset_content_type_when_not_negotiating(mk):
mk(("index.spt", NEGOTIATED_RESOURCE))
request = StubRequest.from_fs("index.spt")
response = Response()
response.headers["Content-Type"] = "never/mind"
actual = get_response(request, response).headers["Content-Type"]
assert actual == "never/mind"
def test_get_response_doesnt_reset_content_type_when_not_negotiating(harness):
harness.fs.www.mk(('index.spt', NEGOTIATED_RESOURCE))
request = harness.make_request(filepath='index.spt', contents=NEGOTIATED_RESOURCE)
response = Response()
response.headers['Content-Type'] = 'never/mind'
actual = get_response(request, response).headers['Content-Type']
assert actual == "never/mind"
def test_get_response_doesnt_reset_content_type_when_not_negotiating():
mk(('index', NEGOTIATED_RESOURCE))
request = StubRequest.from_fs('index')
response = Response()
response.headers['Content-Type'] = 'never/mind'
actual = get_response(request, response).headers['Content-Type']
assert actual == "never/mind", actual
def test_respond_doesnt_reset_content_type_when_not_negotiating(harness):
harness.fs.www.mk(('index.spt', SIMPLATE))
state = _get_state(harness, filepath='index.spt', contents=SIMPLATE)
response = Response()
response.headers['Content-Type'] = 'never/mind'
state['response'] = response
actual = _respond(state).headers['Content-Type']
assert actual == "never/mind"
def test_get_response_doesnt_reset_content_type_when_negotiating(harness):
harness.fs.www.mk(('index.spt', NEGOTIATED_RESOURCE))
state = get_state(harness, filepath='index.spt', contents=NEGOTIATED_RESOURCE)
state['request'].headers['Accept'] = 'text/html'
response = Response()
response.headers['Content-Type'] = 'never/mind'
actual = get_response(state, response).headers['Content-Type']
response = Response()
response.headers['Content-Type'] = 'never/mind'
actual = get_response(state, response).headers['Content-Type']
assert actual == "never/mind"
def oauth_dance(website, qs):
"""Given a querystring, return a dict of user_info.
The querystring should be the querystring that we get from GitHub when
we send the user to the return value of oauth_url above.
See also:
http://developer.github.com/v3/oauth/
"""
log("Doing an OAuth dance with Github.")
data = { 'code': qs['code'].encode('US-ASCII')
, 'client_id': website.github_client_id
, 'client_secret': website.github_client_secret
}
r = requests.post("https://github.com/login/oauth/access_token", data=data)
assert r.status_code == 200, (r.status_code, r.text)
back = dict([pair.split('=') for pair in r.text.split('&')]) # XXX
if 'error' in back:
raise Response(400, back['error'].encode('utf-8'))
assert back.get('token_type', '') == 'bearer', back
access_token = back['access_token']
r = requests.get( "https://api.github.com/user"
, headers={'Authorization': 'token %s' % access_token}
)
assert r.status_code == 200, (r.status_code, r.text)
user_info = json.loads(r.text)
log("Done with OAuth dance with Github for %s (%s)."
% (user_info['login'], user_info['id']))
return user_info
def reject_forgeries(request, csrf_token):
# Assume that anything not defined as 'safe' by RC2616 needs protection.
if request.line.method not in ('GET', 'HEAD', 'OPTIONS', 'TRACE'):
# except webhooks
if request.line.uri.startswith('/callbacks/'):
return
# and requests using HTTP auth
if 'Authorization' in request.headers:
return
# Check non-cookie token for match.
second_token = ""
if request.line.method == "POST":
if isinstance(request.body, dict):
second_token = request.body.get('csrf_token', '')
if second_token == "":
# Fall back to X-CSRF-TOKEN, to make things easier for AJAX,
# and possible for PUT/DELETE.
second_token = request.headers.get('X-CSRF-TOKEN', '')
if not constant_time_compare(second_token, csrf_token):
raise Response(403, "Bad CSRF cookie")
def __init__(self, *args, **kw):
Response.__init__(self, 403, '', **kw)
def __init__(self, code, lazy_body, **kw):
Response.__init__(self, code, '', **kw)
self.lazy_body = lazy_body
def __init__(self, *args):
Response.__init__(self, 400, self.msg.format(*args))
def hidden_files(request):
"""Protect hidden files.
"""
if '/.' in request.fs[len(request.root):]:
raise Response(404)
def test_response_body_can_be_iterable():
response = Response(body=["Greetings, ", "program!"])
expected = ["Greetings, ", "program!"]
actual = response.body
assert actual == expected
def __init__(self, ctype):
Response.__init__(self, code=415, body="Unknown body Content-Type: %s" % ctype)
def allow(self, *methods):
"""Given a list of methods, raise 405 if we don't meet the requirement.
"""
methods = [x.upper() for x in methods]
if self.method not in methods:
raise Response(405, headers={'Allow': ', '.join(methods)})
def canonicalize(path, base, canonical, given):
if given != canonical:
assert canonical.lower() == given.lower() # sanity check
remainder = path[len(base + given):]
newpath = base + canonical + remainder
raise Response(302, headers={"Location": newpath})
def shake_hands(self):
"""Return a handshake response.
"""
handshake = ":".join(
[self.sid, self.heartbeat, self.timeout, self.transports])
return Response(200, handshake)
def hook(request):
"""A hook to return 200 to an 'OPTIONS *' request"""
if request.line.method == "OPTIONS" and request.line.uri == "*":
raise Response(200)
return request
def get(request):
"""Takes a Request object and returns a Response or Transport object.
When we get the request it has socket set to a string, the path part after
*.sock, which is something like 1/websocket/43ef6fe7?foo=bar.
1 protocol (we only support 1)
websocket transport
43ef6fe7 socket id (sid)
?foo=bar querystring
The Socket.IO handshake is a GET request to 1/. We return Response for the
handshake. After the handshake, subsequent messages are to the full URL as
above. We return a Transported instance for actual messages.
"""
# Exit early.
# ===========
if request.socket is None:
return None
# Parse and validate the socket URL.
# ==================================
parts = request.socket.split('/')
nparts = len(parts)
if nparts not in (2, 3):
msg = "Expected 2 or 3 path parts for Socket.IO socket, got %d."
raise Response(400, msg % nparts)
protocol = parts[0]
if protocol != '1':
msg = "Expected Socket.IO protocol version 1, got %s."
raise Response(400, msg % protocol)
# Handshake
# =========
if len(parts) == 2:
path = request.line.uri.path.raw
if path in __channels__:
channel = __channels__[path]
else:
channel = Channel(path, request.website.network_engine.Buffer)
__channels__[path] = channel
socket = Socket(request, channel)
assert socket.sid not in __sockets__ # sanity check
__sockets__[socket.sid] = socket
socket.loop.start()
return socket.shake_hands() # a Response
# More than a handshake.
# ======================
transport = parts[1]
sid = parts[2]
if transport not in TRANSPORTS:
msg = "Expected transport in {%s}, got %s."
msg %= (",".join(TRANSPORTS), transport)
raise Response(400, msg)
if sid not in __sockets__:
msg = "Expected %s in cache, didn't find it"
raise Response(400, msg % sid)
if type(__sockets__[sid]) is Socket:
# This is the first request after a handshake. It's not until this
# point that we know what transport the client wants to use.
Transport = XHRPollingTransport # XXX derp
__sockets__[sid] = Transport(__sockets__[sid])
transport = __sockets__[sid]
return transport
def error():
if 'default' in kw:
return kw['default']
raise Response(400, "invalid base64 input")
def __init__(self, code=400, lazy_body=None, **kw):
Response.__init__(self, code, '', **kw)
if lazy_body:
self.lazy_body = lazy_body
def dispatch(request, pure_dispatch=False):
"""Concretize dispatch_abstract.
This is all side-effecty on the request object, setting, at the least,
request.fs, and at worst other random contents including but not limited
to: request.line.uri.path, request.headers, request.socket
"""
# Handle websockets.
# ==================
request.line.uri.path.decoded, request.socket = extract_socket_info(
request.line.uri.path.decoded)
# Handle URI path parts
pathparts = request.line.uri.path.parts
# Set up the real environment for the dispatcher.
# ===============================================
listnodes = os.listdir
is_leaf = os.path.isfile
traverse = os.path.join
find_index = lambda x: match_index(request.website.indices, x)
noext_matched = lambda x: update_neg_type(request, x)
startdir = request.website.www_root
# Dispatch!
# =========
result = dispatch_abstract(listnodes, is_leaf, traverse, find_index,
noext_matched, startdir, pathparts)
debug(lambda: "dispatch_abstract returned: " + repr(result))
if result.match:
matchbase, matchname = result.match.rsplit(os.path.sep, 1)
if pathparts[-1] != '' and matchname in request.website.indices and \
is_first_index(request.website.indices, matchbase, matchname):
# asked for something that maps to a default index file; redirect to / per issue #175
debug(lambda: "found default index '%s' maps into %r" %
(pathparts[-1], request.website.indices))
uri = request.line.uri
location = uri.path.raw[:-len(pathparts[-1])]
if uri.querystring.raw:
location += '?' + uri.querystring.raw
raise Response(302, headers={'Location': location})
if not pure_dispatch:
# favicon.ico
# ===========
# Serve Aspen's favicon if there's not one.
if request.line.uri.path.raw == '/favicon.ico':
if result.status != DispatchStatus.okay:
path = request.line.uri.path.raw[1:]
request.fs = request.website.find_ours(path)
return
# robots.txt
# ==========
# Don't let robots.txt be handled by anything other than an actual
# robots.txt file
if request.line.uri.path.raw == '/robots.txt':
if result.status != DispatchStatus.missing:
if not result.match.endswith('robots.txt'):
raise Response(404)
# Handle returned states.
# =======================
if result.status == DispatchStatus.okay:
if result.match.endswith('/'): # autoindex
if not request.website.list_directories:
raise Response(404)
autoindex = request.website.ours_or_theirs('autoindex.html.spt')
assert autoindex is not None # sanity check
request.headers['X-Aspen-AutoIndexDir'] = result.match
request.fs = autoindex
return # return so we skip the no-escape check
else: # normal match
request.fs = result.match
for k, v in result.wildcards.iteritems():
request.line.uri.path[k] = v
elif result.status == DispatchStatus.non_leaf: # trailing-slash redirect
uri = request.line.uri
location = uri.path.raw + '/'
if uri.querystring.raw:
location += '?' + uri.querystring.raw
raise Response(302, headers={'Location': location})
elif result.status == DispatchStatus.missing: # 404
raise Response(404)
else:
raise Response(500, "Unknown result status.")
# Protect against escaping the www_root.
# ======================================
if not request.fs.startswith(startdir):
raise Response(404)
def __init__(self):
Response.__init__(self, code=400, body="Possible CRLF Injection detected.")
def __init__(self, header):
Response.__init__(self, code=400, body="Malformed header: %s" % header)
def __init__(self, *args, **kw):
Response.__init__(self, 403, '', **kw)
def inbound(request):
"""Try to serve a 304 for resources under assets/.
"""
uri = request.line.uri
if not uri.startswith('/assets/'):
# Only apply to the assets/ directory.
return request
if version_is_dash(request):
# Special-case a version of '-' to never 304/404 here.
return request
if not version_is_available(request):
# Don't serve one version of a file as if it were another.
raise Response(404)
ims = request.headers.get('If-Modified-Since')
if not ims:
# This client doesn't care about when the file was modified.
return request
if request.fs.endswith('.spt'):
# This is a requests for a dynamic resource. Perhaps in the future
# we'll delegate to such resources to compute a sensible Last-Modified
# or E-Tag, but for now we punt. This is okay, because we expect to
# put our dynamic assets behind a CDN in production.
return request
try:
ims = timegm(parsedate(ims))
except:
# Malformed If-Modified-Since header. Proceed with the request.
return request
last_modified = get_last_modified(request.fs)
if ims < last_modified:
# The file has been modified since. Serve the whole thing.
return request
# Huzzah!
# =======
# We can serve a 304! :D
response = Response(304)
response.headers['Last-Modified'] = format_date_time(last_modified)
response.headers['Cache-Control'] = 'no-cache'
raise response
def export_history(participant,
year,
mode,
key,
back_as='namedtuple',
require_key=False):
db = participant.db
params = dict(id=participant.id, year=year)
out = {}
if mode == 'aggregate':
out['given'] = lambda: db.all("""
SELECT tippee, sum(amount) AS amount
FROM transfers
WHERE tipper = %(id)s
AND extract(year from timestamp) = %(year)s
AND status = 'succeeded'
GROUP BY tippee
""",
params,
back_as=back_as)
out['taken'] = lambda: db.all("""
SELECT team, sum(amount) AS amount
FROM transfers
WHERE tippee = %(id)s
AND context = 'take'
AND extract(year from timestamp) = %(year)s
AND status = 'succeeded'
GROUP BY team
""",
params,
back_as=back_as)
else:
out['exchanges'] = lambda: db.all("""
SELECT timestamp, amount, fee, status, note
FROM exchanges
WHERE participant = %(id)s
AND extract(year from timestamp) = %(year)s
ORDER BY id ASC
""",
params,
back_as=back_as)
out['given'] = lambda: db.all("""
SELECT timestamp, tippee, amount, context
FROM transfers
WHERE tipper = %(id)s
AND extract(year from timestamp) = %(year)s
AND status = 'succeeded'
ORDER BY id ASC
""",
params,
back_as=back_as)
out['taken'] = lambda: db.all("""
SELECT timestamp, team, amount
FROM transfers
WHERE tippee = %(id)s
AND context = 'take'
AND extract(year from timestamp) = %(year)s
AND status = 'succeeded'
ORDER BY id ASC
""",
params,
back_as=back_as)
out['received'] = lambda: db.all("""
SELECT timestamp, amount, context
FROM transfers
WHERE tippee = %(id)s
AND context <> 'take'
AND extract(year from timestamp) = %(year)s
AND status = 'succeeded'
ORDER BY id ASC
""",
params,
back_as=back_as)
if key:
try:
return out[key]()
except KeyError:
raise Response(400, "bad key `%s`" % key)
elif require_key:
raise Response(400, "missing `key` parameter")
else:
return {k: v() for k, v in out.items()}
def test_response_body_can_be_bytestring():
response = Response(body=b"Greetings, program!")
expected = "Greetings, program!"
actual = response.body
assert actual == expected
def __init__(self, *args, **kw):
Response.__init__(self, 400, '', **kw)
self.lazy_body = self.msg
self.args = args
def test_response_body_can_be_unicode():
try:
Response(body=u'Greetings, program!')
except:
assert False, 'expecting no error'
def __init__(self, *args):
Response.__init__(self, 400, self.msg.format(*args))
def __init__(self, extension):
body = "Failure to typecast extension '{0}'".format(extension)
Response.__init__(self, code=404, body=body)
def __init__(self, code=400, lazy_body=None, **kw):
Response.__init__(self, code, '', **kw)
if lazy_body:
self.lazy_body = lazy_body
def authenticate_user_if_possible(request, state, user, _):
"""This signs the user in.
"""
if request.line.uri.startswith('/assets/'):
return
# HTTP auth
if 'Authorization' in request.headers:
header = request.headers['authorization']
if not header.startswith('Basic '):
raise Response(401, 'Unsupported authentication method')
try:
creds = binascii.a2b_base64(header[len('Basic '):]).split(':', 1)
except binascii.Error:
raise Response(400, 'Malformed "Authorization" header')
participant = Participant.authenticate('id', 'password', *creds)
if not participant:
raise Response(401)
return {'user': participant}
# Cookie and form auth
# We want to try cookie auth first, but we want form auth to supersede it
p = None
response = state.setdefault('response', Response())
if SESSION in request.headers.cookie:
creds = request.headers.cookie[SESSION].value.split(':', 1)
p = Participant.authenticate('id', 'session', *creds)
if p:
state['user'] = p
session_p, p = p, None
session_suffix = ''
redirect_url = request.line.uri
if request.method == 'POST':
body = _get_body(request)
if body:
p = sign_in_with_form_data(body, state)
carry_on = body.pop('email-login.carry-on', None)
if not p and carry_on:
p_email = session_p and (session_p.email
or session_p.get_emails()[0].address)
if p_email != carry_on:
state['email-login.carry-on'] = carry_on
raise AuthRequired
redirect_url = body.get('sign-in.back-to') or redirect_url
elif request.method == 'GET' and request.qs.get('log-in.id'):
id, token = request.qs.pop('log-in.id'), request.qs.pop('log-in.token')
p = Participant.authenticate('id', 'session', id, token)
if not p and (not session_p or session_p.id != id):
raise Response(400, _("This login link is expired or invalid."))
else:
qs = '?' + urlencode(request.qs, doseq=True) if request.qs else ''
redirect_url = request.path.raw + qs
session_p = p
session_suffix = '.em'
if p:
if session_p:
session_p.sign_out(response.headers.cookie)
p.sign_in(response.headers.cookie, session_suffix)
state['user'] = p
if request.body.pop('form.repost', None) != 'true':
response.redirect(redirect_url)
def not_found(request, favicon):
if not isfile(request.fs):
if request.path.raw == '/favicon.ico': # special case
request.fs = favicon
else:
raise Response(404)
def test_get_response_gets_response(mk):
mk(('index.spt', NEGOTIATED_RESOURCE))
response = Response()
request = StubRequest.from_fs('index.spt')
actual = get_response(request, response)
assert actual is response
def test_get_response_is_happy_not_to_negotiate(mk):
mk(('index.spt', NEGOTIATED_RESOURCE))
request = StubRequest.from_fs('index.spt')
actual = get_response(request, Response()).body
assert actual == "Greetings, program!\n"
def reject_null_bytes_in_uri(environ):
# https://hackerone.com/reports/262852
if '%00' in environ['PATH_INFO'] + environ.get('QUERY_STRING', ''):
raise Response(400)
def test_get_response_negotiates(mk):
mk(('index.spt', NEGOTIATED_RESOURCE))
request = StubRequest.from_fs('index.spt')
request.headers['Accept'] = 'text/html'
actual = get_response(request, Response()).body
assert actual == "<h1>Greetings, program!</h1>\n"
def __init__(self, code, lazy_body, **kw):
Response.__init__(self, code, '', **kw)
self.lazy_body = lazy_body
def test_get_response_sets_content_type_when_it_negotiates(mk):
mk(('index.spt', NEGOTIATED_RESOURCE))
request = StubRequest.from_fs('index.spt')
request.headers['Accept'] = 'text/html'
actual = get_response(request, Response()).headers['Content-Type']
assert actual == "text/html; charset=UTF-8"
def __init__(self, *args, **kw):
Response.__init__(self, self.code, '', **kw)
self.lazy_body = self.msg
self.args = args
def test_get_response_raises_406_if_need_be(mk):
mk(('index.spt', NEGOTIATED_RESOURCE))
request = StubRequest.from_fs('index.spt')
request.headers['Accept'] = 'cheese/head'
actual = raises(Response, get_response, request, Response()).value.code
assert actual == 406
def inbound(request):
"""Try to serve a 304 for resources under assets/.
"""
uri = request.line.uri
if not uri.startswith('/assets/'):
# Only apply to the assets/ directory.
return request
if version_is_dash(request):
# Special-case a version of '-' to never 304/404 here.
return request
if not version_is_available(request):
# Don't serve one version of a file as if it were another.
raise Response(404)
ims = request.headers.get('If-Modified-Since')
if not ims:
# This client doesn't care about when the file was modified.
return request
if request.fs.endswith('.spt'):
# This is a requests for a dynamic resource. Perhaps in the future
# we'll delegate to such resources to compute a sensible Last-Modified
# or E-Tag, but for now we punt. This is okay, because we expect to
# put our dynamic assets behind a CDN in production.
return request
try:
ims = timegm(parsedate(ims))
except:
# Malformed If-Modified-Since header. Proceed with the request.
return request
last_modified = get_last_modified(request.fs)
if ims < last_modified:
# The file has been modified since. Serve the whole thing.
return request
# Huzzah!
# =======
# We can serve a 304! :D
response = Response(304)
response.headers['Last-Modified'] = format_date_time(last_modified)
response.headers['Cache-Control'] = 'no-cache'
raise response
def inbound(request):
"""Given a Request object, reject it if it's a forgery.
"""
try:
csrf_token = request.headers.cookie.get('csrf_token')
csrf_token = '' if csrf_token is None else csrf_token.value
csrf_token = _sanitize_token(csrf_token)
# Use same token next time
request.context['csrf_token'] = csrf_token
except KeyError:
csrf_token = None
# Generate token and store it in the request, so it's
# available to the view.
request.context['csrf_token'] = _get_new_csrf_key()
# Assume that anything not defined as 'safe' by RC2616 needs protection
if request.line.method not in ('GET', 'HEAD', 'OPTIONS', 'TRACE'):
if _is_secure(request):
# Suppose user visits http://example.com/
# An active network attacker (man-in-the-middle, MITM) sends a
# POST form that targets https://example.com/detonate-bomb/ and
# submits it via JavaScript.
#
# The attacker will need to provide a CSRF cookie and token, but
# that's no problem for a MITM and the session-independent
# nonce we're using. So the MITM can circumvent the CSRF
# protection. This is true for any HTTP connection, but anyone
# using HTTPS expects better! For this reason, for
# https://example.com/ we need additional protection that treats
# http://example.com/ as completely untrusted. Under HTTPS,
# Barth et al. found that the Referer header is missing for
# same-domain requests in only about 0.2% of cases or less, so
# we can use strict Referer checking.
referer = request.headers.get('Referer')
if referer is None:
raise Response(403, REASON_NO_REFERER)
# Note that get_host() includes the port.
good_referer = 'https://%s/' % _get_host(request)
if not same_origin(referer, good_referer):
reason = REASON_BAD_REFERER % (referer, good_referer)
raise Response(403, reason)
if csrf_token is None:
# No CSRF cookie. For POST requests, we insist on a CSRF cookie,
# and in this way we can avoid all CSRF attacks, including login
# CSRF.
raise Response(403, REASON_NO_CSRF_COOKIE)
# Check non-cookie token for match.
request_csrf_token = ""
if request.line.method == "POST":
request_csrf_token = request.body.get('csrf_token', '')
if request_csrf_token == "":
# Fall back to X-CSRF-TOKEN, to make things easier for AJAX,
# and possible for PUT/DELETE.
request_csrf_token = request.headers.get('X-CSRF-TOKEN', '')
if not constant_time_compare(request_csrf_token, csrf_token):
raise Response(403, REASON_BAD_TOKEN)
def get_user_info(login):
"""Get the given user's information from the DB or failing that, github.
:param login:
A unicode string representing a username in github.
:returns:
A dictionary containing github specific information for the user.
"""
typecheck(login, (unicode, UnicodeWithParams))
rec = gittip.db.one( "SELECT user_info FROM elsewhere "
"WHERE platform='github' "
"AND user_info->'login' = %s"
, (login,)
)
if rec is not None:
user_info = rec
else:
url = "https://api.github.com/users/%s"
user_info = requests.get(url % login, params={
'client_id': os.environ.get('GITHUB_CLIENT_ID'),
'client_secret': os.environ.get('GITHUB_CLIENT_SECRET')
})
status = user_info.status_code
content = user_info.text
# Calculate how much of our ratelimit we have consumed
remaining = int(user_info.headers['x-ratelimit-remaining'])
limit = int(user_info.headers['x-ratelimit-limit'])
# thanks to from __future__ import division this is a float
percent_remaining = remaining/limit
log_msg = ''
log_lvl = None
# We want anything 50% or over
if 0.5 <= percent_remaining:
log_msg = ("{0}% of GitHub's ratelimit has been consumed. {1}"
" requests remaining.").format(percent_remaining * 100,
remaining)
if 0.5 <= percent_remaining < 0.8:
log_lvl = logging.WARNING
elif 0.8 <= percent_remaining < 0.95:
log_lvl = logging.ERROR
elif 0.95 <= percent_remaining:
log_lvl = logging.CRITICAL
if log_msg and log_lvl:
log(log_msg, log_lvl)
if status == 200:
user_info = json.loads(content)
elif status == 404:
raise Response(404,
"GitHub identity '{0}' not found.".format(login))
else:
log("Github api responded with {0}: {1}".format(status, content),
level=logging.WARNING)
raise Response(502, "GitHub lookup failed with %d." % status)
return user_info
def _response(self, *args):
from aspen import Response
r = Response(*args)
r.request = self.request
return r