def test_response_cookie():
response = Response()
response.headers.cookie[str('foo')] = str('bar')
def start_response(status, headers):
assert headers[0][0] == str('Set-Cookie')
assert headers[0][1].startswith(str('foo=bar'))
response.to_wsgi({}, start_response, 'utf8')
def test_response_body_as_bytestring_results_in_an_iterable():
response = Response(body=b"Greetings, program!")
def start_response(status, headers):
pass
expected = [b"Greetings, program!"]
actual = list(response.to_wsgi({}, start_response, 'utf8').body)
assert actual == expected
def test_response_to_wsgi():
response = Response(body=b"Greetings, program!")
def start_response(status, headers):
pass
expected = [b"Greetings, program!"]
actual = list(response.to_wsgi({}, start_response, 'utf8').body)
assert actual == expected
def turn_socket_error_into_50X(website,
state,
exception,
_=lambda a: a,
response=None):
# The mangopay module reraises exceptions and stores the original in `__cause__`.
if isinstance(exception, mangopay.exceptions.APIError):
exception = exception.__cause__
# replace the exception with a Response
if isinstance(exception, Timeout) or 'timeout' in str(exception).lower():
response = response or Response()
response.code = 504
elif isinstance(exception, (OSError, ConnectionError)):
response = response or Response()
response.code = 502
else:
return
# log the exception
website.tell_sentry(exception, state, level='warning')
# show a proper error message
response.body = _(
"Processing your request failed because our server was unable to communicate "
"with a service located on another machine. This is a temporary issue, please "
"try again later.")
return {'response': response, 'exception': None}
def test_response_body_as_iterable_comes_through_untouched():
response = Response(body=[b"Greetings, ", b"program!"])
def start_response(status, headers):
pass
expected = [b"Greetings, ", b"program!"]
actual = list(response.to_wsgi({}, start_response, 'utf8').body)
assert actual == expected
def word(mapping, k, pattern=r'^\w+$', unicode=False):
r = mapping[k]
if not r:
raise Response().error(400, "`%s` value %r is empty" % (k, r))
if not re.match(pattern, r, re.UNICODE if unicode else re.ASCII):
raise Response().error(400, "`%s` value %r contains forbidden characters" % (k, r))
return r
def test_response_headers_are_str():
response = Response()
response.headers[b'Location'] = b'somewhere'
def start_response(status, headers):
assert isinstance(headers[0][0], str)
assert isinstance(headers[0][1], str)
response.to_wsgi({}, start_response, 'utf8')
def handle_negotiation_exception(exception):
if isinstance(exception, NotFound):
response = Response(404)
elif isinstance(exception, NegotiationFailure):
response = Response(406, exception.message)
else:
return
return {'response': response, 'exception': None}
def test_response_cookie():
response = Response()
response.headers.cookie[str('foo')] = str('bar')
def start_response(status, headers):
assert headers[0][0] == str('Set-Cookie')
assert headers[0][1].startswith(str('foo=bar'))
response.to_wsgi({}, start_response, 'utf8')
def test_response_headers_are_str():
response = Response()
response.headers[b'Location'] = b'somewhere'
def start_response(status, headers):
assert isinstance(headers[0][0], str)
assert isinstance(headers[0][1], str)
response.to_wsgi({}, start_response, 'utf8')
def test_response_body_as_iterable_comes_through_untouched():
response = Response(body=[b"Greetings, ", b"program!"])
def start_response(status, headers):
pass
expected = [b"Greetings, ", b"program!"]
actual = list(response.to_wsgi({}, start_response, 'utf8').body)
assert actual == expected
def test_response_body_as_bytestring_results_in_an_iterable():
response = Response(body=b"Greetings, program!")
def start_response(status, headers):
pass
expected = [b"Greetings, program!"]
actual = list(response.to_wsgi({}, start_response, 'utf8').body)
assert actual == expected
def test_response_to_wsgi():
response = Response(body=b"Greetings, program!")
def start_response(status, headers):
pass
expected = [b"Greetings, program!"]
actual = list(response.to_wsgi({}, start_response, 'utf8').body)
assert actual == expected
def canonize(request, website):
"""Enforce a certain scheme and hostname.
This is a Pando state chain function to ensure that requests are served on a
certain root URL, even if multiple domains point to the application.
"""
is_callback = request.path.raw.startswith('/callbacks/')
is_healthcheck = request.headers.get(b'User-Agent',
b'').startswith(b'ELB-HealthChecker')
if is_callback or is_healthcheck:
# Don't redirect callbacks
if request.path.raw[-1] == '/' or is_healthcheck:
l = request.line
scheme, netloc, path, query, fragment = urlsplit(l.uri)
assert path[-1] == '/' # sanity check
path = '/callbacks/health.txt' if is_healthcheck else path[:-1]
new_uri = urlunsplit((scheme, netloc, path, query, fragment))
request.line = Line(l.method.raw, new_uri, l.version.raw)
return
scheme = request.headers.get(b'X-Forwarded-Proto', b'http')
try:
request.hostname = host = request.headers[b'Host'].decode('idna')
except UnicodeDecodeError:
request.hostname = host = ''
canonical_host = website.canonical_host
canonical_scheme = website.canonical_scheme
bad_scheme = scheme.decode('ascii', 'replace') != canonical_scheme
bad_host = False
if canonical_host:
if host == canonical_host:
pass
elif host.endswith('.' + canonical_host):
subdomain = host[:-len(canonical_host) - 1]
if subdomain in website.locales:
accept_langs = request.headers.get(b'Accept-Language', b'')
accept_langs = subdomain.encode('idna') + b',' + accept_langs
request.headers[b'Accept-Language'] = accept_langs
else:
bad_host = True
else:
bad_host = True
if bad_scheme or bad_host:
url = '%s://%s' % (canonical_scheme,
canonical_host if bad_host else host)
if request.line.method in ('GET', 'HEAD', 'OPTIONS', 'TRACE'):
# Redirect to a particular path for idempotent methods.
url += request.line.uri.path.raw
if request.line.uri.querystring:
url += '?' + request.line.uri.querystring.raw
else:
# For non-idempotent methods, redirect to homepage.
url += '/'
response = Response()
response.headers[b'Cache-Control'] = b'public, max-age=86400'
response.redirect(url)
def canonize(request, website):
"""Enforce a certain scheme and hostname.
This is a Pando state chain function to ensure that requests are served on a
certain root URL, even if multiple domains point to the application.
"""
if request.path.raw.startswith('/callbacks/'):
# Don't redirect callbacks
if request.path.raw[-1] == '/':
# Remove trailing slash
l = request.line
scheme, netloc, path, query, fragment = urlsplit(l.uri)
assert path[-1] == '/' # sanity check
path = path[:-1]
new_uri = urlunsplit((scheme, netloc, path, query, fragment))
request.line = Line(l.method.raw, new_uri, l.version.raw)
return
scheme = request.headers.get(b'X-Forwarded-Proto', b'http')
try:
request.hostname = host = request.headers[b'Host'].decode('idna')
except UnicodeDecodeError:
request.hostname = host = ''
canonical_host = website.canonical_host
canonical_scheme = website.canonical_scheme
bad_scheme = scheme.decode('ascii', 'replace') != canonical_scheme
bad_host = False
if canonical_host:
if host == canonical_host:
pass
elif host.endswith('.'+canonical_host):
subdomain = host[:-len(canonical_host)-1]
if subdomain in website.locales:
accept_langs = request.headers.get(b'Accept-Language', b'')
accept_langs = subdomain.encode('idna') + b',' + accept_langs
request.headers[b'Accept-Language'] = accept_langs
else:
bad_host = True
else:
bad_host = True
if bad_scheme or bad_host:
url = '%s://%s' % (canonical_scheme, canonical_host if bad_host else host)
if request.line.method in ('GET', 'HEAD', 'OPTIONS', 'TRACE'):
# Redirect to a particular path for idempotent methods.
url += request.line.uri.path.raw
if request.line.uri.querystring:
url += '?' + request.line.uri.querystring.raw
else:
# For non-idempotent methods, redirect to homepage.
url += '/'
response = Response()
response.headers[b'Cache-Control'] = b'public, max-age=86400'
response.redirect(url)
def get_int(d, k, default=None, minimum=None):
r = d.get(k)
if r is None:
return default
try:
r = int(r)
except (ValueError, TypeError):
raise Response().error(400,
"`%s` value %r is not a valid integer" % (k, r))
if minimum is not None and r < minimum:
raise Response().error(
400, "`%s` value %r is less than %i" % (k, r, minimum))
return r
def test_response_headers_protect_against_crlf_injection():
response = Response()
def inject():
response.headers[b'Location'] = b'foo\r\nbar'
raises(CRLFInjection, inject)
def parse_int(o, **kw):
try:
return int(o)
except (ValueError, TypeError):
if 'default' in kw:
return kw['default']
raise Response().error(400, "%r is not a valid integer" % o)
def get_query_id(self, querystring):
token = querystring['access_token']
i = token.rfind('.')
data, data_hash = token[:i], token[i + 1:]
if data_hash != hashlib.md5(data + '.' + self.api_secret).hexdigest():
raise Response(400, 'Invalid hash in access_token')
return querystring['query_id']
def api_error_handler(self, response, is_user_session, domain):
response_text = response.text # for Sentry
status = response.status_code
if status == 404:
raise Response(404, response_text)
if status == 401 and is_user_session:
# https://tools.ietf.org/html/rfc5849#section-3.2
raise TokenExpiredError
if status == 429 and is_user_session:
limit, remaining, reset = self.get_ratelimit_headers(response)
def msg(_, to_age):
if remaining == 0 and reset:
return _(
"You've consumed your quota of requests, you can try again in {0}.",
to_age(reset))
else:
return _(
"You're making requests too fast, please try again later."
)
raise LazyResponse(status, msg)
if status != 200:
logger.error('{} responded with {}:\n{}'.format(
domain, status, response_text))
msg = lambda _: _("{0} returned an error, please try again later.",
domain)
raise LazyResponse(502, msg)
def x_user_info(self, extracted, info, default):
if 'accounts' in info:
accounts = info.get('accounts')
if accounts:
return accounts[0]
raise Response(404)
return default
def get_int(d, k, default=NO_DEFAULT, minimum=None):
try:
r = d[k]
except (KeyError, Response):
if default is NO_DEFAULT:
raise
return default
try:
r = int(r)
except (ValueError, TypeError):
raise Response().error(400,
"`%s` value %r is not a valid integer" % (k, r))
if minimum is not None and r < minimum:
raise Response().error(
400, "`%s` value %r is less than %i" % (k, r, minimum))
return r
def test_set_whence_raised_works():
try:
raise Response(200)
except Response as r:
assert r.whence_raised == (None, None)
r.set_whence_raised()
assert r.whence_raised[0] == 'tests' + os.sep + 'test_response.py'
assert isinstance(r.whence_raised[1], int)
def turn_socket_error_into_50X(website, exception, _=lambda a: a, response=None):
# The mangopay module reraises exceptions and stores the original in `__cause__`.
exception = getattr(exception, '__cause__', exception)
if isinstance(exception, Timeout) or 'timeout' in str(exception).lower():
response = response or Response()
response.code = 504
elif isinstance(exception, (socket.error, ConnectionError)):
response = response or Response()
response.code = 502
else:
return
response.body = _(
"Processing your request failed because our server was unable to communicate "
"with a service located on another machine. This is a temporary issue, please "
"try again later."
)
return {'response': response, 'exception': None}
def get_int(d, k, default=None):
r = d.get(k)
if r is None:
return default
try:
return int(r)
except (ValueError, TypeError):
raise Response().error(400,
"`%s` value %r is not a valid integer" % (k, r))
def get_choice(d, k, choices, default=NO_DEFAULT):
try:
r = d[k]
except (KeyError, Response):
if default is NO_DEFAULT:
raise
return default
if r not in choices:
raise Response().error(400, "`%s` value %r is invalid. Choices: %r" % (k, r, choices))
return r
def get_color(d, k, default=NO_DEFAULT):
try:
r = d[k]
except (KeyError, Response):
if default is NO_DEFAULT:
raise
return default
if not color_re.match(r):
raise Response().error(
400, "`%s` value %r is not a valid hexadecimal color" % (k, r))
return r
def return_500_for_exception(website, exception, response=None):
response = response or Response()
response.code = 500
if website.show_tracebacks:
import traceback
response.body = traceback.format_exc()
else:
response.body = (
"Uh-oh, you've found a serious bug. Sorry for the inconvenience, "
"we'll get it fixed ASAP.")
return {'response': response, 'exception': None}
def parse_boolean(mapping, k, default=NO_DEFAULT):
try:
r = mapping[k].lower()
except (KeyError, Response):
if default is NO_DEFAULT:
raise
return default
if r in TRUEISH:
return True
if r in FALSEISH:
return False
raise Response().error(400, "`%s` value %r is invalid" % (k, r))
def parse_list(mapping, k, cast, default=NO_DEFAULT, sep=','):
try:
r = mapping[k].split(sep)
except (KeyError, Response):
if default is NO_DEFAULT:
raise
return default
try:
r = [cast(v) for v in r]
except (ValueError, TypeError):
raise Response().error(400, "`%s` value %r is invalid" % (k, mapping[k]))
return r
def turn_socket_error_into_50X(website,
state,
exception,
_=str.format,
response=None):
"""Catch network errors and replace them with a 502 or 504 response.
Because network exceptions are often caught and wrapped by libraries, this
function recursively looks at the standard `__cause__` and `__context__`
attributes of exceptions in order to find the initial error.
https://docs.python.org/3/reference/simple_stmts.html#the-raise-statement
https://stackoverflow.com/a/11235957/2729778
"""
for i in range(100):
if isinstance(exception,
Timeout) or 'timeout' in str(exception).lower():
response = response or Response()
response.code = 504
break
elif isinstance(exception, (OSError, ConnectionError)):
response = response or Response()
response.code = 502
break
elif getattr(exception, '__cause__', None):
exception = exception.__cause__
elif getattr(exception, '__context__', None):
exception = exception.__context__
else:
return
# log the exception
website.tell_sentry(exception, state, level='warning')
# show a proper error message
response.body = _(
"Processing your request failed because our server was unable to communicate "
"with a service located on another machine. This is a temporary issue, please "
"try again later.")
return {'response': response, 'exception': None}
def compile_assets(website):
cleanup = []
for spt in find_files(website.www_root+'/assets/', '*.spt'):
filepath = spt[:-4] # /path/to/www/assets/foo.css
if not os.path.exists(filepath):
cleanup.append(filepath)
dispatch_result = DispatchResult(DispatchStatus.okay, spt, {}, "Found.", {}, True)
state = dict(dispatch_result=dispatch_result, response=Response())
state['state'] = state
content = resources.get(website.request_processor, spt).render(state).body
if not isinstance(content, bytes):
content = content.encode('utf8')
tmpfd, tmpfpath = mkstemp(dir='.')
os.write(tmpfd, content)
os.close(tmpfd)
os.rename(tmpfpath, filepath)
if website.env.clean_assets:
atexit.register(lambda: rm_f(*cleanup))
def remove_email_address_from_blacklist(address, user, request):
"""
This function allows anyone to remove an email address from the blacklist,
but with rate limits for non-admins in order to prevent abuse.
"""
with website.db.get_cursor() as cursor:
if not user.is_acting_as('admin'):
source = user.id or request.source
website.db.hit_rate_limit('email.unblacklist.source', source,
TooManyAttempts)
r = cursor.all(
"""
UPDATE email_blacklist
SET ignore_after = current_timestamp
, ignored_by = %(user_id)s
WHERE lower(address) = lower(%(address)s)
AND (ignore_after IS NULL OR ignore_after > current_timestamp)
RETURNING *
""", dict(address=address, user_id=user.id))
if not r:
return
if not user.is_acting_as('admin'):
if any(bl.reason == 'complaint' for bl in r):
raise Response(
403,
("Only admins are allowed to unblock an address which is "
"blacklisted because of a complaint."))
website.db.hit_rate_limit('email.unblacklist.target', address,
TooManyAttempts)
# Mark the matching `email_blacklisted` notifications as read
participant = website.db.Participant.from_email(address)
if participant:
notifications = website.db.all(
"""
SELECT id, context
FROM notifications
WHERE participant = %s
AND event = 'email_blacklisted'
AND is_new
""", (participant.id, ))
for notif in notifications:
context = deserialize(notif.context)
if context['blacklisted_address'].lower() == address.lower():
participant.mark_notification_as_read(notif.id)
def parse_date(mapping, k, default=NO_DEFAULT, sep='-'):
try:
r = mapping[k]
if r:
r = r.split(sep)
elif default is not NO_DEFAULT:
return default
except (KeyError, Response):
if default is NO_DEFAULT:
raise
return default
try:
year, month, day = map(int, r)
# the above raises ValueError if the number of parts isn't 3
# or if any part isn't an integer
r = date(year, month, day)
except (ValueError, TypeError):
raise Response().error(400, "`%s` value %r is invalid" % (k, mapping[k]))
return r
def api_error_handler(self, response, is_user_session):
status = response.status_code
if status == 404:
raise Response(404, response.text)
if status == 429 and is_user_session:
limit, remaining, reset = self.get_ratelimit_headers(response)
def msg(_, to_age):
if remaining == 0 and reset:
return _(
"You've consumed your quota of requests, you can try again in {0}.",
to_age(reset))
else:
return _(
"You're making requests too fast, please try again later."
)
raise LazyResponse(status, msg)
if status != 200:
logger.error('{} api responded with {}:\n{}'.format(
self.name, status, response.text))
msg = lambda _: _("{0} returned an error, please try again later.",
self.display_name)
raise LazyResponse(502, msg)
def __init__(self, *args, **kw):
Response.__init__(self, self.code, '', **kw)
self.lazy_body = self.msg
self.args = args
def __init__(self):
Response.__init__(self, 503, '')
self.html_template = 'templates/no-db.html'
def __init__(self):
Response.__init__(self, 403, '')
self.html_template = 'templates/log-in-required.html'
def __init__(self):
Response.__init__(self, 403, '')
self.html_template = 'templates/no-payins.html'
def __init__(self, ambiguous_string, suggestions):
Response.__init__(self, 400, '')
self.ambiguous_string = ambiguous_string
self.suggestions = suggestions
def __init__(self, code, lazy_body, **kw):
Response.__init__(self, code, '', **kw)
self.lazy_body = lazy_body
def __init__(self, id, class_name):
Response.__init__(self, 400, "Invalid %s ID: %r" % (class_name, id))
def create_response_object(request, website):
response = Response()
response.request = request
response.website = website
return {'response': response}
def export_history(participant,
year,
mode,
key,
back_as='namedtuple',
require_key=False):
db = participant.db
base_url = website.canonical_url + '/~'
params = dict(id=participant.id, year=year, base_url=base_url)
out = {}
if mode == 'aggregate':
out['given'] = lambda: db.all("""
SELECT (%(base_url)s || t.tippee::text) AS donee_url,
min(p.username) AS donee_username, basket_sum(t.amount) AS amount
FROM transfers t
JOIN participants p ON p.id = t.tippee
WHERE t.tipper = %(id)s
AND extract(year from t.timestamp) = %(year)s
AND t.status = 'succeeded'
AND t.context IN ('tip', 'take', 'tip-in-advance', 'take-in-advance')
AND t.refund_ref IS NULL
AND t.virtual IS NOT true
GROUP BY t.tippee
""",
params,
back_as=back_as)
out['reimbursed'] = lambda: db.all("""
SELECT (%(base_url)s || t.tippee::text) AS recipient_url,
min(p.username) AS recipient_username, basket_sum(t.amount) AS amount
FROM transfers t
JOIN participants p ON p.id = t.tippee
WHERE t.tipper = %(id)s
AND extract(year from t.timestamp) = %(year)s
AND t.status = 'succeeded'
AND t.context = 'expense'
GROUP BY t.tippee
""",
params,
back_as=back_as)
out['taken'] = lambda: db.all("""
SELECT (%(base_url)s || t.team::text) AS team_url,
min(p.username) AS team_username, basket_sum(t.amount) AS amount
FROM transfers t
JOIN participants p ON p.id = t.team
WHERE t.tippee = %(id)s
AND t.context IN ('take', 'take-in-advance')
AND extract(year from t.timestamp) = %(year)s
AND t.status = 'succeeded'
AND t.virtual IS NOT true
GROUP BY t.team
""",
params,
back_as=back_as)
else:
out['exchanges'] = lambda: db.all("""
SELECT timestamp, amount, fee, status, note
FROM exchanges
WHERE participant = %(id)s
AND extract(year from timestamp) = %(year)s
ORDER BY id ASC
""",
params,
back_as=back_as)
out['given'] = lambda: db.all("""
SELECT timestamp, (%(base_url)s || t.tippee::text) AS donee_url,
p.username AS donee_username, t.amount, t.context
FROM transfers t
JOIN participants p ON p.id = t.tippee
WHERE t.tipper = %(id)s
AND extract(year from t.timestamp) = %(year)s
AND t.status = 'succeeded'
AND t.virtual IS NOT true
ORDER BY t.id ASC
""",
params,
back_as=back_as)
out['taken'] = lambda: db.all("""
SELECT timestamp, (%(base_url)s || t.team::text) AS team_url,
p.username AS team_username, t.amount
FROM transfers t
JOIN participants p ON p.id = t.team
WHERE t.tippee = %(id)s
AND t.context IN ('take', 'take-in-advance')
AND extract(year from t.timestamp) = %(year)s
AND t.status = 'succeeded'
AND t.virtual IS NOT true
ORDER BY t.id ASC
""",
params,
back_as=back_as)
out['received'] = lambda: db.all("""
SELECT timestamp, amount, context
FROM transfers
WHERE tippee = %(id)s
AND context NOT IN ('take', 'take-in-advance')
AND extract(year from timestamp) = %(year)s
AND status = 'succeeded'
AND virtual IS NOT true
ORDER BY id ASC
""",
params,
back_as=back_as)
if key:
try:
return out[key]()
except KeyError:
raise Response(400, "bad key `%s`" % key)
elif require_key:
raise Response(400, "missing `key` parameter")
else:
return {k: v() for k, v in out.items()}
def __init__(self, *args, **kw):
Response.__init__(self, 429, (
"You have consumed your quota of admin actions. This isn't supposed "
"to happen."
))