def log_traceback_for_5xx(response, traceback=None):
if response.code >= 500:
if traceback:
aspen.log_dammit(traceback)
else:
aspen.log_dammit(response.body)
return {'traceback': None}
def handle_conflict_over_port(exception, website):
"""Be friendly about port conflicts.
The traceback one gets from a port conflict or permission error is not that
friendly. Here's a helper to let the user know (in color?!) that a port
conflict or a permission error is probably the problem. But in case it
isn't (website.start fires the start hook, and maybe the user tries to
connect to a network service in there?), don't fully swallow the exception.
Also, be explicit about the port number. What if they have logging turned
off? Then they won't see the port number in the "Greetings, program!" line.
They definitely won't see it if using an engine like eventlet that binds to
the port early.
"""
if exception.__class__ is not socket.error:
return
if website.network_port is not None:
msg = "Is something already running on port %s? Because ..."
if not aspen.WINDOWS:
if website.network_port < 1024:
if os.geteuid() > 0:
msg = ("Do you have permission to bind to port %s?"
" Because ...")
msg %= website.network_port
if not aspen.WINDOWS:
# Assume we can use ANSI color escapes if not on Windows.
# XXX Maybe a bad assumption if this is going into a log
# file? See also: colorama
msg = '\033[01;33m%s\033[00m' % msg
aspen.log_dammit(msg)
raise
def update_homepage_queries_once(db):
with db.get_cursor() as cursor:
log_dammit("updating homepage queries")
start = time.time()
cursor.execute("DELETE FROM homepage_top_givers")
cursor.execute("""
INSERT INTO homepage_top_givers (username, anonymous, amount, avatar_url, statement, number)
SELECT username, anonymous_giving, giving, avatar_url, statement, number
FROM participants
WHERE is_suspicious IS NOT true
AND last_bill_result = ''
ORDER BY giving DESC
LIMIT 100;
""".strip())
cursor.execute("DELETE FROM homepage_top_receivers")
cursor.execute("""
INSERT INTO homepage_top_receivers (username, anonymous, amount, avatar_url, statement, number)
SELECT username, anonymous_receiving, receiving, avatar_url, statement, number
FROM participants
WHERE is_suspicious IS NOT true
AND claimed_time IS NOT NULL
ORDER BY receiving DESC
LIMIT 100;
""".strip())
end = time.time()
elapsed = end - start
log_dammit("updated homepage queries in %.2f seconds" % elapsed)
def main(argv=None):
"""http://aspen.io/cli/
"""
try:
_main(argv)
except (SystemExit, KeyboardInterrupt):
# Under some (most?) network engines, a SIGINT will be trapped by the
# SIGINT signal handler above. However, gevent does "something" with
# signals and our signal handler never fires. However, we *do* get a
# KeyboardInterrupt here in that case. *shrug*
#
# See: https://github.com/gittip/aspen-python/issues/196
pass
except:
import aspen, aspen.execution, time, traceback
aspen.log_dammit("Oh no! Aspen crashed!")
aspen.log_dammit(traceback.format_exc())
try:
while 1:
aspen.execution.check_all()
time.sleep(1)
except KeyboardInterrupt:
raise SystemExit
def handle_error_nicely(self, request):
"""Try to provide some nice error handling.
"""
try: # nice error messages
tb_1 = traceback.format_exc()
response = sys.exc_info()[1]
if not isinstance(response, Response):
aspen.log_dammit(tb_1)
response = Response(500, tb_1)
elif 200 <= response.code < 300:
return response
response.request = request
self.hooks.outbound_early.run(response)
fs = self.ours_or_theirs(str(response.code) + '.html')
if fs is None:
fs = self.ours_or_theirs('error.html')
if fs is None:
raise
request.fs = fs
request.original_resource = request.resource
request.resource = resources.get(request)
response = request.resource.respond(request, response)
return response
except Response, response: # no nice error simplate available
raise
def __init__(self, env, db, tell_sentry, root):
if self._have_ses(env):
log_dammit("AWS SES is configured! We'll send mail through SES.")
self._mailer = boto3.client(
service_name='ses',
region_name=env.aws_ses_default_region,
aws_access_key_id=env.aws_ses_access_key_id,
aws_secret_access_key=env.aws_ses_secret_access_key)
else:
log_dammit(
"AWS SES is not configured! Mail will be dumped to the console here."
)
self._mailer = ConsoleMailer()
self.db = db
self.tell_sentry = tell_sentry
self.sleep_for = env.email_queue_sleep_for
self.allow_up_to = env.email_queue_allow_up_to
templates = {}
templates_dir = os.path.join(root, 'emails')
assert os.path.isdir(templates_dir)
i = len(templates_dir) + 1
for spt in find_files(templates_dir, '*.spt'):
base_name = spt[i:-4]
templates[base_name] = compile_email_spt(spt)
self._email_templates = templates
def __init__(self, env, db, tell_sentry, root):
if self._have_ses(env):
log_dammit("AWS SES is configured! We'll send mail through SES.")
self._mailer = boto3.client( service_name='ses'
, region_name=env.aws_ses_default_region
, aws_access_key_id=env.aws_ses_access_key_id
, aws_secret_access_key=env.aws_ses_secret_access_key
)
else:
log_dammit("AWS SES is not configured! Mail will be dumped to the console here.")
self._mailer = ConsoleMailer()
self.db = db
self.tell_sentry = tell_sentry
self.sleep_for = env.email_queue_sleep_for
self.allow_up_to = env.email_queue_allow_up_to
self.log_every = env.email_queue_log_metrics_every
templates = {}
templates_dir = os.path.join(root, 'emails')
assert os.path.isdir(templates_dir)
i = len(templates_dir) + 1
for spt in find_files(templates_dir, '*.spt'):
base_name = spt[i:-4]
templates[base_name] = compile_email_spt(spt)
self._email_templates = templates
def log_traceback_for_exception(website, exception):
tb = traceback.format_exc()
aspen.log_dammit(tb)
response = Response(500)
if website.show_tracebacks:
response.body = tb
return {'response': response, 'exception': None}
def handle_error_nicely(self, request):
"""Try to provide some nice error handling.
"""
try: # nice error messages
tb_1 = traceback.format_exc()
response = sys.exc_info()[1]
if not isinstance(response, Response):
aspen.log_dammit(tb_1)
response = Response(500, tb_1)
elif 200 <= response.code < 300:
return response
response.request = request
self.hooks.outbound_early.run(response)
fs = self.ours_or_theirs(str(response.code) + '.html')
if fs is None:
fs = self.ours_or_theirs('error.html')
if fs is None:
raise
request.fs = fs
request.original_resource = request.resource
request.resource = resources.get(request)
response = request.resource.respond(request, response)
return response
except Response, response: # no nice error simplate available
raise
def update_homepage_queries_once(db):
with db.get_cursor() as cursor:
log_dammit("updating homepage queries")
start = time.time()
cursor.execute("DELETE FROM homepage_top_givers")
cursor.execute("""
INSERT INTO homepage_top_givers (username, anonymous, amount, avatar_url, statement, number)
SELECT username, anonymous_giving, giving, avatar_url, statement, number
FROM participants
WHERE is_suspicious IS NOT true
AND last_bill_result = ''
ORDER BY giving DESC
LIMIT 100;
""".strip())
cursor.execute("DELETE FROM homepage_top_receivers")
cursor.execute("""
INSERT INTO homepage_top_receivers (username, anonymous, amount, avatar_url, statement, number)
SELECT username, anonymous_receiving, receiving, avatar_url, statement, number
FROM participants
WHERE is_suspicious IS NOT true
AND claimed_time IS NOT NULL
ORDER BY receiving DESC
LIMIT 100;
""".strip())
end = time.time()
elapsed = end - start
log_dammit("updated homepage queries in %.2f seconds" % elapsed)
def get_csrf_token_from_request(request):
"""Given a Request object, reject it if it's a forgery.
"""
if request.line.uri.startswith('/assets/'): return
if request.line.uri.startswith('/callbacks/'): return
try:
csrf_token = _sanitize_token(
request.headers.cookie['csrf_token'].value)
except KeyError:
csrf_token = None
request.context['csrf_token'] = csrf_token or _get_new_csrf_key()
# Assume that anything not defined as 'safe' by RC2616 needs protection
if request.line.method not in ('GET', 'HEAD', 'OPTIONS', 'TRACE'):
if _is_secure(request):
# Suppose user visits http://example.com/
# An active network attacker (man-in-the-middle, MITM) sends a
# POST form that targets https://example.com/detonate-bomb/ and
# submits it via JavaScript.
#
# The attacker will need to provide a CSRF cookie and token, but
# that's no problem for a MITM and the session-independent
# nonce we're using. So the MITM can circumvent the CSRF
# protection. This is true for any HTTP connection, but anyone
# using HTTPS expects better! For this reason, for
# https://example.com/ we need additional protection that treats
# http://example.com/ as completely untrusted. Under HTTPS,
# Barth et al. found that the Referer header is missing for
# same-domain requests in only about 0.2% of cases or less, so
# we can use strict Referer checking.
referer = request.headers.get('Referer')
if referer is None:
raise Response(403, REASON_NO_REFERER)
good_referer = 'https://%s/' % _get_host(request)
if not same_origin(referer, good_referer):
reason = REASON_BAD_REFERER % (referer, good_referer)
log_dammit(reason)
raise Response(403, reason)
if csrf_token is None:
raise Response(403, REASON_NO_CSRF_COOKIE)
# Check non-cookie token for match.
request_csrf_token = ""
if request.line.method == "POST":
if isinstance(request.body, dict):
request_csrf_token = request.body.get('csrf_token', '')
if request_csrf_token == "":
# Fall back to X-CSRF-TOKEN, to make things easier for AJAX,
# and possible for PUT/DELETE.
request_csrf_token = request.headers.get('X-CSRF-TOKEN', '')
if not constant_time_compare(request_csrf_token, csrf_token):
raise Response(403, REASON_BAD_TOKEN)
def f():
while True:
try:
func()
except Exception, e:
self.website.tell_sentry(e, {})
log_dammit(traceback.format_exc().strip())
sleep(period)
def f():
while True:
try:
func()
except Exception, e:
self.website.tell_sentry(e, {})
log_dammit(traceback.format_exc().strip())
sleep(period)
def update_homepage_queries_once(db):
with db.get_cursor() as cursor:
log_dammit("updating homepage queries")
start = time.time()
cursor.execute("DELETE FROM homepage_top_givers")
cursor.execute("""
INSERT INTO homepage_top_givers (username, anonymous, amount, avatar_url)
SELECT tipper, anonymous_giving, sum(amount) AS amount, avatar_url
FROM ( SELECT DISTINCT ON (tipper, tippee)
amount
, tipper
FROM tips
JOIN participants p ON p.username = tipper
JOIN participants p2 ON p2.username = tippee
JOIN elsewhere ON elsewhere.participant = tippee
WHERE p.last_bill_result = ''
AND p.is_suspicious IS NOT true
AND p2.claimed_time IS NOT NULL
AND elsewhere.is_locked = false
ORDER BY tipper, tippee, mtime DESC
) AS foo
JOIN participants p ON p.username = tipper
WHERE is_suspicious IS NOT true
GROUP BY tipper, anonymous_giving, avatar_url
ORDER BY amount DESC
LIMIT 100;
""".strip())
cursor.execute("DELETE FROM homepage_top_receivers")
cursor.execute("""
INSERT INTO homepage_top_receivers (username, anonymous, amount, avatar_url)
SELECT tippee, anonymous_receiving, sum(amount) AS amount, avatar_url
FROM ( SELECT DISTINCT ON (tipper, tippee)
amount
, tippee
FROM tips
JOIN participants p ON p.username = tipper
JOIN elsewhere ON elsewhere.participant = tippee
WHERE last_bill_result = ''
AND elsewhere.is_locked = false
AND is_suspicious IS NOT true
AND claimed_time IS NOT null
ORDER BY tipper, tippee, mtime DESC
) AS foo
JOIN participants p ON p.username = tippee
WHERE is_suspicious IS NOT true
GROUP BY tippee, anonymous_receiving, avatar_url
ORDER BY amount DESC
LIMIT 100;
""".strip())
end = time.time()
elapsed = end - start
log_dammit("updated homepage queries in %.2f seconds" % elapsed)
def inbound(request):
"""Given a Request object, reject it if it's a forgery.
"""
if request.line.uri.startswith('/assets/'): return
try:
csrf_token = request.headers.cookie.get('csrf_token')
csrf_token = '' if csrf_token is None else csrf_token.value
csrf_token = _sanitize_token(csrf_token)
except KeyError:
csrf_token = _get_new_csrf_key()
request.context['csrf_token'] = csrf_token
# Assume that anything not defined as 'safe' by RC2616 needs protection
if request.line.method not in ('GET', 'HEAD', 'OPTIONS', 'TRACE'):
if _is_secure(request):
# Suppose user visits http://example.com/
# An active network attacker (man-in-the-middle, MITM) sends a
# POST form that targets https://example.com/detonate-bomb/ and
# submits it via JavaScript.
#
# The attacker will need to provide a CSRF cookie and token, but
# that's no problem for a MITM and the session-independent
# nonce we're using. So the MITM can circumvent the CSRF
# protection. This is true for any HTTP connection, but anyone
# using HTTPS expects better! For this reason, for
# https://example.com/ we need additional protection that treats
# http://example.com/ as completely untrusted. Under HTTPS,
# Barth et al. found that the Referer header is missing for
# same-domain requests in only about 0.2% of cases or less, so
# we can use strict Referer checking.
referer = request.headers.get('Referer')
if referer is None:
raise Response(403, REASON_NO_REFERER)
good_referer = 'https://%s/' % _get_host(request)
if not same_origin(referer, good_referer):
reason = REASON_BAD_REFERER % (referer, good_referer)
log_dammit(reason)
raise Response(403, reason)
if csrf_token is None:
raise Response(403, REASON_NO_CSRF_COOKIE)
# Check non-cookie token for match.
request_csrf_token = ""
if request.line.method == "POST":
request_csrf_token = request.body.get('csrf_token', '')
if request_csrf_token == "":
# Fall back to X-CSRF-TOKEN, to make things easier for AJAX,
# and possible for PUT/DELETE.
request_csrf_token = request.headers.get('X-CSRF-TOKEN', '')
if not constant_time_compare(request_csrf_token, csrf_token):
raise Response(403, REASON_BAD_TOKEN)
def update_homepage_queries_once(db):
with db.get_cursor() as cursor:
log_dammit("updating homepage queries")
start = time.time()
cursor.execute("DELETE FROM homepage_top_givers")
cursor.execute("""
INSERT INTO homepage_top_givers (username, anonymous, amount, avatar_url, statement, number)
SELECT tipper, anonymous_giving, sum(amount) AS amount, avatar_url, statement, number
FROM ( SELECT DISTINCT ON (tipper, tippee)
amount
, tipper
FROM tips
JOIN participants p ON p.username = tipper
JOIN participants p2 ON p2.username = tippee
JOIN elsewhere ON elsewhere.participant = tippee
WHERE p.last_bill_result = ''
AND p.is_suspicious IS NOT true
AND p2.claimed_time IS NOT NULL
AND elsewhere.is_locked = false
ORDER BY tipper, tippee, mtime DESC
) AS foo
JOIN participants p ON p.username = tipper
WHERE is_suspicious IS NOT true
GROUP BY tipper, anonymous_giving, avatar_url, statement, number
ORDER BY amount DESC
LIMIT 100;
""".strip())
cursor.execute("DELETE FROM homepage_top_receivers")
cursor.execute("""
INSERT INTO homepage_top_receivers (username, anonymous, amount, avatar_url, statement, number)
SELECT tippee, anonymous_receiving, sum(amount) AS amount, avatar_url, statement, number
FROM ( SELECT DISTINCT ON (tipper, tippee)
amount
, tippee
FROM tips
JOIN participants p ON p.username = tipper
JOIN elsewhere ON elsewhere.participant = tippee
WHERE last_bill_result = ''
AND elsewhere.is_locked = false
AND is_suspicious IS NOT true
AND claimed_time IS NOT null
ORDER BY tipper, tippee, mtime DESC
) AS foo
JOIN participants p ON p.username = tippee
WHERE is_suspicious IS NOT true
GROUP BY tippee, anonymous_receiving, avatar_url, statement, number
ORDER BY amount DESC
LIMIT 100;
""".strip())
end = time.time()
elapsed = end - start
log_dammit("updated homepage queries in %.2f seconds" % elapsed)
def handle_error_at_all(self, tb_1):
tb_2 = traceback.format_exc().strip()
tbs = '\n\n'.join([tb_2, "... while handling ...", tb_1])
aspen.log_dammit(tbs)
if self.show_tracebacks:
response = Response(500, tbs)
else:
response = Response(500)
return response
def handle_error_at_all(self, tb_1):
tb_2 = traceback.format_exc().strip()
tbs = '\n\n'.join([tb_2, "... while handling ...", tb_1])
aspen.log_dammit(tbs)
if self.show_tracebacks:
response = Response(500, tbs)
else:
response = Response(500)
return response
def f():
if period <= 0:
return
sleep = time.sleep
while 1:
try:
func()
except Exception, e:
tell_sentry(e)
log_dammit(traceback.format_exc().strip())
sleep(period)
def tell_sentry(request):
cls, response = sys.exc_info()[:2]
if cls is aspen.Response:
if response.code < 500:
return
kw = {'extra': { "filepath": request.fs
, "request": str(request).splitlines()
}}
exc = sentry.captureException(**kw)
ident = sentry.get_ident(exc)
aspen.log_dammit("Exception reference: " + ident)
def stop(website):
"""Stop the server.
"""
if website is None:
return
aspen.log_dammit("Shutting down Aspen website.")
website.network_engine.stop()
if hasattr(socket, 'AF_UNIX'):
if website.network_sockfam == socket.AF_UNIX:
if os.path.exists(website.network_address):
os.remove(website.network_address)
def tell_sentry(request):
cls, response = sys.exc_info()[:2]
if cls is aspen.Response:
if response.code < 500:
return
kw = {'extra': { "filepath": request.fs
, "request": str(request).splitlines()
}}
exc = client.captureException(**kw)
ident = client.get_ident(exc)
aspen.log_dammit("Exception reference: " + ident)
def update_homepage_queries():
from gittip import utils
while 1:
try:
utils.update_global_stats(website)
utils.update_homepage_queries_once(website.db)
website.db.self_check()
except:
exception = sys.exc_info()[0]
tell_sentry(exception)
tb = traceback.format_exc().strip()
log_dammit(tb)
time.sleep(UPDATE_HOMEPAGE_EVERY)
def update_homepage_queries():
from gittip import utils
while 1:
try:
utils.update_global_stats(website)
utils.update_homepage_queries_once(website.db)
website.db.self_check()
except:
exception = sys.exc_info()[0]
tell_sentry(exception)
tb = traceback.format_exc().strip()
log_dammit(tb)
time.sleep(UPDATE_HOMEPAGE_EVERY)
def update_homepage_queries():
from gittip import utils
while 1:
try:
utils.update_global_stats(website)
utils.update_homepage_queries_once(website.db)
except:
if tell_sentry:
tell_sentry(None)
else:
tb = traceback.format_exc().strip()
log_dammit(tb)
time.sleep(UPDATE_HOMEPAGE_EVERY)
def update_homepage_queries():
from gittip import utils
while 1:
try:
utils.update_global_stats(website)
utils.update_homepage_queries_once(website.db)
except:
if tell_sentry:
tell_sentry(None)
else:
tb = traceback.format_exc().strip()
log_dammit(tb)
time.sleep(UPDATE_HOMEPAGE_EVERY)
def main(argv=None):
"""http://aspen.io/cli/
"""
try:
_main(argv)
except KeyboardInterrupt:
raise SystemExit(75) # 75 == INT
except SystemExit:
pass
except:
import aspen, traceback
aspen.log_dammit("Oh no! Server crashed!")
aspen.log_dammit(traceback.format_exc())
raise SystemExit(1) # 1 == Other
def bind_server_to_port(website):
"""
"""
if hasattr(socket, 'AF_UNIX'):
if website.network_sockfam == socket.AF_UNIX:
if os.path.exists(website.network_address):
aspen.log("Removing stale socket.")
os.remove(website.network_address)
if website.network_port is not None:
welcome = "port %d" % website.network_port
else:
welcome = website.network_address
aspen.log("Starting %s engine." % website.network_engine.name)
website.network_engine.bind()
aspen.log_dammit("Greetings, program! Welcome to %s." % welcome)
def main(argv=None):
"""http://aspen.io/cli/
"""
try:
_main(argv)
except SystemExit:
pass
except:
import aspen, aspen.execution, time, traceback
aspen.log_dammit("Oh no! Aspen crashed!")
aspen.log_dammit(traceback.format_exc())
try:
while 1:
aspen.execution.check_all()
time.sleep(1)
except KeyboardInterrupt:
raise SystemExit
def main(argv=None):
"""http://aspen.io/cli/
"""
try:
_main(argv)
except SystemExit:
pass
except:
import aspen, aspen.execution, time, traceback
aspen.log_dammit("Oh no! Aspen crashed!")
aspen.log_dammit(traceback.format_exc())
try:
while 1:
aspen.execution.check_all()
time.sleep(1)
except KeyboardInterrupt:
raise SystemExit
def f():
if period <= 0:
return
sleep = time.sleep
if exclusive:
cursor = conn.cursor()
try_lock = lambda: cursor.one("SELECT pg_try_advisory_lock(0)")
has_lock = False
while 1:
try:
if exclusive and not has_lock:
has_lock = try_lock()
if not exclusive or has_lock:
func()
except Exception, e:
tell_sentry(e)
log_dammit(traceback.format_exc().strip())
sleep(period)
def f():
if period <= 0:
return
sleep = time.sleep
if exclusive:
cursor = conn.cursor()
try_lock = lambda: cursor.one("SELECT pg_try_advisory_lock(0)")
has_lock = False
while 1:
try:
if exclusive and not has_lock:
has_lock = try_lock()
if not exclusive or has_lock:
func()
except Exception, e:
tell_sentry(e)
log_dammit(traceback.format_exc().strip())
sleep(period)
def handle_error_nicely(self, tb_1, request):
response = sys.exc_info()[1]
if not isinstance(response, Response):
# We have a true Exception; convert it to a Response object.
response = Response(500, tb_1)
response.request = request
if 500 <= response.code < 600:
# Log tracebacks for Reponse(5xx).
aspen.log_dammit(tb_1)
# TODO Switch to the logging module and use something like this:
# log_level = [DEBUG,INFO,WARNING,ERROR][(response.code/100)-2]
# logging.log(log_level, tb_1)
if 200 <= response.code < 300 or response.code == 304:
# The app raised a Response(2xx) or Response(304).
# Act as if nothing happened. This is unusual but allowed.
pass
else:
# Delegate to any error simplate.
# ===============================
rc = str(response.code)
possibles = [ rc + ".html", rc + ".html.spt", "error.html", "error.html.spt" ]
fs = first( self.ours_or_theirs(errpage) for errpage in possibles )
if fs is not None:
request.fs = fs
request.original_resource = request.resource
request.resource = resources.get(request)
response = request.resource.respond(request, response)
return response
def handle_error_nicely(self, tb_1, request):
response = sys.exc_info()[1]
if not isinstance(response, Response):
# We have a true Exception; convert it to a Response object.
response = Response(500, tb_1)
response.request = request
if 500 <= response.code < 600:
# Log tracebacks for Reponse(5xx).
aspen.log_dammit(tb_1)
# TODO Switch to the logging module and use something like this:
# log_level = [DEBUG,INFO,WARNING,ERROR][(response.code/100)-2]
# logging.log(log_level, tb_1)
if 200 <= response.code < 300 or response.code == 304:
# The app raised a Response(2xx) or Response(304).
# Act as if nothing happened. This is unusual but allowed.
pass
else:
# Delegate to any error simplate.
# ===============================
rc = str(response.code)
possibles = [ rc + ".html", rc + ".html.spt", "error.html", "error.html.spt" ]
fs = first( self.ours_or_theirs(errpage) for errpage in possibles )
if fs is not None:
request.fs = fs
request.original_resource = request.resource
request.resource = resources.get(request)
response = request.resource.respond(request, response)
return response
def mail(env, project_root='.'):
if env.aws_ses_access_key_id and env.aws_ses_secret_access_key and env.aws_ses_default_region:
aspen.log_dammit("AWS SES is configured! We'll send mail through SES.")
Participant._mailer = boto3.client(
service_name='ses',
region_name=env.aws_ses_default_region,
aws_access_key_id=env.aws_ses_access_key_id,
aws_secret_access_key=env.aws_ses_secret_access_key)
else:
aspen.log_dammit(
"AWS SES is not configured! Mail will be dumped to the console here."
)
Participant._mailer = ConsoleMailer()
emails = {}
emails_dir = project_root + '/emails/'
i = len(emails_dir)
for spt in find_files(emails_dir, '*.spt'):
base_name = spt[i:-4]
emails[base_name] = compile_email_spt(spt)
Participant._emails = emails
def main(self, argv=None):
"""http://aspen.io/cli/
"""
try:
argv = argv if argv is not None else self.argv
algorithm = self.get_algorithm()
algorithm.run(argv=argv)
except (SystemExit, KeyboardInterrupt):
# Under some (most?) network engines, a SIGINT will be trapped by the
# SIGINT signal handler above. However, gevent does "something" with
# signals and our signal handler never fires. However, we *do* get a
# KeyboardInterrupt here in that case. *shrug*
#
# See: https://github.com/gittip/aspen-python/issues/196
pass
except:
import aspen, traceback
aspen.log_dammit("Oh no! Aspen crashed!")
aspen.log_dammit(traceback.format_exc())
def _do_execv():
"""Re-execute the current process.
This must be called from the main thread, because certain platforms
(OS X) don't allow execv to be called in a child thread very well.
"""
args = sys.argv[:]
aspen.log_dammit("Restarting %s." % ' '.join(args))
if sys.platform[:4] == 'java':
from _systemrestart import SystemRestart
raise SystemRestart
else:
args.insert(0, sys.executable)
if sys.platform == 'win32':
args = ['"%s"' % arg for arg in args]
os.chdir(_startup_cwd)
if max_cloexec_files:
_set_cloexec()
os.execv(sys.executable, args)
def _do_execv():
"""Re-execute the current process.
This must be called from the main thread, because certain platforms
(OS X) don't allow execv to be called in a child thread very well.
"""
args = sys.argv[:]
aspen.log_dammit("Re-executing %s." % ' '.join(args))
if sys.platform[:4] == 'java':
from _systemrestart import SystemRestart
raise SystemRestart
else:
args.insert(0, sys.executable)
if sys.platform == 'win32':
args = ['"%s"' % arg for arg in args]
os.chdir(_startup_cwd)
if max_cloexec_files:
_set_cloexec()
os.execv(sys.executable, args)
def show_renderers(self):
aspen.log_dammit("Renderers (*ed are unavailable, CAPS is default):")
width = max(map(len, self.renderer_factories))
for name, factory in self.renderer_factories.items():
star, error = " ", ""
if isinstance(factory, ImportError):
star = "*"
error = "ImportError: " + factory.args[0]
if name == self.renderer_default:
name = name.upper()
name = name.ljust(width + 2)
aspen.log_dammit(" %s%s%s" % (star, name, error))
default_renderer = self.renderer_factories[self.renderer_default]
if isinstance(default_renderer, ImportError):
msg = "\033[1;31mImportError loading the default renderer, %s:\033[0m"
aspen.log_dammit(msg % self.renderer_default)
sys.excepthook(*default_renderer.info)
raise default_renderer
def SIGQUIT(signum, frame):
aspen.log_dammit("Received QUIT, exiting.")
raise SystemExit
def tell_sentry(exception, request=None):
# Decide if we care.
# ==================
if isinstance(exception, aspen.Response):
if exception.code < 500:
# Only log server errors to Sentry. For responses < 500 we use
# stream-/line-based access logging. See discussion on:
# https://github.com/gittip/www.gittip.com/pull/1560.
return
# Find a user.
# ============
# | is disallowed in usernames, so we can use it here to indicate
# situations in which we can't get a username.
request_context = getattr(request, 'context', None)
user = {}
user_id = 'n/a'
if request_context is None:
username = '******'
else:
user = request.context.get('user', None)
if user is None:
username = '******'
else:
is_anon = getattr(user, 'ANON', None)
if is_anon is None:
username = '******'
elif is_anon:
username = '******'
else:
participant = getattr(user, 'participant', None)
if participant is None:
username = '******'
else:
username = getattr(user.participant, 'username', None)
if username is None:
username = '******'
else:
user_id = user.participant.id
username = username.encode('utf8')
user = {
'id':
user_id,
'is_admin':
user.participant.is_admin,
'is_suspicious':
user.participant.is_suspicious,
'claimed_time':
user.participant.claimed_time.isoformat(),
'url':
'https://www.gittip.com/{}/'.format(username)
}
# Fire off a Sentry call.
# =======================
tags = {'username': username, 'user_id': user_id}
extra = {
'filepath': getattr(request, 'fs', None),
'request': str(request).splitlines(),
'user': user
}
result = sentry.captureException(tags=tags, extra=extra)
# Emit a reference string to stdout.
# ==================================
ident = sentry.get_ident(result)
aspen.log_dammit('Exception reference: ' + ident)
def envvars(website):
missing_keys = []
malformed_values = []
def envvar(key, cast=None):
if key not in os.environ:
missing_keys.append(key)
return ""
value = os.environ[key].decode('ASCII')
if cast is not None:
try:
value = cast(value)
except:
err = str(sys.exc_info()[1])
malformed_values.append((key, err))
return ""
return value
def is_yesish(val):
return val.lower() in ('1', 'true', 'yes')
signin_platforms = [
Twitter(
website.db,
envvar('TWITTER_CONSUMER_KEY'),
envvar('TWITTER_CONSUMER_SECRET'),
envvar('TWITTER_CALLBACK'),
),
GitHub(
website.db,
envvar('GITHUB_CLIENT_ID'),
envvar('GITHUB_CLIENT_SECRET'),
envvar('GITHUB_CALLBACK'),
),
Bitbucket(
website.db,
envvar('BITBUCKET_CONSUMER_KEY'),
envvar('BITBUCKET_CONSUMER_SECRET'),
envvar('BITBUCKET_CALLBACK'),
),
OpenStreetMap(
website.db,
envvar('OPENSTREETMAP_CONSUMER_KEY'),
envvar('OPENSTREETMAP_CONSUMER_SECRET'),
envvar('OPENSTREETMAP_CALLBACK'),
envvar('OPENSTREETMAP_API_URL'),
envvar('OPENSTREETMAP_AUTH_URL'),
),
]
website.signin_platforms = PlatformRegistry(signin_platforms)
AccountElsewhere.signin_platforms_names = tuple(p.name
for p in signin_platforms)
other_platforms = [
Bountysource(
website.db,
None,
envvar('BOUNTYSOURCE_API_SECRET'),
envvar('BOUNTYSOURCE_CALLBACK'),
envvar('BOUNTYSOURCE_API_HOST'),
envvar('BOUNTYSOURCE_WWW_HOST'),
),
Venmo(
website.db,
envvar('VENMO_CLIENT_ID'),
envvar('VENMO_CLIENT_SECRET'),
envvar('VENMO_CALLBACK'),
),
]
platforms = signin_platforms + other_platforms
website.platforms = AccountElsewhere.platforms = PlatformRegistry(
platforms)
website.asset_version_url = envvar('GITTIP_ASSET_VERSION_URL') \
.replace('%version', website.version)
website.asset_url = envvar('GITTIP_ASSET_URL')
website.cache_static = is_yesish(envvar('GITTIP_CACHE_STATIC'))
website.compress_assets = is_yesish(envvar('GITTIP_COMPRESS_ASSETS'))
website.google_analytics_id = envvar('GOOGLE_ANALYTICS_ID')
website.sentry_dsn = envvar('SENTRY_DSN')
website.min_threads = envvar('MIN_THREADS', int)
website.log_busy_threads_every = envvar('LOG_BUSY_THREADS_EVERY', int)
website.log_metrics = is_yesish(envvar('LOG_METRICS'))
if malformed_values:
malformed_values.sort()
these = len(malformed_values) != 1 and 'these' or 'this'
plural = len(malformed_values) != 1 and 's' or ''
aspen.log_dammit("=" * 42)
aspen.log_dammit("Oh no! Gittip.com couldn't understand %s " % these,
"environment variable%s:" % plural)
aspen.log_dammit(" ")
for key, err in malformed_values:
aspen.log_dammit(" {} ({})".format(key, err))
aspen.log_dammit(" ")
aspen.log_dammit("See ./default_local.env for hints.")
aspen.log_dammit("=" * 42)
keys = ', '.join([key for key in malformed_values])
raise BadEnvironment("Malformed envvar{}: {}.".format(plural, keys))
if missing_keys:
missing_keys.sort()
these = len(missing_keys) != 1 and 'these' or 'this'
plural = len(missing_keys) != 1 and 's' or ''
aspen.log_dammit("=" * 42)
aspen.log_dammit("Oh no! Gittip.com needs %s missing " % these,
"environment variable%s:" % plural)
aspen.log_dammit(" ")
for key in missing_keys:
aspen.log_dammit(" " + key)
aspen.log_dammit(" ")
aspen.log_dammit("(Sorry, we must've started looking for ",
"%s since you last updated Gittip!)" % these)
aspen.log_dammit(" ")
aspen.log_dammit("Running Gittip locally? Edit ./local.env.")
aspen.log_dammit("Running the test suite? Edit ./tests/env.")
aspen.log_dammit(" ")
aspen.log_dammit("See ./default_local.env for hints.")
aspen.log_dammit("=" * 42)
keys = ', '.join([key for key in missing_keys])
raise BadEnvironment("Missing envvar{}: {}.".format(plural, keys))
def env():
env = Environment(
DATABASE_URL=unicode,
CANONICAL_HOST=unicode,
CANONICAL_SCHEME=unicode,
DATABASE_MAXCONN=int,
GRATIPAY_ASSET_URL=unicode,
GRATIPAY_CACHE_STATIC=is_yesish,
GRATIPAY_COMPRESS_ASSETS=is_yesish,
BALANCED_API_SECRET=unicode,
GITHUB_CLIENT_ID=unicode,
GITHUB_CLIENT_SECRET=unicode,
GITHUB_CALLBACK=unicode,
BITBUCKET_CONSUMER_KEY=unicode,
BITBUCKET_CONSUMER_SECRET=unicode,
BITBUCKET_CALLBACK=unicode,
TWITTER_CONSUMER_KEY=unicode,
TWITTER_CONSUMER_SECRET=unicode,
TWITTER_CALLBACK=unicode,
FACEBOOK_APP_ID=unicode,
FACEBOOK_APP_SECRET=unicode,
FACEBOOK_CALLBACK=unicode,
GOOGLE_CLIENT_ID=unicode,
GOOGLE_CLIENT_SECRET=unicode,
GOOGLE_CALLBACK=unicode,
BOUNTYSOURCE_API_SECRET=unicode,
BOUNTYSOURCE_CALLBACK=unicode,
BOUNTYSOURCE_API_HOST=unicode,
BOUNTYSOURCE_WWW_HOST=unicode,
VENMO_CLIENT_ID=unicode,
VENMO_CLIENT_SECRET=unicode,
VENMO_CALLBACK=unicode,
OPENSTREETMAP_CONSUMER_KEY=unicode,
OPENSTREETMAP_CONSUMER_SECRET=unicode,
OPENSTREETMAP_CALLBACK=unicode,
OPENSTREETMAP_API_URL=unicode,
OPENSTREETMAP_AUTH_URL=unicode,
UPDATE_GLOBAL_STATS_EVERY=int,
CHECK_DB_EVERY=int,
DEQUEUE_EMAILS_EVERY=int,
OPTIMIZELY_ID=unicode,
SENTRY_DSN=unicode,
LOG_METRICS=is_yesish,
INCLUDE_PIWIK=is_yesish,
MANDRILL_KEY=unicode,
RAISE_SIGNIN_NOTIFICATIONS=is_yesish,
# This is used in our Procfile. (PORT is also used but is provided by
# Heroku; we don't set it ourselves in our app config.)
GUNICORN_OPTS=unicode,
)
# Error Checking
# ==============
if env.malformed:
these = len(env.malformed) != 1 and 'these' or 'this'
plural = len(env.malformed) != 1 and 's' or ''
aspen.log_dammit("=" * 42)
aspen.log_dammit("Oh no! Gratipay.com couldn't understand %s " % these,
"environment variable%s:" % plural)
aspen.log_dammit(" ")
for key, err in env.malformed:
aspen.log_dammit(" {} ({})".format(key, err))
aspen.log_dammit(" ")
aspen.log_dammit("See ./default_local.env for hints.")
aspen.log_dammit("=" * 42)
keys = ', '.join([key for key in env.malformed])
raise BadEnvironment("Malformed envvar{}: {}.".format(plural, keys))
if env.missing:
these = len(env.missing) != 1 and 'these' or 'this'
plural = len(env.missing) != 1 and 's' or ''
aspen.log_dammit("=" * 42)
aspen.log_dammit("Oh no! Gratipay.com needs %s missing " % these,
"environment variable%s:" % plural)
aspen.log_dammit(" ")
for key in env.missing:
aspen.log_dammit(" " + key)
aspen.log_dammit(" ")
aspen.log_dammit("(Sorry, we must've started looking for ",
"%s since you last updated Gratipay!)" % these)
aspen.log_dammit(" ")
aspen.log_dammit("Running Gratipay locally? Edit ./local.env.")
aspen.log_dammit("Running the test suite? Edit ./tests/env.")
aspen.log_dammit(" ")
aspen.log_dammit("See ./default_local.env for hints.")
aspen.log_dammit("=" * 42)
keys = ', '.join([key for key in env.missing])
raise BadEnvironment("Missing envvar{}: {}.".format(plural, keys))
return env
def make_sentry_teller(env):
if not env.sentry_dsn:
aspen.log_dammit("Won't log to Sentry (SENTRY_DSN is empty).")
def noop(*a, **kw):
pass
Participant._tell_sentry = noop
return noop
sentry = raven.Client(env.sentry_dsn)
def tell_sentry(exception, state):
# Decide if we care.
# ==================
if isinstance(exception, aspen.Response):
if exception.code < 500:
# Only log server errors to Sentry. For responses < 500 we use
# stream-/line-based access logging. See discussion on:
# https://github.com/gratipay/gratipay.com/pull/1560.
return
# Find a user.
# ============
# | is disallowed in usernames, so we can use it here to indicate
# situations in which we can't get a username.
user = state.get('user')
user_id = 'n/a'
if user is None:
username = '******'
else:
is_anon = getattr(user, 'ANON', None)
if is_anon is None:
username = '******'
elif is_anon:
username = '******'
else:
participant = getattr(user, 'participant', None)
if participant is None:
username = '******'
else:
username = getattr(user.participant, 'username', None)
if username is None:
username = '******'
else:
user_id = user.participant.id
username = username.encode('utf8')
user = {
'id':
user_id,
'is_admin':
user.participant.is_admin,
'is_suspicious':
user.participant.is_suspicious,
'claimed_time':
user.participant.claimed_time.isoformat(),
'url':
'https://gratipay.com/{}/'.format(username)
}
# Fire off a Sentry call.
# =======================
dispatch_result = state.get('dispatch_result')
request = state.get('request')
tags = {'username': username, 'user_id': user_id}
extra = {
'filepath': getattr(dispatch_result, 'match', None),
'request': str(request).splitlines(),
'user': user
}
result = sentry.captureException(tags=tags, extra=extra)
# Emit a reference string to stdout.
# ==================================
ident = sentry.get_ident(result)
aspen.log_dammit('Exception reference: ' + ident)
Participant._tell_sentry = tell_sentry
return tell_sentry
def update_homepage_queries_once(db):
with db.get_cursor() as cursor:
log_dammit("updating homepage queries")
start = time.time()
cursor.execute("DELETE FROM homepage_top_givers")
cursor.execute("""
INSERT INTO homepage_top_givers
SELECT tipper AS username, anonymous, sum(amount) AS amount
FROM ( SELECT DISTINCT ON (tipper, tippee)
amount
, tipper
FROM tips
JOIN participants p ON p.username = tipper
JOIN participants p2 ON p2.username = tippee
JOIN elsewhere ON elsewhere.participant = tippee
WHERE p.last_bill_result = ''
AND p.is_suspicious IS NOT true
AND p2.claimed_time IS NOT NULL
AND elsewhere.is_locked = false
ORDER BY tipper, tippee, mtime DESC
) AS foo
JOIN participants p ON p.username = tipper
WHERE is_suspicious IS NOT true
GROUP BY tipper, anonymous
ORDER BY amount DESC;
""".strip())
cursor.execute("""
UPDATE homepage_top_givers
SET gravatar_id = ( SELECT user_info->'gravatar_id'
FROM elsewhere
WHERE participant=username
AND platform='github'
)
""")
cursor.execute("""
UPDATE homepage_top_givers
SET twitter_pic = ( SELECT user_info->'profile_image_url_https'
FROM elsewhere
WHERE participant=username
AND platform='twitter'
)
""")
cursor.execute("DELETE FROM homepage_top_receivers")
cursor.execute("""
INSERT INTO homepage_top_receivers
SELECT tippee AS username, claimed_time, sum(amount) AS amount
FROM ( SELECT DISTINCT ON (tipper, tippee)
amount
, tippee
FROM tips
JOIN participants p ON p.username = tipper
JOIN elsewhere ON elsewhere.participant = tippee
WHERE last_bill_result = ''
AND elsewhere.is_locked = false
AND is_suspicious IS NOT true
AND claimed_time IS NOT null
ORDER BY tipper, tippee, mtime DESC
) AS foo
JOIN participants p ON p.username = tippee
WHERE is_suspicious IS NOT true
GROUP BY tippee, claimed_time
ORDER BY amount DESC;
""".strip())
cursor.execute("""
UPDATE homepage_top_receivers
SET gravatar_id = ( SELECT user_info->'gravatar_id'
FROM elsewhere
WHERE participant=username
AND platform='github'
)
""")
cursor.execute("""
UPDATE homepage_top_receivers
SET twitter_pic = ( SELECT user_info->'profile_image_url_https'
FROM elsewhere
WHERE participant=username
AND platform='twitter'
)
""")
end = time.time()
elapsed = end - start
log_dammit("updated homepage queries in %.2f seconds" % elapsed)
def stop(self):
aspen.log_dammit("Shutting down Aspen website.")
self.hooks.shutdown.run(self)
self.network_engine.stop()
def update_homepage_queries_once(db):
with db.get_cursor() as cursor:
log_dammit("updating homepage queries")
start = time.time()
cursor.execute("DELETE FROM homepage_top_givers")
cursor.execute(
"""
INSERT INTO homepage_top_givers
SELECT tipper AS username, anonymous, sum(amount) AS amount
FROM ( SELECT DISTINCT ON (tipper, tippee)
amount
, tipper
FROM tips
JOIN participants p ON p.username = tipper
JOIN participants p2 ON p2.username = tippee
JOIN elsewhere ON elsewhere.participant = tippee
WHERE p.last_bill_result = ''
AND p.is_suspicious IS NOT true
AND p2.claimed_time IS NOT NULL
AND elsewhere.is_locked = false
ORDER BY tipper, tippee, mtime DESC
) AS foo
JOIN participants p ON p.username = tipper
WHERE is_suspicious IS NOT true
GROUP BY tipper, anonymous
ORDER BY amount DESC;
""".strip()
)
cursor.execute(
"""
UPDATE homepage_top_givers
SET gravatar_id = ( SELECT user_info->'gravatar_id'
FROM elsewhere
WHERE participant=username
AND platform='github'
)
"""
)
cursor.execute(
"""
UPDATE homepage_top_givers
SET twitter_pic = ( SELECT user_info->'profile_image_url_https'
FROM elsewhere
WHERE participant=username
AND platform='twitter'
)
"""
)
cursor.execute("DELETE FROM homepage_top_receivers")
cursor.execute(
"""
INSERT INTO homepage_top_receivers
SELECT tippee AS username, claimed_time, sum(amount) AS amount
FROM ( SELECT DISTINCT ON (tipper, tippee)
amount
, tippee
FROM tips
JOIN participants p ON p.username = tipper
JOIN elsewhere ON elsewhere.participant = tippee
WHERE last_bill_result = ''
AND elsewhere.is_locked = false
AND is_suspicious IS NOT true
AND claimed_time IS NOT null
ORDER BY tipper, tippee, mtime DESC
) AS foo
JOIN participants p ON p.username = tippee
WHERE is_suspicious IS NOT true
GROUP BY tippee, claimed_time
ORDER BY amount DESC;
""".strip()
)
cursor.execute(
"""
UPDATE homepage_top_receivers
SET gravatar_id = ( SELECT user_info->'gravatar_id'
FROM elsewhere
WHERE participant=username
AND platform='github'
)
"""
)
cursor.execute(
"""
UPDATE homepage_top_receivers
SET twitter_pic = ( SELECT user_info->'profile_image_url_https'
FROM elsewhere
WHERE participant=username
AND platform='twitter'
)
"""
)
end = time.time()
elapsed = end - start
log_dammit("updated homepage queries in %.2f seconds" % elapsed)
def update_homepage_queries_once(db):
with db.get_cursor() as cursor:
log_dammit("updating homepage queries")
start = time.time()
cursor.execute("""
DROP TABLE IF EXISTS _homepage_new_participants;
CREATE TABLE _homepage_new_participants AS
SELECT username, claimed_time FROM (
SELECT DISTINCT ON (p.username)
p.username
, claimed_time
FROM participants p
JOIN elsewhere e
ON p.username = participant
WHERE claimed_time IS NOT null
AND is_suspicious IS NOT true
) AS foo
ORDER BY claimed_time DESC;
DROP TABLE IF EXISTS _homepage_top_givers;
CREATE TABLE _homepage_top_givers AS
SELECT tipper AS username, anonymous, sum(amount) AS amount
FROM ( SELECT DISTINCT ON (tipper, tippee)
amount
, tipper
FROM tips
JOIN participants p ON p.username = tipper
JOIN participants p2 ON p2.username = tippee
JOIN elsewhere ON elsewhere.participant = tippee
WHERE p.last_bill_result = ''
AND p.is_suspicious IS NOT true
AND p2.claimed_time IS NOT NULL
AND elsewhere.is_locked = false
ORDER BY tipper, tippee, mtime DESC
) AS foo
JOIN participants p ON p.username = tipper
WHERE is_suspicious IS NOT true
GROUP BY tipper, anonymous
ORDER BY amount DESC;
DROP TABLE IF EXISTS _homepage_top_receivers;
CREATE TABLE _homepage_top_receivers AS
SELECT tippee AS username, claimed_time, sum(amount) AS amount
FROM ( SELECT DISTINCT ON (tipper, tippee)
amount
, tippee
FROM tips
JOIN participants p ON p.username = tipper
JOIN elsewhere ON elsewhere.participant = tippee
WHERE last_bill_result = ''
AND elsewhere.is_locked = false
AND is_suspicious IS NOT true
AND claimed_time IS NOT null
ORDER BY tipper, tippee, mtime DESC
) AS foo
JOIN participants p ON p.username = tippee
WHERE is_suspicious IS NOT true
GROUP BY tippee, claimed_time
ORDER BY amount DESC;
DROP TABLE IF EXISTS homepage_new_participants;
ALTER TABLE _homepage_new_participants
RENAME TO homepage_new_participants;
DROP TABLE IF EXISTS homepage_top_givers;
ALTER TABLE _homepage_top_givers
RENAME TO homepage_top_givers;
DROP TABLE IF EXISTS homepage_top_receivers;
ALTER TABLE _homepage_top_receivers
RENAME TO homepage_top_receivers;
""")
end = time.time()
elapsed = end - start
log_dammit("updated homepage queries in %.2f seconds" % elapsed)
def stop(self):
aspen.log_dammit("Shutting down Aspen website.")
self.hooks.run('shutdown', self)
self.network_engine.stop()
def envvars(website):
missing_keys = []
malformed_values = []
def envvar(key, cast=None):
if key not in os.environ:
missing_keys.append(key)
return ""
value = os.environ[key].decode('ASCII')
if cast is not None:
try:
value = cast(value)
except:
err = str(sys.exc_info()[1])
malformed_values.append((key, err))
return ""
return value
def is_yesish(val):
return val.lower() in ('1', 'true', 'yes')
website.bitbucket_consumer_key = envvar('BITBUCKET_CONSUMER_KEY')
website.bitbucket_consumer_secret = envvar('BITBUCKET_CONSUMER_SECRET')
website.bitbucket_callback = envvar('BITBUCKET_CALLBACK')
website.github_client_id = envvar('GITHUB_CLIENT_ID')
website.github_client_secret = envvar('GITHUB_CLIENT_SECRET')
website.github_callback = envvar('GITHUB_CALLBACK')
website.twitter_consumer_key = envvar('TWITTER_CONSUMER_KEY')
website.twitter_consumer_secret = envvar('TWITTER_CONSUMER_SECRET')
website.twitter_access_token = envvar('TWITTER_ACCESS_TOKEN')
website.twitter_access_token_secret = envvar('TWITTER_ACCESS_TOKEN_SECRET')
website.twitter_callback = envvar('TWITTER_CALLBACK')
website.bountysource_www_host = envvar('BOUNTYSOURCE_WWW_HOST')
website.bountysource_api_host = envvar('BOUNTYSOURCE_API_HOST')
website.bountysource_api_secret = envvar('BOUNTYSOURCE_API_SECRET')
website.bountysource_callback = envvar('BOUNTYSOURCE_CALLBACK')
website.css_href = envvar('GITTIP_CSS_HREF') \
.replace('%version', website.version)
website.js_src = envvar('GITTIP_JS_SRC') \
.replace('%version', website.version)
website.cache_static = is_yesish(envvar('GITTIP_CACHE_STATIC'))
website.google_analytics_id = envvar('GOOGLE_ANALYTICS_ID')
website.gauges_id = envvar('GAUGES_ID')
website.sentry_dsn = envvar('SENTRY_DSN')
website.min_threads = envvar('MIN_THREADS', int)
website.log_busy_threads_every = envvar('LOG_BUSY_THREADS_EVERY', int)
website.log_metrics = is_yesish(envvar('LOG_METRICS'))
if malformed_values:
malformed_values.sort()
these = len(malformed_values) != 1 and 'these' or 'this'
plural = len(malformed_values) != 1 and 's' or ''
aspen.log_dammit("=" * 42)
aspen.log_dammit( "Oh no! Gittip.com couldn't understand %s " % these
, "environment variable%s:" % plural
)
aspen.log_dammit(" ")
for key, err in malformed_values:
aspen.log_dammit(" {} ({})".format(key, err))
aspen.log_dammit(" ")
aspen.log_dammit("See ./default_local.env for hints.")
aspen.log_dammit("=" * 42)
raise SystemExit
if missing_keys:
missing_keys.sort()
these = len(missing_keys) != 1 and 'these' or 'this'
plural = len(missing_keys) != 1 and 's' or ''
aspen.log_dammit("=" * 42)
aspen.log_dammit( "Oh no! Gittip.com needs %s missing " % these
, "environment variable%s:" % plural
)
aspen.log_dammit(" ")
for key in missing_keys:
aspen.log_dammit(" " + key)
aspen.log_dammit(" ")
aspen.log_dammit( "(Sorry, we must've started looking for "
, "%s since you last updated Gittip!)" % these
)
aspen.log_dammit(" ")
aspen.log_dammit("Running Gittip locally? Edit ./local.env.")
aspen.log_dammit("Running the test suite? Edit ./tests/env.")
aspen.log_dammit(" ")
aspen.log_dammit("See ./default_local.env for hints.")
aspen.log_dammit("=" * 42)
raise SystemExit
def SIGHUP(signum, frame):
aspen.log_dammit("Received HUP, re-executing.")
execution.execute()
# only thing missing will be the response.body which is not logged.
try:
import http.client as http_client
except ImportError:
# Python 2
import httplib as http_client
http_client.HTTPConnection.debuglevel = 1
# You must initialize logging, otherwise you'll not see debug output.
logging.basicConfig()
logging.getLogger().setLevel(logging.DEBUG)
requests_log = logging.getLogger("requests.packages.urllib3")
requests_log.setLevel(logging.DEBUG)
requests_log.propagate = True
# Provide a decorator to skip tests when marky-markdown is missing.
try:
markdown.render_like_npm('test')
except OSError as exc:
MISSING_MARKY_MARKDOWN = True
log_dammit('Will skip marky-markdown-related tests because:', exc.args[0])
else:
MISSING_MARKY_MARKDOWN = False
def skipif_missing_marky_markdown(func):
return pytest.mark.skipif(MISSING_MARKY_MARKDOWN,
reason="missing marky-markdown")(func)
def SIGHUP(signum, frame):
aspen.log_dammit("Received HUP, restarting.")
restart()
suitable for development and testing. It can be invoked via:
python -m aspen
though even for development you'll likely want to specify a
project root, so a more likely incantation is:
ASPEN_PROJECT_ROOT=/path/to/wherever python -m aspen
For production deployment, you should probably deploy using
a higher performance WSGI server like Gunicorn, uwsgi, Spawning,
or the like.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import os
from aspen import log_dammit
from aspen.website import Website
from wsgiref.simple_server import make_server
if __name__ == '__main__':
website = Website()
port = int(os.environ.get('PORT', '8080'))
server = make_server('0.0.0.0', port, website)
log_dammit("Greetings, program! Welcome to port {0}.".format(port))
server.serve_forever()
except OSError, err:
if err.errno != errno.ENOENT:
raise
raise ConfigurationError(errorstr)
# LOGGING_THRESHOLD
# -----------------
# This is initially set to -1 and not 0 so that we can tell if the user
# changed it programmatically directly before we got here. I do this in
# the testing module, that's really what this is about.
if aspen.logging.LOGGING_THRESHOLD == -1:
aspen.logging.LOGGING_THRESHOLD = self.logging_threshold
# Now that we know the user's desires, we can log appropriately.
aspen.log_dammit(os.linesep.join(msgs))
# project root
if self.project_root is None:
aspen.log_dammit("project_root not configured (no template bases, "
"etc.).")
else:
# canonicalize it
if not os.path.isabs(self.project_root):
aspen.log_dammit("project_root is relative to CWD: '%s'."
% self.project_root)
cwd = safe_getcwd("Could not get a current working "
"directory. You can specify "
"ASPEN_PROJECT_ROOT in the environment, "
"or --project_root on the command line.")
self.project_root = os.path.join(cwd, self.project_root)
def env():
env = Environment(
DATABASE_URL=unicode,
CANONICAL_HOST=unicode,
CANONICAL_SCHEME=unicode,
MIN_THREADS=int,
DATABASE_MAXCONN=int,
GITTIP_ASSET_URL=unicode,
GITTIP_CACHE_STATIC=is_yesish,
GITTIP_COMPRESS_ASSETS=is_yesish,
STRIPE_SECRET_API_KEY=unicode,
STRIPE_PUBLISHABLE_API_KEY=unicode,
BALANCED_API_SECRET=unicode,
#DEBUG = unicode,
GITHUB_CLIENT_ID=unicode,
GITHUB_CLIENT_SECRET=unicode,
GITHUB_CALLBACK=unicode,
BITBUCKET_CONSUMER_KEY=unicode,
BITBUCKET_CONSUMER_SECRET=unicode,
BITBUCKET_CALLBACK=unicode,
TWITTER_CONSUMER_KEY=unicode,
TWITTER_CONSUMER_SECRET=unicode,
TWITTER_CALLBACK=unicode,
BOUNTYSOURCE_API_SECRET=unicode,
BOUNTYSOURCE_CALLBACK=unicode,
BOUNTYSOURCE_API_HOST=unicode,
BOUNTYSOURCE_WWW_HOST=unicode,
VENMO_CLIENT_ID=unicode,
VENMO_CLIENT_SECRET=unicode,
VENMO_CALLBACK=unicode,
OPENSTREETMAP_CONSUMER_KEY=unicode,
OPENSTREETMAP_CONSUMER_SECRET=unicode,
OPENSTREETMAP_CALLBACK=unicode,
OPENSTREETMAP_API_URL=unicode,
OPENSTREETMAP_AUTH_URL=unicode,
NANSWERS_THRESHOLD=int,
UPDATE_HOMEPAGE_EVERY=int,
GOOGLE_ANALYTICS_ID=unicode,
SENTRY_DSN=unicode,
LOG_BUSY_THREADS_EVERY=int,
LOG_METRICS=is_yesish,
MANDRILL_KEY=unicode,
)
# Error Checking
# ==============
if env.malformed:
these = len(env.malformed) != 1 and 'these' or 'this'
plural = len(env.malformed) != 1 and 's' or ''
aspen.log_dammit("=" * 42)
aspen.log_dammit("Oh no! Gittip.com couldn't understand %s " % these,
"environment variable%s:" % plural)
aspen.log_dammit(" ")
for key, err in env.malformed:
aspen.log_dammit(" {} ({})".format(key, err))
aspen.log_dammit(" ")
aspen.log_dammit("See ./default_local.env for hints.")
aspen.log_dammit("=" * 42)
keys = ', '.join([key for key in env.malformed])
raise BadEnvironment("Malformed envvar{}: {}.".format(plural, keys))
if env.missing:
these = len(env.missing) != 1 and 'these' or 'this'
plural = len(env.missing) != 1 and 's' or ''
aspen.log_dammit("=" * 42)
aspen.log_dammit("Oh no! Gittip.com needs %s missing " % these,
"environment variable%s:" % plural)
aspen.log_dammit(" ")
for key in env.missing:
aspen.log_dammit(" " + key)
aspen.log_dammit(" ")
aspen.log_dammit("(Sorry, we must've started looking for ",
"%s since you last updated Gittip!)" % these)
aspen.log_dammit(" ")
aspen.log_dammit("Running Gittip locally? Edit ./local.env.")
aspen.log_dammit("Running the test suite? Edit ./tests/env.")
aspen.log_dammit(" ")
aspen.log_dammit("See ./default_local.env for hints.")
aspen.log_dammit("=" * 42)
keys = ', '.join([key for key in env.missing])
raise BadEnvironment("Missing envvar{}: {}.".format(plural, keys))
return env
def _main(argv):
# Do imports.
# ===========
# These are in here so that if you Ctrl-C during an import, the
# KeyboardInterrupt is caught and ignored. Yes, that's how much I care.
# No, I don't care enough to put aspen/__init__.py in here too.
import os
import signal
import socket
import sys
import traceback
import aspen
from aspen import execution
from aspen.website import Website
# Set up signal handling.
# =======================
def SIGHUP(signum, frame):
aspen.log_dammit("Received HUP, re-executing.")
execution.execute()
if not aspen.WINDOWS:
signal.signal(signal.SIGHUP, SIGHUP)
def SIGINT(signum, frame):
aspen.log_dammit("Received INT, exiting.")
raise SystemExit
signal.signal(signal.SIGINT, SIGINT)
def SIGQUIT(signum, frame):
aspen.log_dammit("Received QUIT, exiting.")
raise SystemExit
if not aspen.WINDOWS:
signal.signal(signal.SIGQUIT, SIGQUIT)
# Website
# =======
# User-developers get this website object inside of their resources and
# hooks. It provides access to configuration information in addition to
# being a WSGI callable and holding the request/response handling
# logic. See aspen/website.py
if argv is None:
argv = sys.argv[1:]
website = Website(argv)
# Start serving the website.
# ==========================
# This amounts to binding the requested socket, with logging and
# restarting as needed. Wrap the whole thing in a try/except to
# do some cleanup on shutdown.
try:
if hasattr(socket, 'AF_UNIX'):
if website.network_sockfam == socket.AF_UNIX:
if os.path.exists(website.network_address):
aspen.log("Removing stale socket.")
os.remove(website.network_address)
if website.network_port is not None:
welcome = "port %d" % website.network_port
else:
welcome = website.network_address
aspen.log("Starting %s engine." % website.network_engine.name)
website.network_engine.bind()
aspen.log_dammit("Greetings, program! Welcome to %s." % welcome)
if website.changes_reload:
aspen.log("Aspen will restart when configuration scripts or "
"Python modules change.")
execution.install(website)
website.start()
except socket.error:
# Be friendly about port conflicts.
# =================================
# The traceback one gets from a port conflict or permission error
# is not that friendly. Here's a helper to let the user know (in
# color?!) that a port conflict or a permission error is probably
# the problem. But in case it isn't (website.start fires the start
# hook, and maybe the user tries to connect to a network service in
# there?), don't fully swallow the exception. Also, be explicit
# about the port number. What if they have logging turned off? Then
# they won't see the port number in the "Greetings, program!" line.
# They definitely won't see it if using an engine like eventlet
# that binds to the port early.
if website.network_port is not None:
msg = "Is something already running on port %s? Because ..."
if not aspen.WINDOWS:
if website.network_port < 1024:
if os.geteuid() > 0:
msg = ("Do you have permission to bind to port %s?"
" Because ...")
msg %= website.network_port
if not aspen.WINDOWS:
# Assume we can use ANSI color escapes if not on Windows.
# XXX Maybe a bad assumption if this is going into a log
# file? See also: colorama
msg = '\033[01;33m%s\033[00m' % msg
aspen.log_dammit(msg)
raise
except (KeyboardInterrupt, SystemExit):
raise # Don't bother logging these.
except:
aspen.log_dammit(traceback.format_exc())
finally:
if hasattr(socket, 'AF_UNIX'):
if website.network_sockfam == socket.AF_UNIX:
if os.path.exists(website.network_address):
os.remove(website.network_address)
website.stop()
def envvars(website):
missing_keys = []
def envvar(key):
if key not in os.environ:
missing_keys.append(key)
return ""
return os.environ[key].decode('ASCII')
def is_yesish(val):
return val.lower() in ('1', 'true', 'yes')
website.bitbucket_consumer_key = envvar('BITBUCKET_CONSUMER_KEY')
website.bitbucket_consumer_secret = envvar('BITBUCKET_CONSUMER_SECRET')
website.bitbucket_callback = envvar('BITBUCKET_CALLBACK')
website.github_client_id = envvar('GITHUB_CLIENT_ID')
website.github_client_secret = envvar('GITHUB_CLIENT_SECRET')
website.github_callback = envvar('GITHUB_CALLBACK')
website.twitter_consumer_key = envvar('TWITTER_CONSUMER_KEY')
website.twitter_consumer_secret = envvar('TWITTER_CONSUMER_SECRET')
website.twitter_access_token = envvar('TWITTER_ACCESS_TOKEN')
website.twitter_access_token_secret = envvar('TWITTER_ACCESS_TOKEN_SECRET')
website.twitter_callback = envvar('TWITTER_CALLBACK')
website.bountysource_www_host = envvar('BOUNTYSOURCE_WWW_HOST')
website.bountysource_api_host = envvar('BOUNTYSOURCE_API_HOST')
website.bountysource_api_secret = envvar('BOUNTYSOURCE_API_SECRET')
website.bountysource_callback = envvar('BOUNTYSOURCE_CALLBACK')
website.css_href = envvar('GITTIP_CSS_HREF') \
.replace('%version', website.version)
website.cache_static = is_yesish(envvar('GITTIP_CACHE_STATIC'))
if missing_keys:
missing_keys.sort()
these = len(missing_keys) != 1 and 'these' or 'this'
plural = len(missing_keys) != 1 and 's' or ''
aspen.log_dammit("=" * 42)
aspen.log_dammit("Oh no! Gittip.com needs %s missing " % these,
"environment variable%s:" % plural)
aspen.log_dammit(" ")
for key in missing_keys:
aspen.log_dammit(" " + key)
aspen.log_dammit(" ")
aspen.log_dammit("(Sorry, we must've started looking for ",
"%s since you last updated Gittip!)" % these)
aspen.log_dammit(" ")
aspen.log_dammit("Running Gittip locally? Edit ./local.env.")
aspen.log_dammit("Running the test suite? Edit ./tests/env.")
aspen.log_dammit(" ")
aspen.log_dammit("See ./default_local.env for hints.")
aspen.log_dammit("=" * 42)
raise SystemExit
def envvars(website):
missing_keys = []
def envvar(key):
if key not in os.environ:
missing_keys.append(key)
return ""
return os.environ[key].decode('ASCII')
def is_yesish(val):
return val.lower() in ('1', 'true', 'yes')
website.bitbucket_consumer_key = envvar('BITBUCKET_CONSUMER_KEY')
website.bitbucket_consumer_secret = envvar('BITBUCKET_CONSUMER_SECRET')
website.bitbucket_callback = envvar('BITBUCKET_CALLBACK')
website.github_client_id = envvar('GITHUB_CLIENT_ID')
website.github_client_secret = envvar('GITHUB_CLIENT_SECRET')
website.github_callback = envvar('GITHUB_CALLBACK')
website.twitter_consumer_key = envvar('TWITTER_CONSUMER_KEY')
website.twitter_consumer_secret = envvar('TWITTER_CONSUMER_SECRET')
website.twitter_access_token = envvar('TWITTER_ACCESS_TOKEN')
website.twitter_access_token_secret = envvar('TWITTER_ACCESS_TOKEN_SECRET')
website.twitter_callback = envvar('TWITTER_CALLBACK')
website.bountysource_www_host = envvar('BOUNTYSOURCE_WWW_HOST')
website.bountysource_api_host = envvar('BOUNTYSOURCE_API_HOST')
website.bountysource_api_secret = envvar('BOUNTYSOURCE_API_SECRET')
website.bountysource_callback = envvar('BOUNTYSOURCE_CALLBACK')
website.css_href = envvar('GITTIP_CSS_HREF') \
.replace('%version', website.version)
website.js_src = envvar('GITTIP_JS_SRC') \
.replace('%version', website.version)
website.cache_static = is_yesish(envvar('GITTIP_CACHE_STATIC'))
website.google_analytics_id = envvar('GOOGLE_ANALYTICS_ID')
website.gauges_id = envvar('GAUGES_ID')
if missing_keys:
missing_keys.sort()
these = len(missing_keys) != 1 and 'these' or 'this'
plural = len(missing_keys) != 1 and 's' or ''
aspen.log_dammit("=" * 42)
aspen.log_dammit( "Oh no! Gittip.com needs %s missing " % these
, "environment variable%s:" % plural
)
aspen.log_dammit(" ")
for key in missing_keys:
aspen.log_dammit(" " + key)
aspen.log_dammit(" ")
aspen.log_dammit( "(Sorry, we must've started looking for "
, "%s since you last updated Gittip!)" % these
)
aspen.log_dammit(" ")
aspen.log_dammit("Running Gittip locally? Edit ./local.env.")
aspen.log_dammit("Running the test suite? Edit ./tests/env.")
aspen.log_dammit(" ")
aspen.log_dammit("See ./default_local.env for hints.")
aspen.log_dammit("=" * 42)
raise SystemExit
def env():
env = Environment(
AWS_SES_ACCESS_KEY_ID=unicode,
AWS_SES_SECRET_ACCESS_KEY=unicode,
AWS_SES_DEFAULT_REGION=unicode,
BASE_URL=unicode,
DATABASE_URL=unicode,
DATABASE_MAXCONN=int,
CRYPTO_KEYS=unicode,
GRATIPAY_ASSET_URL=unicode,
GRATIPAY_CACHE_STATIC=is_yesish,
GRATIPAY_COMPRESS_ASSETS=is_yesish,
BALANCED_API_SECRET=unicode,
BRAINTREE_SANDBOX_MODE=is_yesish,
BRAINTREE_MERCHANT_ID=unicode,
BRAINTREE_PUBLIC_KEY=unicode,
BRAINTREE_PRIVATE_KEY=unicode,
GITHUB_CLIENT_ID=unicode,
GITHUB_CLIENT_SECRET=unicode,
GITHUB_CALLBACK=unicode,
BITBUCKET_CONSUMER_KEY=unicode,
BITBUCKET_CONSUMER_SECRET=unicode,
BITBUCKET_CALLBACK=unicode,
TWITTER_CONSUMER_KEY=unicode,
TWITTER_CONSUMER_SECRET=unicode,
TWITTER_CALLBACK=unicode,
FACEBOOK_APP_ID=unicode,
FACEBOOK_APP_SECRET=unicode,
FACEBOOK_CALLBACK=unicode,
GOOGLE_CLIENT_ID=unicode,
GOOGLE_CLIENT_SECRET=unicode,
GOOGLE_CALLBACK=unicode,
BOUNTYSOURCE_API_SECRET=unicode,
BOUNTYSOURCE_CALLBACK=unicode,
BOUNTYSOURCE_API_HOST=unicode,
BOUNTYSOURCE_WWW_HOST=unicode,
VENMO_CLIENT_ID=unicode,
VENMO_CLIENT_SECRET=unicode,
VENMO_CALLBACK=unicode,
OPENSTREETMAP_CONSUMER_KEY=unicode,
OPENSTREETMAP_CONSUMER_SECRET=unicode,
OPENSTREETMAP_CALLBACK=unicode,
OPENSTREETMAP_API_URL=unicode,
OPENSTREETMAP_AUTH_URL=unicode,
UPDATE_CTA_EVERY=int,
CHECK_DB_EVERY=int,
CHECK_NPM_SYNC_EVERY=int,
EMAIL_QUEUE_FLUSH_EVERY=int,
EMAIL_QUEUE_SLEEP_FOR=int,
EMAIL_QUEUE_ALLOW_UP_TO=int,
EMAIL_QUEUE_LOG_METRICS_EVERY=int,
OPTIMIZELY_ID=unicode,
SENTRY_DSN=unicode,
LOG_METRICS=is_yesish,
INCLUDE_PIWIK=is_yesish,
PROJECT_REVIEW_REPO=unicode,
PROJECT_REVIEW_USERNAME=unicode,
PROJECT_REVIEW_TOKEN=unicode,
RAISE_SIGNIN_NOTIFICATIONS=is_yesish,
GUNICORN_OPTS=unicode,
)
# Error Checking
# ==============
if env.malformed:
these = len(env.malformed) != 1 and 'these' or 'this'
plural = len(env.malformed) != 1 and 's' or ''
aspen.log_dammit("=" * 42)
aspen.log_dammit("Oh no! Gratipay.com couldn't understand %s " % these,
"environment variable%s:" % plural)
aspen.log_dammit(" ")
for key, err in env.malformed:
aspen.log_dammit(" {} ({})".format(key, err))
aspen.log_dammit(" ")
aspen.log_dammit("See ./default_local.env for hints.")
aspen.log_dammit("=" * 42)
keys = ', '.join([key for key, value in env.malformed])
raise BadEnvironment("Malformed envvar{}: {}.".format(plural, keys))
if env.missing:
these = len(env.missing) != 1 and 'these' or 'this'
plural = len(env.missing) != 1 and 's' or ''
aspen.log_dammit("=" * 42)
aspen.log_dammit("Oh no! Gratipay.com needs %s missing " % these,
"environment variable%s:" % plural)
aspen.log_dammit(" ")
for key in env.missing:
aspen.log_dammit(" " + key)
aspen.log_dammit(" ")
aspen.log_dammit("(Sorry, we must've started looking for ",
"%s since you last updated Gratipay!)" % these)
aspen.log_dammit(" ")
aspen.log_dammit("Running Gratipay locally? Edit ./local.env.")
aspen.log_dammit("Running the test suite? Edit ./tests/env.")
aspen.log_dammit(" ")
aspen.log_dammit("See ./default_local.env for hints.")
aspen.log_dammit("=" * 42)
keys = ', '.join([key for key in env.missing])
raise BadEnvironment("Missing envvar{}: {}.".format(plural, keys))
return env
