Пример #1
0
def log_traceback_for_5xx(response, traceback=None):
    if response.code >= 500:
        if traceback:
            aspen.log_dammit(traceback)
        else:
            aspen.log_dammit(response.body)
    return {'traceback': None}
Пример #2
0
def handle_conflict_over_port(exception, website):
    """Be friendly about port conflicts.

    The traceback one gets from a port conflict or permission error is not that
    friendly. Here's a helper to let the user know (in color?!) that a port
    conflict or a permission error is probably the problem. But in case it
    isn't (website.start fires the start hook, and maybe the user tries to
    connect to a network service in there?), don't fully swallow the exception.
    Also, be explicit about the port number. What if they have logging turned
    off? Then they won't see the port number in the "Greetings, program!" line.
    They definitely won't see it if using an engine like eventlet that binds to
    the port early.

    """
    if exception.__class__ is not socket.error:
        return

    if website.network_port is not None:
        msg = "Is something already running on port %s? Because ..."
        if not aspen.WINDOWS:
            if website.network_port < 1024:
                if os.geteuid() > 0:
                    msg = ("Do you have permission to bind to port %s?"
                           " Because ...")
        msg %= website.network_port
        if not aspen.WINDOWS:
            # Assume we can use ANSI color escapes if not on Windows.
            # XXX Maybe a bad assumption if this is going into a log
            # file? See also: colorama
            msg = '\033[01;33m%s\033[00m' % msg
        aspen.log_dammit(msg)
    raise
Пример #3
0
def update_homepage_queries_once(db):
    with db.get_cursor() as cursor:
        log_dammit("updating homepage queries")
        start = time.time()
        cursor.execute("DELETE FROM homepage_top_givers")
        cursor.execute("""

        INSERT INTO homepage_top_givers (username, anonymous, amount, avatar_url, statement, number)
            SELECT username, anonymous_giving, giving, avatar_url, statement, number
              FROM participants
             WHERE is_suspicious IS NOT true
               AND last_bill_result = ''
          ORDER BY giving DESC
             LIMIT 100;

        """.strip())

        cursor.execute("DELETE FROM homepage_top_receivers")
        cursor.execute("""

        INSERT INTO homepage_top_receivers (username, anonymous, amount, avatar_url, statement, number)
            SELECT username, anonymous_receiving, receiving, avatar_url, statement, number
              FROM participants
             WHERE is_suspicious IS NOT true
               AND claimed_time IS NOT NULL
          ORDER BY receiving DESC
             LIMIT 100;

        """.strip())

        end = time.time()
        elapsed = end - start
        log_dammit("updated homepage queries in %.2f seconds" % elapsed)
Пример #4
0
def main(argv=None):
    """http://aspen.io/cli/
    """
    try:
        _main(argv)
    except (SystemExit, KeyboardInterrupt):

        # Under some (most?) network engines, a SIGINT will be trapped by the
        # SIGINT signal handler above. However, gevent does "something" with
        # signals and our signal handler never fires. However, we *do* get a
        # KeyboardInterrupt here in that case. *shrug*
        #
        # See: https://github.com/gittip/aspen-python/issues/196

        pass
    except:
        import aspen, aspen.execution, time, traceback
        aspen.log_dammit("Oh no! Aspen crashed!")
        aspen.log_dammit(traceback.format_exc())
        try:
            while 1:
                aspen.execution.check_all()
                time.sleep(1)
        except KeyboardInterrupt:
            raise SystemExit
Пример #5
0
 def handle_error_nicely(self, request):
     """Try to provide some nice error handling.
     """
     try:                        # nice error messages
         tb_1 = traceback.format_exc()
         response = sys.exc_info()[1]
         if not isinstance(response, Response):
             aspen.log_dammit(tb_1)
             response = Response(500, tb_1)
         elif 200 <= response.code < 300:
             return response
         response.request = request
         self.hooks.outbound_early.run(response)
         fs = self.ours_or_theirs(str(response.code) + '.html')
         if fs is None:
             fs = self.ours_or_theirs('error.html')
         if fs is None:
             raise
         request.fs = fs
         request.original_resource = request.resource
         request.resource = resources.get(request)
         response = request.resource.respond(request, response)
         return response
     except Response, response:  # no nice error simplate available
         raise
Пример #6
0
    def __init__(self, env, db, tell_sentry, root):
        if self._have_ses(env):
            log_dammit("AWS SES is configured! We'll send mail through SES.")
            self._mailer = boto3.client(
                service_name='ses',
                region_name=env.aws_ses_default_region,
                aws_access_key_id=env.aws_ses_access_key_id,
                aws_secret_access_key=env.aws_ses_secret_access_key)
        else:
            log_dammit(
                "AWS SES is not configured! Mail will be dumped to the console here."
            )
            self._mailer = ConsoleMailer()

        self.db = db
        self.tell_sentry = tell_sentry
        self.sleep_for = env.email_queue_sleep_for
        self.allow_up_to = env.email_queue_allow_up_to

        templates = {}
        templates_dir = os.path.join(root, 'emails')
        assert os.path.isdir(templates_dir)
        i = len(templates_dir) + 1
        for spt in find_files(templates_dir, '*.spt'):
            base_name = spt[i:-4]
            templates[base_name] = compile_email_spt(spt)
        self._email_templates = templates
Пример #7
0
    def __init__(self, env, db, tell_sentry, root):
        if self._have_ses(env):
            log_dammit("AWS SES is configured! We'll send mail through SES.")
            self._mailer = boto3.client( service_name='ses'
                                       , region_name=env.aws_ses_default_region
                                       , aws_access_key_id=env.aws_ses_access_key_id
                                       , aws_secret_access_key=env.aws_ses_secret_access_key
                                        )
        else:
            log_dammit("AWS SES is not configured! Mail will be dumped to the console here.")
            self._mailer = ConsoleMailer()

        self.db = db
        self.tell_sentry = tell_sentry
        self.sleep_for = env.email_queue_sleep_for
        self.allow_up_to = env.email_queue_allow_up_to
        self.log_every = env.email_queue_log_metrics_every

        templates = {}
        templates_dir = os.path.join(root, 'emails')
        assert os.path.isdir(templates_dir)
        i = len(templates_dir) + 1
        for spt in find_files(templates_dir, '*.spt'):
            base_name = spt[i:-4]
            templates[base_name] = compile_email_spt(spt)
        self._email_templates = templates
Пример #8
0
def log_traceback_for_exception(website, exception):
    tb = traceback.format_exc()
    aspen.log_dammit(tb)
    response = Response(500)
    if website.show_tracebacks:
        response.body = tb
    return {'response': response, 'exception': None}
Пример #9
0
 def handle_error_nicely(self, request):
     """Try to provide some nice error handling.
     """
     try:  # nice error messages
         tb_1 = traceback.format_exc()
         response = sys.exc_info()[1]
         if not isinstance(response, Response):
             aspen.log_dammit(tb_1)
             response = Response(500, tb_1)
         elif 200 <= response.code < 300:
             return response
         response.request = request
         self.hooks.outbound_early.run(response)
         fs = self.ours_or_theirs(str(response.code) + '.html')
         if fs is None:
             fs = self.ours_or_theirs('error.html')
         if fs is None:
             raise
         request.fs = fs
         request.original_resource = request.resource
         request.resource = resources.get(request)
         response = request.resource.respond(request, response)
         return response
     except Response, response:  # no nice error simplate available
         raise
Пример #10
0
def update_homepage_queries_once(db):
    with db.get_cursor() as cursor:
        log_dammit("updating homepage queries")
        start = time.time()
        cursor.execute("DELETE FROM homepage_top_givers")
        cursor.execute("""

        INSERT INTO homepage_top_givers (username, anonymous, amount, avatar_url, statement, number)
            SELECT username, anonymous_giving, giving, avatar_url, statement, number
              FROM participants
             WHERE is_suspicious IS NOT true
               AND last_bill_result = ''
          ORDER BY giving DESC
             LIMIT 100;

        """.strip())

        cursor.execute("DELETE FROM homepage_top_receivers")
        cursor.execute("""

        INSERT INTO homepage_top_receivers (username, anonymous, amount, avatar_url, statement, number)
            SELECT username, anonymous_receiving, receiving, avatar_url, statement, number
              FROM participants
             WHERE is_suspicious IS NOT true
               AND claimed_time IS NOT NULL
          ORDER BY receiving DESC
             LIMIT 100;

        """.strip())

        end = time.time()
        elapsed = end - start
        log_dammit("updated homepage queries in %.2f seconds" % elapsed)
Пример #11
0
def get_csrf_token_from_request(request):
    """Given a Request object, reject it if it's a forgery.
    """
    if request.line.uri.startswith('/assets/'): return
    if request.line.uri.startswith('/callbacks/'): return

    try:
        csrf_token = _sanitize_token(
            request.headers.cookie['csrf_token'].value)
    except KeyError:
        csrf_token = None

    request.context['csrf_token'] = csrf_token or _get_new_csrf_key()

    # Assume that anything not defined as 'safe' by RC2616 needs protection
    if request.line.method not in ('GET', 'HEAD', 'OPTIONS', 'TRACE'):

        if _is_secure(request):
            # Suppose user visits http://example.com/
            # An active network attacker (man-in-the-middle, MITM) sends a
            # POST form that targets https://example.com/detonate-bomb/ and
            # submits it via JavaScript.
            #
            # The attacker will need to provide a CSRF cookie and token, but
            # that's no problem for a MITM and the session-independent
            # nonce we're using. So the MITM can circumvent the CSRF
            # protection. This is true for any HTTP connection, but anyone
            # using HTTPS expects better! For this reason, for
            # https://example.com/ we need additional protection that treats
            # http://example.com/ as completely untrusted. Under HTTPS,
            # Barth et al. found that the Referer header is missing for
            # same-domain requests in only about 0.2% of cases or less, so
            # we can use strict Referer checking.
            referer = request.headers.get('Referer')
            if referer is None:
                raise Response(403, REASON_NO_REFERER)

            good_referer = 'https://%s/' % _get_host(request)
            if not same_origin(referer, good_referer):
                reason = REASON_BAD_REFERER % (referer, good_referer)
                log_dammit(reason)
                raise Response(403, reason)

        if csrf_token is None:
            raise Response(403, REASON_NO_CSRF_COOKIE)

        # Check non-cookie token for match.
        request_csrf_token = ""
        if request.line.method == "POST":
            if isinstance(request.body, dict):
                request_csrf_token = request.body.get('csrf_token', '')

        if request_csrf_token == "":
            # Fall back to X-CSRF-TOKEN, to make things easier for AJAX,
            # and possible for PUT/DELETE.
            request_csrf_token = request.headers.get('X-CSRF-TOKEN', '')

        if not constant_time_compare(request_csrf_token, csrf_token):
            raise Response(403, REASON_BAD_TOKEN)
Пример #12
0
 def f():
     while True:
         try:
             func()
         except Exception, e:
             self.website.tell_sentry(e, {})
             log_dammit(traceback.format_exc().strip())
         sleep(period)
Пример #13
0
 def f():
     while True:
         try:
             func()
         except Exception, e:
             self.website.tell_sentry(e, {})
             log_dammit(traceback.format_exc().strip())
         sleep(period)
Пример #14
0
def update_homepage_queries_once(db):
    with db.get_cursor() as cursor:
        log_dammit("updating homepage queries")
        start = time.time()
        cursor.execute("DELETE FROM homepage_top_givers")
        cursor.execute("""

        INSERT INTO homepage_top_givers (username, anonymous, amount, avatar_url)
            SELECT tipper, anonymous_giving, sum(amount) AS amount, avatar_url
              FROM (    SELECT DISTINCT ON (tipper, tippee)
                               amount
                             , tipper
                          FROM tips
                          JOIN participants p ON p.username = tipper
                          JOIN participants p2 ON p2.username = tippee
                          JOIN elsewhere ON elsewhere.participant = tippee
                         WHERE p.last_bill_result = ''
                           AND p.is_suspicious IS NOT true
                           AND p2.claimed_time IS NOT NULL
                           AND elsewhere.is_locked = false
                      ORDER BY tipper, tippee, mtime DESC
                      ) AS foo
              JOIN participants p ON p.username = tipper
             WHERE is_suspicious IS NOT true
          GROUP BY tipper, anonymous_giving, avatar_url
          ORDER BY amount DESC
             LIMIT 100;

        """.strip())

        cursor.execute("DELETE FROM homepage_top_receivers")
        cursor.execute("""

        INSERT INTO homepage_top_receivers (username, anonymous, amount, avatar_url)
            SELECT tippee, anonymous_receiving, sum(amount) AS amount, avatar_url
              FROM (    SELECT DISTINCT ON (tipper, tippee)
                               amount
                             , tippee
                          FROM tips
                          JOIN participants p ON p.username = tipper
                          JOIN elsewhere ON elsewhere.participant = tippee
                         WHERE last_bill_result = ''
                           AND elsewhere.is_locked = false
                           AND is_suspicious IS NOT true
                           AND claimed_time IS NOT null
                      ORDER BY tipper, tippee, mtime DESC
                      ) AS foo
              JOIN participants p ON p.username = tippee
             WHERE is_suspicious IS NOT true
          GROUP BY tippee, anonymous_receiving, avatar_url
          ORDER BY amount DESC
             LIMIT 100;

        """.strip())

        end = time.time()
        elapsed = end - start
        log_dammit("updated homepage queries in %.2f seconds" % elapsed)
def inbound(request):
    """Given a Request object, reject it if it's a forgery.
    """
    if request.line.uri.startswith('/assets/'): return

    try:
        csrf_token = request.headers.cookie.get('csrf_token')
        csrf_token = '' if csrf_token is None else csrf_token.value
        csrf_token = _sanitize_token(csrf_token)
    except KeyError:
        csrf_token = _get_new_csrf_key()

    request.context['csrf_token'] = csrf_token

    # Assume that anything not defined as 'safe' by RC2616 needs protection
    if request.line.method not in ('GET', 'HEAD', 'OPTIONS', 'TRACE'):

        if _is_secure(request):
            # Suppose user visits http://example.com/
            # An active network attacker (man-in-the-middle, MITM) sends a
            # POST form that targets https://example.com/detonate-bomb/ and
            # submits it via JavaScript.
            #
            # The attacker will need to provide a CSRF cookie and token, but
            # that's no problem for a MITM and the session-independent
            # nonce we're using. So the MITM can circumvent the CSRF
            # protection. This is true for any HTTP connection, but anyone
            # using HTTPS expects better! For this reason, for
            # https://example.com/ we need additional protection that treats
            # http://example.com/ as completely untrusted. Under HTTPS,
            # Barth et al. found that the Referer header is missing for
            # same-domain requests in only about 0.2% of cases or less, so
            # we can use strict Referer checking.
            referer = request.headers.get('Referer')
            if referer is None:
                raise Response(403, REASON_NO_REFERER)

            good_referer = 'https://%s/' % _get_host(request)
            if not same_origin(referer, good_referer):
                reason = REASON_BAD_REFERER % (referer, good_referer)
                log_dammit(reason)
                raise Response(403, reason)

        if csrf_token is None:
            raise Response(403, REASON_NO_CSRF_COOKIE)

        # Check non-cookie token for match.
        request_csrf_token = ""
        if request.line.method == "POST":
            request_csrf_token = request.body.get('csrf_token', '')

        if request_csrf_token == "":
            # Fall back to X-CSRF-TOKEN, to make things easier for AJAX,
            # and possible for PUT/DELETE.
            request_csrf_token = request.headers.get('X-CSRF-TOKEN', '')

        if not constant_time_compare(request_csrf_token, csrf_token):
            raise Response(403, REASON_BAD_TOKEN)
Пример #16
0
def update_homepage_queries_once(db):
    with db.get_cursor() as cursor:
        log_dammit("updating homepage queries")
        start = time.time()
        cursor.execute("DELETE FROM homepage_top_givers")
        cursor.execute("""

        INSERT INTO homepage_top_givers (username, anonymous, amount, avatar_url, statement, number)
            SELECT tipper, anonymous_giving, sum(amount) AS amount, avatar_url, statement, number
              FROM (    SELECT DISTINCT ON (tipper, tippee)
                               amount
                             , tipper
                          FROM tips
                          JOIN participants p ON p.username = tipper
                          JOIN participants p2 ON p2.username = tippee
                          JOIN elsewhere ON elsewhere.participant = tippee
                         WHERE p.last_bill_result = ''
                           AND p.is_suspicious IS NOT true
                           AND p2.claimed_time IS NOT NULL
                           AND elsewhere.is_locked = false
                      ORDER BY tipper, tippee, mtime DESC
                      ) AS foo
              JOIN participants p ON p.username = tipper
             WHERE is_suspicious IS NOT true
          GROUP BY tipper, anonymous_giving, avatar_url, statement, number
          ORDER BY amount DESC
             LIMIT 100;

        """.strip())

        cursor.execute("DELETE FROM homepage_top_receivers")
        cursor.execute("""

        INSERT INTO homepage_top_receivers (username, anonymous, amount, avatar_url, statement, number)
            SELECT tippee, anonymous_receiving, sum(amount) AS amount, avatar_url, statement, number
              FROM (    SELECT DISTINCT ON (tipper, tippee)
                               amount
                             , tippee
                          FROM tips
                          JOIN participants p ON p.username = tipper
                          JOIN elsewhere ON elsewhere.participant = tippee
                         WHERE last_bill_result = ''
                           AND elsewhere.is_locked = false
                           AND is_suspicious IS NOT true
                           AND claimed_time IS NOT null
                      ORDER BY tipper, tippee, mtime DESC
                      ) AS foo
              JOIN participants p ON p.username = tippee
             WHERE is_suspicious IS NOT true
          GROUP BY tippee, anonymous_receiving, avatar_url, statement, number
          ORDER BY amount DESC
             LIMIT 100;

        """.strip())

        end = time.time()
        elapsed = end - start
        log_dammit("updated homepage queries in %.2f seconds" % elapsed)
Пример #17
0
 def handle_error_at_all(self, tb_1):
     tb_2 = traceback.format_exc().strip()
     tbs = '\n\n'.join([tb_2, "... while handling ...", tb_1])
     aspen.log_dammit(tbs)
     if self.show_tracebacks:
         response = Response(500, tbs)
     else:
         response = Response(500)
     return response
Пример #18
0
 def handle_error_at_all(self, tb_1):
     tb_2 = traceback.format_exc().strip()
     tbs = '\n\n'.join([tb_2, "... while handling ...", tb_1])
     aspen.log_dammit(tbs)
     if self.show_tracebacks:
         response = Response(500, tbs)
     else:
         response = Response(500)
     return response
Пример #19
0
 def f():
     if period <= 0:
         return
     sleep = time.sleep
     while 1:
         try:
             func()
         except Exception, e:
             tell_sentry(e)
             log_dammit(traceback.format_exc().strip())
         sleep(period)
Пример #20
0
        def tell_sentry(request):
            cls, response = sys.exc_info()[:2]
            if cls is aspen.Response:
                if response.code < 500:
                    return

            kw = {'extra': { "filepath": request.fs
                           , "request": str(request).splitlines()
                            }}
            exc = sentry.captureException(**kw)
            ident = sentry.get_ident(exc)
            aspen.log_dammit("Exception reference: " + ident)
Пример #21
0
def stop(website):
    """Stop the server.
    """
    if website is None:
        return

    aspen.log_dammit("Shutting down Aspen website.")
    website.network_engine.stop()
    if hasattr(socket, 'AF_UNIX'):
        if website.network_sockfam == socket.AF_UNIX:
            if os.path.exists(website.network_address):
                os.remove(website.network_address)
Пример #22
0
    def tell_sentry(request):
        cls, response = sys.exc_info()[:2]
        if cls is aspen.Response:
            if response.code < 500:
                return

        kw = {'extra': { "filepath": request.fs
                       , "request": str(request).splitlines()
                        }}
        exc = client.captureException(**kw)
        ident = client.get_ident(exc)
        aspen.log_dammit("Exception reference: " + ident)
Пример #23
0
def update_homepage_queries():
    from gittip import utils
    while 1:
        try:
            utils.update_global_stats(website)
            utils.update_homepage_queries_once(website.db)
            website.db.self_check()
        except:
            exception = sys.exc_info()[0]
            tell_sentry(exception)
            tb = traceback.format_exc().strip()
            log_dammit(tb)
        time.sleep(UPDATE_HOMEPAGE_EVERY)
def update_homepage_queries():
    from gittip import utils
    while 1:
        try:
            utils.update_global_stats(website)
            utils.update_homepage_queries_once(website.db)
            website.db.self_check()
        except:
            exception = sys.exc_info()[0]
            tell_sentry(exception)
            tb = traceback.format_exc().strip()
            log_dammit(tb)
        time.sleep(UPDATE_HOMEPAGE_EVERY)
Пример #25
0
def update_homepage_queries():
    from gittip import utils
    while 1:
        try:
            utils.update_global_stats(website)
            utils.update_homepage_queries_once(website.db)
        except:
            if tell_sentry:
                tell_sentry(None)
            else:
                tb = traceback.format_exc().strip()
                log_dammit(tb)
        time.sleep(UPDATE_HOMEPAGE_EVERY)
Пример #26
0
def update_homepage_queries():
    from gittip import utils

    while 1:
        try:
            utils.update_global_stats(website)
            utils.update_homepage_queries_once(website.db)
        except:
            if tell_sentry:
                tell_sentry(None)
            else:
                tb = traceback.format_exc().strip()
                log_dammit(tb)
        time.sleep(UPDATE_HOMEPAGE_EVERY)
Пример #27
0
def main(argv=None):
    """http://aspen.io/cli/
    """
    try:
        _main(argv)
    except KeyboardInterrupt:
        raise SystemExit(75)  # 75 == INT
    except SystemExit:
        pass
    except:
        import aspen, traceback

        aspen.log_dammit("Oh no! Server crashed!")
        aspen.log_dammit(traceback.format_exc())
        raise SystemExit(1)  #  1 == Other
Пример #28
0
def bind_server_to_port(website):
    """
    """
    if hasattr(socket, 'AF_UNIX'):
        if website.network_sockfam == socket.AF_UNIX:
            if os.path.exists(website.network_address):
                aspen.log("Removing stale socket.")
                os.remove(website.network_address)
    if website.network_port is not None:
        welcome = "port %d" % website.network_port
    else:
        welcome = website.network_address
    aspen.log("Starting %s engine." % website.network_engine.name)
    website.network_engine.bind()
    aspen.log_dammit("Greetings, program! Welcome to %s." % welcome)
Пример #29
0
def main(argv=None):
    """http://aspen.io/cli/
    """
    try:
        _main(argv)
    except SystemExit:
        pass
    except:
        import aspen, aspen.execution, time, traceback
        aspen.log_dammit("Oh no! Aspen crashed!")
        aspen.log_dammit(traceback.format_exc())
        try:
            while 1:
                aspen.execution.check_all()
                time.sleep(1)
        except KeyboardInterrupt:
            raise SystemExit
Пример #30
0
def main(argv=None):
    """http://aspen.io/cli/
    """
    try:
        _main(argv)
    except SystemExit:
        pass
    except:
        import aspen, aspen.execution, time, traceback
        aspen.log_dammit("Oh no! Aspen crashed!")
        aspen.log_dammit(traceback.format_exc())
        try:
            while 1:
                aspen.execution.check_all()
                time.sleep(1)
        except KeyboardInterrupt:
            raise SystemExit
Пример #31
0
 def f():
     if period <= 0:
         return
     sleep = time.sleep
     if exclusive:
         cursor = conn.cursor()
         try_lock = lambda: cursor.one("SELECT pg_try_advisory_lock(0)")
     has_lock = False
     while 1:
         try:
             if exclusive and not has_lock:
                 has_lock = try_lock()
             if not exclusive or has_lock:
                 func()
         except Exception, e:
             tell_sentry(e)
             log_dammit(traceback.format_exc().strip())
         sleep(period)
Пример #32
0
 def f():
     if period <= 0:
         return
     sleep = time.sleep
     if exclusive:
         cursor = conn.cursor()
         try_lock = lambda: cursor.one("SELECT pg_try_advisory_lock(0)")
     has_lock = False
     while 1:
         try:
             if exclusive and not has_lock:
                 has_lock = try_lock()
             if not exclusive or has_lock:
                 func()
         except Exception, e:
             tell_sentry(e)
             log_dammit(traceback.format_exc().strip())
         sleep(period)
Пример #33
0
    def handle_error_nicely(self, tb_1, request):

        response = sys.exc_info()[1]

        if not isinstance(response, Response):

            # We have a true Exception; convert it to a Response object.

            response = Response(500, tb_1)
            response.request = request

        if 500 <= response.code < 600:
            # Log tracebacks for Reponse(5xx).
            aspen.log_dammit(tb_1)

            # TODO Switch to the logging module and use something like this:
            # log_level = [DEBUG,INFO,WARNING,ERROR][(response.code/100)-2]
            # logging.log(log_level, tb_1)

        if 200 <= response.code < 300 or response.code == 304:

            # The app raised a Response(2xx) or Response(304).
            # Act as if nothing happened. This is unusual but allowed.

            pass

        else:

            # Delegate to any error simplate.
            # ===============================

            rc = str(response.code)
            possibles = [ rc + ".html", rc + ".html.spt", "error.html", "error.html.spt" ]
            fs = first( self.ours_or_theirs(errpage) for errpage in possibles )

            if fs is not None:
                request.fs = fs
                request.original_resource = request.resource
                request.resource = resources.get(request)
                response = request.resource.respond(request, response)

        return response
Пример #34
0
    def handle_error_nicely(self, tb_1, request):

        response = sys.exc_info()[1]

        if not isinstance(response, Response):

            # We have a true Exception; convert it to a Response object.

            response = Response(500, tb_1)
            response.request = request

        if 500 <= response.code < 600:
            # Log tracebacks for Reponse(5xx).
            aspen.log_dammit(tb_1)

            # TODO Switch to the logging module and use something like this:
            # log_level = [DEBUG,INFO,WARNING,ERROR][(response.code/100)-2]
            # logging.log(log_level, tb_1)

        if 200 <= response.code < 300 or response.code == 304:

            # The app raised a Response(2xx) or Response(304).
            # Act as if nothing happened. This is unusual but allowed.

            pass

        else:

            # Delegate to any error simplate.
            # ===============================

            rc = str(response.code)
            possibles = [ rc + ".html", rc + ".html.spt", "error.html", "error.html.spt" ]
            fs = first( self.ours_or_theirs(errpage) for errpage in possibles )

            if fs is not None:
                request.fs = fs
                request.original_resource = request.resource
                request.resource = resources.get(request)
                response = request.resource.respond(request, response)

        return response
Пример #35
0
def mail(env, project_root='.'):
    if env.aws_ses_access_key_id and env.aws_ses_secret_access_key and env.aws_ses_default_region:
        aspen.log_dammit("AWS SES is configured! We'll send mail through SES.")
        Participant._mailer = boto3.client(
            service_name='ses',
            region_name=env.aws_ses_default_region,
            aws_access_key_id=env.aws_ses_access_key_id,
            aws_secret_access_key=env.aws_ses_secret_access_key)
    else:
        aspen.log_dammit(
            "AWS SES is not configured! Mail will be dumped to the console here."
        )
        Participant._mailer = ConsoleMailer()
    emails = {}
    emails_dir = project_root + '/emails/'
    i = len(emails_dir)
    for spt in find_files(emails_dir, '*.spt'):
        base_name = spt[i:-4]
        emails[base_name] = compile_email_spt(spt)
    Participant._emails = emails
Пример #36
0
    def main(self, argv=None):
        """http://aspen.io/cli/
        """
        try:
            argv = argv if argv is not None else self.argv
            algorithm = self.get_algorithm()
            algorithm.run(argv=argv)
        except (SystemExit, KeyboardInterrupt):

            # Under some (most?) network engines, a SIGINT will be trapped by the
            # SIGINT signal handler above. However, gevent does "something" with
            # signals and our signal handler never fires. However, we *do* get a
            # KeyboardInterrupt here in that case. *shrug*
            #
            # See: https://github.com/gittip/aspen-python/issues/196

            pass
        except:
            import aspen, traceback
            aspen.log_dammit("Oh no! Aspen crashed!")
            aspen.log_dammit(traceback.format_exc())
Пример #37
0
def _do_execv():
    """Re-execute the current process.
    
    This must be called from the main thread, because certain platforms
    (OS X) don't allow execv to be called in a child thread very well.
    """
    args = sys.argv[:]
    aspen.log_dammit("Restarting %s." % ' '.join(args))
    
    if sys.platform[:4] == 'java':
        from _systemrestart import SystemRestart
        raise SystemRestart
    else:
        args.insert(0, sys.executable)
        if sys.platform == 'win32':
            args = ['"%s"' % arg for arg in args]

        os.chdir(_startup_cwd)
        if max_cloexec_files:
            _set_cloexec()
        os.execv(sys.executable, args)
Пример #38
0
def _do_execv():
    """Re-execute the current process.

    This must be called from the main thread, because certain platforms
    (OS X) don't allow execv to be called in a child thread very well.

    """
    args = sys.argv[:]
    aspen.log_dammit("Re-executing %s." % ' '.join(args))

    if sys.platform[:4] == 'java':
        from _systemrestart import SystemRestart
        raise SystemRestart
    else:
        args.insert(0, sys.executable)
        if sys.platform == 'win32':
            args = ['"%s"' % arg for arg in args]

        os.chdir(_startup_cwd)
        if max_cloexec_files:
            _set_cloexec()
        os.execv(sys.executable, args)
Пример #39
0
    def show_renderers(self):
        aspen.log_dammit("Renderers (*ed are unavailable, CAPS is default):")
        width = max(map(len, self.renderer_factories))
        for name, factory in self.renderer_factories.items():
            star, error = " ", ""
            if isinstance(factory, ImportError):
                star = "*"
                error = "ImportError: " + factory.args[0]
            if name == self.renderer_default:
                name = name.upper()
            name = name.ljust(width + 2)
            aspen.log_dammit(" %s%s%s" % (star, name, error))

        default_renderer = self.renderer_factories[self.renderer_default]
        if isinstance(default_renderer, ImportError):
            msg = "\033[1;31mImportError loading the default renderer, %s:\033[0m"
            aspen.log_dammit(msg % self.renderer_default)
            sys.excepthook(*default_renderer.info)
            raise default_renderer
Пример #40
0
 def SIGQUIT(signum, frame):
     aspen.log_dammit("Received QUIT, exiting.")
     raise SystemExit
Пример #41
0
    def tell_sentry(exception, request=None):

        # Decide if we care.
        # ==================

        if isinstance(exception, aspen.Response):

            if exception.code < 500:

                # Only log server errors to Sentry. For responses < 500 we use
                # stream-/line-based access logging. See discussion on:

                # https://github.com/gittip/www.gittip.com/pull/1560.

                return

        # Find a user.
        # ============
        # | is disallowed in usernames, so we can use it here to indicate
        # situations in which we can't get a username.

        request_context = getattr(request, 'context', None)
        user = {}
        user_id = 'n/a'
        if request_context is None:
            username = '******'
        else:
            user = request.context.get('user', None)
            if user is None:
                username = '******'
            else:
                is_anon = getattr(user, 'ANON', None)
                if is_anon is None:
                    username = '******'
                elif is_anon:
                    username = '******'
                else:
                    participant = getattr(user, 'participant', None)
                    if participant is None:
                        username = '******'
                    else:
                        username = getattr(user.participant, 'username', None)
                        if username is None:
                            username = '******'
                        else:
                            user_id = user.participant.id
                            username = username.encode('utf8')
                            user = {
                                'id':
                                user_id,
                                'is_admin':
                                user.participant.is_admin,
                                'is_suspicious':
                                user.participant.is_suspicious,
                                'claimed_time':
                                user.participant.claimed_time.isoformat(),
                                'url':
                                'https://www.gittip.com/{}/'.format(username)
                            }

        # Fire off a Sentry call.
        # =======================

        tags = {'username': username, 'user_id': user_id}
        extra = {
            'filepath': getattr(request, 'fs', None),
            'request': str(request).splitlines(),
            'user': user
        }
        result = sentry.captureException(tags=tags, extra=extra)

        # Emit a reference string to stdout.
        # ==================================

        ident = sentry.get_ident(result)
        aspen.log_dammit('Exception reference: ' + ident)
Пример #42
0
def envvars(website):

    missing_keys = []
    malformed_values = []

    def envvar(key, cast=None):
        if key not in os.environ:
            missing_keys.append(key)
            return ""
        value = os.environ[key].decode('ASCII')
        if cast is not None:
            try:
                value = cast(value)
            except:
                err = str(sys.exc_info()[1])
                malformed_values.append((key, err))
                return ""
        return value

    def is_yesish(val):
        return val.lower() in ('1', 'true', 'yes')

    signin_platforms = [
        Twitter(
            website.db,
            envvar('TWITTER_CONSUMER_KEY'),
            envvar('TWITTER_CONSUMER_SECRET'),
            envvar('TWITTER_CALLBACK'),
        ),
        GitHub(
            website.db,
            envvar('GITHUB_CLIENT_ID'),
            envvar('GITHUB_CLIENT_SECRET'),
            envvar('GITHUB_CALLBACK'),
        ),
        Bitbucket(
            website.db,
            envvar('BITBUCKET_CONSUMER_KEY'),
            envvar('BITBUCKET_CONSUMER_SECRET'),
            envvar('BITBUCKET_CALLBACK'),
        ),
        OpenStreetMap(
            website.db,
            envvar('OPENSTREETMAP_CONSUMER_KEY'),
            envvar('OPENSTREETMAP_CONSUMER_SECRET'),
            envvar('OPENSTREETMAP_CALLBACK'),
            envvar('OPENSTREETMAP_API_URL'),
            envvar('OPENSTREETMAP_AUTH_URL'),
        ),
    ]
    website.signin_platforms = PlatformRegistry(signin_platforms)
    AccountElsewhere.signin_platforms_names = tuple(p.name
                                                    for p in signin_platforms)
    other_platforms = [
        Bountysource(
            website.db,
            None,
            envvar('BOUNTYSOURCE_API_SECRET'),
            envvar('BOUNTYSOURCE_CALLBACK'),
            envvar('BOUNTYSOURCE_API_HOST'),
            envvar('BOUNTYSOURCE_WWW_HOST'),
        ),
        Venmo(
            website.db,
            envvar('VENMO_CLIENT_ID'),
            envvar('VENMO_CLIENT_SECRET'),
            envvar('VENMO_CALLBACK'),
        ),
    ]
    platforms = signin_platforms + other_platforms
    website.platforms = AccountElsewhere.platforms = PlatformRegistry(
        platforms)

    website.asset_version_url = envvar('GITTIP_ASSET_VERSION_URL') \
                                      .replace('%version', website.version)
    website.asset_url = envvar('GITTIP_ASSET_URL')
    website.cache_static = is_yesish(envvar('GITTIP_CACHE_STATIC'))
    website.compress_assets = is_yesish(envvar('GITTIP_COMPRESS_ASSETS'))

    website.google_analytics_id = envvar('GOOGLE_ANALYTICS_ID')
    website.sentry_dsn = envvar('SENTRY_DSN')

    website.min_threads = envvar('MIN_THREADS', int)
    website.log_busy_threads_every = envvar('LOG_BUSY_THREADS_EVERY', int)
    website.log_metrics = is_yesish(envvar('LOG_METRICS'))

    if malformed_values:
        malformed_values.sort()
        these = len(malformed_values) != 1 and 'these' or 'this'
        plural = len(malformed_values) != 1 and 's' or ''
        aspen.log_dammit("=" * 42)
        aspen.log_dammit("Oh no! Gittip.com couldn't understand %s " % these,
                         "environment variable%s:" % plural)
        aspen.log_dammit(" ")
        for key, err in malformed_values:
            aspen.log_dammit("  {} ({})".format(key, err))
        aspen.log_dammit(" ")
        aspen.log_dammit("See ./default_local.env for hints.")

        aspen.log_dammit("=" * 42)
        keys = ', '.join([key for key in malformed_values])
        raise BadEnvironment("Malformed envvar{}: {}.".format(plural, keys))

    if missing_keys:
        missing_keys.sort()
        these = len(missing_keys) != 1 and 'these' or 'this'
        plural = len(missing_keys) != 1 and 's' or ''
        aspen.log_dammit("=" * 42)
        aspen.log_dammit("Oh no! Gittip.com needs %s missing " % these,
                         "environment variable%s:" % plural)
        aspen.log_dammit(" ")
        for key in missing_keys:
            aspen.log_dammit("  " + key)
        aspen.log_dammit(" ")
        aspen.log_dammit("(Sorry, we must've started looking for ",
                         "%s since you last updated Gittip!)" % these)
        aspen.log_dammit(" ")
        aspen.log_dammit("Running Gittip locally? Edit ./local.env.")
        aspen.log_dammit("Running the test suite? Edit ./tests/env.")
        aspen.log_dammit(" ")
        aspen.log_dammit("See ./default_local.env for hints.")

        aspen.log_dammit("=" * 42)
        keys = ', '.join([key for key in missing_keys])
        raise BadEnvironment("Missing envvar{}: {}.".format(plural, keys))
Пример #43
0
def env():
    env = Environment(
        DATABASE_URL=unicode,
        CANONICAL_HOST=unicode,
        CANONICAL_SCHEME=unicode,
        DATABASE_MAXCONN=int,
        GRATIPAY_ASSET_URL=unicode,
        GRATIPAY_CACHE_STATIC=is_yesish,
        GRATIPAY_COMPRESS_ASSETS=is_yesish,
        BALANCED_API_SECRET=unicode,
        GITHUB_CLIENT_ID=unicode,
        GITHUB_CLIENT_SECRET=unicode,
        GITHUB_CALLBACK=unicode,
        BITBUCKET_CONSUMER_KEY=unicode,
        BITBUCKET_CONSUMER_SECRET=unicode,
        BITBUCKET_CALLBACK=unicode,
        TWITTER_CONSUMER_KEY=unicode,
        TWITTER_CONSUMER_SECRET=unicode,
        TWITTER_CALLBACK=unicode,
        FACEBOOK_APP_ID=unicode,
        FACEBOOK_APP_SECRET=unicode,
        FACEBOOK_CALLBACK=unicode,
        GOOGLE_CLIENT_ID=unicode,
        GOOGLE_CLIENT_SECRET=unicode,
        GOOGLE_CALLBACK=unicode,
        BOUNTYSOURCE_API_SECRET=unicode,
        BOUNTYSOURCE_CALLBACK=unicode,
        BOUNTYSOURCE_API_HOST=unicode,
        BOUNTYSOURCE_WWW_HOST=unicode,
        VENMO_CLIENT_ID=unicode,
        VENMO_CLIENT_SECRET=unicode,
        VENMO_CALLBACK=unicode,
        OPENSTREETMAP_CONSUMER_KEY=unicode,
        OPENSTREETMAP_CONSUMER_SECRET=unicode,
        OPENSTREETMAP_CALLBACK=unicode,
        OPENSTREETMAP_API_URL=unicode,
        OPENSTREETMAP_AUTH_URL=unicode,
        UPDATE_GLOBAL_STATS_EVERY=int,
        CHECK_DB_EVERY=int,
        DEQUEUE_EMAILS_EVERY=int,
        OPTIMIZELY_ID=unicode,
        SENTRY_DSN=unicode,
        LOG_METRICS=is_yesish,
        INCLUDE_PIWIK=is_yesish,
        MANDRILL_KEY=unicode,
        RAISE_SIGNIN_NOTIFICATIONS=is_yesish,

        # This is used in our Procfile. (PORT is also used but is provided by
        # Heroku; we don't set it ourselves in our app config.)
        GUNICORN_OPTS=unicode,
    )

    # Error Checking
    # ==============

    if env.malformed:
        these = len(env.malformed) != 1 and 'these' or 'this'
        plural = len(env.malformed) != 1 and 's' or ''
        aspen.log_dammit("=" * 42)
        aspen.log_dammit("Oh no! Gratipay.com couldn't understand %s " % these,
                         "environment variable%s:" % plural)
        aspen.log_dammit(" ")
        for key, err in env.malformed:
            aspen.log_dammit("  {} ({})".format(key, err))
        aspen.log_dammit(" ")
        aspen.log_dammit("See ./default_local.env for hints.")

        aspen.log_dammit("=" * 42)
        keys = ', '.join([key for key in env.malformed])
        raise BadEnvironment("Malformed envvar{}: {}.".format(plural, keys))

    if env.missing:
        these = len(env.missing) != 1 and 'these' or 'this'
        plural = len(env.missing) != 1 and 's' or ''
        aspen.log_dammit("=" * 42)
        aspen.log_dammit("Oh no! Gratipay.com needs %s missing " % these,
                         "environment variable%s:" % plural)
        aspen.log_dammit(" ")
        for key in env.missing:
            aspen.log_dammit("  " + key)
        aspen.log_dammit(" ")
        aspen.log_dammit("(Sorry, we must've started looking for ",
                         "%s since you last updated Gratipay!)" % these)
        aspen.log_dammit(" ")
        aspen.log_dammit("Running Gratipay locally? Edit ./local.env.")
        aspen.log_dammit("Running the test suite? Edit ./tests/env.")
        aspen.log_dammit(" ")
        aspen.log_dammit("See ./default_local.env for hints.")

        aspen.log_dammit("=" * 42)
        keys = ', '.join([key for key in env.missing])
        raise BadEnvironment("Missing envvar{}: {}.".format(plural, keys))

    return env
Пример #44
0
def make_sentry_teller(env):
    if not env.sentry_dsn:
        aspen.log_dammit("Won't log to Sentry (SENTRY_DSN is empty).")

        def noop(*a, **kw):
            pass

        Participant._tell_sentry = noop
        return noop

    sentry = raven.Client(env.sentry_dsn)

    def tell_sentry(exception, state):

        # Decide if we care.
        # ==================

        if isinstance(exception, aspen.Response):

            if exception.code < 500:

                # Only log server errors to Sentry. For responses < 500 we use
                # stream-/line-based access logging. See discussion on:

                # https://github.com/gratipay/gratipay.com/pull/1560.

                return

        # Find a user.
        # ============
        # | is disallowed in usernames, so we can use it here to indicate
        # situations in which we can't get a username.

        user = state.get('user')
        user_id = 'n/a'
        if user is None:
            username = '******'
        else:
            is_anon = getattr(user, 'ANON', None)
            if is_anon is None:
                username = '******'
            elif is_anon:
                username = '******'
            else:
                participant = getattr(user, 'participant', None)
                if participant is None:
                    username = '******'
                else:
                    username = getattr(user.participant, 'username', None)
                    if username is None:
                        username = '******'
                    else:
                        user_id = user.participant.id
                        username = username.encode('utf8')
                        user = {
                            'id':
                            user_id,
                            'is_admin':
                            user.participant.is_admin,
                            'is_suspicious':
                            user.participant.is_suspicious,
                            'claimed_time':
                            user.participant.claimed_time.isoformat(),
                            'url':
                            'https://gratipay.com/{}/'.format(username)
                        }

        # Fire off a Sentry call.
        # =======================

        dispatch_result = state.get('dispatch_result')
        request = state.get('request')
        tags = {'username': username, 'user_id': user_id}
        extra = {
            'filepath': getattr(dispatch_result, 'match', None),
            'request': str(request).splitlines(),
            'user': user
        }
        result = sentry.captureException(tags=tags, extra=extra)

        # Emit a reference string to stdout.
        # ==================================

        ident = sentry.get_ident(result)
        aspen.log_dammit('Exception reference: ' + ident)

    Participant._tell_sentry = tell_sentry
    return tell_sentry
Пример #45
0
def update_homepage_queries_once(db):
    with db.get_cursor() as cursor:
        log_dammit("updating homepage queries")
        start = time.time()
        cursor.execute("DELETE FROM homepage_top_givers")
        cursor.execute("""

        INSERT INTO homepage_top_givers
            SELECT tipper AS username, anonymous, sum(amount) AS amount
              FROM (    SELECT DISTINCT ON (tipper, tippee)
                               amount
                             , tipper
                          FROM tips
                          JOIN participants p ON p.username = tipper
                          JOIN participants p2 ON p2.username = tippee
                          JOIN elsewhere ON elsewhere.participant = tippee
                         WHERE p.last_bill_result = ''
                           AND p.is_suspicious IS NOT true
                           AND p2.claimed_time IS NOT NULL
                           AND elsewhere.is_locked = false
                      ORDER BY tipper, tippee, mtime DESC
                      ) AS foo
              JOIN participants p ON p.username = tipper
             WHERE is_suspicious IS NOT true
          GROUP BY tipper, anonymous
          ORDER BY amount DESC;

        """.strip())
        cursor.execute("""

        UPDATE homepage_top_givers
           SET gravatar_id = ( SELECT user_info->'gravatar_id'
                                 FROM elsewhere
                                WHERE participant=username
                                  AND platform='github'
                              )
        """)
        cursor.execute("""

        UPDATE homepage_top_givers
           SET twitter_pic = ( SELECT user_info->'profile_image_url_https'
                                 FROM elsewhere
                                WHERE participant=username
                                  AND platform='twitter'
                              )
        """)

        cursor.execute("DELETE FROM homepage_top_receivers")
        cursor.execute("""

        INSERT INTO homepage_top_receivers
            SELECT tippee AS username, claimed_time, sum(amount) AS amount
              FROM (    SELECT DISTINCT ON (tipper, tippee)
                               amount
                             , tippee
                          FROM tips
                          JOIN participants p ON p.username = tipper
                          JOIN elsewhere ON elsewhere.participant = tippee
                         WHERE last_bill_result = ''
                           AND elsewhere.is_locked = false
                           AND is_suspicious IS NOT true
                           AND claimed_time IS NOT null
                      ORDER BY tipper, tippee, mtime DESC
                      ) AS foo
              JOIN participants p ON p.username = tippee
             WHERE is_suspicious IS NOT true
          GROUP BY tippee, claimed_time
          ORDER BY amount DESC;

        """.strip())
        cursor.execute("""

        UPDATE homepage_top_receivers
           SET gravatar_id = ( SELECT user_info->'gravatar_id'
                                 FROM elsewhere
                                WHERE participant=username
                                  AND platform='github'
                              )
        """)
        cursor.execute("""

        UPDATE homepage_top_receivers
           SET twitter_pic = ( SELECT user_info->'profile_image_url_https'
                                 FROM elsewhere
                                WHERE participant=username
                                  AND platform='twitter'
                              )
        """)
        end = time.time()
        elapsed = end - start
        log_dammit("updated homepage queries in %.2f seconds" % elapsed)
Пример #46
0
 def stop(self):
     aspen.log_dammit("Shutting down Aspen website.")
     self.hooks.shutdown.run(self)
     self.network_engine.stop()
Пример #47
0
def update_homepage_queries_once(db):
    with db.get_cursor() as cursor:
        log_dammit("updating homepage queries")
        start = time.time()
        cursor.execute("DELETE FROM homepage_top_givers")
        cursor.execute(
            """

        INSERT INTO homepage_top_givers
            SELECT tipper AS username, anonymous, sum(amount) AS amount
              FROM (    SELECT DISTINCT ON (tipper, tippee)
                               amount
                             , tipper
                          FROM tips
                          JOIN participants p ON p.username = tipper
                          JOIN participants p2 ON p2.username = tippee
                          JOIN elsewhere ON elsewhere.participant = tippee
                         WHERE p.last_bill_result = ''
                           AND p.is_suspicious IS NOT true
                           AND p2.claimed_time IS NOT NULL
                           AND elsewhere.is_locked = false
                      ORDER BY tipper, tippee, mtime DESC
                      ) AS foo
              JOIN participants p ON p.username = tipper
             WHERE is_suspicious IS NOT true
          GROUP BY tipper, anonymous
          ORDER BY amount DESC;

        """.strip()
        )
        cursor.execute(
            """

        UPDATE homepage_top_givers
           SET gravatar_id = ( SELECT user_info->'gravatar_id'
                                 FROM elsewhere
                                WHERE participant=username
                                  AND platform='github'
                              )
        """
        )
        cursor.execute(
            """

        UPDATE homepage_top_givers
           SET twitter_pic = ( SELECT user_info->'profile_image_url_https'
                                 FROM elsewhere
                                WHERE participant=username
                                  AND platform='twitter'
                              )
        """
        )

        cursor.execute("DELETE FROM homepage_top_receivers")
        cursor.execute(
            """

        INSERT INTO homepage_top_receivers
            SELECT tippee AS username, claimed_time, sum(amount) AS amount
              FROM (    SELECT DISTINCT ON (tipper, tippee)
                               amount
                             , tippee
                          FROM tips
                          JOIN participants p ON p.username = tipper
                          JOIN elsewhere ON elsewhere.participant = tippee
                         WHERE last_bill_result = ''
                           AND elsewhere.is_locked = false
                           AND is_suspicious IS NOT true
                           AND claimed_time IS NOT null
                      ORDER BY tipper, tippee, mtime DESC
                      ) AS foo
              JOIN participants p ON p.username = tippee
             WHERE is_suspicious IS NOT true
          GROUP BY tippee, claimed_time
          ORDER BY amount DESC;

        """.strip()
        )
        cursor.execute(
            """

        UPDATE homepage_top_receivers
           SET gravatar_id = ( SELECT user_info->'gravatar_id'
                                 FROM elsewhere
                                WHERE participant=username
                                  AND platform='github'
                              )
        """
        )
        cursor.execute(
            """

        UPDATE homepage_top_receivers
           SET twitter_pic = ( SELECT user_info->'profile_image_url_https'
                                 FROM elsewhere
                                WHERE participant=username
                                  AND platform='twitter'
                              )
        """
        )
        end = time.time()
        elapsed = end - start
        log_dammit("updated homepage queries in %.2f seconds" % elapsed)
Пример #48
0
def update_homepage_queries_once(db):
    with db.get_cursor() as cursor:
        log_dammit("updating homepage queries")
        start = time.time()
        cursor.execute("""

        DROP TABLE IF EXISTS _homepage_new_participants;
        CREATE TABLE _homepage_new_participants AS
              SELECT username, claimed_time FROM (
                  SELECT DISTINCT ON (p.username)
                         p.username
                       , claimed_time
                    FROM participants p
                    JOIN elsewhere e
                      ON p.username = participant
                   WHERE claimed_time IS NOT null
                     AND is_suspicious IS NOT true
                     ) AS foo
            ORDER BY claimed_time DESC;

        DROP TABLE IF EXISTS _homepage_top_givers;
        CREATE TABLE _homepage_top_givers AS
            SELECT tipper AS username, anonymous, sum(amount) AS amount
              FROM (    SELECT DISTINCT ON (tipper, tippee)
                               amount
                             , tipper
                          FROM tips
                          JOIN participants p ON p.username = tipper
                          JOIN participants p2 ON p2.username = tippee
                          JOIN elsewhere ON elsewhere.participant = tippee
                         WHERE p.last_bill_result = ''
                           AND p.is_suspicious IS NOT true
                           AND p2.claimed_time IS NOT NULL
                           AND elsewhere.is_locked = false
                      ORDER BY tipper, tippee, mtime DESC
                      ) AS foo
              JOIN participants p ON p.username = tipper
             WHERE is_suspicious IS NOT true
          GROUP BY tipper, anonymous
          ORDER BY amount DESC;

        DROP TABLE IF EXISTS _homepage_top_receivers;
        CREATE TABLE _homepage_top_receivers AS
            SELECT tippee AS username, claimed_time, sum(amount) AS amount
              FROM (    SELECT DISTINCT ON (tipper, tippee)
                               amount
                             , tippee
                          FROM tips
                          JOIN participants p ON p.username = tipper
                          JOIN elsewhere ON elsewhere.participant = tippee
                         WHERE last_bill_result = ''
                           AND elsewhere.is_locked = false
                           AND is_suspicious IS NOT true
                           AND claimed_time IS NOT null
                      ORDER BY tipper, tippee, mtime DESC
                      ) AS foo
              JOIN participants p ON p.username = tippee
             WHERE is_suspicious IS NOT true
          GROUP BY tippee, claimed_time
          ORDER BY amount DESC;

        DROP TABLE IF EXISTS homepage_new_participants;
        ALTER TABLE _homepage_new_participants
          RENAME TO homepage_new_participants;

        DROP TABLE IF EXISTS homepage_top_givers;
        ALTER TABLE _homepage_top_givers
          RENAME TO homepage_top_givers;

        DROP TABLE IF EXISTS homepage_top_receivers;
        ALTER TABLE _homepage_top_receivers
          RENAME TO homepage_top_receivers;

        """)
        end = time.time()
        elapsed = end - start
        log_dammit("updated homepage queries in %.2f seconds" % elapsed)
Пример #49
0
 def stop(self):
     aspen.log_dammit("Shutting down Aspen website.")
     self.hooks.run('shutdown', self)
     self.network_engine.stop()
Пример #50
0
def envvars(website):

    missing_keys = []
    malformed_values = []

    def envvar(key, cast=None):
        if key not in os.environ:
            missing_keys.append(key)
            return ""
        value = os.environ[key].decode('ASCII')
        if cast is not None:
            try:
                value = cast(value)
            except:
                err = str(sys.exc_info()[1])
                malformed_values.append((key, err))
                return ""
        return value

    def is_yesish(val):
        return val.lower() in ('1', 'true', 'yes')

    website.bitbucket_consumer_key = envvar('BITBUCKET_CONSUMER_KEY')
    website.bitbucket_consumer_secret = envvar('BITBUCKET_CONSUMER_SECRET')
    website.bitbucket_callback = envvar('BITBUCKET_CALLBACK')

    website.github_client_id = envvar('GITHUB_CLIENT_ID')
    website.github_client_secret = envvar('GITHUB_CLIENT_SECRET')
    website.github_callback = envvar('GITHUB_CALLBACK')

    website.twitter_consumer_key = envvar('TWITTER_CONSUMER_KEY')
    website.twitter_consumer_secret = envvar('TWITTER_CONSUMER_SECRET')
    website.twitter_access_token = envvar('TWITTER_ACCESS_TOKEN')
    website.twitter_access_token_secret = envvar('TWITTER_ACCESS_TOKEN_SECRET')
    website.twitter_callback = envvar('TWITTER_CALLBACK')

    website.bountysource_www_host = envvar('BOUNTYSOURCE_WWW_HOST')
    website.bountysource_api_host = envvar('BOUNTYSOURCE_API_HOST')
    website.bountysource_api_secret = envvar('BOUNTYSOURCE_API_SECRET')
    website.bountysource_callback = envvar('BOUNTYSOURCE_CALLBACK')

    website.css_href = envvar('GITTIP_CSS_HREF') \
                                          .replace('%version', website.version)
    website.js_src = envvar('GITTIP_JS_SRC') \
                                          .replace('%version', website.version)
    website.cache_static = is_yesish(envvar('GITTIP_CACHE_STATIC'))

    website.google_analytics_id = envvar('GOOGLE_ANALYTICS_ID')
    website.gauges_id = envvar('GAUGES_ID')
    website.sentry_dsn = envvar('SENTRY_DSN')

    website.min_threads = envvar('MIN_THREADS', int)
    website.log_busy_threads_every = envvar('LOG_BUSY_THREADS_EVERY', int)
    website.log_metrics = is_yesish(envvar('LOG_METRICS'))

    if malformed_values:
        malformed_values.sort()
        these = len(malformed_values) != 1 and 'these' or 'this'
        plural = len(malformed_values) != 1 and 's' or ''
        aspen.log_dammit("=" * 42)
        aspen.log_dammit( "Oh no! Gittip.com couldn't understand %s " % these
                        , "environment variable%s:" % plural
                         )
        aspen.log_dammit(" ")
        for key, err in malformed_values:
            aspen.log_dammit("  {} ({})".format(key, err))
        aspen.log_dammit(" ")
        aspen.log_dammit("See ./default_local.env for hints.")

        aspen.log_dammit("=" * 42)
        raise SystemExit

    if missing_keys:
        missing_keys.sort()
        these = len(missing_keys) != 1 and 'these' or 'this'
        plural = len(missing_keys) != 1 and 's' or ''
        aspen.log_dammit("=" * 42)
        aspen.log_dammit( "Oh no! Gittip.com needs %s missing " % these
                        , "environment variable%s:" % plural
                         )
        aspen.log_dammit(" ")
        for key in missing_keys:
            aspen.log_dammit("  " + key)
        aspen.log_dammit(" ")
        aspen.log_dammit( "(Sorry, we must've started looking for "
                        , "%s since you last updated Gittip!)" % these
                         )
        aspen.log_dammit(" ")
        aspen.log_dammit("Running Gittip locally? Edit ./local.env.")
        aspen.log_dammit("Running the test suite? Edit ./tests/env.")
        aspen.log_dammit(" ")
        aspen.log_dammit("See ./default_local.env for hints.")

        aspen.log_dammit("=" * 42)
        raise SystemExit
Пример #51
0
 def SIGHUP(signum, frame):
     aspen.log_dammit("Received HUP, re-executing.")
     execution.execute()
Пример #52
0
    # only thing missing will be the response.body which is not logged.
    try:
        import http.client as http_client
    except ImportError:
        # Python 2
        import httplib as http_client
    http_client.HTTPConnection.debuglevel = 1

    # You must initialize logging, otherwise you'll not see debug output.
    logging.basicConfig()
    logging.getLogger().setLevel(logging.DEBUG)
    requests_log = logging.getLogger("requests.packages.urllib3")
    requests_log.setLevel(logging.DEBUG)
    requests_log.propagate = True


# Provide a decorator to skip tests when marky-markdown is missing.

try:
    markdown.render_like_npm('test')
except OSError as exc:
    MISSING_MARKY_MARKDOWN = True
    log_dammit('Will skip marky-markdown-related tests because:', exc.args[0])
else:
    MISSING_MARKY_MARKDOWN = False


def skipif_missing_marky_markdown(func):
    return pytest.mark.skipif(MISSING_MARKY_MARKDOWN,
                              reason="missing marky-markdown")(func)
Пример #53
0
def SIGHUP(signum, frame):
    aspen.log_dammit("Received HUP, restarting.")
    restart()
Пример #54
0
suitable for development and testing.  It can be invoked via:

    python -m aspen

though even for development you'll likely want to specify a
project root, so a more likely incantation is:

    ASPEN_PROJECT_ROOT=/path/to/wherever python -m aspen

For production deployment, you should probably deploy using
a higher performance WSGI server like Gunicorn, uwsgi, Spawning,
or the like.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import os
from aspen import log_dammit
from aspen.website import Website
from wsgiref.simple_server import make_server


if __name__ == '__main__':
    website = Website()
    port = int(os.environ.get('PORT', '8080'))
    server = make_server('0.0.0.0', port, website)
    log_dammit("Greetings, program! Welcome to port {0}.".format(port))
    server.serve_forever()
Пример #55
0
            except OSError, err:
                if err.errno != errno.ENOENT:
                    raise
                raise ConfigurationError(errorstr)



        # LOGGING_THRESHOLD
        # -----------------
        # This is initially set to -1 and not 0 so that we can tell if the user
        # changed it programmatically directly before we got here. I do this in
        # the testing module, that's really what this is about.
        if aspen.logging.LOGGING_THRESHOLD == -1:
            aspen.logging.LOGGING_THRESHOLD = self.logging_threshold
        # Now that we know the user's desires, we can log appropriately.
        aspen.log_dammit(os.linesep.join(msgs))

        # project root
        if self.project_root is None:
            aspen.log_dammit("project_root not configured (no template bases, "
                             "etc.).")
        else:
            # canonicalize it
            if not os.path.isabs(self.project_root):
                aspen.log_dammit("project_root is relative to CWD: '%s'."
                                 % self.project_root)
                cwd = safe_getcwd("Could not get a current working "
                                  "directory. You can specify "
                                  "ASPEN_PROJECT_ROOT in the environment, "
                                  "or --project_root on the command line.")
                self.project_root = os.path.join(cwd, self.project_root)
Пример #56
0
def env():
    env = Environment(
        DATABASE_URL=unicode,
        CANONICAL_HOST=unicode,
        CANONICAL_SCHEME=unicode,
        MIN_THREADS=int,
        DATABASE_MAXCONN=int,
        GITTIP_ASSET_URL=unicode,
        GITTIP_CACHE_STATIC=is_yesish,
        GITTIP_COMPRESS_ASSETS=is_yesish,
        STRIPE_SECRET_API_KEY=unicode,
        STRIPE_PUBLISHABLE_API_KEY=unicode,
        BALANCED_API_SECRET=unicode,
        #DEBUG                           = unicode,
        GITHUB_CLIENT_ID=unicode,
        GITHUB_CLIENT_SECRET=unicode,
        GITHUB_CALLBACK=unicode,
        BITBUCKET_CONSUMER_KEY=unicode,
        BITBUCKET_CONSUMER_SECRET=unicode,
        BITBUCKET_CALLBACK=unicode,
        TWITTER_CONSUMER_KEY=unicode,
        TWITTER_CONSUMER_SECRET=unicode,
        TWITTER_CALLBACK=unicode,
        BOUNTYSOURCE_API_SECRET=unicode,
        BOUNTYSOURCE_CALLBACK=unicode,
        BOUNTYSOURCE_API_HOST=unicode,
        BOUNTYSOURCE_WWW_HOST=unicode,
        VENMO_CLIENT_ID=unicode,
        VENMO_CLIENT_SECRET=unicode,
        VENMO_CALLBACK=unicode,
        OPENSTREETMAP_CONSUMER_KEY=unicode,
        OPENSTREETMAP_CONSUMER_SECRET=unicode,
        OPENSTREETMAP_CALLBACK=unicode,
        OPENSTREETMAP_API_URL=unicode,
        OPENSTREETMAP_AUTH_URL=unicode,
        NANSWERS_THRESHOLD=int,
        UPDATE_HOMEPAGE_EVERY=int,
        GOOGLE_ANALYTICS_ID=unicode,
        SENTRY_DSN=unicode,
        LOG_BUSY_THREADS_EVERY=int,
        LOG_METRICS=is_yesish,
        MANDRILL_KEY=unicode,
    )

    # Error Checking
    # ==============

    if env.malformed:
        these = len(env.malformed) != 1 and 'these' or 'this'
        plural = len(env.malformed) != 1 and 's' or ''
        aspen.log_dammit("=" * 42)
        aspen.log_dammit("Oh no! Gittip.com couldn't understand %s " % these,
                         "environment variable%s:" % plural)
        aspen.log_dammit(" ")
        for key, err in env.malformed:
            aspen.log_dammit("  {} ({})".format(key, err))
        aspen.log_dammit(" ")
        aspen.log_dammit("See ./default_local.env for hints.")

        aspen.log_dammit("=" * 42)
        keys = ', '.join([key for key in env.malformed])
        raise BadEnvironment("Malformed envvar{}: {}.".format(plural, keys))

    if env.missing:
        these = len(env.missing) != 1 and 'these' or 'this'
        plural = len(env.missing) != 1 and 's' or ''
        aspen.log_dammit("=" * 42)
        aspen.log_dammit("Oh no! Gittip.com needs %s missing " % these,
                         "environment variable%s:" % plural)
        aspen.log_dammit(" ")
        for key in env.missing:
            aspen.log_dammit("  " + key)
        aspen.log_dammit(" ")
        aspen.log_dammit("(Sorry, we must've started looking for ",
                         "%s since you last updated Gittip!)" % these)
        aspen.log_dammit(" ")
        aspen.log_dammit("Running Gittip locally? Edit ./local.env.")
        aspen.log_dammit("Running the test suite? Edit ./tests/env.")
        aspen.log_dammit(" ")
        aspen.log_dammit("See ./default_local.env for hints.")

        aspen.log_dammit("=" * 42)
        keys = ', '.join([key for key in env.missing])
        raise BadEnvironment("Missing envvar{}: {}.".format(plural, keys))

    return env
Пример #57
0
def _main(argv):

    # Do imports.
    # ===========
    # These are in here so that if you Ctrl-C during an import, the
    # KeyboardInterrupt is caught and ignored. Yes, that's how much I care.
    # No, I don't care enough to put aspen/__init__.py in here too.

    import os
    import signal
    import socket
    import sys
    import traceback

    import aspen
    from aspen import execution
    from aspen.website import Website


    # Set up signal handling.
    # =======================

    def SIGHUP(signum, frame):
        aspen.log_dammit("Received HUP, re-executing.")
        execution.execute()
    if not aspen.WINDOWS:
        signal.signal(signal.SIGHUP, SIGHUP)

    def SIGINT(signum, frame):
        aspen.log_dammit("Received INT, exiting.")
        raise SystemExit
    signal.signal(signal.SIGINT, SIGINT)


    def SIGQUIT(signum, frame):
        aspen.log_dammit("Received QUIT, exiting.")
        raise SystemExit
    if not aspen.WINDOWS:
        signal.signal(signal.SIGQUIT, SIGQUIT)


    # Website
    # =======
    # User-developers get this website object inside of their resources and
    # hooks. It provides access to configuration information in addition to
    # being a WSGI callable and holding the request/response handling
    # logic. See aspen/website.py

    if argv is None:
        argv = sys.argv[1:]
    website = Website(argv)


    # Start serving the website.
    # ==========================
    # This amounts to binding the requested socket, with logging and
    # restarting as needed. Wrap the whole thing in a try/except to
    # do some cleanup on shutdown.

    try:
        if hasattr(socket, 'AF_UNIX'):
            if website.network_sockfam == socket.AF_UNIX:
                if os.path.exists(website.network_address):
                    aspen.log("Removing stale socket.")
                    os.remove(website.network_address)
        if website.network_port is not None:
            welcome = "port %d" % website.network_port
        else:
            welcome = website.network_address
        aspen.log("Starting %s engine." % website.network_engine.name)
        website.network_engine.bind()
        aspen.log_dammit("Greetings, program! Welcome to %s." % welcome)
        if website.changes_reload:
            aspen.log("Aspen will restart when configuration scripts or "
                      "Python modules change.")
            execution.install(website)
        website.start()

    except socket.error:

        # Be friendly about port conflicts.
        # =================================

        # The traceback one gets from a port conflict or permission error
        # is not that friendly. Here's a helper to let the user know (in
        # color?!) that a port conflict or a permission error is probably
        # the problem. But in case it isn't (website.start fires the start
        # hook, and maybe the user tries to connect to a network service in
        # there?), don't fully swallow the exception. Also, be explicit
        # about the port number. What if they have logging turned off? Then
        # they won't see the port number in the "Greetings, program!" line.
        # They definitely won't see it if using an engine like eventlet
        # that binds to the port early.

        if website.network_port is not None:
            msg = "Is something already running on port %s? Because ..."
            if not aspen.WINDOWS:
                if website.network_port < 1024:
                    if os.geteuid() > 0:
                        msg = ("Do you have permission to bind to port %s?"
                               " Because ...")
            msg %= website.network_port
            if not aspen.WINDOWS:
                # Assume we can use ANSI color escapes if not on Windows.
                # XXX Maybe a bad assumption if this is going into a log
                # file? See also: colorama
                msg = '\033[01;33m%s\033[00m' % msg
            aspen.log_dammit(msg)
        raise
    except (KeyboardInterrupt, SystemExit):
        raise  # Don't bother logging these.
    except:
        aspen.log_dammit(traceback.format_exc())
    finally:
        if hasattr(socket, 'AF_UNIX'):
            if website.network_sockfam == socket.AF_UNIX:
                if os.path.exists(website.network_address):
                    os.remove(website.network_address)
        website.stop()
Пример #58
0
def envvars(website):

    missing_keys = []

    def envvar(key):
        if key not in os.environ:
            missing_keys.append(key)
            return ""
        return os.environ[key].decode('ASCII')

    def is_yesish(val):
        return val.lower() in ('1', 'true', 'yes')

    website.bitbucket_consumer_key = envvar('BITBUCKET_CONSUMER_KEY')
    website.bitbucket_consumer_secret = envvar('BITBUCKET_CONSUMER_SECRET')
    website.bitbucket_callback = envvar('BITBUCKET_CALLBACK')

    website.github_client_id = envvar('GITHUB_CLIENT_ID')
    website.github_client_secret = envvar('GITHUB_CLIENT_SECRET')
    website.github_callback = envvar('GITHUB_CALLBACK')

    website.twitter_consumer_key = envvar('TWITTER_CONSUMER_KEY')
    website.twitter_consumer_secret = envvar('TWITTER_CONSUMER_SECRET')
    website.twitter_access_token = envvar('TWITTER_ACCESS_TOKEN')
    website.twitter_access_token_secret = envvar('TWITTER_ACCESS_TOKEN_SECRET')
    website.twitter_callback = envvar('TWITTER_CALLBACK')

    website.bountysource_www_host = envvar('BOUNTYSOURCE_WWW_HOST')
    website.bountysource_api_host = envvar('BOUNTYSOURCE_API_HOST')
    website.bountysource_api_secret = envvar('BOUNTYSOURCE_API_SECRET')
    website.bountysource_callback = envvar('BOUNTYSOURCE_CALLBACK')

    website.css_href = envvar('GITTIP_CSS_HREF') \
                                          .replace('%version', website.version)
    website.cache_static = is_yesish(envvar('GITTIP_CACHE_STATIC'))

    if missing_keys:
        missing_keys.sort()
        these = len(missing_keys) != 1 and 'these' or 'this'
        plural = len(missing_keys) != 1 and 's' or ''
        aspen.log_dammit("=" * 42)
        aspen.log_dammit("Oh no! Gittip.com needs %s missing " % these,
                         "environment variable%s:" % plural)
        aspen.log_dammit(" ")
        for key in missing_keys:
            aspen.log_dammit("  " + key)
        aspen.log_dammit(" ")
        aspen.log_dammit("(Sorry, we must've started looking for ",
                         "%s since you last updated Gittip!)" % these)
        aspen.log_dammit(" ")
        aspen.log_dammit("Running Gittip locally? Edit ./local.env.")
        aspen.log_dammit("Running the test suite? Edit ./tests/env.")
        aspen.log_dammit(" ")
        aspen.log_dammit("See ./default_local.env for hints.")

        aspen.log_dammit("=" * 42)
        raise SystemExit
Пример #59
0
def envvars(website):

    missing_keys = []

    def envvar(key):
        if key not in os.environ:
            missing_keys.append(key)
            return ""
        return os.environ[key].decode('ASCII')

    def is_yesish(val):
        return val.lower() in ('1', 'true', 'yes')

    website.bitbucket_consumer_key = envvar('BITBUCKET_CONSUMER_KEY')
    website.bitbucket_consumer_secret = envvar('BITBUCKET_CONSUMER_SECRET')
    website.bitbucket_callback = envvar('BITBUCKET_CALLBACK')

    website.github_client_id = envvar('GITHUB_CLIENT_ID')
    website.github_client_secret = envvar('GITHUB_CLIENT_SECRET')
    website.github_callback = envvar('GITHUB_CALLBACK')

    website.twitter_consumer_key = envvar('TWITTER_CONSUMER_KEY')
    website.twitter_consumer_secret = envvar('TWITTER_CONSUMER_SECRET')
    website.twitter_access_token = envvar('TWITTER_ACCESS_TOKEN')
    website.twitter_access_token_secret = envvar('TWITTER_ACCESS_TOKEN_SECRET')
    website.twitter_callback = envvar('TWITTER_CALLBACK')

    website.bountysource_www_host = envvar('BOUNTYSOURCE_WWW_HOST')
    website.bountysource_api_host = envvar('BOUNTYSOURCE_API_HOST')
    website.bountysource_api_secret = envvar('BOUNTYSOURCE_API_SECRET')
    website.bountysource_callback = envvar('BOUNTYSOURCE_CALLBACK')

    website.css_href = envvar('GITTIP_CSS_HREF') \
                                          .replace('%version', website.version)
    website.js_src = envvar('GITTIP_JS_SRC') \
                                          .replace('%version', website.version)
    website.cache_static = is_yesish(envvar('GITTIP_CACHE_STATIC'))

    website.google_analytics_id = envvar('GOOGLE_ANALYTICS_ID')
    website.gauges_id = envvar('GAUGES_ID')

    if missing_keys:
        missing_keys.sort()
        these = len(missing_keys) != 1 and 'these' or 'this'
        plural = len(missing_keys) != 1 and 's' or ''
        aspen.log_dammit("=" * 42)
        aspen.log_dammit( "Oh no! Gittip.com needs %s missing " % these
                        , "environment variable%s:" % plural
                         )
        aspen.log_dammit(" ")
        for key in missing_keys:
            aspen.log_dammit("  " + key)
        aspen.log_dammit(" ")
        aspen.log_dammit( "(Sorry, we must've started looking for "
                        , "%s since you last updated Gittip!)" % these
                         )
        aspen.log_dammit(" ")
        aspen.log_dammit("Running Gittip locally? Edit ./local.env.")
        aspen.log_dammit("Running the test suite? Edit ./tests/env.")
        aspen.log_dammit(" ")
        aspen.log_dammit("See ./default_local.env for hints.")

        aspen.log_dammit("=" * 42)
        raise SystemExit
Пример #60
0
def env():
    env = Environment(
        AWS_SES_ACCESS_KEY_ID=unicode,
        AWS_SES_SECRET_ACCESS_KEY=unicode,
        AWS_SES_DEFAULT_REGION=unicode,
        BASE_URL=unicode,
        DATABASE_URL=unicode,
        DATABASE_MAXCONN=int,
        CRYPTO_KEYS=unicode,
        GRATIPAY_ASSET_URL=unicode,
        GRATIPAY_CACHE_STATIC=is_yesish,
        GRATIPAY_COMPRESS_ASSETS=is_yesish,
        BALANCED_API_SECRET=unicode,
        BRAINTREE_SANDBOX_MODE=is_yesish,
        BRAINTREE_MERCHANT_ID=unicode,
        BRAINTREE_PUBLIC_KEY=unicode,
        BRAINTREE_PRIVATE_KEY=unicode,
        GITHUB_CLIENT_ID=unicode,
        GITHUB_CLIENT_SECRET=unicode,
        GITHUB_CALLBACK=unicode,
        BITBUCKET_CONSUMER_KEY=unicode,
        BITBUCKET_CONSUMER_SECRET=unicode,
        BITBUCKET_CALLBACK=unicode,
        TWITTER_CONSUMER_KEY=unicode,
        TWITTER_CONSUMER_SECRET=unicode,
        TWITTER_CALLBACK=unicode,
        FACEBOOK_APP_ID=unicode,
        FACEBOOK_APP_SECRET=unicode,
        FACEBOOK_CALLBACK=unicode,
        GOOGLE_CLIENT_ID=unicode,
        GOOGLE_CLIENT_SECRET=unicode,
        GOOGLE_CALLBACK=unicode,
        BOUNTYSOURCE_API_SECRET=unicode,
        BOUNTYSOURCE_CALLBACK=unicode,
        BOUNTYSOURCE_API_HOST=unicode,
        BOUNTYSOURCE_WWW_HOST=unicode,
        VENMO_CLIENT_ID=unicode,
        VENMO_CLIENT_SECRET=unicode,
        VENMO_CALLBACK=unicode,
        OPENSTREETMAP_CONSUMER_KEY=unicode,
        OPENSTREETMAP_CONSUMER_SECRET=unicode,
        OPENSTREETMAP_CALLBACK=unicode,
        OPENSTREETMAP_API_URL=unicode,
        OPENSTREETMAP_AUTH_URL=unicode,
        UPDATE_CTA_EVERY=int,
        CHECK_DB_EVERY=int,
        CHECK_NPM_SYNC_EVERY=int,
        EMAIL_QUEUE_FLUSH_EVERY=int,
        EMAIL_QUEUE_SLEEP_FOR=int,
        EMAIL_QUEUE_ALLOW_UP_TO=int,
        EMAIL_QUEUE_LOG_METRICS_EVERY=int,
        OPTIMIZELY_ID=unicode,
        SENTRY_DSN=unicode,
        LOG_METRICS=is_yesish,
        INCLUDE_PIWIK=is_yesish,
        PROJECT_REVIEW_REPO=unicode,
        PROJECT_REVIEW_USERNAME=unicode,
        PROJECT_REVIEW_TOKEN=unicode,
        RAISE_SIGNIN_NOTIFICATIONS=is_yesish,
        GUNICORN_OPTS=unicode,
    )

    # Error Checking
    # ==============

    if env.malformed:
        these = len(env.malformed) != 1 and 'these' or 'this'
        plural = len(env.malformed) != 1 and 's' or ''
        aspen.log_dammit("=" * 42)
        aspen.log_dammit("Oh no! Gratipay.com couldn't understand %s " % these,
                         "environment variable%s:" % plural)
        aspen.log_dammit(" ")
        for key, err in env.malformed:
            aspen.log_dammit("  {} ({})".format(key, err))
        aspen.log_dammit(" ")
        aspen.log_dammit("See ./default_local.env for hints.")

        aspen.log_dammit("=" * 42)
        keys = ', '.join([key for key, value in env.malformed])
        raise BadEnvironment("Malformed envvar{}: {}.".format(plural, keys))

    if env.missing:
        these = len(env.missing) != 1 and 'these' or 'this'
        plural = len(env.missing) != 1 and 's' or ''
        aspen.log_dammit("=" * 42)
        aspen.log_dammit("Oh no! Gratipay.com needs %s missing " % these,
                         "environment variable%s:" % plural)
        aspen.log_dammit(" ")
        for key in env.missing:
            aspen.log_dammit("  " + key)
        aspen.log_dammit(" ")
        aspen.log_dammit("(Sorry, we must've started looking for ",
                         "%s since you last updated Gratipay!)" % these)
        aspen.log_dammit(" ")
        aspen.log_dammit("Running Gratipay locally? Edit ./local.env.")
        aspen.log_dammit("Running the test suite? Edit ./tests/env.")
        aspen.log_dammit(" ")
        aspen.log_dammit("See ./default_local.env for hints.")

        aspen.log_dammit("=" * 42)
        keys = ', '.join([key for key in env.missing])
        raise BadEnvironment("Missing envvar{}: {}.".format(plural, keys))

    return env