Пример #1
0
 def test_timeout_exceeded(self) -> None:
     try:
         timeout(1, lambda: self.sleep_x_seconds_y_times(0.1, 50))
         raise AssertionError("Failed to raise a timeout")
     except TimeoutExpired as exc:
         tb = traceback.format_tb(exc.__traceback__)
         self.assertIn("in sleep_x_seconds_y_times", tb[-1])
         self.assertIn("time.sleep(x)", tb[-1])
Пример #2
0
 def test_timeout_raises(self) -> None:
     try:
         timeout(1, lambda: self.something_exceptional())
         raise AssertionError("Failed to raise an exception")
     except ValueError as exc:
         tb = traceback.format_tb(exc.__traceback__)
         self.assertIn("in something_exceptional", tb[-1])
         self.assertIn("raise ValueError", tb[-1])
Пример #3
0
 def test_timeout_warn(self) -> None:
     # If the sleep is long enough, it will outlast the attempts to
     # kill it
     with self.assertLogs(level="WARNING") as m:
         try:
             timeout(1, lambda: self.sleep_x_seconds_y_times(5, 1))
             raise AssertionError("Failed to raise a timeout")
         except TimeoutExpired as exc:
             tb = traceback.format_tb(exc.__traceback__)
             self.assertNotIn("in sleep_x_seconds_y_times", tb[-1])
             self.assertIn("raise TimeoutExpired", tb[-1])
     self.assertEqual(m.output,
                      ["WARNING:root:Failed to time out backend thread"])
Пример #4
0
def do_convert(md, realm_domain=None, message=None):
    """Convert Markdown to HTML, with Zulip-specific settings and hacks."""
    from zerver.models import get_active_user_dicts_in_realm, UserProfile

    if message:
        maybe_update_realm_filters(message.get_realm().domain)

    if realm_domain in md_engines:
        _md_engine = md_engines[realm_domain]
    else:
        _md_engine = md_engines["default"]
    # Reset the parser; otherwise it will get slower over time.
    _md_engine.reset()

    global current_message
    current_message = message

    # Pre-fetch data from the DB that is used in the bugdown thread
    global db_data
    if message:
        realm_users = get_active_user_dicts_in_realm(message.get_realm())

        db_data = {
            'realm_alert_words':
            alert_words.alert_words_in_realm(message.get_realm()),
            'full_names':
            dict((user['full_name'].lower(), user) for user in realm_users),
            'short_names':
            dict((user['short_name'].lower(), user) for user in realm_users),
            'emoji':
            message.get_realm().get_emoji()
        }

    try:
        # Spend at most 5 seconds rendering.
        # Sometimes Python-Markdown is really slow; see
        # https://trac.zulip.net/ticket/345
        return timeout(5, _md_engine.convert, md)
    except:
        from zerver.lib.actions import internal_send_message

        cleaned = _sanitize_for_log(md)

        # Output error to log as well as sending a zulip and email
        logging.getLogger('').error(
            'Exception in Markdown parser: %sInput (sanitized) was: %s' %
            (traceback.format_exc(), cleaned))
        subject = "Markdown parser failure on %s" % (platform.node(), )
        if settings.ERROR_BOT is not None:
            internal_send_message(
                settings.ERROR_BOT, "stream", "errors", subject,
                "Markdown parser failed, email sent with details.")
        mail.mail_admins(subject,
                         "Failed message: %s\n\n%s\n\n" %
                         (cleaned, traceback.format_exc()),
                         fail_silently=False)
        return None
    finally:
        current_message = None
        db_data = None
Пример #5
0
def fetch_tweet_data(tweet_id):
    # type: (text_type) -> Optional[Dict[text_type, Any]]
    if settings.TEST_SUITE:
        from . import testing_mocks
        res = testing_mocks.twitter(tweet_id)
    else:
        creds = {
            'consumer_key': settings.TWITTER_CONSUMER_KEY,
            'consumer_secret': settings.TWITTER_CONSUMER_SECRET,
            'access_token_key': settings.TWITTER_ACCESS_TOKEN_KEY,
            'access_token_secret': settings.TWITTER_ACCESS_TOKEN_SECRET,
        }
        if not all(creds.values()):
            return None

        try:
            api = twitter.Api(**creds)
            # Sometimes Twitter hangs on responses.  Timing out here
            # will cause the Tweet to go through as-is with no inline
            # preview, rather than having the message be rejected
            # entirely. This timeout needs to be less than our overall
            # formatting timeout.
            tweet = timeout(3, api.GetStatus, tweet_id)
            res = tweet.AsDict()
            res['media'] = tweet.media  # AsDict does not include media
        except AttributeError:
            logging.error(
                'Unable to load twitter api, you may have the wrong '
                'library installed, see https://github.com/zulip/zulip/issues/86'
            )
            return None
        except TimeoutExpired as e:
            # We'd like to try again later and not cache the bad result,
            # so we need to re-raise the exception (just as though
            # we were being rate-limited)
            raise
        except twitter.TwitterError as e:
            t = e.args[0]
            if len(t) == 1 and ('code' in t[0]) and (t[0]['code'] == 34):
                # Code 34 means that the message doesn't exist; return
                # None so that we will cache the error
                return None
            elif len(t) == 1 and ('code' in t[0]) and (t[0]['code'] == 88
                                                       or t[0]['code'] == 130):
                # Code 88 means that we were rate-limited and 130
                # means Twitter is having capacity issues; either way
                # just raise the error so we don't cache None and will
                # try again later.
                raise
            else:
                # It's not clear what to do in cases of other errors,
                # but for now it seems reasonable to log at error
                # level (so that we get notified), but then cache the
                # failure to proceed with our usual work
                logging.error(traceback.format_exc())
                return None
    return res
Пример #6
0
def fetch_tweet_data(tweet_id):
    if settings.TEST_SUITE:
        from . import testing_mocks

        res = testing_mocks.twitter(tweet_id)
    else:
        creds = {
            "consumer_key": settings.TWITTER_CONSUMER_KEY,
            "consumer_secret": settings.TWITTER_CONSUMER_SECRET,
            "access_token_key": settings.TWITTER_ACCESS_TOKEN_KEY,
            "access_token_secret": settings.TWITTER_ACCESS_TOKEN_SECRET,
        }
        if not all(creds.values()):
            return None

        try:
            api = twitter.Api(**creds)
            # Sometimes Twitter hangs on responses.  Timing out here
            # will cause the Tweet to go through as-is with no inline
            # preview, rather than having the message be rejected
            # entirely. This timeout needs to be less than our overall
            # formatting timeout.
            tweet = timeout(3, api.GetStatus, tweet_id)
            res = tweet.AsDict()
            res["media"] = tweet.media  # AsDict does not include media
        except AttributeError:
            logging.error(
                "Unable to load twitter api, you may have the wrong "
                "library installed, see https://github.com/zulip/zulip/issues/86"
            )
            return None
        except TimeoutExpired as e:
            # We'd like to try again later and not cache the bad result,
            # so we need to re-raise the exception (just as though
            # we were being rate-limited)
            raise
        except twitter.TwitterError as e:
            t = e.args[0]
            if len(t) == 1 and ("code" in t[0]) and (t[0]["code"] == 34):
                # Code 34 means that the message doesn't exist; return
                # None so that we will cache the error
                return None
            elif len(t) == 1 and ("code" in t[0]) and (t[0]["code"] == 88 or t[0]["code"] == 130):
                # Code 88 means that we were rate-limited and 130
                # means Twitter is having capacity issues; either way
                # just raise the error so we don't cache None and will
                # try again later.
                raise
            else:
                # It's not clear what to do in cases of other errors,
                # but for now it seems reasonable to log at error
                # level (so that we get notified), but then cache the
                # failure to proceed with our usual work
                logging.error(traceback.format_exc())
                return None
    return res
Пример #7
0
def do_convert(md, realm_domain=None, message=None, possible_words=None):
    # type: (markdown.Markdown, Optional[text_type], Optional[Message], Optional[Set[text_type]]) -> Optional[text_type]
    """Convert Markdown to HTML, with Zulip-specific settings and hacks."""
    from zerver.models import get_active_user_dicts_in_realm, UserProfile

    if message:
        maybe_update_realm_filters(message.get_realm().domain)

    if realm_domain in md_engines:
        _md_engine = md_engines[realm_domain]
    else:
        _md_engine = md_engines["default"]
    # Reset the parser; otherwise it will get slower over time.
    _md_engine.reset()

    global current_message
    current_message = message

    # Pre-fetch data from the DB that is used in the bugdown thread
    global db_data
    if message:
        realm_users = get_active_user_dicts_in_realm(message.get_realm())

        if possible_words is None:
            possible_words = set() # Set[text_type]

        db_data = {'possible_words':    possible_words,
                   'full_names':        dict((user['full_name'].lower(), user) for user in realm_users),
                   'short_names':       dict((user['short_name'].lower(), user) for user in realm_users),
                   'emoji':             message.get_realm().get_emoji()}

    try:
        # Spend at most 5 seconds rendering.
        # Sometimes Python-Markdown is really slow; see
        # https://trac.zulip.net/ticket/345
        return timeout(5, _md_engine.convert, md)
    except:
        from zerver.lib.actions import internal_send_message

        cleaned = _sanitize_for_log(md)

        # Output error to log as well as sending a zulip and email
        log_bugdown_error('Exception in Markdown parser: %sInput (sanitized) was: %s'
            % (traceback.format_exc(), cleaned))
        subject = "Markdown parser failure on %s" % (platform.node(),)
        if settings.ERROR_BOT is not None:
            internal_send_message(settings.ERROR_BOT, "stream",
                    "errors", subject, "Markdown parser failed, email sent with details.")
        mail.mail_admins(subject, "Failed message: %s\n\n%s\n\n" % (
                                    cleaned, traceback.format_exc()),
                         fail_silently=False)
        raise BugdownRenderingException()
    finally:
        current_message = None
        db_data = None
Пример #8
0
 def test_timeout_returns(self) -> None:
     ret = timeout(1, lambda: 42)
     self.assertEqual(ret, 42)