def test_exponential_retrier(self): num_retries = 5 def make_crappy_function(fail_start=1, fail_end=5): """Make a function that iterates from zero to infinity. However when it is iterating in the range [fail_start,fail_end] it will throw a ValueError exception but still increment """ side_effects = [0] def ret(): ret = side_effects[0] side_effects[0] += 1 if ret >= fail_start and ret <= fail_end: raise ValueError("foo %d" % ret) return ret return ret crappy_function = make_crappy_function() with self.check_exponential_backoff_sleep_times(500, 0): # first call to crappy_function should return zero without # any retrying self.assertEquals(0, utils.exponential_retrier( crappy_function, max_retries=num_retries)) with self.check_exponential_backoff_sleep_times(500, num_retries): # this should return 6 as we will retry 5 times self.assertEquals(6, utils.exponential_retrier( crappy_function, max_retries=num_retries)) with self.check_exponential_backoff_sleep_times(500, 0): self.assertEqual(7, utils.exponential_retrier( crappy_function, max_retries=num_retries)) with self.check_exponential_backoff_sleep_times(1, num_retries): # make sure this will fail in exponential_retrier and # test that last exception is re_thrown crappy_function = make_crappy_function(fail_start=0, fail_end=1000) error = None try: utils.exponential_retrier(crappy_function, retry_min_wait_ms=1, max_retries=num_retries) except ValueError as e: # test that exception caught here has proper stack trace self.assertTrue(any(map( lambda x: x[3] == "raise ValueError(\"foo %d\" % ret)", traceback.extract_tb(sys.exc_traceback)))) error = e self.assertEquals(error.message, "foo %d" % num_retries) with patch('time.sleep'): # check that exception is rethrown if we pass # exception filter that returns False if exception # is ValueError crappy_function = make_crappy_function(fail_start=0, fail_end=0) def exception_filter(exception): return type(exception) is not ValueError error = None try: utils.exponential_retrier(crappy_function, retry_min_wait_ms=1, max_retries=100000, exception_filter=exception_filter) except ValueError as e: error = e self.assertEquals(error.message, "foo 0")
def _send_queued_mail(test=False): """sends mail from the mail queue to smtplib for delivery. Also, on successes, empties the mail queue and adds all emails to the sent_mail list.""" from r2.lib.pages import Share, Mail_Opt uids_to_clear = [] if not test: session = smtplib.SMTP(g.smtp_server) else: session = None def sendmail_multiplexer(email): """Use mailgun for password resets. Use old sendmail for everything else """ if email.kind == Email.Kind.RESET_PASSWORD: _sendmail_using_mailgun(email, test) else: _sendmail(email, session, test) try: for email in Email.get_unsent(datetime.datetime.now(pytz.UTC)): uids_to_clear.append(email.uid) should_queue = email.should_queue() # check only on sharing that the mail is invalid if not test: if email.kind == Email.Kind.SHARE: if should_queue: email.body = Share( username=email.from_name(), msg_hash=email.msg_hash, link=email.thing, body=email.body).render(style="email") else: email.set_sent(rejected=True) continue elif email.kind == Email.Kind.OPTOUT: email.body = Mail_Opt(msg_hash=email.msg_hash, leave=True).render(style="email") elif email.kind == Email.Kind.OPTIN: email.body = Mail_Opt(msg_hash=email.msg_hash, leave=False).render(style="email") # handle unknown types here elif not email.body: print("Rejecting email with empty body from %r and to %r" % (email.fr_addr, email.to_addr)) email.set_sent(rejected=True) continue exponential_retrier(lambda: sendmail_multiplexer(email), should_retry_exception) g.log.info("Sent email from %r to %r", email.fr_addr, email.to_addr) except: # Log exceptions here and re-throw to make sure we are not swallowing # elsewhere g.log.exception("Unable to deliver email") raise finally: g.stats.flush() if not test: session.quit() # always perform clear_queue_by_uids, even if we have an # unhandled exception if len(uids_to_clear) > 0: Email.handler.clear_queue_by_uids(uids_to_clear)
def test_exponential_retrier(self): num_retries = 5 def make_crappy_function(fail_start=1, fail_end=5): """Make a function that iterates from zero to infinity. However when it is iterating in the range [fail_start,fail_end] it will throw a ValueError exception but still increment """ side_effects = [0] def ret(): ret = side_effects[0] side_effects[0] += 1 if ret >= fail_start and ret <= fail_end: raise ValueError("foo %d" % ret) return ret return ret crappy_function = make_crappy_function() with self.check_exponential_backoff_sleep_times(500, 0): # first call to crappy_function should return zero without # any retrying self.assertEquals( 0, utils.exponential_retrier(crappy_function, max_retries=num_retries)) with self.check_exponential_backoff_sleep_times(500, num_retries): # this should return 6 as we will retry 5 times self.assertEquals( 6, utils.exponential_retrier(crappy_function, max_retries=num_retries)) with self.check_exponential_backoff_sleep_times(500, 0): self.assertEqual( 7, utils.exponential_retrier(crappy_function, max_retries=num_retries)) with self.check_exponential_backoff_sleep_times(1, num_retries): # make sure this will fail in exponential_retrier and # test that last exception is re_thrown crappy_function = make_crappy_function(fail_start=0, fail_end=1000) error = None try: utils.exponential_retrier(crappy_function, retry_min_wait_ms=1, max_retries=num_retries) except ValueError as e: # test that exception caught here has proper stack trace self.assertTrue( any( map( lambda x: x[3] == "raise ValueError(\"foo %d\" % ret)", traceback.extract_tb(sys.exc_traceback)))) error = e self.assertEquals(error.message, "foo %d" % num_retries) with patch('time.sleep'): # check that exception is rethrown if we pass # exception filter that returns False if exception # is ValueError crappy_function = make_crappy_function(fail_start=0, fail_end=0) def exception_filter(exception): return type(exception) is not ValueError error = None try: utils.exponential_retrier(crappy_function, retry_min_wait_ms=1, max_retries=100000, exception_filter=exception_filter) except ValueError as e: error = e self.assertEquals(error.message, "foo 0")
def _send_queued_mail(test=False): """sends mail from the mail queue to smtplib for delivery. Also, on successes, empties the mail queue and adds all emails to the sent_mail list.""" from r2.lib.pages import Share, Mail_Opt uids_to_clear = [] if not test: session = smtplib.SMTP(g.smtp_server) else: session = None def sendmail_multiplexer(email): """Use mailgun for password resets. Use old sendmail for everything else """ if email.kind == Email.Kind.RESET_PASSWORD: _sendmail_using_mailgun(email, test) else: _sendmail(email, session, test) try: for email in Email.get_unsent(datetime.datetime.now(pytz.UTC)): uids_to_clear.append(email.uid) should_queue = email.should_queue() # check only on sharing that the mail is invalid if not test: if email.kind == Email.Kind.SHARE: if should_queue: email.body = Share(username=email.from_name(), msg_hash=email.msg_hash, link=email.thing, body=email.body).render( style="email") else: email.set_sent(rejected=True) continue elif email.kind == Email.Kind.OPTOUT: email.body = Mail_Opt(msg_hash=email.msg_hash, leave=True).render(style="email") elif email.kind == Email.Kind.OPTIN: email.body = Mail_Opt(msg_hash=email.msg_hash, leave=False).render(style="email") # handle unknown types here elif not email.body: print("Rejecting email with empty body from %r and to %r" % (email.fr_addr, email.to_addr)) email.set_sent(rejected=True) continue exponential_retrier( lambda: sendmail_multiplexer(email), should_retry_exception) g.log.info("Sent email from %r to %r", email.fr_addr, email.to_addr) except: # Log exceptions here and re-throw to make sure we are not swallowing # elsewhere g.log.exception("Unable to deliver email") raise finally: g.stats.flush() if not test: session.quit() # always perform clear_queue_by_uids, even if we have an # unhandled exception if len(uids_to_clear) > 0: Email.handler.clear_queue_by_uids(uids_to_clear)