def texst_retry_exceeds_max_backoff(self): opts = util.RetryOptions(timedelta(microseconds=10), timedelta(microseconds=10), 1000, 3) start = time.time() with self.assertRaises(util.RetryMaxAttemptsError): util.retry_with_backoff(opts, lambda: util.RetryStatus.CONTINUE) end = time.time() self.assertLess(end - start, 1.0, "max backoff not respected: 1000 attempts took %ss" % (end - start))
def test_retry_exceeds_max_attempts(self): opts = util.RetryOptions(timedelta(microseconds=10), timedelta(seconds=1), 2, 3) retries = [0] def fn(): retries[0] += 1 return util.RetryStatus.CONTINUE with self.assertRaises(util.RetryMaxAttemptsError): util.retry_with_backoff(opts, fn) self.assertEqual(retries[0], 3)
def test_retry(self): opts = util.RetryOptions(timedelta(microseconds=10), timedelta(seconds=1), 2, 10) retries = [0] def fn(): retries[0] += 1 if retries[0] >= 3: return util.RetryStatus.BREAK return util.RetryStatus.CONTINUE util.retry_with_backoff(opts, fn) self.assertEqual(retries[0], 3)
def test_retry_reset(self): opts = util.RetryOptions(timedelta(microseconds=10), timedelta(seconds=1), 2, 1) # Backoff loop has 1 allowed retry; we always return RESET, so just # make sure we get to 2 retries and then break. count = [0] def fn(): count[0] += 1 if count[0] == 2: return util.RetryStatus.BREAK return util.RetryStatus.RESET util.retry_with_backoff(opts, fn) self.assertEqual(count[0], 2)
def send(self, call): """Send call to Cockroach via an HTTP post. HTTP response codes which are retryable are retried with backoff in a loop using the default retry options. Other errors sending HTTP request are retried indefinitely using the same client command ID to avoid reporting failure when in fact the command may have gone through and been executed successfully. We retry here to eventually get through with the same client command ID and be given the cached response. """ retry_opts = http_retry_options.copy() retry_opts.tag = "http %s" % call.method.name def retryable(): try: resp = self._post(call) except Exception: # Assume all errors sending request are retryable. logging.warning("failed to send HTTP request or read its response", exc_info=True) return RetryStatus.CONTINUE else: if resp.status_code != 200: logging.warning("failed to send HTTP request with status code %d", resp.status_code) # See if we can retry based on HTTP response code. if resp.status_code in (429, 503, 504): # Retry on service unavailable and request timeout. # TODO: respect the Retry-After header if present. return RetryStatus.CONTINUE else: resp.raise_for_status() else: # On a successful ost, we're done with retry loop. return RetryStatus.BREAK try: retry_with_backoff(retry_opts, retryable) except Exception as e: # TODO: Are there any non-generic errors we need to handle here? # Is it better to let exceptions escape instead of stuffing them into # the reply? call.reply.header.error.message = str(e) return call.reply
def test_retry_function_raises_error(self): opts = util.RetryOptions(timedelta(microseconds=10), timedelta(seconds=1), 2) with self.assertRaises(ZeroDivisionError): util.retry_with_backoff(opts, lambda: 1/0)
def run_transaction(self, opts, retryable): """RunTransaction executes retryable in the context of a distributed transaction. The ``retryable`` argument is a function which takes one parameter, a `KV` object. The transaction is automatically aborted if retryable returns any error aside from recoverable internal errors, and is automatically committed otherwise. retryable should have no side effects which could cause problems in the event it must be run more than once. The opts struct contains transaction settings. Calling run_transaction on the transactional KV client which is supplied to the retryable function is an error. """ if isinstance(self._sender, TxnSender): raise Exception("cannot invoke run_transaction on an already-transactional client") # Create a new KV for the transaction using a transactional KV sender. txn_sender = TxnSender(self._sender, opts) txn_kv = KV(txn_sender, user=self.user, user_priority=self.user_priority) # Run retriable in a loop until we encounter a success or error condition this # loop isn't capable of handling. retry_opts = txn_retry_options.copy() retry_opts.tag = opts.name def callback(): txn_sender.txn_end = False # always reset before [re]starting txn try: retryable(txn_kv) if not txn_sender.txn_end: # If there were no errors running retryable, commit the txn. # This may block waiting for outstanding writes to complete in # case retryable didn't -- we need the most recent of all response # timestamps in order to commit. txn_kv.call(Methods.EndTransaction, api_pb2.EndTransactionRequest(commit=True)) except errors.ReadWithinUncertaintyIntervalError: # Retry immediately on read within uncertainty interval. return util.RetryStatus.RESET except errors.TransactionAbortedError: # If the transaction was aborted, the TxnSender will have created # a new txn. We allow backoff/retry in this case. return util.RetryStatus.CONTINUE except errors.TransactionPushError: # Backoff and retry on failure to push a conflicting transaction. return util.RetryStatus.CONTINUE except errors.TransactionRetryError: # Return RESET for an immediate retry (as in the case of an SSI txn # whose timestamp was pushed. return util.RetryStatus.RESET # All other errors are allowed to escape, aborting the retry loop. return util.RetryStatus.BREAK try: util.retry_with_backoff(retry_opts, callback) except Exception as e: if not txn_sender.txn_end: try: txn_kv.call(Methods.EndTransaction, api_pb2.EndTransactionRequest(commit=False)) except Exception: logging.error("failure aborting transaction; abort caused by %s", e, exc_info=True) raise