def texst_retry_exceeds_max_backoff(self):
opts = util.RetryOptions(timedelta(microseconds=10), timedelta(microseconds=10), 1000, 3)
start = time.time()
with self.assertRaises(util.RetryMaxAttemptsError):
util.retry_with_backoff(opts, lambda: util.RetryStatus.CONTINUE)
end = time.time()
self.assertLess(end - start, 1.0,
"max backoff not respected: 1000 attempts took %ss" % (end - start))
def test_retry_exceeds_max_attempts(self):
opts = util.RetryOptions(timedelta(microseconds=10), timedelta(seconds=1), 2, 3)
retries = [0]
def fn():
retries[0] += 1
return util.RetryStatus.CONTINUE
with self.assertRaises(util.RetryMaxAttemptsError):
util.retry_with_backoff(opts, fn)
self.assertEqual(retries[0], 3)
def test_retry(self):
opts = util.RetryOptions(timedelta(microseconds=10), timedelta(seconds=1), 2, 10)
retries = [0]
def fn():
retries[0] += 1
if retries[0] >= 3:
return util.RetryStatus.BREAK
return util.RetryStatus.CONTINUE
util.retry_with_backoff(opts, fn)
self.assertEqual(retries[0], 3)
def test_retry_reset(self):
opts = util.RetryOptions(timedelta(microseconds=10), timedelta(seconds=1), 2, 1)
# Backoff loop has 1 allowed retry; we always return RESET, so just
# make sure we get to 2 retries and then break.
count = [0]
def fn():
count[0] += 1
if count[0] == 2:
return util.RetryStatus.BREAK
return util.RetryStatus.RESET
util.retry_with_backoff(opts, fn)
self.assertEqual(count[0], 2)
def send(self, call):
"""Send call to Cockroach via an HTTP post.
HTTP response codes which are retryable are retried with
backoff in a loop using the default retry options. Other
errors sending HTTP request are retried indefinitely using the
same client command ID to avoid reporting failure when in fact
the command may have gone through and been executed
successfully. We retry here to eventually get through with the
same client command ID and be given the cached response.
"""
retry_opts = http_retry_options.copy()
retry_opts.tag = "http %s" % call.method.name
def retryable():
try:
resp = self._post(call)
except Exception:
# Assume all errors sending request are retryable.
logging.warning("failed to send HTTP request or read its response", exc_info=True)
return RetryStatus.CONTINUE
else:
if resp.status_code != 200:
logging.warning("failed to send HTTP request with status code %d", resp.status_code)
# See if we can retry based on HTTP response code.
if resp.status_code in (429, 503, 504):
# Retry on service unavailable and request timeout.
# TODO: respect the Retry-After header if present.
return RetryStatus.CONTINUE
else:
resp.raise_for_status()
else:
# On a successful ost, we're done with retry loop.
return RetryStatus.BREAK
try:
retry_with_backoff(retry_opts, retryable)
except Exception as e:
# TODO: Are there any non-generic errors we need to handle here?
# Is it better to let exceptions escape instead of stuffing them into
# the reply?
call.reply.header.error.message = str(e)
return call.reply
def test_retry_function_raises_error(self):
opts = util.RetryOptions(timedelta(microseconds=10), timedelta(seconds=1), 2)
with self.assertRaises(ZeroDivisionError):
util.retry_with_backoff(opts, lambda: 1/0)
def run_transaction(self, opts, retryable):
"""RunTransaction executes retryable in the context of a distributed transaction.
The ``retryable`` argument is a function which takes one parameter, a `KV`
object.
The transaction is automatically aborted if retryable
returns any error aside from recoverable internal errors, and is
automatically committed otherwise. retryable should have no side
effects which could cause problems in the event it must be run more
than once. The opts struct contains transaction settings.
Calling run_transaction on the transactional KV client which is
supplied to the retryable function is an error.
"""
if isinstance(self._sender, TxnSender):
raise Exception("cannot invoke run_transaction on an already-transactional client")
# Create a new KV for the transaction using a transactional KV sender.
txn_sender = TxnSender(self._sender, opts)
txn_kv = KV(txn_sender, user=self.user, user_priority=self.user_priority)
# Run retriable in a loop until we encounter a success or error condition this
# loop isn't capable of handling.
retry_opts = txn_retry_options.copy()
retry_opts.tag = opts.name
def callback():
txn_sender.txn_end = False # always reset before [re]starting txn
try:
retryable(txn_kv)
if not txn_sender.txn_end:
# If there were no errors running retryable, commit the txn.
# This may block waiting for outstanding writes to complete in
# case retryable didn't -- we need the most recent of all response
# timestamps in order to commit.
txn_kv.call(Methods.EndTransaction,
api_pb2.EndTransactionRequest(commit=True))
except errors.ReadWithinUncertaintyIntervalError:
# Retry immediately on read within uncertainty interval.
return util.RetryStatus.RESET
except errors.TransactionAbortedError:
# If the transaction was aborted, the TxnSender will have created
# a new txn. We allow backoff/retry in this case.
return util.RetryStatus.CONTINUE
except errors.TransactionPushError:
# Backoff and retry on failure to push a conflicting transaction.
return util.RetryStatus.CONTINUE
except errors.TransactionRetryError:
# Return RESET for an immediate retry (as in the case of an SSI txn
# whose timestamp was pushed.
return util.RetryStatus.RESET
# All other errors are allowed to escape, aborting the retry loop.
return util.RetryStatus.BREAK
try:
util.retry_with_backoff(retry_opts, callback)
except Exception as e:
if not txn_sender.txn_end:
try:
txn_kv.call(Methods.EndTransaction,
api_pb2.EndTransactionRequest(commit=False))
except Exception:
logging.error("failure aborting transaction; abort caused by %s",
e, exc_info=True)
raise
