def test_async_with_retries_succeeds_after_backoff_on_transient_error(
self):
attempt_times = []
fake_time = test_util.FakeTime()
# Setup: Server which passes on the third attempt.
def handler(request, context):
attempt_times.append(fake_time.time())
if len(attempt_times) < 3:
context.abort(grpc.StatusCode.UNAVAILABLE, "foo")
return make_response(request.nonce)
server = TestGrpcServer(handler)
with server.run() as client:
# Execute `async_call_with_retries` against the scripted server.
future = grpc_util.async_call_with_retries(
client.TestRpc,
make_request(42),
clock=fake_time,
)
# Expect:
# 1) The response contains the expected value.
# 2) The number of attempts and delays between them.
self.assertEqual(make_response(42), future.result(2))
self.assertLen(attempt_times, 3)
self.assertBetween(attempt_times[1] - attempt_times[0], 2, 4)
self.assertBetween(attempt_times[2] - attempt_times[1], 4, 8)
def test_async_with_retries_fails_after_backoff_on_nonpermanent_error(
self):
attempt_times = []
fake_time = test_util.FakeTime()
# Setup: Server which always fails with an UNAVAILABLE error.
def handler(_, context):
attempt_times.append(fake_time.time())
context.abort(grpc.StatusCode.UNAVAILABLE,
f"just a sec {len(attempt_times)}.")
server = TestGrpcServer(handler)
with server.run() as client:
# Execute `async_call_with_retries` against the scripted server.
future = grpc_util.async_call_with_retries(
client.TestRpc,
make_request(42),
clock=fake_time,
)
# Expect that the future raises an Exception which is the right
# type and carries the right message.
with self.assertRaises(grpc.RpcError) as raised:
future.result(2)
self.assertEqual(grpc.StatusCode.UNAVAILABLE,
raised.exception.code())
self.assertEqual("just a sec 5.", raised.exception.details())
# Verify the number of attempts and delays between them.
self.assertLen(attempt_times, 5)
self.assertBetween(attempt_times[1] - attempt_times[0], 2, 4)
self.assertBetween(attempt_times[2] - attempt_times[1], 4, 8)
self.assertBetween(attempt_times[3] - attempt_times[2], 8, 16)
self.assertBetween(attempt_times[4] - attempt_times[3], 16, 32)
def test_call_with_retries_succeeds_after_backoff_on_transient_error(self):
attempt_times = []
fake_time = test_util.FakeTime()
def handler(request, context):
attempt_times.append(fake_time.time())
if len(attempt_times) < 3:
context.abort(grpc.StatusCode.UNAVAILABLE, "foo")
return make_response(request.nonce)
server = TestGrpcServer(handler)
with server.run() as client:
response = grpc_util.call_with_retries(
client.TestRpc, make_request(42), fake_time)
self.assertEqual(make_response(42), response)
self.assertLen(attempt_times, 3)
self.assertBetween(attempt_times[1] - attempt_times[0], 2, 4)
self.assertBetween(attempt_times[2] - attempt_times[1], 4, 8)
def test_call_with_retries_fails_after_backoff_on_nonpermanent_error(self):
attempt_times = []
fake_time = test_util.FakeTime()
def handler(_, context):
attempt_times.append(fake_time.time())
context.abort(grpc.StatusCode.UNAVAILABLE, "foo")
server = TestGrpcServer(handler)
with server.run() as client:
with self.assertRaises(grpc.RpcError) as raised:
grpc_util.call_with_retries(client.TestRpc, make_request(42), fake_time)
self.assertEqual(grpc.StatusCode.UNAVAILABLE, raised.exception.code())
self.assertEqual("foo", raised.exception.details())
self.assertLen(attempt_times, 5)
self.assertBetween(attempt_times[1] - attempt_times[0], 2, 4)
self.assertBetween(attempt_times[2] - attempt_times[1], 4, 8)
self.assertBetween(attempt_times[3] - attempt_times[2], 8, 16)
self.assertBetween(attempt_times[4] - attempt_times[3], 16, 32)
