def test_retry_times_out_no_response(self, mock_time):
mock_time.return_value = 1
retry = RetryOptions([_FAKE_STATUS_CODE_1],
BackoffSettings(0, 0, 0, 0, 0, 0, 0))
settings = _CallSettings(timeout=0, retry=retry)
my_callable = api_callable.create_api_call(lambda: None, settings)
self.assertRaises(RetryError, my_callable, None)
def test_no_retry_if_no_codes(self, mock_time):
retry = RetryOptions([], BackoffSettings(1, 2, 3, 4, 5, 6, 7))
mock_call = mock.Mock()
mock_call.side_effect = CustomException('', _FAKE_STATUS_CODE_1)
mock_time.return_value = 0
settings = CallSettings(timeout=0, retry=retry)
my_callable = api_callable.create_api_call(mock_call, settings)
self.assertRaises(CustomException, my_callable, None)
self.assertEqual(mock_call.call_count, 1)
def test_retry_aborts_on_unexpected_exception(self, mock_exc_to_code,
mock_time):
mock_exc_to_code.side_effect = lambda e: e.code
retry = RetryOptions([_FAKE_STATUS_CODE_1],
BackoffSettings(0, 0, 0, 0, 0, 0, 1))
mock_call = mock.Mock()
mock_call.side_effect = CustomException('', _FAKE_STATUS_CODE_2)
mock_time.return_value = 0
settings = _CallSettings(timeout=0, retry=retry)
my_callable = api_callable.create_api_call(mock_call, settings)
self.assertRaises(Exception, my_callable, None)
self.assertEqual(mock_call.call_count, 1)
def test_retry_aborts_simple(self, mock_exc_to_code, mock_time):
def fake_call(dummy_request, dummy_timeout):
raise CustomException('', _FAKE_STATUS_CODE_1)
retry = RetryOptions([_FAKE_STATUS_CODE_1],
BackoffSettings(0, 0, 0, 0, 0, 0, 1))
mock_time.side_effect = [0, 2]
mock_exc_to_code.side_effect = lambda e: e.code
settings = _CallSettings(timeout=0, retry=retry)
my_callable = api_callable.create_api_call(fake_call, settings)
try:
my_callable(None)
except RetryError as exc:
self.assertIsInstance(exc.cause, CustomException)
def test_retry(self, mock_exc_to_code, mock_time):
mock_exc_to_code.side_effect = lambda e: e.code
to_attempt = 3
retry = RetryOptions([_FAKE_STATUS_CODE_1],
BackoffSettings(0, 0, 0, 0, 0, 0, 1))
# Succeeds on the to_attempt'th call, and never again afterward
mock_call = mock.Mock()
mock_call.side_effect = ([CustomException('', _FAKE_STATUS_CODE_1)] *
(to_attempt - 1) + [mock.DEFAULT])
mock_call.return_value = 1729
mock_time.return_value = 0
settings = _CallSettings(timeout=0, retry=retry)
my_callable = api_callable.create_api_call(mock_call, settings)
self.assertEqual(my_callable(None), 1729)
self.assertEqual(mock_call.call_count, to_attempt)
def test_retryable_with_timeout(self, mock_time, mock_exc_to_code):
mock_time.return_value = 1
mock_exc_to_code.side_effect = lambda e: e.code
mock_call = mock.Mock()
mock_call.side_effect = [CustomException('', _FAKE_STATUS_CODE_1),
mock.DEFAULT]
mock_call.return_value = 1729
retry_options = RetryOptions(
[_FAKE_STATUS_CODE_1],
BackoffSettings(0, 0, 0, 0, 0, 0, 0))
my_callable = retry.retryable(mock_call, retry_options)
self.assertRaises(errors.RetryError, my_callable)
self.assertEqual(0, mock_call.call_count)
def test_retry_times_out_simple(self, mock_exc_to_code, mock_time):
mock_exc_to_code.side_effect = lambda e: e.code
to_attempt = 3
retry = RetryOptions([_FAKE_STATUS_CODE_1],
BackoffSettings(0, 0, 0, 0, 0, 0, 1))
mock_call = mock.Mock()
mock_call.side_effect = CustomException('', _FAKE_STATUS_CODE_1)
mock_time.side_effect = ([0] * to_attempt + [2])
settings = _CallSettings(timeout=0, retry=retry)
my_callable = api_callable.create_api_call(mock_call, settings)
try:
my_callable(None)
except RetryError as exc:
self.assertIsInstance(exc.cause, CustomException)
self.assertEqual(mock_call.call_count, to_attempt)
def test_retryable_without_timeout(self, mock_time, mock_exc_to_code):
mock_time.return_value = 0
mock_exc_to_code.side_effect = lambda e: e.code
to_attempt = 3
mock_call = mock.Mock()
mock_call.side_effect = ([CustomException('', _FAKE_STATUS_CODE_1)] *
(to_attempt - 1) + [mock.DEFAULT])
mock_call.return_value = 1729
retry_options = RetryOptions(
[_FAKE_STATUS_CODE_1],
BackoffSettings(0, 0, 0, None, None, None, None))
my_callable = retry.retryable(mock_call, retry_options)
self.assertEqual(my_callable(None), 1729)
self.assertEqual(to_attempt, mock_call.call_count)
def test_retryable_when_no_codes(self, mock_time, mock_exc_to_code):
mock_time.return_value = 0
mock_exc_to_code.side_effect = lambda e: e.code
mock_call = mock.Mock()
mock_call.side_effect = [CustomException('', _FAKE_STATUS_CODE_1),
mock.DEFAULT]
mock_call.return_value = 1729
retry_options = RetryOptions(
[],
BackoffSettings(0, 0, 0, 0, 0, 0, 1))
my_callable = retry.retryable(mock_call, retry_options)
try:
my_callable(None)
self.fail('Should not have been reached')
except errors.RetryError as exc:
self.assertIsInstance(exc.cause, CustomException)
self.assertEqual(1, mock_call.call_count)
def test_retry_exponential_backoff(self, mock_exc_to_code, mock_time,
mock_sleep):
# pylint: disable=too-many-locals
mock_exc_to_code.side_effect = lambda e: e.code
MILLIS_PER_SEC = 1000
mock_time.return_value = 0
def incr_time(secs):
mock_time.return_value += secs
def api_call(dummy_request, timeout, **dummy_kwargs):
incr_time(timeout)
raise CustomException(str(timeout), _FAKE_STATUS_CODE_1)
mock_call = mock.Mock()
mock_sleep.side_effect = incr_time
mock_call.side_effect = api_call
params = BackoffSettings(3, 2, 24, 5, 2, 80, 2500)
retry = RetryOptions([_FAKE_STATUS_CODE_1], params)
settings = CallSettings(timeout=0, retry=retry)
my_callable = api_callable.create_api_call(mock_call, settings)
try:
my_callable(None)
except RetryError as exc:
self.assertIsInstance(exc.cause, CustomException)
self.assertGreaterEqual(mock_time(),
params.total_timeout_millis / MILLIS_PER_SEC)
# Very rough bounds
calls_lower_bound = params.total_timeout_millis / (
params.max_retry_delay_millis + params.max_rpc_timeout_millis)
self.assertGreater(mock_call.call_count, calls_lower_bound)
calls_upper_bound = (params.total_timeout_millis /
params.initial_retry_delay_millis)
self.assertLess(mock_call.call_count, calls_upper_bound)
def test_retryable_exponential_backoff(
self, mock_time, mock_sleep, mock_exc_to_code):
def incr_time(secs):
mock_time.return_value += secs
def api_call(timeout):
incr_time(timeout)
raise CustomException(str(timeout), _FAKE_STATUS_CODE_1)
mock_time.return_value = 0
mock_sleep.side_effect = incr_time
mock_exc_to_code.side_effect = lambda e: e.code
mock_call = mock.Mock()
mock_call.side_effect = api_call
params = BackoffSettings(3, 2, 24, 5, 2, 80, 2500)
retry_options = RetryOptions([_FAKE_STATUS_CODE_1], params)
my_callable = retry.retryable(mock_call, retry_options)
try:
my_callable()
self.fail('Should not have been reached')
except errors.RetryError as exc:
self.assertIsInstance(exc.cause, CustomException)
self.assertGreaterEqual(mock_time(),
params.total_timeout_millis / _MILLIS_PER_SEC)
# Very rough bounds
calls_lower_bound = params.total_timeout_millis / (
params.max_retry_delay_millis + params.max_rpc_timeout_millis)
self.assertGreater(mock_call.call_count, calls_lower_bound)
calls_upper_bound = (params.total_timeout_millis /
params.initial_retry_delay_millis)
self.assertLess(mock_call.call_count, calls_upper_bound)
def _construct_retry(method_config, retry_codes, retry_params, retry_names):
"""Helper for ``construct_settings()``.
Args:
method_config: A dictionary representing a single ``methods`` entry of the
standard API client config file. (See ``construct_settings()`` for
information on this yaml.)
retry_codes: A dictionary parsed from the ``retry_codes`` entry
of the standard API client config file. (See ``construct_settings()``
for information on this yaml.)
retry_params: A dictionary parsed from the ``retry_params`` entry
of the standard API client config file. (See ``construct_settings()``
for information on this yaml.)
retry_names: A dictionary mapping the string names used in the
standard API client config file to API response status codes.
Returns:
A RetryOptions object, or None.
"""
if method_config is None:
return None
codes = None
if retry_codes and 'retry_codes_name' in method_config:
codes_name = method_config['retry_codes_name']
if codes_name in retry_codes and retry_codes[codes_name]:
codes = [retry_names[name] for name in retry_codes[codes_name]]
else:
codes = []
backoff_settings = None
if retry_params and 'retry_params_name' in method_config:
params_name = method_config['retry_params_name']
if params_name and params_name in retry_params:
backoff_settings = BackoffSettings(**retry_params[params_name])
return RetryOptions(retry_codes=codes, backoff_settings=backoff_settings)
def test_read_rows_retry_timeout(self):
from google.cloud._testing import _Monkey
from tests.unit._testing import _CustomFakeStub
from google.cloud.bigtable.row_data import PartialRowsData
from google.cloud.bigtable import retry as MUT
from google.cloud.bigtable.retry import ReadRowsIterator
from google.gax import BackoffSettings
from google.gax.errors import RetryError
from grpc import StatusCode, RpcError
import time
client = _Client()
instance = _Instance(self.INSTANCE_NAME, client=client)
table = self._make_one(self.TABLE_ID, instance)
# Create request_pb
request_pb = object() # Returned by our mock.
mock_created = []
def mock_create_row_request(table_name, **kwargs):
mock_created.append((table_name, kwargs))
return request_pb
# Create a slow response iterator to cause a timeout
class MockTimeoutError(RpcError):
def code(self):
return StatusCode.DEADLINE_EXCEEDED
def _wait_then_raise():
time.sleep(0.1)
raise MockTimeoutError()
# Patch the stub used by the API method. The stub should create a new
# slow_iterator every time its queried.
def make_slow_iterator():
return (_wait_then_raise() for i in range(10))
client._data_stub = stub = _CustomFakeStub(make_slow_iterator)
# Set to timeout before RPC completes
test_backoff_settings = BackoffSettings(
initial_retry_delay_millis=10,
retry_delay_multiplier=0.3,
max_retry_delay_millis=30000,
initial_rpc_timeout_millis=1000,
rpc_timeout_multiplier=1.0,
max_rpc_timeout_millis=25 * 60 * 1000,
total_timeout_millis=1000)
start_key = b'start-key'
end_key = b'end-key'
filter_obj = object()
limit = 22
with _Monkey(MUT, _create_row_request=mock_create_row_request):
# Verify that a RetryError is thrown on read.
result = table.read_rows(start_key=start_key,
end_key=end_key,
filter_=filter_obj,
limit=limit,
backoff_settings=test_backoff_settings)
with self.assertRaises(RetryError):
result.consume_next()
table_admin_messages_v2_pb2)
from google.cloud.bigtable._generated import (table_pb2 as table_v2_pb2)
from google.cloud.bigtable.column_family import _gc_rule_from_pb
from google.cloud.bigtable.column_family import ColumnFamily
from google.cloud.bigtable.row import AppendRow
from google.cloud.bigtable.row import ConditionalRow
from google.cloud.bigtable.row import DirectRow
from google.cloud.bigtable.row_data import PartialRowsData
from google.gax import RetryOptions, BackoffSettings
from google.cloud.bigtable.retry import ReadRowsIterator, _create_row_request
from grpc import StatusCode
BACKOFF_SETTINGS = BackoffSettings(initial_retry_delay_millis=10,
retry_delay_multiplier=1.3,
max_retry_delay_millis=30000,
initial_rpc_timeout_millis=25 * 60 * 1000,
rpc_timeout_multiplier=1.0,
max_rpc_timeout_millis=25 * 60 * 1000,
total_timeout_millis=30 * 60 * 1000)
RETRY_CODES = [
StatusCode.DEADLINE_EXCEEDED, StatusCode.ABORTED, StatusCode.INTERNAL,
StatusCode.UNAVAILABLE
]
# Maximum number of mutations in bulk (MutateRowsRequest message):
# (https://cloud.google.com/bigtable/docs/reference/data/rpc/
# google.bigtable.v2#google.bigtable.v2.MutateRowRequest)
_MAX_BULK_MUTATIONS = 100000
