def test_retry_times_out_no_response(self, mock_time): mock_time.return_value = 1 retry = RetryOptions([_FAKE_STATUS_CODE_1], BackoffSettings(0, 0, 0, 0, 0, 0, 0)) settings = _CallSettings(timeout=0, retry=retry) my_callable = api_callable.create_api_call(lambda: None, settings) self.assertRaises(RetryError, my_callable, None)
def test_no_retry_if_no_codes(self, mock_time): retry = RetryOptions([], BackoffSettings(1, 2, 3, 4, 5, 6, 7)) mock_call = mock.Mock() mock_call.side_effect = CustomException('', _FAKE_STATUS_CODE_1) mock_time.return_value = 0 settings = CallSettings(timeout=0, retry=retry) my_callable = api_callable.create_api_call(mock_call, settings) self.assertRaises(CustomException, my_callable, None) self.assertEqual(mock_call.call_count, 1)
def test_retry_aborts_on_unexpected_exception(self, mock_exc_to_code, mock_time): mock_exc_to_code.side_effect = lambda e: e.code retry = RetryOptions([_FAKE_STATUS_CODE_1], BackoffSettings(0, 0, 0, 0, 0, 0, 1)) mock_call = mock.Mock() mock_call.side_effect = CustomException('', _FAKE_STATUS_CODE_2) mock_time.return_value = 0 settings = _CallSettings(timeout=0, retry=retry) my_callable = api_callable.create_api_call(mock_call, settings) self.assertRaises(Exception, my_callable, None) self.assertEqual(mock_call.call_count, 1)
def test_retry_aborts_simple(self, mock_exc_to_code, mock_time): def fake_call(dummy_request, dummy_timeout): raise CustomException('', _FAKE_STATUS_CODE_1) retry = RetryOptions([_FAKE_STATUS_CODE_1], BackoffSettings(0, 0, 0, 0, 0, 0, 1)) mock_time.side_effect = [0, 2] mock_exc_to_code.side_effect = lambda e: e.code settings = _CallSettings(timeout=0, retry=retry) my_callable = api_callable.create_api_call(fake_call, settings) try: my_callable(None) except RetryError as exc: self.assertIsInstance(exc.cause, CustomException)
def test_retry(self, mock_exc_to_code, mock_time): mock_exc_to_code.side_effect = lambda e: e.code to_attempt = 3 retry = RetryOptions([_FAKE_STATUS_CODE_1], BackoffSettings(0, 0, 0, 0, 0, 0, 1)) # Succeeds on the to_attempt'th call, and never again afterward mock_call = mock.Mock() mock_call.side_effect = ([CustomException('', _FAKE_STATUS_CODE_1)] * (to_attempt - 1) + [mock.DEFAULT]) mock_call.return_value = 1729 mock_time.return_value = 0 settings = _CallSettings(timeout=0, retry=retry) my_callable = api_callable.create_api_call(mock_call, settings) self.assertEqual(my_callable(None), 1729) self.assertEqual(mock_call.call_count, to_attempt)
def test_retryable_with_timeout(self, mock_time, mock_exc_to_code): mock_time.return_value = 1 mock_exc_to_code.side_effect = lambda e: e.code mock_call = mock.Mock() mock_call.side_effect = [CustomException('', _FAKE_STATUS_CODE_1), mock.DEFAULT] mock_call.return_value = 1729 retry_options = RetryOptions( [_FAKE_STATUS_CODE_1], BackoffSettings(0, 0, 0, 0, 0, 0, 0)) my_callable = retry.retryable(mock_call, retry_options) self.assertRaises(errors.RetryError, my_callable) self.assertEqual(0, mock_call.call_count)
def test_retry_times_out_simple(self, mock_exc_to_code, mock_time): mock_exc_to_code.side_effect = lambda e: e.code to_attempt = 3 retry = RetryOptions([_FAKE_STATUS_CODE_1], BackoffSettings(0, 0, 0, 0, 0, 0, 1)) mock_call = mock.Mock() mock_call.side_effect = CustomException('', _FAKE_STATUS_CODE_1) mock_time.side_effect = ([0] * to_attempt + [2]) settings = _CallSettings(timeout=0, retry=retry) my_callable = api_callable.create_api_call(mock_call, settings) try: my_callable(None) except RetryError as exc: self.assertIsInstance(exc.cause, CustomException) self.assertEqual(mock_call.call_count, to_attempt)
def test_retryable_without_timeout(self, mock_time, mock_exc_to_code): mock_time.return_value = 0 mock_exc_to_code.side_effect = lambda e: e.code to_attempt = 3 mock_call = mock.Mock() mock_call.side_effect = ([CustomException('', _FAKE_STATUS_CODE_1)] * (to_attempt - 1) + [mock.DEFAULT]) mock_call.return_value = 1729 retry_options = RetryOptions( [_FAKE_STATUS_CODE_1], BackoffSettings(0, 0, 0, None, None, None, None)) my_callable = retry.retryable(mock_call, retry_options) self.assertEqual(my_callable(None), 1729) self.assertEqual(to_attempt, mock_call.call_count)
def test_retryable_when_no_codes(self, mock_time, mock_exc_to_code): mock_time.return_value = 0 mock_exc_to_code.side_effect = lambda e: e.code mock_call = mock.Mock() mock_call.side_effect = [CustomException('', _FAKE_STATUS_CODE_1), mock.DEFAULT] mock_call.return_value = 1729 retry_options = RetryOptions( [], BackoffSettings(0, 0, 0, 0, 0, 0, 1)) my_callable = retry.retryable(mock_call, retry_options) try: my_callable(None) self.fail('Should not have been reached') except errors.RetryError as exc: self.assertIsInstance(exc.cause, CustomException) self.assertEqual(1, mock_call.call_count)
def test_retry_exponential_backoff(self, mock_exc_to_code, mock_time, mock_sleep): # pylint: disable=too-many-locals mock_exc_to_code.side_effect = lambda e: e.code MILLIS_PER_SEC = 1000 mock_time.return_value = 0 def incr_time(secs): mock_time.return_value += secs def api_call(dummy_request, timeout, **dummy_kwargs): incr_time(timeout) raise CustomException(str(timeout), _FAKE_STATUS_CODE_1) mock_call = mock.Mock() mock_sleep.side_effect = incr_time mock_call.side_effect = api_call params = BackoffSettings(3, 2, 24, 5, 2, 80, 2500) retry = RetryOptions([_FAKE_STATUS_CODE_1], params) settings = CallSettings(timeout=0, retry=retry) my_callable = api_callable.create_api_call(mock_call, settings) try: my_callable(None) except RetryError as exc: self.assertIsInstance(exc.cause, CustomException) self.assertGreaterEqual(mock_time(), params.total_timeout_millis / MILLIS_PER_SEC) # Very rough bounds calls_lower_bound = params.total_timeout_millis / ( params.max_retry_delay_millis + params.max_rpc_timeout_millis) self.assertGreater(mock_call.call_count, calls_lower_bound) calls_upper_bound = (params.total_timeout_millis / params.initial_retry_delay_millis) self.assertLess(mock_call.call_count, calls_upper_bound)
def test_retryable_exponential_backoff( self, mock_time, mock_sleep, mock_exc_to_code): def incr_time(secs): mock_time.return_value += secs def api_call(timeout): incr_time(timeout) raise CustomException(str(timeout), _FAKE_STATUS_CODE_1) mock_time.return_value = 0 mock_sleep.side_effect = incr_time mock_exc_to_code.side_effect = lambda e: e.code mock_call = mock.Mock() mock_call.side_effect = api_call params = BackoffSettings(3, 2, 24, 5, 2, 80, 2500) retry_options = RetryOptions([_FAKE_STATUS_CODE_1], params) my_callable = retry.retryable(mock_call, retry_options) try: my_callable() self.fail('Should not have been reached') except errors.RetryError as exc: self.assertIsInstance(exc.cause, CustomException) self.assertGreaterEqual(mock_time(), params.total_timeout_millis / _MILLIS_PER_SEC) # Very rough bounds calls_lower_bound = params.total_timeout_millis / ( params.max_retry_delay_millis + params.max_rpc_timeout_millis) self.assertGreater(mock_call.call_count, calls_lower_bound) calls_upper_bound = (params.total_timeout_millis / params.initial_retry_delay_millis) self.assertLess(mock_call.call_count, calls_upper_bound)
def _construct_retry(method_config, retry_codes, retry_params, retry_names): """Helper for ``construct_settings()``. Args: method_config: A dictionary representing a single ``methods`` entry of the standard API client config file. (See ``construct_settings()`` for information on this yaml.) retry_codes: A dictionary parsed from the ``retry_codes`` entry of the standard API client config file. (See ``construct_settings()`` for information on this yaml.) retry_params: A dictionary parsed from the ``retry_params`` entry of the standard API client config file. (See ``construct_settings()`` for information on this yaml.) retry_names: A dictionary mapping the string names used in the standard API client config file to API response status codes. Returns: A RetryOptions object, or None. """ if method_config is None: return None codes = None if retry_codes and 'retry_codes_name' in method_config: codes_name = method_config['retry_codes_name'] if codes_name in retry_codes and retry_codes[codes_name]: codes = [retry_names[name] for name in retry_codes[codes_name]] else: codes = [] backoff_settings = None if retry_params and 'retry_params_name' in method_config: params_name = method_config['retry_params_name'] if params_name and params_name in retry_params: backoff_settings = BackoffSettings(**retry_params[params_name]) return RetryOptions(retry_codes=codes, backoff_settings=backoff_settings)
def test_read_rows_retry_timeout(self): from google.cloud._testing import _Monkey from tests.unit._testing import _CustomFakeStub from google.cloud.bigtable.row_data import PartialRowsData from google.cloud.bigtable import retry as MUT from google.cloud.bigtable.retry import ReadRowsIterator from google.gax import BackoffSettings from google.gax.errors import RetryError from grpc import StatusCode, RpcError import time client = _Client() instance = _Instance(self.INSTANCE_NAME, client=client) table = self._make_one(self.TABLE_ID, instance) # Create request_pb request_pb = object() # Returned by our mock. mock_created = [] def mock_create_row_request(table_name, **kwargs): mock_created.append((table_name, kwargs)) return request_pb # Create a slow response iterator to cause a timeout class MockTimeoutError(RpcError): def code(self): return StatusCode.DEADLINE_EXCEEDED def _wait_then_raise(): time.sleep(0.1) raise MockTimeoutError() # Patch the stub used by the API method. The stub should create a new # slow_iterator every time its queried. def make_slow_iterator(): return (_wait_then_raise() for i in range(10)) client._data_stub = stub = _CustomFakeStub(make_slow_iterator) # Set to timeout before RPC completes test_backoff_settings = BackoffSettings( initial_retry_delay_millis=10, retry_delay_multiplier=0.3, max_retry_delay_millis=30000, initial_rpc_timeout_millis=1000, rpc_timeout_multiplier=1.0, max_rpc_timeout_millis=25 * 60 * 1000, total_timeout_millis=1000) start_key = b'start-key' end_key = b'end-key' filter_obj = object() limit = 22 with _Monkey(MUT, _create_row_request=mock_create_row_request): # Verify that a RetryError is thrown on read. result = table.read_rows(start_key=start_key, end_key=end_key, filter_=filter_obj, limit=limit, backoff_settings=test_backoff_settings) with self.assertRaises(RetryError): result.consume_next()
table_admin_messages_v2_pb2) from google.cloud.bigtable._generated import (table_pb2 as table_v2_pb2) from google.cloud.bigtable.column_family import _gc_rule_from_pb from google.cloud.bigtable.column_family import ColumnFamily from google.cloud.bigtable.row import AppendRow from google.cloud.bigtable.row import ConditionalRow from google.cloud.bigtable.row import DirectRow from google.cloud.bigtable.row_data import PartialRowsData from google.gax import RetryOptions, BackoffSettings from google.cloud.bigtable.retry import ReadRowsIterator, _create_row_request from grpc import StatusCode BACKOFF_SETTINGS = BackoffSettings(initial_retry_delay_millis=10, retry_delay_multiplier=1.3, max_retry_delay_millis=30000, initial_rpc_timeout_millis=25 * 60 * 1000, rpc_timeout_multiplier=1.0, max_rpc_timeout_millis=25 * 60 * 1000, total_timeout_millis=30 * 60 * 1000) RETRY_CODES = [ StatusCode.DEADLINE_EXCEEDED, StatusCode.ABORTED, StatusCode.INTERNAL, StatusCode.UNAVAILABLE ] # Maximum number of mutations in bulk (MutateRowsRequest message): # (https://cloud.google.com/bigtable/docs/reference/data/rpc/ # google.bigtable.v2#google.bigtable.v2.MutateRowRequest) _MAX_BULK_MUTATIONS = 100000