Ejemplo n.º 1
0
 def test_settings_merge_options2(self):
     retry = RetryOptions(None, None)
     options = CallOptions(retry=retry)
     settings = _CallSettings(
         timeout=9, page_descriptor=None, retry=RetryOptions(None, None))
     final = settings.merge(options)
     self.assertEqual(final.timeout, 9)
     self.assertIsNone(final.page_descriptor)
     self.assertEqual(final.retry, retry)
Ejemplo n.º 2
0
def _merge_retry_options(retry_options, overrides):
    """Helper for ``construct_settings()``.

    Takes two retry options, and merges them into a single RetryOption instance.

    Args:
      retry_options: The base RetryOptions.
      overrides: The RetryOptions used for overriding ``retry``. Use the values
        if it is not None. If entire ``overrides`` is None, ignore the base
        retry and return None.

    Returns:
      The merged RetryOptions, or None if it will be canceled.
    """
    if overrides is None:
        return None

    if overrides.retry_codes is None and overrides.backoff_settings is None:
        return retry_options

    codes = retry_options.retry_codes
    if overrides.retry_codes is not None:
        codes = overrides.retry_codes
    backoff_settings = retry_options.backoff_settings
    if overrides.backoff_settings is not None:
        backoff_settings = overrides.backoff_settings

    return RetryOptions(retry_codes=codes, backoff_settings=backoff_settings)
Ejemplo n.º 3
0
 def test_settings_merge_options_page_streaming(self):
     retry = RetryOptions(None, None)
     options = CallOptions(timeout=46, is_page_streaming=False)
     settings = CallSettings(timeout=9, retry=retry)
     final = settings.merge(options)
     self.assertEqual(final.timeout, 46)
     self.assertIsNone(final.page_descriptor)
     self.assertEqual(final.retry, retry)
Ejemplo n.º 4
0
    def test_retry_times_out_no_response(self, mock_time):
        mock_time.return_value = 1
        retry = RetryOptions([_FAKE_STATUS_CODE_1],
                             BackoffSettings(0, 0, 0, 0, 0, 0, 0))
        settings = _CallSettings(timeout=0, retry=retry)
        my_callable = api_callable.create_api_call(lambda: None, settings)

        self.assertRaises(RetryError, my_callable, None)
Ejemplo n.º 5
0
    def test_no_retry_if_no_codes(self, mock_time):
        retry = RetryOptions([], BackoffSettings(1, 2, 3, 4, 5, 6, 7))

        mock_call = mock.Mock()
        mock_call.side_effect = CustomException('', _FAKE_STATUS_CODE_1)
        mock_time.return_value = 0

        settings = CallSettings(timeout=0, retry=retry)
        my_callable = api_callable.create_api_call(mock_call, settings)
        self.assertRaises(CustomException, my_callable, None)
        self.assertEqual(mock_call.call_count, 1)
Ejemplo n.º 6
0
 def test_settings_merge_options_page_streaming(self):
     retry = RetryOptions(None, None)
     page_descriptor = object()
     options = CallOptions(timeout=46, page_token=INITIAL_PAGE)
     settings = _CallSettings(timeout=9, retry=retry,
                              page_descriptor=page_descriptor)
     final = settings.merge(options)
     self.assertEqual(final.timeout, 46)
     self.assertEqual(final.page_descriptor, page_descriptor)
     self.assertEqual(final.page_token, INITIAL_PAGE)
     self.assertFalse(final.flatten_pages)
     self.assertEqual(final.retry, retry)
Ejemplo n.º 7
0
 def test_retry_aborts_on_unexpected_exception(self, mock_exc_to_code,
                                               mock_time):
     mock_exc_to_code.side_effect = lambda e: e.code
     retry = RetryOptions([_FAKE_STATUS_CODE_1],
                          BackoffSettings(0, 0, 0, 0, 0, 0, 1))
     mock_call = mock.Mock()
     mock_call.side_effect = CustomException('', _FAKE_STATUS_CODE_2)
     mock_time.return_value = 0
     settings = _CallSettings(timeout=0, retry=retry)
     my_callable = api_callable.create_api_call(mock_call, settings)
     self.assertRaises(Exception, my_callable, None)
     self.assertEqual(mock_call.call_count, 1)
Ejemplo n.º 8
0
    def test_retry_aborts_simple(self, mock_exc_to_code, mock_time):
        def fake_call(dummy_request, dummy_timeout):
            raise CustomException('', _FAKE_STATUS_CODE_1)

        retry = RetryOptions([_FAKE_STATUS_CODE_1],
                             BackoffSettings(0, 0, 0, 0, 0, 0, 1))
        mock_time.side_effect = [0, 2]
        mock_exc_to_code.side_effect = lambda e: e.code
        settings = _CallSettings(timeout=0, retry=retry)
        my_callable = api_callable.create_api_call(fake_call, settings)

        try:
            my_callable(None)
        except RetryError as exc:
            self.assertIsInstance(exc.cause, CustomException)
Ejemplo n.º 9
0
    def test_retry(self, mock_exc_to_code, mock_time):
        mock_exc_to_code.side_effect = lambda e: e.code
        to_attempt = 3
        retry = RetryOptions([_FAKE_STATUS_CODE_1],
                             BackoffSettings(0, 0, 0, 0, 0, 0, 1))

        # Succeeds on the to_attempt'th call, and never again afterward
        mock_call = mock.Mock()
        mock_call.side_effect = ([CustomException('', _FAKE_STATUS_CODE_1)] *
                                 (to_attempt - 1) + [mock.DEFAULT])
        mock_call.return_value = 1729
        mock_time.return_value = 0
        settings = _CallSettings(timeout=0, retry=retry)
        my_callable = api_callable.create_api_call(mock_call, settings)
        self.assertEqual(my_callable(None), 1729)
        self.assertEqual(mock_call.call_count, to_attempt)
Ejemplo n.º 10
0
    def read_rows(self,
                  start_key=None,
                  end_key=None,
                  limit=None,
                  filter_=None,
                  end_inclusive=False,
                  backoff_settings=None):
        """Read rows from this table.

        :type start_key: bytes
        :param start_key: (Optional) The beginning of a range of row keys to
                          read from. The range will include ``start_key``. If
                          left empty, will be interpreted as the empty string.

        :type end_key: bytes
        :param end_key: (Optional) The end of a range of row keys to read from.
                        The range will not include ``end_key``. If left empty,
                        will be interpreted as an infinite string.

        :type limit: int
        :param limit: (Optional) The read will terminate after committing to N
                      rows' worth of results. The default (zero) is to return
                      all results.

        :type filter_: :class:`.RowFilter`
        :param filter_: (Optional) The filter to apply to the contents of the
                        specified row(s). If unset, reads every column in
                        each row.

        :type end_inclusive: bool
        :param end_inclusive: (Optional) Whether the ``end_key`` should be
                      considered inclusive. The default is False (exclusive).

        :rtype: :class:`.PartialRowsData`
        :returns: A :class:`.PartialRowsData` convenience wrapper for consuming
                  the streamed results.
        """
        client = self._instance._client
        if backoff_settings is None:
            backoff_settings = BACKOFF_SETTINGS
        RETRY_OPTIONS = RetryOptions(retry_codes=RETRY_CODES,
                                     backoff_settings=backoff_settings)

        retrying_iterator = ReadRowsIterator(client, self.name, start_key,
                                             end_key, filter_, limit,
                                             end_inclusive, RETRY_OPTIONS)
        return PartialRowsData(retrying_iterator)
Ejemplo n.º 11
0
    def test_retryable_with_timeout(self, mock_time, mock_exc_to_code):
        mock_time.return_value = 1
        mock_exc_to_code.side_effect = lambda e: e.code

        mock_call = mock.Mock()
        mock_call.side_effect = [CustomException('', _FAKE_STATUS_CODE_1),
                                 mock.DEFAULT]
        mock_call.return_value = 1729

        retry_options = RetryOptions(
            [_FAKE_STATUS_CODE_1],
            BackoffSettings(0, 0, 0, 0, 0, 0, 0))

        my_callable = retry.retryable(mock_call, retry_options)

        self.assertRaises(errors.RetryError, my_callable)
        self.assertEqual(0, mock_call.call_count)
Ejemplo n.º 12
0
    def test_retry_times_out_simple(self, mock_exc_to_code, mock_time):
        mock_exc_to_code.side_effect = lambda e: e.code
        to_attempt = 3
        retry = RetryOptions([_FAKE_STATUS_CODE_1],
                             BackoffSettings(0, 0, 0, 0, 0, 0, 1))
        mock_call = mock.Mock()
        mock_call.side_effect = CustomException('', _FAKE_STATUS_CODE_1)
        mock_time.side_effect = ([0] * to_attempt + [2])
        settings = _CallSettings(timeout=0, retry=retry)
        my_callable = api_callable.create_api_call(mock_call, settings)

        try:
            my_callable(None)
        except RetryError as exc:
            self.assertIsInstance(exc.cause, CustomException)

        self.assertEqual(mock_call.call_count, to_attempt)
Ejemplo n.º 13
0
    def test_retryable_without_timeout(self, mock_time, mock_exc_to_code):
        mock_time.return_value = 0
        mock_exc_to_code.side_effect = lambda e: e.code

        to_attempt = 3
        mock_call = mock.Mock()
        mock_call.side_effect = ([CustomException('', _FAKE_STATUS_CODE_1)] *
                                 (to_attempt - 1) + [mock.DEFAULT])
        mock_call.return_value = 1729

        retry_options = RetryOptions(
            [_FAKE_STATUS_CODE_1],
            BackoffSettings(0, 0, 0, None, None, None, None))

        my_callable = retry.retryable(mock_call, retry_options)

        self.assertEqual(my_callable(None), 1729)
        self.assertEqual(to_attempt, mock_call.call_count)
Ejemplo n.º 14
0
    def test_retry_exponential_backoff(self, mock_exc_to_code, mock_time,
                                       mock_sleep):
        # pylint: disable=too-many-locals
        mock_exc_to_code.side_effect = lambda e: e.code
        MILLIS_PER_SEC = 1000
        mock_time.return_value = 0

        def incr_time(secs):
            mock_time.return_value += secs

        def api_call(dummy_request, timeout, **dummy_kwargs):
            incr_time(timeout)
            raise CustomException(str(timeout), _FAKE_STATUS_CODE_1)

        mock_call = mock.Mock()
        mock_sleep.side_effect = incr_time
        mock_call.side_effect = api_call

        params = BackoffSettings(3, 2, 24, 5, 2, 80, 2500)
        retry = RetryOptions([_FAKE_STATUS_CODE_1], params)
        settings = CallSettings(timeout=0, retry=retry)
        my_callable = api_callable.create_api_call(mock_call, settings)

        try:
            my_callable(None)
        except RetryError as exc:
            self.assertIsInstance(exc.cause, CustomException)

        self.assertGreaterEqual(mock_time(),
                                params.total_timeout_millis / MILLIS_PER_SEC)

        # Very rough bounds
        calls_lower_bound = params.total_timeout_millis / (
            params.max_retry_delay_millis + params.max_rpc_timeout_millis)
        self.assertGreater(mock_call.call_count, calls_lower_bound)

        calls_upper_bound = (params.total_timeout_millis /
                             params.initial_retry_delay_millis)
        self.assertLess(mock_call.call_count, calls_upper_bound)
Ejemplo n.º 15
0
    def test_retryable_when_no_codes(self, mock_time, mock_exc_to_code):
        mock_time.return_value = 0
        mock_exc_to_code.side_effect = lambda e: e.code

        mock_call = mock.Mock()
        mock_call.side_effect = [CustomException('', _FAKE_STATUS_CODE_1),
                                 mock.DEFAULT]
        mock_call.return_value = 1729

        retry_options = RetryOptions(
            [],
            BackoffSettings(0, 0, 0, 0, 0, 0, 1))

        my_callable = retry.retryable(mock_call, retry_options)

        try:
            my_callable(None)
            self.fail('Should not have been reached')
        except errors.RetryError as exc:
            self.assertIsInstance(exc.cause, CustomException)

        self.assertEqual(1, mock_call.call_count)
Ejemplo n.º 16
0
    def test_retryable_exponential_backoff(
            self, mock_time, mock_sleep, mock_exc_to_code):
        def incr_time(secs):
            mock_time.return_value += secs

        def api_call(timeout):
            incr_time(timeout)
            raise CustomException(str(timeout), _FAKE_STATUS_CODE_1)

        mock_time.return_value = 0
        mock_sleep.side_effect = incr_time
        mock_exc_to_code.side_effect = lambda e: e.code

        mock_call = mock.Mock()
        mock_call.side_effect = api_call

        params = BackoffSettings(3, 2, 24, 5, 2, 80, 2500)
        retry_options = RetryOptions([_FAKE_STATUS_CODE_1], params)

        my_callable = retry.retryable(mock_call, retry_options)

        try:
            my_callable()
            self.fail('Should not have been reached')
        except errors.RetryError as exc:
            self.assertIsInstance(exc.cause, CustomException)

        self.assertGreaterEqual(mock_time(),
                                params.total_timeout_millis / _MILLIS_PER_SEC)

        # Very rough bounds
        calls_lower_bound = params.total_timeout_millis / (
            params.max_retry_delay_millis + params.max_rpc_timeout_millis)
        self.assertGreater(mock_call.call_count, calls_lower_bound)

        calls_upper_bound = (params.total_timeout_millis /
                             params.initial_retry_delay_millis)
        self.assertLess(mock_call.call_count, calls_upper_bound)
Ejemplo n.º 17
0
def _construct_retry(method_config, retry_codes, retry_params, retry_names):
    """Helper for ``construct_settings()``.

    Args:
      method_config: A dictionary representing a single ``methods`` entry of the
        standard API client config file. (See ``construct_settings()`` for
        information on this yaml.)
      retry_codes: A dictionary parsed from the ``retry_codes`` entry
        of the standard API client config file. (See ``construct_settings()``
        for information on this yaml.)
      retry_params: A dictionary parsed from the ``retry_params`` entry
        of the standard API client config file. (See ``construct_settings()``
        for information on this yaml.)
      retry_names: A dictionary mapping the string names used in the
        standard API client config file to API response status codes.

    Returns:
      A RetryOptions object, or None.
    """
    if method_config is None:
        return None

    codes = None
    if retry_codes and 'retry_codes_name' in method_config:
        codes_name = method_config['retry_codes_name']
        if codes_name in retry_codes and retry_codes[codes_name]:
            codes = [retry_names[name] for name in retry_codes[codes_name]]
        else:
            codes = []

    backoff_settings = None
    if retry_params and 'retry_params_name' in method_config:
        params_name = method_config['retry_params_name']
        if params_name and params_name in retry_params:
            backoff_settings = BackoffSettings(**retry_params[params_name])

    return RetryOptions(retry_codes=codes, backoff_settings=backoff_settings)
Ejemplo n.º 18
0
 def test_cannot_construct_bad_options(self):
     self.assertRaises(ValueError,
                       CallOptions,
                       timeout=47,
                       retry=RetryOptions(None, None))