def test_catch_error(self): def abortion_error_func(*dummy_args, **dummy_kwargs): raise CustomException(None, None) def other_error_func(*dummy_args, **dummy_kwargs): raise AnotherException gax_error_callable = api_callable.create_api_call( abortion_error_func, _CallSettings()) self.assertRaises(GaxError, gax_error_callable, None) other_error_callable = api_callable.create_api_call( other_error_func, _CallSettings()) self.assertRaises(AnotherException, other_error_callable, None)
def test_call_kwargs(self): settings = _CallSettings(kwargs={'key': 'value'}) my_callable = api_callable.create_api_call( lambda _req, _timeout, **kwargs: kwargs['key'], settings) self.assertEqual(my_callable(None), 'value') self.assertEqual(my_callable(None, CallOptions(key='updated')), 'updated')
def test_settings_merge_options1(self): options = CallOptions(timeout=46) settings = _CallSettings(timeout=9, page_descriptor=None, retry=None) final = settings.merge(options) self.assertEqual(final.timeout, 46) self.assertIsNone(final.retry) self.assertIsNone(final.page_descriptor)
def test_retry_times_out_no_response(self, mock_time): mock_time.return_value = 1 retry = RetryOptions([_FAKE_STATUS_CODE_1], BackoffSettings(0, 0, 0, 0, 0, 0, 0)) settings = _CallSettings(timeout=0, retry=retry) my_callable = api_callable.create_api_call(lambda: None, settings) self.assertRaises(RetryError, my_callable, None)
def test_retry_times_out_no_response(self, mock_time): mock_time.return_value = 1 retry = RetryOptions( [_FAKE_STATUS_CODE_1], BackoffSettings(0, 0, 0, 0, 0, 0, 0)) settings = _CallSettings(timeout=0, retry=retry) my_callable = api_callable.create_api_call(lambda: None, settings) self.assertRaises(RetryError, my_callable, None)
def test_settings_merge_options2(self): retry = RetryOptions(None, None) options = CallOptions(retry=retry) settings = _CallSettings( timeout=9, page_descriptor=None, retry=RetryOptions(None, None)) final = settings.merge(options) self.assertEqual(final.timeout, 9) self.assertIsNone(final.page_descriptor) self.assertEqual(final.retry, retry)
def test_settings_merge_none(self): settings = _CallSettings( timeout=23, page_descriptor=object(), bundler=object(), retry=object()) final = settings.merge(None) self.assertEqual(final.timeout, settings.timeout) self.assertEqual(final.retry, settings.retry) self.assertEqual(final.page_descriptor, settings.page_descriptor) self.assertEqual(final.bundler, settings.bundler) self.assertEqual(final.bundle_descriptor, settings.bundle_descriptor)
def test_no_retry_if_no_codes(self, mock_time): retry = RetryOptions([], BackoffSettings(1, 2, 3, 4, 5, 6, 7)) mock_call = mock.Mock() mock_call.side_effect = CustomException('', _FAKE_STATUS_CODE_1) mock_time.return_value = 0 settings = _CallSettings(timeout=0, retry=retry) my_callable = api_callable.create_api_call(mock_call, settings) self.assertRaises(CustomException, my_callable, None) self.assertEqual(mock_call.call_count, 1)
def test_wrap_value_error(self): invalid_attribute_exc = grpc.RpcError() invalid_attribute_exc.code = lambda: grpc.StatusCode.INVALID_ARGUMENT def value_error_func(*dummy_args, **dummy_kwargs): raise invalid_attribute_exc value_error_callable = api_callable.create_api_call( value_error_func, _CallSettings()) self.assertRaises(ValueError, value_error_callable, None)
def test_settings_merge_options_page_streaming(self): retry = RetryOptions(None, None) page_descriptor = object() options = CallOptions(timeout=46, page_token=INITIAL_PAGE) settings = _CallSettings(timeout=9, retry=retry, page_descriptor=page_descriptor) final = settings.merge(options) self.assertEqual(final.timeout, 46) self.assertEqual(final.page_descriptor, page_descriptor) self.assertEqual(final.page_token, INITIAL_PAGE) self.assertFalse(final.flatten_pages) self.assertEqual(final.retry, retry)
def test_retry_aborts_on_unexpected_exception(self, mock_exc_to_code, mock_time): mock_exc_to_code.side_effect = lambda e: e.code retry = RetryOptions([_FAKE_STATUS_CODE_1], BackoffSettings(0, 0, 0, 0, 0, 0, 1)) mock_call = mock.Mock() mock_call.side_effect = CustomException('', _FAKE_STATUS_CODE_2) mock_time.return_value = 0 settings = _CallSettings(timeout=0, retry=retry) my_callable = api_callable.create_api_call(mock_call, settings) self.assertRaises(Exception, my_callable, None) self.assertEqual(mock_call.call_count, 1)
def test_retry_aborts_on_unexpected_exception( self, mock_exc_to_code, mock_time): mock_exc_to_code.side_effect = lambda e: e.code retry = RetryOptions( [_FAKE_STATUS_CODE_1], BackoffSettings(0, 0, 0, 0, 0, 0, 1)) mock_call = mock.Mock() mock_call.side_effect = CustomException('', _FAKE_STATUS_CODE_2) mock_time.return_value = 0 settings = _CallSettings(timeout=0, retry=retry) my_callable = api_callable.create_api_call(mock_call, settings) self.assertRaises(Exception, my_callable, None) self.assertEqual(mock_call.call_count, 1)
def test_retry_aborts_simple(self, mock_exc_to_code, mock_time): def fake_call(dummy_request, dummy_timeout): raise CustomException('', _FAKE_STATUS_CODE_1) retry = RetryOptions([_FAKE_STATUS_CODE_1], BackoffSettings(0, 0, 0, 0, 0, 0, 1)) mock_time.side_effect = [0, 2] mock_exc_to_code.side_effect = lambda e: e.code settings = _CallSettings(timeout=0, retry=retry) my_callable = api_callable.create_api_call(fake_call, settings) try: my_callable(None) except RetryError as exc: self.assertIsInstance(exc.cause, CustomException)
def test_retry_aborts_simple(self, mock_exc_to_code, mock_time): def fake_call(dummy_request, dummy_timeout): raise CustomException('', _FAKE_STATUS_CODE_1) retry = RetryOptions( [_FAKE_STATUS_CODE_1], BackoffSettings(0, 0, 0, 0, 0, 0, 1)) mock_time.side_effect = [0, 2] mock_exc_to_code.side_effect = lambda e: e.code settings = _CallSettings(timeout=0, retry=retry) my_callable = api_callable.create_api_call(fake_call, settings) try: my_callable(None) except RetryError as exc: self.assertIsInstance(exc.cause, CustomException)
def test_retry(self, mock_exc_to_code, mock_time): mock_exc_to_code.side_effect = lambda e: e.code to_attempt = 3 retry = RetryOptions([_FAKE_STATUS_CODE_1], BackoffSettings(0, 0, 0, 0, 0, 0, 1)) # Succeeds on the to_attempt'th call, and never again afterward mock_call = mock.Mock() mock_call.side_effect = ([CustomException('', _FAKE_STATUS_CODE_1)] * (to_attempt - 1) + [mock.DEFAULT]) mock_call.return_value = 1729 mock_time.return_value = 0 settings = _CallSettings(timeout=0, retry=retry) my_callable = api_callable.create_api_call(mock_call, settings) self.assertEqual(my_callable(None), 1729) self.assertEqual(mock_call.call_count, to_attempt)
def test_retry(self, mock_exc_to_code, mock_time): mock_exc_to_code.side_effect = lambda e: e.code to_attempt = 3 retry = RetryOptions( [_FAKE_STATUS_CODE_1], BackoffSettings(0, 0, 0, 0, 0, 0, 1)) # Succeeds on the to_attempt'th call, and never again afterward mock_call = mock.Mock() mock_call.side_effect = ([CustomException('', _FAKE_STATUS_CODE_1)] * (to_attempt - 1) + [mock.DEFAULT]) mock_call.return_value = 1729 mock_time.return_value = 0 settings = _CallSettings(timeout=0, retry=retry) my_callable = api_callable.create_api_call(mock_call, settings) self.assertEqual(my_callable(None), 1729) self.assertEqual(mock_call.call_count, to_attempt)
def test_retry_times_out_simple(self, mock_exc_to_code, mock_time): mock_exc_to_code.side_effect = lambda e: e.code to_attempt = 3 retry = RetryOptions([_FAKE_STATUS_CODE_1], BackoffSettings(0, 0, 0, 0, 0, 0, 1)) mock_call = mock.Mock() mock_call.side_effect = CustomException('', _FAKE_STATUS_CODE_1) mock_time.side_effect = ([0] * to_attempt + [2]) settings = _CallSettings(timeout=0, retry=retry) my_callable = api_callable.create_api_call(mock_call, settings) try: my_callable(None) except RetryError as exc: self.assertIsInstance(exc.cause, CustomException) self.assertEqual(mock_call.call_count, to_attempt)
def test_retry_times_out_simple(self, mock_exc_to_code, mock_time): mock_exc_to_code.side_effect = lambda e: e.code to_attempt = 3 retry = RetryOptions( [_FAKE_STATUS_CODE_1], BackoffSettings(0, 0, 0, 0, 0, 0, 1)) mock_call = mock.Mock() mock_call.side_effect = CustomException('', _FAKE_STATUS_CODE_1) mock_time.side_effect = ([0] * to_attempt + [2]) settings = _CallSettings(timeout=0, retry=retry) my_callable = api_callable.create_api_call(mock_call, settings) try: my_callable(None) except RetryError as exc: self.assertIsInstance(exc.cause, CustomException) self.assertEqual(mock_call.call_count, to_attempt)
def test_wrap_value_error(self): from google.gax.errors import InvalidArgumentError invalid_attribute_exc = grpc.RpcError() invalid_attribute_exc.code = lambda: grpc.StatusCode.INVALID_ARGUMENT def value_error_func(*dummy_args, **dummy_kwargs): raise invalid_attribute_exc value_error_callable = api_callable.create_api_call( value_error_func, _CallSettings()) with self.assertRaises(ValueError) as exc_info: value_error_callable(None) self.assertIsInstance(exc_info.exception, InvalidArgumentError) self.assertEqual(exc_info.exception.args, (u'RPC failed', )) self.assertIs(exc_info.exception.cause, invalid_attribute_exc)
def test_call_merge_options_metadata(self): settings_kwargs = { 'key': 'value', 'metadata': [('key1', 'val1'), ('key2', 'val2')] } settings = _CallSettings(kwargs=settings_kwargs) my_callable = api_callable.create_api_call( lambda _req, _timeout, **kwargs: kwargs, settings) # Merge empty options, settings.kwargs['metadata'] remain unchanged expected_kwargs = settings_kwargs self.assertEqual(my_callable(None), expected_kwargs) # Override an existing key in settings.kwargs['metadata'] expected_kwargs = { 'key': 'value', 'metadata': [('key1', '_val1'), ('key2', 'val2')] } self.assertEqual( my_callable(None, CallOptions(metadata=[('key1', '_val1')])), expected_kwargs) # Add a new key in settings.kwargs['metadata'] expected_kwargs = { 'key': 'value', 'metadata': [('key3', 'val3'), ('key1', 'val1'), ('key2', 'val2')] } self.assertEqual( my_callable(None, CallOptions(metadata=[('key3', 'val3')])), expected_kwargs) # Do all: add a new key and override an existing one in # settings.kwargs['metadata'] expected_kwargs = { 'key': 'value', 'metadata': [('key3', 'val3'), ('key2', '_val2'), ('key1', 'val1')] } self.assertEqual( my_callable( None, CallOptions(metadata=[('key3', 'val3'), ('key2', '_val2')])), expected_kwargs)
def test_bundling(self): # pylint: disable=abstract-method, too-few-public-methods class BundlingRequest(object): def __init__(self, elements=None): self.elements = elements fake_grpc_func_descriptor = BundleDescriptor('elements', []) bundler = bundling.Executor(BundleOptions(element_count_threshold=8)) def my_func(request, dummy_timeout): return len(request.elements) settings = _CallSettings( bundler=bundler, bundle_descriptor=fake_grpc_func_descriptor, timeout=0) my_callable = api_callable.create_api_call(my_func, settings) first = my_callable(BundlingRequest([0] * 3)) self.assertIsInstance(first, bundling.Event) self.assertIsNone(first.result) # pylint: disable=no-member second = my_callable(BundlingRequest([0] * 5)) self.assertEqual(second.result, 8) # pylint: disable=no-member
def test_bundling(self): # pylint: disable=abstract-method, too-few-public-methods class BundlingRequest(object): def __init__(self, elements=None): self.elements = elements fake_grpc_func_descriptor = BundleDescriptor('elements', []) bundler = bundling.Executor(BundleOptions(element_count_threshold=8)) def my_func(request, dummy_timeout): return len(request.elements) settings = _CallSettings(bundler=bundler, bundle_descriptor=fake_grpc_func_descriptor, timeout=0) my_callable = api_callable.create_api_call(my_func, settings) first = my_callable(BundlingRequest([0] * 3)) self.assertIsInstance(first, bundling.Event) self.assertIsNone(first.result) # pylint: disable=no-member second = my_callable(BundlingRequest([0] * 5)) self.assertEqual(second.result, 8) # pylint: disable=no-member
def test_retry_exponential_backoff(self, mock_exc_to_code, mock_time, mock_sleep): # pylint: disable=too-many-locals mock_exc_to_code.side_effect = lambda e: e.code MILLIS_PER_SEC = 1000 mock_time.return_value = 0 def incr_time(secs): mock_time.return_value += secs def api_call(dummy_request, timeout, **dummy_kwargs): incr_time(timeout) raise CustomException(str(timeout), _FAKE_STATUS_CODE_1) mock_call = mock.Mock() mock_sleep.side_effect = incr_time mock_call.side_effect = api_call params = BackoffSettings(3, 2, 24, 5, 2, 80, 2500) retry = RetryOptions([_FAKE_STATUS_CODE_1], params) settings = _CallSettings(timeout=0, retry=retry) my_callable = api_callable.create_api_call(mock_call, settings) try: my_callable(None) except RetryError as exc: self.assertIsInstance(exc.cause, CustomException) self.assertGreaterEqual(mock_time(), params.total_timeout_millis / MILLIS_PER_SEC) # Very rough bounds calls_lower_bound = params.total_timeout_millis / ( params.max_retry_delay_millis + params.max_rpc_timeout_millis) self.assertGreater(mock_call.call_count, calls_lower_bound) calls_upper_bound = (params.total_timeout_millis / params.initial_retry_delay_millis) self.assertLess(mock_call.call_count, calls_upper_bound)
def test_call_api_call(self): settings = _CallSettings() my_callable = api_callable.create_api_call( lambda _req, _timeout: 42, settings) self.assertEqual(my_callable(None), 42)
def test_bundling_page_streaming_error(self): settings = _CallSettings( page_descriptor=object(), bundle_descriptor=object(), bundler=object()) with self.assertRaises(ValueError): api_callable.create_api_call(lambda _req, _timeout: 42, settings)
def test_page_streaming(self): # A mock grpc function that page streams a list of consecutive # integers, returning `page_size` integers with each call and using # the next integer to return as the page token, until `pages_to_stream` # pages have been returned. # pylint:disable=too-many-locals page_size = 3 pages_to_stream = 5 # pylint: disable=abstract-method, too-few-public-methods class PageStreamingRequest(object): def __init__(self, page_token=0): self.page_token = page_token class PageStreamingResponse(object): def __init__(self, nums=(), next_page_token=0): self.nums = nums self.next_page_token = next_page_token fake_grpc_func_descriptor = PageDescriptor( 'page_token', 'next_page_token', 'nums') def grpc_return_value(request, *dummy_args, **dummy_kwargs): start = int(request.page_token) if start > 0 and start < page_size * pages_to_stream: return PageStreamingResponse( nums=list(range(start, start + page_size)), next_page_token=start + page_size) elif start >= page_size * pages_to_stream: return PageStreamingResponse() else: return PageStreamingResponse(nums=list(range(page_size)), next_page_token=page_size) with mock.patch('grpc.UnaryUnaryMultiCallable') as mock_grpc: mock_grpc.side_effect = grpc_return_value settings = _CallSettings( page_descriptor=fake_grpc_func_descriptor, timeout=0) my_callable = api_callable.create_api_call( mock_grpc, settings=settings) self.assertEqual(list(my_callable(PageStreamingRequest())), list(range(page_size * pages_to_stream))) unflattened_option = CallOptions(page_token=INITIAL_PAGE) # Expect a list of pages_to_stream pages, each of size page_size, # plus one empty page expected = [list(range(page_size * n, page_size * (n + 1))) for n in range(pages_to_stream)] + [()] self.assertEqual(list(my_callable(PageStreamingRequest(), unflattened_option)), expected) pages_already_read = 2 explicit_page_token_option = CallOptions( page_token=str(page_size * pages_already_read)) # Expect a list of pages_to_stream pages, each of size page_size, # plus one empty page, minus the pages_already_read expected = [list(range(page_size * n, page_size * (n + 1))) for n in range(pages_already_read, pages_to_stream)] expected += [()] self.assertEqual(list(my_callable(PageStreamingRequest(), explicit_page_token_option)), expected)
def test_page_streaming(self): # A mock grpc function that page streams a list of consecutive # integers, returning `page_size` integers with each call and using # the next integer to return as the page token, until `pages_to_stream` # pages have been returned. # pylint:disable=too-many-locals page_size = 3 pages_to_stream = 5 # pylint: disable=abstract-method, too-few-public-methods class PageStreamingRequest(object): def __init__(self, page_token=0): self.page_token = page_token class PageStreamingResponse(object): def __init__(self, nums=(), next_page_token=0): self.nums = nums self.next_page_token = next_page_token fake_grpc_func_descriptor = PageDescriptor('page_token', 'next_page_token', 'nums') def grpc_return_value(request, *dummy_args, **dummy_kwargs): start = int(request.page_token) if start > 0 and start < page_size * pages_to_stream: return PageStreamingResponse(nums=list( range(start, start + page_size)), next_page_token=start + page_size) elif start >= page_size * pages_to_stream: return PageStreamingResponse() else: return PageStreamingResponse(nums=list(range(page_size)), next_page_token=page_size) with mock.patch('grpc.UnaryUnaryMultiCallable') as mock_grpc: mock_grpc.side_effect = grpc_return_value settings = _CallSettings(page_descriptor=fake_grpc_func_descriptor, timeout=0) my_callable = api_callable.create_api_call(mock_grpc, settings=settings) self.assertEqual(list(my_callable(PageStreamingRequest())), list(range(page_size * pages_to_stream))) unflattened_option = CallOptions(page_token=INITIAL_PAGE) # Expect a list of pages_to_stream pages, each of size page_size, # plus one empty page expected = [ list(range(page_size * n, page_size * (n + 1))) for n in range(pages_to_stream) ] + [()] self.assertEqual( list(my_callable(PageStreamingRequest(), unflattened_option)), expected) pages_already_read = 2 explicit_page_token_option = CallOptions( page_token=str(page_size * pages_already_read)) # Expect a list of pages_to_stream pages, each of size page_size, # plus one empty page, minus the pages_already_read expected = [ list(range(page_size * n, page_size * (n + 1))) for n in range(pages_already_read, pages_to_stream) ] expected += [()] self.assertEqual( list( my_callable(PageStreamingRequest(), explicit_page_token_option)), expected)
def test_bundling_page_streaming_error(self): settings = _CallSettings(page_descriptor=object(), bundle_descriptor=object(), bundler=object()) with self.assertRaises(ValueError): api_callable.create_api_call(lambda _req, _timeout: 42, settings)
def construct_settings(service_name, client_config, config_override, retry_names, bundle_descriptors=None, page_descriptors=None, kwargs=None): """Constructs a dictionary mapping method names to _CallSettings. The ``client_config`` parameter is parsed from a client configuration JSON file of the form: .. code-block:: json { "interfaces": { "google.fake.v1.ServiceName": { "retry_codes": { "idempotent": ["UNAVAILABLE", "DEADLINE_EXCEEDED"], "non_idempotent": [] }, "retry_params": { "default": { "initial_retry_delay_millis": 100, "retry_delay_multiplier": 1.2, "max_retry_delay_millis": 1000, "initial_rpc_timeout_millis": 2000, "rpc_timeout_multiplier": 1.5, "max_rpc_timeout_millis": 30000, "total_timeout_millis": 45000 } }, "methods": { "CreateFoo": { "retry_codes_name": "idempotent", "retry_params_name": "default", "timeout_millis": 30000 }, "Publish": { "retry_codes_name": "non_idempotent", "retry_params_name": "default", "bundling": { "element_count_threshold": 40, "element_count_limit": 200, "request_byte_threshold": 90000, "request_byte_limit": 100000, "delay_threshold_millis": 100 } } } } } } Args: service_name: The fully-qualified name of this service, used as a key into the client config file (in the example above, this value should be ``google.fake.v1.ServiceName``). client_config: A dictionary parsed from the standard API client config file. bundle_descriptors: A dictionary of method names to BundleDescriptor objects for methods that are bundling-enabled. page_descriptors: A dictionary of method names to PageDescriptor objects for methods that are page streaming-enabled. config_override: A dictionary in the same structure of client_config to override the settings. Usually client_config is supplied from the default config and config_override will be specified by users. retry_names: A dictionary mapping the strings referring to response status codes to the Python objects representing those codes. kwargs: The keyword arguments to be passed to the API calls. Raises: KeyError: If the configuration for the service in question cannot be located in the provided ``client_config``. """ # pylint: disable=too-many-locals defaults = {} bundle_descriptors = bundle_descriptors or {} page_descriptors = page_descriptors or {} kwargs = kwargs or {} try: service_config = client_config['interfaces'][service_name] except KeyError: raise KeyError('Client configuration not found for service: {}'.format( service_name)) overrides = config_override.get('interfaces', {}).get(service_name, {}) for method in service_config.get('methods'): method_config = service_config['methods'][method] overriding_method = overrides.get('methods', {}).get(method, {}) snake_name = _upper_camel_to_lower_under(method) if overriding_method and overriding_method.get('timeout_millis'): timeout = overriding_method['timeout_millis'] else: timeout = method_config['timeout_millis'] timeout /= _MILLIS_PER_SECOND bundle_descriptor = bundle_descriptors.get(snake_name) bundling_config = method_config.get('bundling', None) if overriding_method and 'bundling' in overriding_method: bundling_config = overriding_method['bundling'] bundler = _construct_bundling(bundling_config, bundle_descriptor) retry_options = _merge_retry_options( _construct_retry(method_config, service_config['retry_codes'], service_config['retry_params'], retry_names), _construct_retry(overriding_method, overrides.get('retry_codes'), overrides.get('retry_params'), retry_names)) defaults[snake_name] = _CallSettings( timeout=timeout, retry=retry_options, page_descriptor=page_descriptors.get(snake_name), bundler=bundler, bundle_descriptor=bundle_descriptor, kwargs=kwargs) return defaults
def test_call_override(self): settings = _CallSettings(timeout=10) my_callable = api_callable.create_api_call( lambda _req, timeout: timeout, settings) self.assertEqual(my_callable(None, CallOptions(timeout=20)), 20)
def test_call_api_call(self): settings = _CallSettings() my_callable = api_callable.create_api_call(lambda _req, _timeout: 42, settings) self.assertEqual(my_callable(None), 42)
def construct_settings( service_name, client_config, config_override, retry_names, bundle_descriptors=None, page_descriptors=None, metrics_headers=(), kwargs=None): """Constructs a dictionary mapping method names to _CallSettings. The ``client_config`` parameter is parsed from a client configuration JSON file of the form: .. code-block:: json { "interfaces": { "google.fake.v1.ServiceName": { "retry_codes": { "idempotent": ["UNAVAILABLE", "DEADLINE_EXCEEDED"], "non_idempotent": [] }, "retry_params": { "default": { "initial_retry_delay_millis": 100, "retry_delay_multiplier": 1.2, "max_retry_delay_millis": 1000, "initial_rpc_timeout_millis": 2000, "rpc_timeout_multiplier": 1.5, "max_rpc_timeout_millis": 30000, "total_timeout_millis": 45000 } }, "methods": { "CreateFoo": { "retry_codes_name": "idempotent", "retry_params_name": "default", "timeout_millis": 30000 }, "Publish": { "retry_codes_name": "non_idempotent", "retry_params_name": "default", "bundling": { "element_count_threshold": 40, "element_count_limit": 200, "request_byte_threshold": 90000, "request_byte_limit": 100000, "delay_threshold_millis": 100 } } } } } } Args: service_name (str): The fully-qualified name of this service, used as a key into the client config file (in the example above, this value would be ``google.fake.v1.ServiceName``). client_config (dict): A dictionary parsed from the standard API client config file. bundle_descriptors (Mapping[str, BundleDescriptor]): A dictionary of method names to BundleDescriptor objects for methods that are bundling-enabled. page_descriptors (Mapping[str, PageDescriptor]): A dictionary of method names to PageDescriptor objects for methods that are page streaming-enabled. config_override (str): A dictionary in the same structure of client_config to override the settings. Usually client_config is supplied from the default config and config_override will be specified by users. retry_names (Mapping[str, object]): A dictionary mapping the strings referring to response status codes to the Python objects representing those codes. metrics_headers (Mapping[str, str]): Dictionary of headers to be passed for analytics. Sent as a dictionary; eventually becomes a space-separated string (e.g. 'foo/1.0.0 bar/3.14.1'). kwargs (dict): The keyword arguments to be passed to the API calls. Returns: dict: A dictionary mapping method names to _CallSettings. Raises: KeyError: If the configuration for the service in question cannot be located in the provided ``client_config``. """ # pylint: disable=too-many-locals # pylint: disable=protected-access defaults = {} bundle_descriptors = bundle_descriptors or {} page_descriptors = page_descriptors or {} kwargs = kwargs or {} # Sanity check: It is possible that we got this far but some headers # were specified with an older library, which sends them as... # kwargs={'metadata': [('x-goog-api-client', 'foo/1.0 bar/3.0')]} # # Note: This is the final format we will send down to GRPC shortly. # # Remove any x-goog-api-client header that may have been present # in the metadata list. if 'metadata' in kwargs: kwargs['metadata'] = [value for value in kwargs['metadata'] if value[0].lower() != 'x-goog-api-client'] # Fill out the metrics headers with GAX and GRPC info, and convert # to a string in the format that the GRPC layer expects. kwargs.setdefault('metadata', []) kwargs['metadata'].append( ('x-goog-api-client', metrics.stringify(metrics.fill(metrics_headers))) ) try: service_config = client_config['interfaces'][service_name] except KeyError: raise KeyError('Client configuration not found for service: {}' .format(service_name)) overrides = config_override.get('interfaces', {}).get(service_name, {}) for method in service_config.get('methods'): method_config = service_config['methods'][method] overriding_method = overrides.get('methods', {}).get(method, {}) snake_name = _upper_camel_to_lower_under(method) if overriding_method and overriding_method.get('timeout_millis'): timeout = overriding_method['timeout_millis'] else: timeout = method_config['timeout_millis'] timeout /= _MILLIS_PER_SECOND bundle_descriptor = bundle_descriptors.get(snake_name) bundling_config = method_config.get('bundling', None) if overriding_method and 'bundling' in overriding_method: bundling_config = overriding_method['bundling'] bundler = _construct_bundling(bundling_config, bundle_descriptor) retry_options = _merge_retry_options( _construct_retry(method_config, service_config['retry_codes'], service_config['retry_params'], retry_names), _construct_retry(overriding_method, overrides.get('retry_codes'), overrides.get('retry_params'), retry_names)) defaults[snake_name] = gax._CallSettings( timeout=timeout, retry=retry_options, page_descriptor=page_descriptors.get(snake_name), bundler=bundler, bundle_descriptor=bundle_descriptor, kwargs=kwargs) return defaults
def construct_settings(service_name, client_config, config_override, retry_names, bundle_descriptors=None, page_descriptors=None, metrics_headers=(), kwargs=None): """Constructs a dictionary mapping method names to _CallSettings. The ``client_config`` parameter is parsed from a client configuration JSON file of the form: .. code-block:: json { "interfaces": { "google.fake.v1.ServiceName": { "retry_codes": { "idempotent": ["UNAVAILABLE", "DEADLINE_EXCEEDED"], "non_idempotent": [] }, "retry_params": { "default": { "initial_retry_delay_millis": 100, "retry_delay_multiplier": 1.2, "max_retry_delay_millis": 1000, "initial_rpc_timeout_millis": 2000, "rpc_timeout_multiplier": 1.5, "max_rpc_timeout_millis": 30000, "total_timeout_millis": 45000 } }, "methods": { "CreateFoo": { "retry_codes_name": "idempotent", "retry_params_name": "default", "timeout_millis": 30000 }, "Publish": { "retry_codes_name": "non_idempotent", "retry_params_name": "default", "bundling": { "element_count_threshold": 40, "element_count_limit": 200, "request_byte_threshold": 90000, "request_byte_limit": 100000, "delay_threshold_millis": 100 } } } } } } Args: service_name (str): The fully-qualified name of this service, used as a key into the client config file (in the example above, this value would be ``google.fake.v1.ServiceName``). client_config (dict): A dictionary parsed from the standard API client config file. bundle_descriptors (Mapping[str, BundleDescriptor]): A dictionary of method names to BundleDescriptor objects for methods that are bundling-enabled. page_descriptors (Mapping[str, PageDescriptor]): A dictionary of method names to PageDescriptor objects for methods that are page streaming-enabled. config_override (str): A dictionary in the same structure of client_config to override the settings. Usually client_config is supplied from the default config and config_override will be specified by users. retry_names (Mapping[str, object]): A dictionary mapping the strings referring to response status codes to the Python objects representing those codes. metrics_headers (Mapping[str, str]): Dictionary of headers to be passed for analytics. Sent as a dictionary; eventually becomes a space-separated string (e.g. 'foo/1.0.0 bar/3.14.1'). kwargs (dict): The keyword arguments to be passed to the API calls. Returns: dict: A dictionary mapping method names to _CallSettings. Raises: KeyError: If the configuration for the service in question cannot be located in the provided ``client_config``. """ # pylint: disable=too-many-locals # pylint: disable=protected-access defaults = {} bundle_descriptors = bundle_descriptors or {} page_descriptors = page_descriptors or {} kwargs = kwargs or {} # Sanity check: It is possible that we got this far but some headers # were specified with an older library, which sends them as... # kwargs={'metadata': [('x-goog-api-client', 'foo/1.0 bar/3.0')]} # # Note: This is the final format we will send down to GRPC shortly. # # Remove any x-goog-api-client header that may have been present # in the metadata list. if 'metadata' in kwargs: kwargs['metadata'] = [ value for value in kwargs['metadata'] if value[0].lower() != 'x-goog-api-client' ] # Fill out the metrics headers with GAX and GRPC info, and convert # to a string in the format that the GRPC layer expects. kwargs.setdefault('metadata', []) kwargs['metadata'].append( ('x-goog-api-client', metrics.stringify(metrics.fill(metrics_headers)))) try: service_config = client_config['interfaces'][service_name] except KeyError: raise KeyError('Client configuration not found for service: {}'.format( service_name)) overrides = config_override.get('interfaces', {}).get(service_name, {}) for method in service_config.get('methods'): method_config = service_config['methods'][method] overriding_method = overrides.get('methods', {}).get(method, {}) snake_name = _upper_camel_to_lower_under(method) if overriding_method and overriding_method.get('timeout_millis'): timeout = overriding_method['timeout_millis'] else: timeout = method_config['timeout_millis'] timeout /= _MILLIS_PER_SECOND bundle_descriptor = bundle_descriptors.get(snake_name) bundling_config = method_config.get('bundling', None) if overriding_method and 'bundling' in overriding_method: bundling_config = overriding_method['bundling'] bundler = _construct_bundling(bundling_config, bundle_descriptor) retry_options = _merge_retry_options( _construct_retry(method_config, service_config['retry_codes'], service_config['retry_params'], retry_names), _construct_retry(overriding_method, overrides.get('retry_codes'), overrides.get('retry_params'), retry_names)) defaults[snake_name] = gax._CallSettings( timeout=timeout, retry=retry_options, page_descriptor=page_descriptors.get(snake_name), bundler=bundler, bundle_descriptor=bundle_descriptor, kwargs=kwargs) return defaults
def construct_settings(service_name, client_config, config_override, retry_names, bundle_descriptors=None, page_descriptors=None, metrics_headers=(), kwargs=None): """Constructs a dictionary mapping method names to _CallSettings. The ``client_config`` parameter is parsed from a client configuration JSON file of the form: .. code-block:: json { "interfaces": { "google.fake.v1.ServiceName": { "retry_codes": { "idempotent": ["UNAVAILABLE", "DEADLINE_EXCEEDED"], "non_idempotent": [] }, "retry_params": { "default": { "initial_retry_delay_millis": 100, "retry_delay_multiplier": 1.2, "max_retry_delay_millis": 1000, "initial_rpc_timeout_millis": 2000, "rpc_timeout_multiplier": 1.5, "max_rpc_timeout_millis": 30000, "total_timeout_millis": 45000 } }, "methods": { "CreateFoo": { "retry_codes_name": "idempotent", "retry_params_name": "default", "timeout_millis": 30000 }, "Publish": { "retry_codes_name": "non_idempotent", "retry_params_name": "default", "bundling": { "element_count_threshold": 40, "element_count_limit": 200, "request_byte_threshold": 90000, "request_byte_limit": 100000, "delay_threshold_millis": 100 } } } } } } Args: service_name: The fully-qualified name of this service, used as a key into the client config file (in the example above, this value should be ``google.fake.v1.ServiceName``). client_config: A dictionary parsed from the standard API client config file. bundle_descriptors: A dictionary of method names to BundleDescriptor objects for methods that are bundling-enabled. page_descriptors: A dictionary of method names to PageDescriptor objects for methods that are page streaming-enabled. config_override: A dictionary in the same structure of client_config to override the settings. Usually client_config is supplied from the default config and config_override will be specified by users. retry_names: A dictionary mapping the strings referring to response status codes to the Python objects representing those codes. metrics_headers: Dictionary of headers to be passed for analytics. Sent as a dictionary; eventually becomes a space-separated string (e.g. 'foo/1.0.0 bar/3.14.1'). kwargs: The keyword arguments to be passed to the API calls. Raises: KeyError: If the configuration for the service in question cannot be located in the provided ``client_config``. """ # pylint: disable=too-many-locals # pylint: disable=protected-access defaults = {} bundle_descriptors = bundle_descriptors or {} metrics_headers = collections.OrderedDict(metrics_headers) page_descriptors = page_descriptors or {} kwargs = kwargs or {} # Add the language header to metrics. metrics_headers['gl-python'] = platform.python_version() # Sanity check: It is possible that we got this far but some headers # were specified with an older library, which sends them as... # kwargs={'metadata': [('x-goog-api-client', 'foo/1.0 bar/3.0')]} # # Note: This is the final format we will send down to GRPC shortly. # # Remove any x-goog-api-client header that may have been present # in the metadata list. if 'metadata' in kwargs: kwargs['metadata'] = [ value for value in kwargs['metadata'] if value[0].lower() != 'x-goog-api-client' ] # Add the GAX and GRPC headers to our metrics. # pylint: disable=no-member grpc_version = pkg_resources.get_distribution('grpcio').version # pylint: enable=no-member metrics_headers['gax'] = gax.__version__ metrics_headers['grpc'] = grpc_version # A/B key-value pairs belong at the end of the metadata string; # shift any that appear to the end of the dictionary. ab_keys = [k for k in metrics_headers.keys() if k.startswith('gl-ab')] for key in ab_keys: value = metrics_headers.pop(key) metrics_headers[key] = value # Okay, now add our new and complete metadata into the old # kwargs format: # kwargs={'metadata': [('x-goog-api-client', 'gax/x.y.z grpc/x.y.z')]} # # This is how it will be sent to the underlying GRPC layer. header = ' '.join(['%s/%s' % (k, v) for k, v in metrics_headers.items()]) kwargs.setdefault('metadata', []) kwargs['metadata'].append(('x-goog-api-client', header)) try: service_config = client_config['interfaces'][service_name] except KeyError: raise KeyError('Client configuration not found for service: {}'.format( service_name)) overrides = config_override.get('interfaces', {}).get(service_name, {}) for method in service_config.get('methods'): method_config = service_config['methods'][method] overriding_method = overrides.get('methods', {}).get(method, {}) snake_name = _upper_camel_to_lower_under(method) if overriding_method and overriding_method.get('timeout_millis'): timeout = overriding_method['timeout_millis'] else: timeout = method_config['timeout_millis'] timeout /= _MILLIS_PER_SECOND bundle_descriptor = bundle_descriptors.get(snake_name) bundling_config = method_config.get('bundling', None) if overriding_method and 'bundling' in overriding_method: bundling_config = overriding_method['bundling'] bundler = _construct_bundling(bundling_config, bundle_descriptor) retry_options = _merge_retry_options( _construct_retry(method_config, service_config['retry_codes'], service_config['retry_params'], retry_names), _construct_retry(overriding_method, overrides.get('retry_codes'), overrides.get('retry_params'), retry_names)) defaults[snake_name] = gax._CallSettings( timeout=timeout, retry=retry_options, page_descriptor=page_descriptors.get(snake_name), bundler=bundler, bundle_descriptor=bundle_descriptor, kwargs=kwargs) return defaults