def test_catch_error(self):
def abortion_error_func(*dummy_args, **dummy_kwargs):
raise CustomException(None, None)
def other_error_func(*dummy_args, **dummy_kwargs):
raise AnotherException
gax_error_callable = api_callable.create_api_call(
abortion_error_func, _CallSettings())
self.assertRaises(GaxError, gax_error_callable, None)
other_error_callable = api_callable.create_api_call(
other_error_func, _CallSettings())
self.assertRaises(AnotherException, other_error_callable, None)
def test_catch_error(self):
def abortion_error_func(*dummy_args, **dummy_kwargs):
raise CustomException(None, None)
def other_error_func(*dummy_args, **dummy_kwargs):
raise AnotherException
gax_error_callable = api_callable.create_api_call(
abortion_error_func, _CallSettings())
self.assertRaises(GaxError, gax_error_callable, None)
other_error_callable = api_callable.create_api_call(
other_error_func, _CallSettings())
self.assertRaises(AnotherException, other_error_callable, None)
def test_call_kwargs(self):
settings = _CallSettings(kwargs={'key': 'value'})
my_callable = api_callable.create_api_call(
lambda _req, _timeout, **kwargs: kwargs['key'], settings)
self.assertEqual(my_callable(None), 'value')
self.assertEqual(my_callable(None, CallOptions(key='updated')),
'updated')
def test_settings_merge_options1(self):
options = CallOptions(timeout=46)
settings = _CallSettings(timeout=9, page_descriptor=None, retry=None)
final = settings.merge(options)
self.assertEqual(final.timeout, 46)
self.assertIsNone(final.retry)
self.assertIsNone(final.page_descriptor)
def test_call_kwargs(self):
settings = _CallSettings(kwargs={'key': 'value'})
my_callable = api_callable.create_api_call(
lambda _req, _timeout, **kwargs: kwargs['key'], settings)
self.assertEqual(my_callable(None), 'value')
self.assertEqual(my_callable(None, CallOptions(key='updated')),
'updated')
def test_retry_times_out_no_response(self, mock_time):
mock_time.return_value = 1
retry = RetryOptions([_FAKE_STATUS_CODE_1],
BackoffSettings(0, 0, 0, 0, 0, 0, 0))
settings = _CallSettings(timeout=0, retry=retry)
my_callable = api_callable.create_api_call(lambda: None, settings)
self.assertRaises(RetryError, my_callable, None)
def test_retry_times_out_no_response(self, mock_time):
mock_time.return_value = 1
retry = RetryOptions(
[_FAKE_STATUS_CODE_1],
BackoffSettings(0, 0, 0, 0, 0, 0, 0))
settings = _CallSettings(timeout=0, retry=retry)
my_callable = api_callable.create_api_call(lambda: None, settings)
self.assertRaises(RetryError, my_callable, None)
def test_settings_merge_options2(self):
retry = RetryOptions(None, None)
options = CallOptions(retry=retry)
settings = _CallSettings(
timeout=9, page_descriptor=None, retry=RetryOptions(None, None))
final = settings.merge(options)
self.assertEqual(final.timeout, 9)
self.assertIsNone(final.page_descriptor)
self.assertEqual(final.retry, retry)
def test_settings_merge_none(self):
settings = _CallSettings(
timeout=23, page_descriptor=object(), bundler=object(),
retry=object())
final = settings.merge(None)
self.assertEqual(final.timeout, settings.timeout)
self.assertEqual(final.retry, settings.retry)
self.assertEqual(final.page_descriptor, settings.page_descriptor)
self.assertEqual(final.bundler, settings.bundler)
self.assertEqual(final.bundle_descriptor, settings.bundle_descriptor)
def test_no_retry_if_no_codes(self, mock_time):
retry = RetryOptions([], BackoffSettings(1, 2, 3, 4, 5, 6, 7))
mock_call = mock.Mock()
mock_call.side_effect = CustomException('', _FAKE_STATUS_CODE_1)
mock_time.return_value = 0
settings = _CallSettings(timeout=0, retry=retry)
my_callable = api_callable.create_api_call(mock_call, settings)
self.assertRaises(CustomException, my_callable, None)
self.assertEqual(mock_call.call_count, 1)
def test_wrap_value_error(self):
invalid_attribute_exc = grpc.RpcError()
invalid_attribute_exc.code = lambda: grpc.StatusCode.INVALID_ARGUMENT
def value_error_func(*dummy_args, **dummy_kwargs):
raise invalid_attribute_exc
value_error_callable = api_callable.create_api_call(
value_error_func, _CallSettings())
self.assertRaises(ValueError, value_error_callable, None)
def test_no_retry_if_no_codes(self, mock_time):
retry = RetryOptions([], BackoffSettings(1, 2, 3, 4, 5, 6, 7))
mock_call = mock.Mock()
mock_call.side_effect = CustomException('', _FAKE_STATUS_CODE_1)
mock_time.return_value = 0
settings = _CallSettings(timeout=0, retry=retry)
my_callable = api_callable.create_api_call(mock_call, settings)
self.assertRaises(CustomException, my_callable, None)
self.assertEqual(mock_call.call_count, 1)
def test_wrap_value_error(self):
invalid_attribute_exc = grpc.RpcError()
invalid_attribute_exc.code = lambda: grpc.StatusCode.INVALID_ARGUMENT
def value_error_func(*dummy_args, **dummy_kwargs):
raise invalid_attribute_exc
value_error_callable = api_callable.create_api_call(
value_error_func, _CallSettings())
self.assertRaises(ValueError, value_error_callable, None)
def test_settings_merge_options_page_streaming(self):
retry = RetryOptions(None, None)
page_descriptor = object()
options = CallOptions(timeout=46, page_token=INITIAL_PAGE)
settings = _CallSettings(timeout=9, retry=retry,
page_descriptor=page_descriptor)
final = settings.merge(options)
self.assertEqual(final.timeout, 46)
self.assertEqual(final.page_descriptor, page_descriptor)
self.assertEqual(final.page_token, INITIAL_PAGE)
self.assertFalse(final.flatten_pages)
self.assertEqual(final.retry, retry)
def test_retry_aborts_on_unexpected_exception(self, mock_exc_to_code,
mock_time):
mock_exc_to_code.side_effect = lambda e: e.code
retry = RetryOptions([_FAKE_STATUS_CODE_1],
BackoffSettings(0, 0, 0, 0, 0, 0, 1))
mock_call = mock.Mock()
mock_call.side_effect = CustomException('', _FAKE_STATUS_CODE_2)
mock_time.return_value = 0
settings = _CallSettings(timeout=0, retry=retry)
my_callable = api_callable.create_api_call(mock_call, settings)
self.assertRaises(Exception, my_callable, None)
self.assertEqual(mock_call.call_count, 1)
def test_retry_aborts_on_unexpected_exception(
self, mock_exc_to_code, mock_time):
mock_exc_to_code.side_effect = lambda e: e.code
retry = RetryOptions(
[_FAKE_STATUS_CODE_1],
BackoffSettings(0, 0, 0, 0, 0, 0, 1))
mock_call = mock.Mock()
mock_call.side_effect = CustomException('', _FAKE_STATUS_CODE_2)
mock_time.return_value = 0
settings = _CallSettings(timeout=0, retry=retry)
my_callable = api_callable.create_api_call(mock_call, settings)
self.assertRaises(Exception, my_callable, None)
self.assertEqual(mock_call.call_count, 1)
def test_retry_aborts_simple(self, mock_exc_to_code, mock_time):
def fake_call(dummy_request, dummy_timeout):
raise CustomException('', _FAKE_STATUS_CODE_1)
retry = RetryOptions([_FAKE_STATUS_CODE_1],
BackoffSettings(0, 0, 0, 0, 0, 0, 1))
mock_time.side_effect = [0, 2]
mock_exc_to_code.side_effect = lambda e: e.code
settings = _CallSettings(timeout=0, retry=retry)
my_callable = api_callable.create_api_call(fake_call, settings)
try:
my_callable(None)
except RetryError as exc:
self.assertIsInstance(exc.cause, CustomException)
def test_retry_aborts_simple(self, mock_exc_to_code, mock_time):
def fake_call(dummy_request, dummy_timeout):
raise CustomException('', _FAKE_STATUS_CODE_1)
retry = RetryOptions(
[_FAKE_STATUS_CODE_1],
BackoffSettings(0, 0, 0, 0, 0, 0, 1))
mock_time.side_effect = [0, 2]
mock_exc_to_code.side_effect = lambda e: e.code
settings = _CallSettings(timeout=0, retry=retry)
my_callable = api_callable.create_api_call(fake_call, settings)
try:
my_callable(None)
except RetryError as exc:
self.assertIsInstance(exc.cause, CustomException)
def test_retry(self, mock_exc_to_code, mock_time):
mock_exc_to_code.side_effect = lambda e: e.code
to_attempt = 3
retry = RetryOptions([_FAKE_STATUS_CODE_1],
BackoffSettings(0, 0, 0, 0, 0, 0, 1))
# Succeeds on the to_attempt'th call, and never again afterward
mock_call = mock.Mock()
mock_call.side_effect = ([CustomException('', _FAKE_STATUS_CODE_1)] *
(to_attempt - 1) + [mock.DEFAULT])
mock_call.return_value = 1729
mock_time.return_value = 0
settings = _CallSettings(timeout=0, retry=retry)
my_callable = api_callable.create_api_call(mock_call, settings)
self.assertEqual(my_callable(None), 1729)
self.assertEqual(mock_call.call_count, to_attempt)
def test_retry(self, mock_exc_to_code, mock_time):
mock_exc_to_code.side_effect = lambda e: e.code
to_attempt = 3
retry = RetryOptions(
[_FAKE_STATUS_CODE_1],
BackoffSettings(0, 0, 0, 0, 0, 0, 1))
# Succeeds on the to_attempt'th call, and never again afterward
mock_call = mock.Mock()
mock_call.side_effect = ([CustomException('', _FAKE_STATUS_CODE_1)] *
(to_attempt - 1) + [mock.DEFAULT])
mock_call.return_value = 1729
mock_time.return_value = 0
settings = _CallSettings(timeout=0, retry=retry)
my_callable = api_callable.create_api_call(mock_call, settings)
self.assertEqual(my_callable(None), 1729)
self.assertEqual(mock_call.call_count, to_attempt)
def test_retry_times_out_simple(self, mock_exc_to_code, mock_time):
mock_exc_to_code.side_effect = lambda e: e.code
to_attempt = 3
retry = RetryOptions([_FAKE_STATUS_CODE_1],
BackoffSettings(0, 0, 0, 0, 0, 0, 1))
mock_call = mock.Mock()
mock_call.side_effect = CustomException('', _FAKE_STATUS_CODE_1)
mock_time.side_effect = ([0] * to_attempt + [2])
settings = _CallSettings(timeout=0, retry=retry)
my_callable = api_callable.create_api_call(mock_call, settings)
try:
my_callable(None)
except RetryError as exc:
self.assertIsInstance(exc.cause, CustomException)
self.assertEqual(mock_call.call_count, to_attempt)
def test_retry_times_out_simple(self, mock_exc_to_code, mock_time):
mock_exc_to_code.side_effect = lambda e: e.code
to_attempt = 3
retry = RetryOptions(
[_FAKE_STATUS_CODE_1],
BackoffSettings(0, 0, 0, 0, 0, 0, 1))
mock_call = mock.Mock()
mock_call.side_effect = CustomException('', _FAKE_STATUS_CODE_1)
mock_time.side_effect = ([0] * to_attempt + [2])
settings = _CallSettings(timeout=0, retry=retry)
my_callable = api_callable.create_api_call(mock_call, settings)
try:
my_callable(None)
except RetryError as exc:
self.assertIsInstance(exc.cause, CustomException)
self.assertEqual(mock_call.call_count, to_attempt)
def test_wrap_value_error(self):
from google.gax.errors import InvalidArgumentError
invalid_attribute_exc = grpc.RpcError()
invalid_attribute_exc.code = lambda: grpc.StatusCode.INVALID_ARGUMENT
def value_error_func(*dummy_args, **dummy_kwargs):
raise invalid_attribute_exc
value_error_callable = api_callable.create_api_call(
value_error_func, _CallSettings())
with self.assertRaises(ValueError) as exc_info:
value_error_callable(None)
self.assertIsInstance(exc_info.exception, InvalidArgumentError)
self.assertEqual(exc_info.exception.args, (u'RPC failed', ))
self.assertIs(exc_info.exception.cause, invalid_attribute_exc)
def test_call_merge_options_metadata(self):
settings_kwargs = {
'key': 'value',
'metadata': [('key1', 'val1'), ('key2', 'val2')]
}
settings = _CallSettings(kwargs=settings_kwargs)
my_callable = api_callable.create_api_call(
lambda _req, _timeout, **kwargs: kwargs, settings)
# Merge empty options, settings.kwargs['metadata'] remain unchanged
expected_kwargs = settings_kwargs
self.assertEqual(my_callable(None), expected_kwargs)
# Override an existing key in settings.kwargs['metadata']
expected_kwargs = {
'key': 'value',
'metadata': [('key1', '_val1'), ('key2', 'val2')]
}
self.assertEqual(
my_callable(None, CallOptions(metadata=[('key1', '_val1')])),
expected_kwargs)
# Add a new key in settings.kwargs['metadata']
expected_kwargs = {
'key': 'value',
'metadata': [('key3', 'val3'), ('key1', 'val1'), ('key2', 'val2')]
}
self.assertEqual(
my_callable(None, CallOptions(metadata=[('key3', 'val3')])),
expected_kwargs)
# Do all: add a new key and override an existing one in
# settings.kwargs['metadata']
expected_kwargs = {
'key': 'value',
'metadata': [('key3', 'val3'), ('key2', '_val2'), ('key1', 'val1')]
}
self.assertEqual(
my_callable(
None,
CallOptions(metadata=[('key3', 'val3'), ('key2', '_val2')])),
expected_kwargs)
def test_bundling(self):
# pylint: disable=abstract-method, too-few-public-methods
class BundlingRequest(object):
def __init__(self, elements=None):
self.elements = elements
fake_grpc_func_descriptor = BundleDescriptor('elements', [])
bundler = bundling.Executor(BundleOptions(element_count_threshold=8))
def my_func(request, dummy_timeout):
return len(request.elements)
settings = _CallSettings(
bundler=bundler, bundle_descriptor=fake_grpc_func_descriptor,
timeout=0)
my_callable = api_callable.create_api_call(my_func, settings)
first = my_callable(BundlingRequest([0] * 3))
self.assertIsInstance(first, bundling.Event)
self.assertIsNone(first.result) # pylint: disable=no-member
second = my_callable(BundlingRequest([0] * 5))
self.assertEqual(second.result, 8) # pylint: disable=no-member
def test_bundling(self):
# pylint: disable=abstract-method, too-few-public-methods
class BundlingRequest(object):
def __init__(self, elements=None):
self.elements = elements
fake_grpc_func_descriptor = BundleDescriptor('elements', [])
bundler = bundling.Executor(BundleOptions(element_count_threshold=8))
def my_func(request, dummy_timeout):
return len(request.elements)
settings = _CallSettings(bundler=bundler,
bundle_descriptor=fake_grpc_func_descriptor,
timeout=0)
my_callable = api_callable.create_api_call(my_func, settings)
first = my_callable(BundlingRequest([0] * 3))
self.assertIsInstance(first, bundling.Event)
self.assertIsNone(first.result) # pylint: disable=no-member
second = my_callable(BundlingRequest([0] * 5))
self.assertEqual(second.result, 8) # pylint: disable=no-member
def test_retry_exponential_backoff(self, mock_exc_to_code, mock_time,
mock_sleep):
# pylint: disable=too-many-locals
mock_exc_to_code.side_effect = lambda e: e.code
MILLIS_PER_SEC = 1000
mock_time.return_value = 0
def incr_time(secs):
mock_time.return_value += secs
def api_call(dummy_request, timeout, **dummy_kwargs):
incr_time(timeout)
raise CustomException(str(timeout), _FAKE_STATUS_CODE_1)
mock_call = mock.Mock()
mock_sleep.side_effect = incr_time
mock_call.side_effect = api_call
params = BackoffSettings(3, 2, 24, 5, 2, 80, 2500)
retry = RetryOptions([_FAKE_STATUS_CODE_1], params)
settings = _CallSettings(timeout=0, retry=retry)
my_callable = api_callable.create_api_call(mock_call, settings)
try:
my_callable(None)
except RetryError as exc:
self.assertIsInstance(exc.cause, CustomException)
self.assertGreaterEqual(mock_time(),
params.total_timeout_millis / MILLIS_PER_SEC)
# Very rough bounds
calls_lower_bound = params.total_timeout_millis / (
params.max_retry_delay_millis + params.max_rpc_timeout_millis)
self.assertGreater(mock_call.call_count, calls_lower_bound)
calls_upper_bound = (params.total_timeout_millis /
params.initial_retry_delay_millis)
self.assertLess(mock_call.call_count, calls_upper_bound)
def test_retry_exponential_backoff(self, mock_exc_to_code, mock_time,
mock_sleep):
# pylint: disable=too-many-locals
mock_exc_to_code.side_effect = lambda e: e.code
MILLIS_PER_SEC = 1000
mock_time.return_value = 0
def incr_time(secs):
mock_time.return_value += secs
def api_call(dummy_request, timeout, **dummy_kwargs):
incr_time(timeout)
raise CustomException(str(timeout), _FAKE_STATUS_CODE_1)
mock_call = mock.Mock()
mock_sleep.side_effect = incr_time
mock_call.side_effect = api_call
params = BackoffSettings(3, 2, 24, 5, 2, 80, 2500)
retry = RetryOptions([_FAKE_STATUS_CODE_1], params)
settings = _CallSettings(timeout=0, retry=retry)
my_callable = api_callable.create_api_call(mock_call, settings)
try:
my_callable(None)
except RetryError as exc:
self.assertIsInstance(exc.cause, CustomException)
self.assertGreaterEqual(mock_time(),
params.total_timeout_millis / MILLIS_PER_SEC)
# Very rough bounds
calls_lower_bound = params.total_timeout_millis / (
params.max_retry_delay_millis + params.max_rpc_timeout_millis)
self.assertGreater(mock_call.call_count, calls_lower_bound)
calls_upper_bound = (params.total_timeout_millis /
params.initial_retry_delay_millis)
self.assertLess(mock_call.call_count, calls_upper_bound)
def test_call_api_call(self):
settings = _CallSettings()
my_callable = api_callable.create_api_call(
lambda _req, _timeout: 42, settings)
self.assertEqual(my_callable(None), 42)
def test_bundling_page_streaming_error(self):
settings = _CallSettings(
page_descriptor=object(), bundle_descriptor=object(),
bundler=object())
with self.assertRaises(ValueError):
api_callable.create_api_call(lambda _req, _timeout: 42, settings)
def test_page_streaming(self):
# A mock grpc function that page streams a list of consecutive
# integers, returning `page_size` integers with each call and using
# the next integer to return as the page token, until `pages_to_stream`
# pages have been returned.
# pylint:disable=too-many-locals
page_size = 3
pages_to_stream = 5
# pylint: disable=abstract-method, too-few-public-methods
class PageStreamingRequest(object):
def __init__(self, page_token=0):
self.page_token = page_token
class PageStreamingResponse(object):
def __init__(self, nums=(), next_page_token=0):
self.nums = nums
self.next_page_token = next_page_token
fake_grpc_func_descriptor = PageDescriptor(
'page_token', 'next_page_token', 'nums')
def grpc_return_value(request, *dummy_args, **dummy_kwargs):
start = int(request.page_token)
if start > 0 and start < page_size * pages_to_stream:
return PageStreamingResponse(
nums=list(range(start,
start + page_size)),
next_page_token=start + page_size)
elif start >= page_size * pages_to_stream:
return PageStreamingResponse()
else:
return PageStreamingResponse(nums=list(range(page_size)),
next_page_token=page_size)
with mock.patch('grpc.UnaryUnaryMultiCallable') as mock_grpc:
mock_grpc.side_effect = grpc_return_value
settings = _CallSettings(
page_descriptor=fake_grpc_func_descriptor, timeout=0)
my_callable = api_callable.create_api_call(
mock_grpc, settings=settings)
self.assertEqual(list(my_callable(PageStreamingRequest())),
list(range(page_size * pages_to_stream)))
unflattened_option = CallOptions(page_token=INITIAL_PAGE)
# Expect a list of pages_to_stream pages, each of size page_size,
# plus one empty page
expected = [list(range(page_size * n, page_size * (n + 1)))
for n in range(pages_to_stream)] + [()]
self.assertEqual(list(my_callable(PageStreamingRequest(),
unflattened_option)),
expected)
pages_already_read = 2
explicit_page_token_option = CallOptions(
page_token=str(page_size * pages_already_read))
# Expect a list of pages_to_stream pages, each of size page_size,
# plus one empty page, minus the pages_already_read
expected = [list(range(page_size * n, page_size * (n + 1)))
for n in range(pages_already_read, pages_to_stream)]
expected += [()]
self.assertEqual(list(my_callable(PageStreamingRequest(),
explicit_page_token_option)),
expected)
def test_page_streaming(self):
# A mock grpc function that page streams a list of consecutive
# integers, returning `page_size` integers with each call and using
# the next integer to return as the page token, until `pages_to_stream`
# pages have been returned.
# pylint:disable=too-many-locals
page_size = 3
pages_to_stream = 5
# pylint: disable=abstract-method, too-few-public-methods
class PageStreamingRequest(object):
def __init__(self, page_token=0):
self.page_token = page_token
class PageStreamingResponse(object):
def __init__(self, nums=(), next_page_token=0):
self.nums = nums
self.next_page_token = next_page_token
fake_grpc_func_descriptor = PageDescriptor('page_token',
'next_page_token', 'nums')
def grpc_return_value(request, *dummy_args, **dummy_kwargs):
start = int(request.page_token)
if start > 0 and start < page_size * pages_to_stream:
return PageStreamingResponse(nums=list(
range(start, start + page_size)),
next_page_token=start + page_size)
elif start >= page_size * pages_to_stream:
return PageStreamingResponse()
else:
return PageStreamingResponse(nums=list(range(page_size)),
next_page_token=page_size)
with mock.patch('grpc.UnaryUnaryMultiCallable') as mock_grpc:
mock_grpc.side_effect = grpc_return_value
settings = _CallSettings(page_descriptor=fake_grpc_func_descriptor,
timeout=0)
my_callable = api_callable.create_api_call(mock_grpc,
settings=settings)
self.assertEqual(list(my_callable(PageStreamingRequest())),
list(range(page_size * pages_to_stream)))
unflattened_option = CallOptions(page_token=INITIAL_PAGE)
# Expect a list of pages_to_stream pages, each of size page_size,
# plus one empty page
expected = [
list(range(page_size * n, page_size * (n + 1)))
for n in range(pages_to_stream)
] + [()]
self.assertEqual(
list(my_callable(PageStreamingRequest(), unflattened_option)),
expected)
pages_already_read = 2
explicit_page_token_option = CallOptions(
page_token=str(page_size * pages_already_read))
# Expect a list of pages_to_stream pages, each of size page_size,
# plus one empty page, minus the pages_already_read
expected = [
list(range(page_size * n, page_size * (n + 1)))
for n in range(pages_already_read, pages_to_stream)
]
expected += [()]
self.assertEqual(
list(
my_callable(PageStreamingRequest(),
explicit_page_token_option)), expected)
def test_bundling_page_streaming_error(self):
settings = _CallSettings(page_descriptor=object(),
bundle_descriptor=object(),
bundler=object())
with self.assertRaises(ValueError):
api_callable.create_api_call(lambda _req, _timeout: 42, settings)
def construct_settings(service_name,
client_config,
config_override,
retry_names,
bundle_descriptors=None,
page_descriptors=None,
kwargs=None):
"""Constructs a dictionary mapping method names to _CallSettings.
The ``client_config`` parameter is parsed from a client configuration JSON
file of the form:
.. code-block:: json
{
"interfaces": {
"google.fake.v1.ServiceName": {
"retry_codes": {
"idempotent": ["UNAVAILABLE", "DEADLINE_EXCEEDED"],
"non_idempotent": []
},
"retry_params": {
"default": {
"initial_retry_delay_millis": 100,
"retry_delay_multiplier": 1.2,
"max_retry_delay_millis": 1000,
"initial_rpc_timeout_millis": 2000,
"rpc_timeout_multiplier": 1.5,
"max_rpc_timeout_millis": 30000,
"total_timeout_millis": 45000
}
},
"methods": {
"CreateFoo": {
"retry_codes_name": "idempotent",
"retry_params_name": "default",
"timeout_millis": 30000
},
"Publish": {
"retry_codes_name": "non_idempotent",
"retry_params_name": "default",
"bundling": {
"element_count_threshold": 40,
"element_count_limit": 200,
"request_byte_threshold": 90000,
"request_byte_limit": 100000,
"delay_threshold_millis": 100
}
}
}
}
}
}
Args:
service_name: The fully-qualified name of this service, used as a key into
the client config file (in the example above, this value should be
``google.fake.v1.ServiceName``).
client_config: A dictionary parsed from the standard API client config
file.
bundle_descriptors: A dictionary of method names to BundleDescriptor
objects for methods that are bundling-enabled.
page_descriptors: A dictionary of method names to PageDescriptor objects
for methods that are page streaming-enabled.
config_override: A dictionary in the same structure of client_config to
override the settings. Usually client_config is supplied from the
default config and config_override will be specified by users.
retry_names: A dictionary mapping the strings referring to response status
codes to the Python objects representing those codes.
kwargs: The keyword arguments to be passed to the API calls.
Raises:
KeyError: If the configuration for the service in question cannot be
located in the provided ``client_config``.
"""
# pylint: disable=too-many-locals
defaults = {}
bundle_descriptors = bundle_descriptors or {}
page_descriptors = page_descriptors or {}
kwargs = kwargs or {}
try:
service_config = client_config['interfaces'][service_name]
except KeyError:
raise KeyError('Client configuration not found for service: {}'.format(
service_name))
overrides = config_override.get('interfaces', {}).get(service_name, {})
for method in service_config.get('methods'):
method_config = service_config['methods'][method]
overriding_method = overrides.get('methods', {}).get(method, {})
snake_name = _upper_camel_to_lower_under(method)
if overriding_method and overriding_method.get('timeout_millis'):
timeout = overriding_method['timeout_millis']
else:
timeout = method_config['timeout_millis']
timeout /= _MILLIS_PER_SECOND
bundle_descriptor = bundle_descriptors.get(snake_name)
bundling_config = method_config.get('bundling', None)
if overriding_method and 'bundling' in overriding_method:
bundling_config = overriding_method['bundling']
bundler = _construct_bundling(bundling_config, bundle_descriptor)
retry_options = _merge_retry_options(
_construct_retry(method_config, service_config['retry_codes'],
service_config['retry_params'], retry_names),
_construct_retry(overriding_method, overrides.get('retry_codes'),
overrides.get('retry_params'), retry_names))
defaults[snake_name] = _CallSettings(
timeout=timeout,
retry=retry_options,
page_descriptor=page_descriptors.get(snake_name),
bundler=bundler,
bundle_descriptor=bundle_descriptor,
kwargs=kwargs)
return defaults
def test_call_override(self):
settings = _CallSettings(timeout=10)
my_callable = api_callable.create_api_call(
lambda _req, timeout: timeout, settings)
self.assertEqual(my_callable(None, CallOptions(timeout=20)), 20)
def test_call_override(self):
settings = _CallSettings(timeout=10)
my_callable = api_callable.create_api_call(
lambda _req, timeout: timeout, settings)
self.assertEqual(my_callable(None, CallOptions(timeout=20)), 20)
def test_call_api_call(self):
settings = _CallSettings()
my_callable = api_callable.create_api_call(lambda _req, _timeout: 42,
settings)
self.assertEqual(my_callable(None), 42)
def construct_settings(
service_name, client_config, config_override,
retry_names, bundle_descriptors=None, page_descriptors=None,
metrics_headers=(), kwargs=None):
"""Constructs a dictionary mapping method names to _CallSettings.
The ``client_config`` parameter is parsed from a client configuration JSON
file of the form:
.. code-block:: json
{
"interfaces": {
"google.fake.v1.ServiceName": {
"retry_codes": {
"idempotent": ["UNAVAILABLE", "DEADLINE_EXCEEDED"],
"non_idempotent": []
},
"retry_params": {
"default": {
"initial_retry_delay_millis": 100,
"retry_delay_multiplier": 1.2,
"max_retry_delay_millis": 1000,
"initial_rpc_timeout_millis": 2000,
"rpc_timeout_multiplier": 1.5,
"max_rpc_timeout_millis": 30000,
"total_timeout_millis": 45000
}
},
"methods": {
"CreateFoo": {
"retry_codes_name": "idempotent",
"retry_params_name": "default",
"timeout_millis": 30000
},
"Publish": {
"retry_codes_name": "non_idempotent",
"retry_params_name": "default",
"bundling": {
"element_count_threshold": 40,
"element_count_limit": 200,
"request_byte_threshold": 90000,
"request_byte_limit": 100000,
"delay_threshold_millis": 100
}
}
}
}
}
}
Args:
service_name (str): The fully-qualified name of this service, used as a
key into the client config file (in the example above, this value
would be ``google.fake.v1.ServiceName``).
client_config (dict): A dictionary parsed from the standard API client
config file.
bundle_descriptors (Mapping[str, BundleDescriptor]): A dictionary of
method names to BundleDescriptor objects for methods that are
bundling-enabled.
page_descriptors (Mapping[str, PageDescriptor]): A dictionary of method
names to PageDescriptor objects for methods that are page
streaming-enabled.
config_override (str): A dictionary in the same structure of
client_config to override the settings. Usually client_config is
supplied from the default config and config_override will be
specified by users.
retry_names (Mapping[str, object]): A dictionary mapping the strings
referring to response status codes to the Python objects representing
those codes.
metrics_headers (Mapping[str, str]): Dictionary of headers to be passed
for analytics. Sent as a dictionary; eventually becomes a
space-separated string (e.g. 'foo/1.0.0 bar/3.14.1').
kwargs (dict): The keyword arguments to be passed to the API calls.
Returns:
dict: A dictionary mapping method names to _CallSettings.
Raises:
KeyError: If the configuration for the service in question cannot be
located in the provided ``client_config``.
"""
# pylint: disable=too-many-locals
# pylint: disable=protected-access
defaults = {}
bundle_descriptors = bundle_descriptors or {}
page_descriptors = page_descriptors or {}
kwargs = kwargs or {}
# Sanity check: It is possible that we got this far but some headers
# were specified with an older library, which sends them as...
# kwargs={'metadata': [('x-goog-api-client', 'foo/1.0 bar/3.0')]}
#
# Note: This is the final format we will send down to GRPC shortly.
#
# Remove any x-goog-api-client header that may have been present
# in the metadata list.
if 'metadata' in kwargs:
kwargs['metadata'] = [value for value in kwargs['metadata']
if value[0].lower() != 'x-goog-api-client']
# Fill out the metrics headers with GAX and GRPC info, and convert
# to a string in the format that the GRPC layer expects.
kwargs.setdefault('metadata', [])
kwargs['metadata'].append(
('x-goog-api-client', metrics.stringify(metrics.fill(metrics_headers)))
)
try:
service_config = client_config['interfaces'][service_name]
except KeyError:
raise KeyError('Client configuration not found for service: {}'
.format(service_name))
overrides = config_override.get('interfaces', {}).get(service_name, {})
for method in service_config.get('methods'):
method_config = service_config['methods'][method]
overriding_method = overrides.get('methods', {}).get(method, {})
snake_name = _upper_camel_to_lower_under(method)
if overriding_method and overriding_method.get('timeout_millis'):
timeout = overriding_method['timeout_millis']
else:
timeout = method_config['timeout_millis']
timeout /= _MILLIS_PER_SECOND
bundle_descriptor = bundle_descriptors.get(snake_name)
bundling_config = method_config.get('bundling', None)
if overriding_method and 'bundling' in overriding_method:
bundling_config = overriding_method['bundling']
bundler = _construct_bundling(bundling_config, bundle_descriptor)
retry_options = _merge_retry_options(
_construct_retry(method_config, service_config['retry_codes'],
service_config['retry_params'], retry_names),
_construct_retry(overriding_method, overrides.get('retry_codes'),
overrides.get('retry_params'), retry_names))
defaults[snake_name] = gax._CallSettings(
timeout=timeout, retry=retry_options,
page_descriptor=page_descriptors.get(snake_name),
bundler=bundler, bundle_descriptor=bundle_descriptor,
kwargs=kwargs)
return defaults
def construct_settings(service_name,
client_config,
config_override,
retry_names,
bundle_descriptors=None,
page_descriptors=None,
metrics_headers=(),
kwargs=None):
"""Constructs a dictionary mapping method names to _CallSettings.
The ``client_config`` parameter is parsed from a client configuration JSON
file of the form:
.. code-block:: json
{
"interfaces": {
"google.fake.v1.ServiceName": {
"retry_codes": {
"idempotent": ["UNAVAILABLE", "DEADLINE_EXCEEDED"],
"non_idempotent": []
},
"retry_params": {
"default": {
"initial_retry_delay_millis": 100,
"retry_delay_multiplier": 1.2,
"max_retry_delay_millis": 1000,
"initial_rpc_timeout_millis": 2000,
"rpc_timeout_multiplier": 1.5,
"max_rpc_timeout_millis": 30000,
"total_timeout_millis": 45000
}
},
"methods": {
"CreateFoo": {
"retry_codes_name": "idempotent",
"retry_params_name": "default",
"timeout_millis": 30000
},
"Publish": {
"retry_codes_name": "non_idempotent",
"retry_params_name": "default",
"bundling": {
"element_count_threshold": 40,
"element_count_limit": 200,
"request_byte_threshold": 90000,
"request_byte_limit": 100000,
"delay_threshold_millis": 100
}
}
}
}
}
}
Args:
service_name (str): The fully-qualified name of this service, used as a
key into the client config file (in the example above, this value
would be ``google.fake.v1.ServiceName``).
client_config (dict): A dictionary parsed from the standard API client
config file.
bundle_descriptors (Mapping[str, BundleDescriptor]): A dictionary of
method names to BundleDescriptor objects for methods that are
bundling-enabled.
page_descriptors (Mapping[str, PageDescriptor]): A dictionary of method
names to PageDescriptor objects for methods that are page
streaming-enabled.
config_override (str): A dictionary in the same structure of
client_config to override the settings. Usually client_config is
supplied from the default config and config_override will be
specified by users.
retry_names (Mapping[str, object]): A dictionary mapping the strings
referring to response status codes to the Python objects representing
those codes.
metrics_headers (Mapping[str, str]): Dictionary of headers to be passed
for analytics. Sent as a dictionary; eventually becomes a
space-separated string (e.g. 'foo/1.0.0 bar/3.14.1').
kwargs (dict): The keyword arguments to be passed to the API calls.
Returns:
dict: A dictionary mapping method names to _CallSettings.
Raises:
KeyError: If the configuration for the service in question cannot be
located in the provided ``client_config``.
"""
# pylint: disable=too-many-locals
# pylint: disable=protected-access
defaults = {}
bundle_descriptors = bundle_descriptors or {}
page_descriptors = page_descriptors or {}
kwargs = kwargs or {}
# Sanity check: It is possible that we got this far but some headers
# were specified with an older library, which sends them as...
# kwargs={'metadata': [('x-goog-api-client', 'foo/1.0 bar/3.0')]}
#
# Note: This is the final format we will send down to GRPC shortly.
#
# Remove any x-goog-api-client header that may have been present
# in the metadata list.
if 'metadata' in kwargs:
kwargs['metadata'] = [
value for value in kwargs['metadata']
if value[0].lower() != 'x-goog-api-client'
]
# Fill out the metrics headers with GAX and GRPC info, and convert
# to a string in the format that the GRPC layer expects.
kwargs.setdefault('metadata', [])
kwargs['metadata'].append(
('x-goog-api-client',
metrics.stringify(metrics.fill(metrics_headers))))
try:
service_config = client_config['interfaces'][service_name]
except KeyError:
raise KeyError('Client configuration not found for service: {}'.format(
service_name))
overrides = config_override.get('interfaces', {}).get(service_name, {})
for method in service_config.get('methods'):
method_config = service_config['methods'][method]
overriding_method = overrides.get('methods', {}).get(method, {})
snake_name = _upper_camel_to_lower_under(method)
if overriding_method and overriding_method.get('timeout_millis'):
timeout = overriding_method['timeout_millis']
else:
timeout = method_config['timeout_millis']
timeout /= _MILLIS_PER_SECOND
bundle_descriptor = bundle_descriptors.get(snake_name)
bundling_config = method_config.get('bundling', None)
if overriding_method and 'bundling' in overriding_method:
bundling_config = overriding_method['bundling']
bundler = _construct_bundling(bundling_config, bundle_descriptor)
retry_options = _merge_retry_options(
_construct_retry(method_config, service_config['retry_codes'],
service_config['retry_params'], retry_names),
_construct_retry(overriding_method, overrides.get('retry_codes'),
overrides.get('retry_params'), retry_names))
defaults[snake_name] = gax._CallSettings(
timeout=timeout,
retry=retry_options,
page_descriptor=page_descriptors.get(snake_name),
bundler=bundler,
bundle_descriptor=bundle_descriptor,
kwargs=kwargs)
return defaults
def construct_settings(service_name,
client_config,
config_override,
retry_names,
bundle_descriptors=None,
page_descriptors=None,
metrics_headers=(),
kwargs=None):
"""Constructs a dictionary mapping method names to _CallSettings.
The ``client_config`` parameter is parsed from a client configuration JSON
file of the form:
.. code-block:: json
{
"interfaces": {
"google.fake.v1.ServiceName": {
"retry_codes": {
"idempotent": ["UNAVAILABLE", "DEADLINE_EXCEEDED"],
"non_idempotent": []
},
"retry_params": {
"default": {
"initial_retry_delay_millis": 100,
"retry_delay_multiplier": 1.2,
"max_retry_delay_millis": 1000,
"initial_rpc_timeout_millis": 2000,
"rpc_timeout_multiplier": 1.5,
"max_rpc_timeout_millis": 30000,
"total_timeout_millis": 45000
}
},
"methods": {
"CreateFoo": {
"retry_codes_name": "idempotent",
"retry_params_name": "default",
"timeout_millis": 30000
},
"Publish": {
"retry_codes_name": "non_idempotent",
"retry_params_name": "default",
"bundling": {
"element_count_threshold": 40,
"element_count_limit": 200,
"request_byte_threshold": 90000,
"request_byte_limit": 100000,
"delay_threshold_millis": 100
}
}
}
}
}
}
Args:
service_name: The fully-qualified name of this service, used as a key into
the client config file (in the example above, this value should be
``google.fake.v1.ServiceName``).
client_config: A dictionary parsed from the standard API client config
file.
bundle_descriptors: A dictionary of method names to BundleDescriptor
objects for methods that are bundling-enabled.
page_descriptors: A dictionary of method names to PageDescriptor objects
for methods that are page streaming-enabled.
config_override: A dictionary in the same structure of client_config to
override the settings. Usually client_config is supplied from the
default config and config_override will be specified by users.
retry_names: A dictionary mapping the strings referring to response status
codes to the Python objects representing those codes.
metrics_headers: Dictionary of headers to be passed for analytics.
Sent as a dictionary; eventually becomes a space-separated string
(e.g. 'foo/1.0.0 bar/3.14.1').
kwargs: The keyword arguments to be passed to the API calls.
Raises:
KeyError: If the configuration for the service in question cannot be
located in the provided ``client_config``.
"""
# pylint: disable=too-many-locals
# pylint: disable=protected-access
defaults = {}
bundle_descriptors = bundle_descriptors or {}
metrics_headers = collections.OrderedDict(metrics_headers)
page_descriptors = page_descriptors or {}
kwargs = kwargs or {}
# Add the language header to metrics.
metrics_headers['gl-python'] = platform.python_version()
# Sanity check: It is possible that we got this far but some headers
# were specified with an older library, which sends them as...
# kwargs={'metadata': [('x-goog-api-client', 'foo/1.0 bar/3.0')]}
#
# Note: This is the final format we will send down to GRPC shortly.
#
# Remove any x-goog-api-client header that may have been present
# in the metadata list.
if 'metadata' in kwargs:
kwargs['metadata'] = [
value for value in kwargs['metadata']
if value[0].lower() != 'x-goog-api-client'
]
# Add the GAX and GRPC headers to our metrics.
# pylint: disable=no-member
grpc_version = pkg_resources.get_distribution('grpcio').version
# pylint: enable=no-member
metrics_headers['gax'] = gax.__version__
metrics_headers['grpc'] = grpc_version
# A/B key-value pairs belong at the end of the metadata string;
# shift any that appear to the end of the dictionary.
ab_keys = [k for k in metrics_headers.keys() if k.startswith('gl-ab')]
for key in ab_keys:
value = metrics_headers.pop(key)
metrics_headers[key] = value
# Okay, now add our new and complete metadata into the old
# kwargs format:
# kwargs={'metadata': [('x-goog-api-client', 'gax/x.y.z grpc/x.y.z')]}
#
# This is how it will be sent to the underlying GRPC layer.
header = ' '.join(['%s/%s' % (k, v) for k, v in metrics_headers.items()])
kwargs.setdefault('metadata', [])
kwargs['metadata'].append(('x-goog-api-client', header))
try:
service_config = client_config['interfaces'][service_name]
except KeyError:
raise KeyError('Client configuration not found for service: {}'.format(
service_name))
overrides = config_override.get('interfaces', {}).get(service_name, {})
for method in service_config.get('methods'):
method_config = service_config['methods'][method]
overriding_method = overrides.get('methods', {}).get(method, {})
snake_name = _upper_camel_to_lower_under(method)
if overriding_method and overriding_method.get('timeout_millis'):
timeout = overriding_method['timeout_millis']
else:
timeout = method_config['timeout_millis']
timeout /= _MILLIS_PER_SECOND
bundle_descriptor = bundle_descriptors.get(snake_name)
bundling_config = method_config.get('bundling', None)
if overriding_method and 'bundling' in overriding_method:
bundling_config = overriding_method['bundling']
bundler = _construct_bundling(bundling_config, bundle_descriptor)
retry_options = _merge_retry_options(
_construct_retry(method_config, service_config['retry_codes'],
service_config['retry_params'], retry_names),
_construct_retry(overriding_method, overrides.get('retry_codes'),
overrides.get('retry_params'), retry_names))
defaults[snake_name] = gax._CallSettings(
timeout=timeout,
retry=retry_options,
page_descriptor=page_descriptors.get(snake_name),
bundler=bundler,
bundle_descriptor=bundle_descriptor,
kwargs=kwargs)
return defaults
