def test_settings_merge_options1(self):
options = CallOptions(timeout=46)
settings = CallSettings(timeout=9, page_descriptor=None, retry=None)
final = settings.merge(options)
self.assertEqual(final.timeout, 46)
self.assertIsNone(final.retry)
self.assertIsNone(final.page_descriptor)
def test_settings_merge_options1(self):
options = CallOptions(timeout=46)
settings = CallSettings(timeout=9, page_descriptor=None, retry=None)
final = settings.merge(options)
self.assertEqual(final.timeout, 46)
self.assertIsNone(final.retry)
self.assertIsNone(final.page_descriptor)
def test_settings_merge_options_page_streaming(self):
retry = RetryOptions(None, None)
options = CallOptions(timeout=46, is_page_streaming=False)
settings = CallSettings(timeout=9, retry=retry)
final = settings.merge(options)
self.assertEqual(final.timeout, 46)
self.assertIsNone(final.page_descriptor)
self.assertEqual(final.retry, retry)
def test_settings_merge_options2(self):
retry = RetryOptions(None, None)
options = CallOptions(retry=retry)
settings = CallSettings(
timeout=9, page_descriptor=None, retry=RetryOptions(None, None))
final = settings.merge(options)
self.assertEqual(final.timeout, 9)
self.assertIsNone(final.page_descriptor)
self.assertEqual(final.retry, retry)
def test_settings_merge_none(self):
settings = CallSettings(
timeout=23, page_descriptor=object(), bundler=object(),
retry=object())
final = settings.merge(None)
self.assertEqual(final.timeout, settings.timeout)
self.assertEqual(final.retry, settings.retry)
self.assertEqual(final.page_descriptor, settings.page_descriptor)
self.assertEqual(final.bundler, settings.bundler)
self.assertEqual(final.bundle_descriptor, settings.bundle_descriptor)
def test_settings_merge_options2(self):
retry = RetryOptions(None, None)
options = CallOptions(retry=retry)
settings = CallSettings(timeout=9,
page_descriptor=None,
retry=RetryOptions(None, None))
final = settings.merge(options)
self.assertEqual(final.timeout, 9)
self.assertIsNone(final.page_descriptor)
self.assertEqual(final.retry, retry)
def test_settings_merge_none(self):
settings = CallSettings(
timeout=23, page_descriptor=object(), bundler=object(),
retry=object())
final = settings.merge(None)
self.assertEqual(final.timeout, settings.timeout)
self.assertEqual(final.retry, settings.retry)
self.assertEqual(final.page_descriptor, settings.page_descriptor)
self.assertEqual(final.bundler, settings.bundler)
self.assertEqual(final.bundle_descriptor, settings.bundle_descriptor)
def test_settings_merge_options_page_streaming(self):
retry = RetryOptions(None, None)
page_descriptor = object()
options = CallOptions(timeout=46, page_token=INITIAL_PAGE)
settings = CallSettings(timeout=9, retry=retry,
page_descriptor=page_descriptor)
final = settings.merge(options)
self.assertEqual(final.timeout, 46)
self.assertEqual(final.page_descriptor, page_descriptor)
self.assertEqual(final.page_token, INITIAL_PAGE)
self.assertFalse(final.flatten_pages)
self.assertEqual(final.retry, retry)
def test_settings_merge_options_page_streaming(self):
retry = RetryOptions(None, None)
page_descriptor = object()
options = CallOptions(timeout=46, page_token=INITIAL_PAGE)
settings = CallSettings(timeout=9,
retry=retry,
page_descriptor=page_descriptor)
final = settings.merge(options)
self.assertEqual(final.timeout, 46)
self.assertEqual(final.page_descriptor, page_descriptor)
self.assertEqual(final.page_token, INITIAL_PAGE)
self.assertFalse(final.flatten_pages)
self.assertEqual(final.retry, retry)
def test_catch_error(self):
def abortion_error_func(*dummy_args, **dummy_kwargs):
raise CustomException(None, None)
def other_error_func(*dummy_args, **dummy_kwargs):
raise AnotherException
gax_error_callable = api_callable.create_api_call(
abortion_error_func, CallSettings())
self.assertRaises(GaxError, gax_error_callable, None)
other_error_callable = api_callable.create_api_call(
other_error_func, CallSettings())
self.assertRaises(AnotherException, other_error_callable, None)
def test_call_kwargs(self):
settings = CallSettings(kwargs={'key': 'value'})
my_callable = api_callable.create_api_call(
lambda _req, _timeout, **kwargs: kwargs['key'], settings)
self.assertEqual(my_callable(None), 'value')
self.assertEqual(my_callable(None, CallOptions(key='updated')),
'updated')
def test_retry_times_out_no_response(self, mock_time):
mock_time.return_value = 1
retry = RetryOptions([_FAKE_STATUS_CODE_1],
BackoffSettings(0, 0, 0, 0, 0, 0, 0))
settings = CallSettings(timeout=0, retry=retry)
my_callable = api_callable.create_api_call(lambda: None, settings)
self.assertRaises(RetryError, my_callable, None)
def test_no_retry_if_no_codes(self, mock_time):
retry = RetryOptions([], BackoffSettings(1, 2, 3, 4, 5, 6, 7))
mock_call = mock.Mock()
mock_call.side_effect = CustomException('', _FAKE_STATUS_CODE_1)
mock_time.return_value = 0
settings = CallSettings(timeout=0, retry=retry)
my_callable = api_callable.create_api_call(mock_call, settings)
self.assertRaises(CustomException, my_callable, None)
self.assertEqual(mock_call.call_count, 1)
def test_retry_aborts_on_unexpected_exception(self, mock_exc_to_code,
mock_time):
mock_exc_to_code.side_effect = lambda e: e.code
retry = RetryOptions([_FAKE_STATUS_CODE_1],
BackoffSettings(0, 0, 0, 0, 0, 0, 1))
mock_call = mock.Mock()
mock_call.side_effect = CustomException('', _FAKE_STATUS_CODE_2)
mock_time.return_value = 0
settings = CallSettings(timeout=0, retry=retry)
my_callable = api_callable.create_api_call(mock_call, settings)
self.assertRaises(Exception, my_callable, None)
self.assertEqual(mock_call.call_count, 1)
def test_retry_aborts_simple(self, mock_exc_to_code, mock_time):
def fake_call(dummy_request, dummy_timeout):
raise CustomException('', _FAKE_STATUS_CODE_1)
retry = RetryOptions([_FAKE_STATUS_CODE_1],
BackoffSettings(0, 0, 0, 0, 0, 0, 1))
mock_time.side_effect = [0, 2]
mock_exc_to_code.side_effect = lambda e: e.code
settings = CallSettings(timeout=0, retry=retry)
my_callable = api_callable.create_api_call(fake_call, settings)
try:
my_callable(None)
except RetryError as exc:
self.assertIsInstance(exc.cause, CustomException)
def test_retry(self, mock_exc_to_code, mock_time):
mock_exc_to_code.side_effect = lambda e: e.code
to_attempt = 3
retry = RetryOptions([_FAKE_STATUS_CODE_1],
BackoffSettings(0, 0, 0, 0, 0, 0, 1))
# Succeeds on the to_attempt'th call, and never again afterward
mock_call = mock.Mock()
mock_call.side_effect = ([CustomException('', _FAKE_STATUS_CODE_1)] *
(to_attempt - 1) + [mock.DEFAULT])
mock_call.return_value = 1729
mock_time.return_value = 0
settings = CallSettings(timeout=0, retry=retry)
my_callable = api_callable.create_api_call(mock_call, settings)
self.assertEqual(my_callable(None), 1729)
self.assertEqual(mock_call.call_count, to_attempt)
def test_retry_times_out_simple(self, mock_exc_to_code, mock_time):
mock_exc_to_code.side_effect = lambda e: e.code
to_attempt = 3
retry = RetryOptions([_FAKE_STATUS_CODE_1],
BackoffSettings(0, 0, 0, 0, 0, 0, 1))
mock_call = mock.Mock()
mock_call.side_effect = CustomException('', _FAKE_STATUS_CODE_1)
mock_time.side_effect = ([0] * to_attempt + [2])
settings = CallSettings(timeout=0, retry=retry)
my_callable = api_callable.create_api_call(mock_call, settings)
try:
my_callable(None)
except RetryError as exc:
self.assertIsInstance(exc.cause, CustomException)
self.assertEqual(mock_call.call_count, to_attempt)
def test_page_streaming(self):
# A mock grpc function that page streams a list of consecutive
# integers, returning `page_size` integers with each call and using
# the next integer to return as the page token, until `pages_to_stream`
# pages have been returned.
page_size = 3
pages_to_stream = 5
# pylint: disable=abstract-method, too-few-public-methods
class PageStreamingRequest(object):
def __init__(self, page_token=0):
self.page_token = page_token
class PageStreamingResponse(object):
def __init__(self, nums=(), next_page_token=0):
self.nums = nums
self.next_page_token = next_page_token
fake_grpc_func_descriptor = PageDescriptor(
'page_token', 'next_page_token', 'nums')
def grpc_return_value(request, *dummy_args, **dummy_kwargs):
if (request.page_token > 0 and
request.page_token < page_size * pages_to_stream):
return PageStreamingResponse(
nums=iter(range(request.page_token,
request.page_token + page_size)),
next_page_token=request.page_token + page_size)
elif request.page_token >= page_size * pages_to_stream:
return PageStreamingResponse()
else:
return PageStreamingResponse(nums=iter(range(page_size)),
next_page_token=page_size)
with mock.patch('grpc.framework.crust.implementations.'
'_UnaryUnaryMultiCallable') as mock_grpc:
mock_grpc.side_effect = grpc_return_value
settings = CallSettings(
page_descriptor=fake_grpc_func_descriptor, timeout=0)
my_callable = api_callable.create_api_call(mock_grpc, settings=settings)
self.assertEqual(list(my_callable(PageStreamingRequest())),
list(range(page_size * pages_to_stream)))
def test_bundling(self):
# pylint: disable=abstract-method, too-few-public-methods
class BundlingRequest(object):
def __init__(self, elements=None):
self.elements = elements
fake_grpc_func_descriptor = BundleDescriptor('elements', [])
bundler = bundling.Executor(BundleOptions(element_count_threshold=8))
def my_func(request, dummy_timeout):
return len(request.elements)
settings = CallSettings(
bundler=bundler, bundle_descriptor=fake_grpc_func_descriptor,
timeout=0)
my_callable = api_callable.create_api_call(my_func, settings)
first = my_callable(BundlingRequest([0] * 3))
self.assertIsInstance(first, bundling.Event)
self.assertIsNone(first.result) # pylint: disable=no-member
second = my_callable(BundlingRequest([0] * 5))
self.assertEquals(second.result, 8) # pylint: disable=no-member
def test_retry_exponential_backoff(self, mock_exc_to_code, mock_time,
mock_sleep):
# pylint: disable=too-many-locals
mock_exc_to_code.side_effect = lambda e: e.code
MILLIS_PER_SEC = 1000
mock_time.return_value = 0
def incr_time(secs):
mock_time.return_value += secs
def api_call(dummy_request, timeout, **dummy_kwargs):
incr_time(timeout)
raise CustomException(str(timeout), _FAKE_STATUS_CODE_1)
mock_call = mock.Mock()
mock_sleep.side_effect = incr_time
mock_call.side_effect = api_call
params = BackoffSettings(3, 2, 24, 5, 2, 80, 2500)
retry = RetryOptions([_FAKE_STATUS_CODE_1], params)
settings = CallSettings(timeout=0, retry=retry)
my_callable = api_callable.create_api_call(mock_call, settings)
try:
my_callable(None)
except RetryError as exc:
self.assertIsInstance(exc.cause, CustomException)
self.assertGreaterEqual(mock_time(),
params.total_timeout_millis / MILLIS_PER_SEC)
# Very rough bounds
calls_lower_bound = params.total_timeout_millis / (
params.max_retry_delay_millis + params.max_rpc_timeout_millis)
self.assertGreater(mock_call.call_count, calls_lower_bound)
calls_upper_bound = (params.total_timeout_millis /
params.initial_retry_delay_millis)
self.assertLess(mock_call.call_count, calls_upper_bound)
def test_call_override(self):
settings = CallSettings(timeout=10)
my_callable = api_callable.create_api_call(
lambda _req, timeout: timeout, settings)
self.assertEqual(my_callable(None, CallOptions(timeout=20)), 20)
def test_call_api_call(self):
settings = CallSettings()
my_callable = api_callable.create_api_call(
lambda _req, _timeout: 42, settings)
self.assertEqual(my_callable(None), 42)
def test_page_streaming(self):
# A mock grpc function that page streams a list of consecutive
# integers, returning `page_size` integers with each call and using
# the next integer to return as the page token, until `pages_to_stream`
# pages have been returned.
# pylint:disable=too-many-locals
page_size = 3
pages_to_stream = 5
# pylint: disable=abstract-method, too-few-public-methods
class PageStreamingRequest(object):
def __init__(self, page_token=0):
self.page_token = page_token
class PageStreamingResponse(object):
def __init__(self, nums=(), next_page_token=0):
self.nums = nums
self.next_page_token = next_page_token
fake_grpc_func_descriptor = PageDescriptor('page_token',
'next_page_token', 'nums')
def grpc_return_value(request, *dummy_args, **dummy_kwargs):
start = int(request.page_token)
if start > 0 and start < page_size * pages_to_stream:
return PageStreamingResponse(nums=list(
range(start, start + page_size)),
next_page_token=start + page_size)
elif start >= page_size * pages_to_stream:
return PageStreamingResponse()
else:
return PageStreamingResponse(nums=list(range(page_size)),
next_page_token=page_size)
with mock.patch('grpc.UnaryUnaryMultiCallable') as mock_grpc:
mock_grpc.side_effect = grpc_return_value
settings = CallSettings(page_descriptor=fake_grpc_func_descriptor,
timeout=0)
my_callable = api_callable.create_api_call(mock_grpc,
settings=settings)
self.assertEqual(list(my_callable(PageStreamingRequest())),
list(range(page_size * pages_to_stream)))
unflattened_option = CallOptions(page_token=INITIAL_PAGE)
# Expect a list of pages_to_stream pages, each of size page_size,
# plus one empty page
expected = [
list(range(page_size * n, page_size * (n + 1)))
for n in range(pages_to_stream)
] + [()]
self.assertEqual(
list(my_callable(PageStreamingRequest(), unflattened_option)),
expected)
pages_already_read = 2
explicit_page_token_option = CallOptions(
page_token=str(page_size * pages_already_read))
# Expect a list of pages_to_stream pages, each of size page_size,
# plus one empty page, minus the pages_already_read
expected = [
list(range(page_size * n, page_size * (n + 1)))
for n in range(pages_already_read, pages_to_stream)
]
expected += [()]
self.assertEqual(
list(
my_callable(PageStreamingRequest(),
explicit_page_token_option)), expected)
def test_bundling_page_streaming_error(self):
settings = CallSettings(
page_descriptor=object(), bundle_descriptor=object(),
bundler=object())
with self.assertRaises(ValueError):
api_callable.create_api_call(lambda _req, _timeout: 42, settings)