def test_write_mutations_throttle_delay_retryable_error(
self, unused_sleep):
mock_batch = MagicMock()
mock_batch.commit.side_effect = [
exceptions.DeadlineExceeded('retryable'), None
]
mock_throttler = MagicMock()
rpc_stats_callback = MagicMock()
# First try: throttle once [True, False]
# Second try: no throttle [False]
mock_throttler.throttle_request.side_effect = [True, False, False]
mutate = datastoreio._Mutate.DatastoreMutateFn(lambda: None)
mutate._batch = mock_batch
mutate._batch_elements = []
mutate._client = MagicMock()
mutate.write_mutations(mock_throttler,
rpc_stats_callback,
throttle_delay=0)
rpc_stats_callback.assert_has_calls([
call(successes=1),
call(throttled_secs=ANY),
call(errors=1),
],
any_order=True)
self.assertEqual(3, rpc_stats_callback.call_count)
def test_on_exception_deadline_exceeded():
policy = create_policy()
details = 'Bad thing happened. Time out, go sit in the corner.'
exc = exceptions.DeadlineExceeded(details)
assert policy.on_exception(exc) is True
def test_create_method_configs():
method_configs = config.parse_method_configs(INTERFACE_CONFIG)
retry, timeout = method_configs["AnnotateVideo"]
assert retry._predicate(exceptions.DeadlineExceeded(None))
assert retry._predicate(exceptions.ServiceUnavailable(None))
assert retry._initial == 1.0
assert retry._multiplier == 2.5
assert retry._maximum == 120.0
assert retry._deadline == 600.0
assert timeout._initial == 120.0
assert timeout._multiplier == 1.0
assert timeout._maximum == 120.0
retry, timeout = method_configs["Other"]
assert retry._predicate(exceptions.FailedPrecondition(None))
assert retry._initial == 1.0
assert retry._multiplier == 1.0
assert retry._maximum == 1.0
assert retry._deadline == 1.0
assert timeout._initial == 1.0
assert timeout._multiplier == 1.0
assert timeout._maximum == 1.0
retry, timeout = method_configs["Plain"]
assert retry is None
assert timeout._timeout == 30.0
def __call__(self, *args, **kwargs):
self._state += 1
if self._state % 3 == 0:
raise gexc.RetryError("", "")
if self._state % 3 == 1:
raise gexc.DeadlineExceeded("")
if self._state % 3 == 2:
return self.pull_response
def test_write_mutations_metric_on_failure(self):
MetricsEnvironment.process_wide_container().reset()
mock_batch = MagicMock()
mock_batch.commit.side_effect = [
exceptions.DeadlineExceeded("Deadline Exceeded"), []
]
mock_throttler = MagicMock()
rpc_stats_callback = MagicMock()
mock_throttler.throttle_request.return_value = False
mutate = datastoreio._Mutate.DatastoreMutateFn("my_project")
mutate._batch = mock_batch
mutate._batch_elements = []
mutate._client = MagicMock()
mutate.write_mutations(mock_throttler, rpc_stats_callback, throttle_delay=0)
self.verify_write_call_metric("my_project", "", "deadline_exceeded", 1)
self.verify_write_call_metric("my_project", "", "ok", 1)
def test_write_mutations_reconstruct_on_error(self):
mock_batch = MagicMock()
mock_batch.begin.side_effect = [None, ValueError]
mock_batch.commit.side_effect = [exceptions.DeadlineExceeded('retryable'),
None]
mock_throttler = MagicMock()
rpc_stats_callback = MagicMock()
mock_throttler.throttle_request.return_value = []
mutate = datastoreio._Mutate.DatastoreMutateFn(lambda: None)
mutate._batch = mock_batch
mutate._client = MagicMock()
mutate._batch_elements = [None]
mock_add_to_batch = MagicMock()
mutate.add_to_batch = mock_add_to_batch
mutate.write_mutations(mock_throttler, rpc_stats_callback)
rpc_stats_callback.assert_has_calls([
call(successes=1),
])
self.assertEqual(1, mock_add_to_batch.call_count)
def test_QueryFn_metric_on_failure(self):
MetricsEnvironment.process_wide_container().reset()
with patch.object(helper, 'get_client',
return_value=self._mock_client):
self._mock_query.project = self._PROJECT
self._mock_query.namespace = self._NAMESPACE
_query_fn = ReadFromDatastore._QueryFn()
client_query = self._mock_query._to_client_query()
# Test with exception
client_query.fetch.side_effect = [
exceptions.DeadlineExceeded("Deadline exceed")
]
list(_query_fn.process(self._mock_query))
self.verify_read_call_metric(self._PROJECT, self._NAMESPACE,
"deadline_exceeded", 1)
# Test success
client_query.fetch.side_effect = [[]]
list(_query_fn.process(self._mock_query))
self.verify_read_call_metric(self._PROJECT, self._NAMESPACE, "ok",
1)
def test_write_mutations_throttle_delay_retryable_error(self):
mock_batch = mock.MagicMock()
mock_batch.commit.side_effect = [
exceptions.DeadlineExceeded('retryable'), None
]
mock_throttler = mock.MagicMock()
rpc_stats_callback = mock.MagicMock()
# First try: throttle once [True, False]
# Second try: no throttle [False]
mock_throttler.throttle_request.side_effect = [True, False, False]
helper.write_mutations(mock_batch,
mock_throttler,
rpc_stats_callback,
throttle_delay=0)
rpc_stats_callback.assert_has_calls([
mock.call(successes=1),
mock.call(throttled_secs=mock.ANY),
mock.call(errors=1),
],
any_order=True)
self.assertEqual(3, rpc_stats_callback.call_count)
