def test__should_recover_true():
manager = make_manager()
details = "UNAVAILABLE. Service taking nap."
exc = exceptions.ServiceUnavailable(details)
assert manager._should_recover(exc) is True
def test_api_retry_5xx_errors(client_constructor):
grpc_client = setup_mock_(client_constructor)
grpc_client.get_quantum_program.side_effect = exceptions.ServiceUnavailable('internal error')
client = EngineClient(max_retry_delay_seconds=0.3)
with pytest.raises(TimeoutError, match='Reached max retry attempts.*internal error'):
client.get_program('proj', 'prog', False)
assert grpc_client.get_quantum_program.call_count == 3
def test__commit_with_retry_success_third_attempt(_sleep):
from google.api_core import exceptions
from google.cloud.firestore_v1.services.firestore import client as firestore_client
from google.cloud.firestore_v1.transaction import _commit_with_retry
# Create a minimal fake GAPIC with a dummy result.
firestore_api = mock.create_autospec(firestore_client.FirestoreClient,
instance=True)
# Make sure the first two requests fail and the third succeeds.
firestore_api.commit.side_effect = [
exceptions.ServiceUnavailable("Server sleepy."),
exceptions.ServiceUnavailable("Server groggy."),
mock.sentinel.commit_response,
]
# Attach the fake GAPIC to a real client.
client = _make_client("outside")
client._firestore_api_internal = firestore_api
# Call function and check result.
txn_id = b"the-world\x00"
commit_response = _commit_with_retry(client, mock.sentinel.write_pbs,
txn_id)
assert commit_response is mock.sentinel.commit_response
# Verify mocks used.
# Ensure _sleep is called after commit failures, with intervals of 1 and 2 seconds
assert _sleep.call_count == 2
_sleep.assert_any_call(1.0)
_sleep.assert_any_call(2.0)
# commit() called same way 3 times.
commit_call = mock.call(
request={
"database": client._database_string,
"writes": mock.sentinel.write_pbs,
"transaction": txn_id,
},
metadata=client._rpc_metadata,
)
assert firestore_api.commit.mock_calls == [
commit_call, commit_call, commit_call
]
def test_api_retry_times(client_constructor, mock_sleep):
grpc_client = setup_mock_(client_constructor)
grpc_client.get_quantum_program.side_effect = exceptions.ServiceUnavailable('internal error')
client = EngineClient(max_retry_delay_seconds=0.3)
with pytest.raises(TimeoutError, match='Reached max retry attempts.*internal error'):
client.get_program('proj', 'prog', False)
assert grpc_client.get_quantum_program.call_count == 3
assert len(mock_sleep.call_args_list) == 2
assert all(x == y for (x, _), y in zip(mock_sleep.call_args_list, [(0.1,), (0.2,)]))
def test_api_retry_5xx_errors(client_constructor):
client = mock.Mock()
client_constructor.return_value = client
client.get_quantum_program.side_effect = exceptions.ServiceUnavailable(
'internal error')
engine = cg.Engine(project_id='project-id')
with pytest.raises(TimeoutError,
match='Reached max retry attempts.*internal error'):
engine.max_retry_delay = 1 # 1 second
engine.get_program('foo')
assert client.get_quantum_program.call_count > 1
async def test_success_third_attempt(self, _sleep):
from google.api_core import exceptions
# Create a minimal fake GAPIC with a dummy result.
firestore_api = AsyncMock()
# Make sure the first two requests fail and the third succeeds.
firestore_api.commit.side_effect = [
exceptions.ServiceUnavailable("Server sleepy."),
exceptions.ServiceUnavailable("Server groggy."),
mock.sentinel.commit_response,
]
# Attach the fake GAPIC to a real client.
client = _make_client("outside")
client._firestore_api_internal = firestore_api
# Call function and check result.
txn_id = b"the-world\x00"
commit_response = await self._call_fut(client, mock.sentinel.write_pbs, txn_id)
self.assertIs(commit_response, mock.sentinel.commit_response)
# Verify mocks used.
# Ensure _sleep is called after commit failures, with intervals of 1 and 2 seconds
self.assertEqual(_sleep.call_count, 2)
_sleep.assert_any_call(1.0)
_sleep.assert_any_call(2.0)
# commit() called same way 3 times.
commit_call = mock.call(
request={
"database": client._database_string,
"writes": mock.sentinel.write_pbs,
"transaction": txn_id,
},
metadata=client._rpc_metadata,
)
self.assertEqual(
firestore_api.commit.mock_calls, [commit_call, commit_call, commit_call]
)
def test_success_third_attempt(self, _sleep):
from google.api_core import exceptions
from google.cloud.firestore_v1.gapic import firestore_client
# Create a minimal fake GAPIC with a dummy result.
firestore_api = mock.create_autospec(
firestore_client.FirestoreClient, instance=True
)
# Make sure the first two requests fail and the third succeeds.
firestore_api.commit.side_effect = [
exceptions.ServiceUnavailable("Server sleepy."),
exceptions.ServiceUnavailable("Server groggy."),
mock.sentinel.commit_response,
]
# Attach the fake GAPIC to a real client.
client = _make_client("outside")
client._firestore_api_internal = firestore_api
# Call function and check result.
txn_id = b"the-world\x00"
commit_response = self._call_fut(client, mock.sentinel.write_pbs, txn_id)
self.assertIs(commit_response, mock.sentinel.commit_response)
# Verify mocks used.
self.assertEqual(_sleep.call_count, 2)
_sleep.assert_any_call(1.0)
_sleep.assert_any_call(2.0)
# commit() called same way 3 times.
commit_call = mock.call(
client._database_string,
mock.sentinel.write_pbs,
transaction=txn_id,
metadata=client._rpc_metadata,
)
self.assertEqual(
firestore_api.commit.mock_calls, [commit_call, commit_call, commit_call]
)
def test_retry_nonidempotent(echo):
# Define our error and OK responses.
err = exceptions.ServiceUnavailable(message='whups')
ok = showcase_v1alpha3.EchoResponse(content='foo')
server = mock.Mock(side_effect=(err, err, ok))
# Mock the transport to send back the error responses followed by a
# success response.
transport = type(echo).get_transport_class()
with mock.patch.object(
transport, 'echo',
new_callable=mock.PropertyMock(return_value=server)):
with mock.patch.object(time, 'sleep'):
response = echo.echo({'content': 'bar'})
assert response.content == 'foo'
assert server.call_count == 3
def test_do_not_disclose_cache_contents(begin_transaction, client_context):
"""Regression test for #482.
https://github.com/googleapis/python-ndb/issues/482
"""
begin_transaction.side_effect = core_exceptions.ServiceUnavailable("Spurious Error")
client_context.cache["hello dad"] = "i'm in jail"
@ndb.transactional()
def callback():
pass
with pytest.raises(Exception) as error_info:
callback()
error = error_info.value
message = "".join(traceback.format_exception_only(type(error), error))
assert "hello dad" not in message
def test_consumer_expected_error(self, caplog):
caplog.set_level(logging.DEBUG)
bidi_rpc = mock.create_autospec(bidi.BidiRpc, instance=True)
bidi_rpc.is_active = True
bidi_rpc.recv.side_effect = exceptions.ServiceUnavailable('Gone away')
on_response = mock.Mock(spec=['__call__'])
consumer = bidi.BackgroundConsumer(bidi_rpc, on_response)
consumer.start()
# Wait for the consumer's thread to exit.
while consumer.is_active:
pass
on_response.assert_not_called()
bidi_rpc.recv.assert_called_once()
assert 'caught error' in caplog.text
def test_retry_idempotent(identity):
# Define our error and OK responses.
err409 = exceptions.Aborted(message='derp de derp')
err503 = exceptions.ServiceUnavailable(message='whups')
errwtf = exceptions.Unknown(message='huh?')
ok = showcase_v1alpha3.User(name='users/0', display_name='Guido')
server = mock.Mock(side_effect=(err409, err503, errwtf, ok))
# Mock the transport to send back the error responses followed by a
# success response.
transport = type(identity).get_transport_class()
with mock.patch.object(
transport,
'get_user',
new_callable=mock.PropertyMock(return_value=server)):
with mock.patch.object(time, 'sleep'):
response = identity.get_user({'name': 'users/0'})
assert response.name == 'users/0'
assert response.display_name == 'Guido'
assert server.call_count == 4
def retry_grpc(num_retries: int, fn: Callable[..., ReturnType], *args: Any,
**kwargs: Any) -> ReturnType:
"""Retries a function call some number of times"""
time_to_sleep = random.uniform(5, RETRY_SLEEP)
for i in range(num_retries + 1):
try:
return fn(*args, **kwargs)
except exceptions.InternalServerError as e:
if i == num_retries:
raise
if 'GOAWAY' in str(e) or 'Deadline Exceeded' in str(e):
logging.exception('Received exception: ')
if environment.in_gae():
logging.warning('Sleeping %.2f seconds and retrying',
time_to_sleep)
time.sleep(time_to_sleep)
continue
else:
raise
raise exceptions.ServiceUnavailable(
f"Function unsuccessful {num_retries + 1} times")
def test_failure_second_attempt(self, _sleep):
from google.api_core import exceptions
from google.cloud.firestore_v1.services.firestore import (
client as firestore_client, )
# Create a minimal fake GAPIC with a dummy result.
firestore_api = mock.create_autospec(firestore_client.FirestoreClient,
instance=True)
# Make sure the first request fails retry-able and second
# fails non-retryable.
exc1 = exceptions.ServiceUnavailable("Come back next time.")
exc2 = exceptions.InternalServerError("Server on fritz.")
firestore_api.commit.side_effect = [exc1, exc2]
# Attach the fake GAPIC to a real client.
client = _make_client("peanut-butter")
client._firestore_api_internal = firestore_api
# Call function and check result.
txn_id = b"the-journey-when-and-where-well-go"
with self.assertRaises(exceptions.InternalServerError) as exc_info:
self._call_fut(client, mock.sentinel.write_pbs, txn_id)
self.assertIs(exc_info.exception, exc2)
# Verify mocks used.
_sleep.assert_called_once_with(1.0)
# commit() called same way 2 times.
commit_call = mock.call(
request={
"database": client._database_string,
"writes": mock.sentinel.write_pbs,
"transaction": txn_id,
},
metadata=client._rpc_metadata,
)
self.assertEqual(firestore_api.commit.mock_calls,
[commit_call, commit_call])
async def test__commit_with_retry_failure_second_attempt(_sleep):
from google.api_core import exceptions
from google.cloud.firestore_v1.async_transaction import _commit_with_retry
# Create a minimal fake GAPIC with a dummy result.
firestore_api = AsyncMock()
# Make sure the first request fails retry-able and second
# fails non-retryable.
exc1 = exceptions.ServiceUnavailable("Come back next time.")
exc2 = exceptions.InternalServerError("Server on fritz.")
firestore_api.commit.side_effect = [exc1, exc2]
# Attach the fake GAPIC to a real client.
client = _make_client("peanut-butter")
client._firestore_api_internal = firestore_api
# Call function and check result.
txn_id = b"the-journey-when-and-where-well-go"
with pytest.raises(exceptions.InternalServerError) as exc_info:
await _commit_with_retry(client, mock.sentinel.write_pbs, txn_id)
assert exc_info.value is exc2
# Verify mocks used.
_sleep.assert_called_once_with(1.0)
# commit() called same way 2 times.
commit_call = mock.call(
request={
"database": client._database_string,
"writes": mock.sentinel.write_pbs,
"transaction": txn_id,
},
metadata=client._rpc_metadata,
)
assert firestore_api.commit.mock_calls == [commit_call, commit_call]
def test_if_transient_error():
assert retry.if_transient_error(exceptions.InternalServerError(""))
assert retry.if_transient_error(exceptions.TooManyRequests(""))
assert retry.if_transient_error(exceptions.ServiceUnavailable(""))
assert not retry.if_transient_error(exceptions.InvalidArgument(""))
def test_unavailable(core_retry):
error = core_exceptions.ServiceUnavailable("testing")
core_retry.if_transient_error.return_value = False
assert _retry.is_transient_error(error) is True
core_retry.if_transient_error.assert_called_once_with(error)
