def test__transactional__maybe_commit_failure_cannot_retry(): from google.api_core import exceptions wrapped = _make__transactional(mock.sentinel.callable_) txn_id = b"failed-but-not-retryable" transaction = _make_transaction_pb(txn_id) transaction._id = txn_id # We won't call ``begin()``. wrapped.current_id = txn_id # We won't call ``_pre_commit()``. wrapped.retry_id = txn_id # We won't call ``_pre_commit()``. # Actually force the ``commit`` to fail. exc = exceptions.InternalServerError("Real bad thing") firestore_api = transaction._client._firestore_api firestore_api.commit.side_effect = exc with pytest.raises(exceptions.InternalServerError) as exc_info: wrapped._maybe_commit(transaction) assert exc_info.value is exc assert transaction._id == txn_id assert wrapped.current_id == txn_id assert wrapped.retry_id == txn_id # Verify mocks. firestore_api.begin_transaction.assert_not_called() firestore_api.rollback.assert_not_called() firestore_api.commit.assert_called_once_with( request={ "database": transaction._client._database_string, "writes": [], "transaction": txn_id, }, metadata=transaction._client._rpc_metadata, )
async def test_wrap_method_with_overriding_retry_deadline( utcnow, unused_sleep): fake_call = grpc_helpers_async.FakeUnaryUnaryCall(42) method = mock.Mock( spec=aio.UnaryUnaryMultiCallable, side_effect=([exceptions.InternalServerError(None)] * 4) + [fake_call], ) default_retry = retry_async.AsyncRetry() default_timeout = timeout.ExponentialTimeout(deadline=60) wrapped_method = gapic_v1.method_async.wrap_method(method, default_retry, default_timeout) # Overriding only the retry's deadline should also override the timeout's # deadline. result = await wrapped_method(retry=default_retry.with_deadline(30)) assert result == 42 timeout_args = [call[1]["timeout"] for call in method.call_args_list] assert timeout_args == [5.0, 10.0, 20.0, 26.0, 25.0] assert utcnow.call_count == ( 1 + 1 # Compute wait_for timeout in retry_async + 5 # First to set the deadline. + 5 # One for each min(timeout, maximum, (DEADLINE - NOW).seconds) )
def test__rollback_failure(self): from google.api_core import exceptions from google.cloud.firestore_v1.gapic import firestore_client # Create a minimal fake GAPIC with a dummy failure. firestore_api = mock.create_autospec( firestore_client.FirestoreClient, instance=True ) exc = exceptions.InternalServerError("Fire during rollback.") firestore_api.rollback.side_effect = exc # Attach the fake GAPIC to a real client. client = _make_client() client._firestore_api_internal = firestore_api # Actually make a transaction and roll it back. transaction = self._make_one(client) txn_id = b"roll-bad-server" transaction._id = txn_id with self.assertRaises(exceptions.InternalServerError) as exc_info: transaction._rollback() self.assertIs(exc_info.exception, exc) self.assertIsNone(transaction._id) self.assertEqual(transaction._write_pbs, []) # Verify the called mock. firestore_api.rollback.assert_called_once_with( client._database_string, txn_id, metadata=client._rpc_metadata )
async def test__rollback_failure(self): from google.api_core import exceptions # Create a minimal fake GAPIC with a dummy failure. firestore_api = AsyncMock() exc = exceptions.InternalServerError("Fire during rollback.") firestore_api.rollback.side_effect = exc # Attach the fake GAPIC to a real client. client = _make_client() client._firestore_api_internal = firestore_api # Actually make a transaction and roll it back. transaction = self._make_one(client) txn_id = b"roll-bad-server" transaction._id = txn_id with self.assertRaises(exceptions.InternalServerError) as exc_info: await transaction._rollback() self.assertIs(exc_info.exception, exc) self.assertIsNone(transaction._id) self.assertEqual(transaction._write_pbs, []) # Verify the called mock. firestore_api.rollback.assert_called_once_with( request={"database": client._database_string, "transaction": txn_id}, metadata=client._rpc_metadata, )
async def test_failure_second_attempt(self, _sleep): from google.api_core import exceptions # Create a minimal fake GAPIC with a dummy result. firestore_api = AsyncMock() # Make sure the first request fails retry-able and second # fails non-retryable. exc1 = exceptions.ServiceUnavailable("Come back next time.") exc2 = exceptions.InternalServerError("Server on fritz.") firestore_api.commit.side_effect = [exc1, exc2] # Attach the fake GAPIC to a real client. client = _make_client("peanut-butter") client._firestore_api_internal = firestore_api # Call function and check result. txn_id = b"the-journey-when-and-where-well-go" with self.assertRaises(exceptions.InternalServerError) as exc_info: await self._call_fut(client, mock.sentinel.write_pbs, txn_id) self.assertIs(exc_info.exception, exc2) # Verify mocks used. _sleep.assert_called_once_with(1.0) # commit() called same way 2 times. commit_call = mock.call( request={ "database": client._database_string, "writes": mock.sentinel.write_pbs, "transaction": txn_id, }, metadata=client._rpc_metadata, ) self.assertEqual(firestore_api.commit.mock_calls, [commit_call, commit_call])
def test_transaction__rollback_failure(): from google.api_core import exceptions from google.cloud.firestore_v1.services.firestore import client as firestore_client # Create a minimal fake GAPIC with a dummy failure. firestore_api = mock.create_autospec(firestore_client.FirestoreClient, instance=True) exc = exceptions.InternalServerError("Fire during rollback.") firestore_api.rollback.side_effect = exc # Attach the fake GAPIC to a real client. client = _make_client() client._firestore_api_internal = firestore_api # Actually make a transaction and roll it back. transaction = _make_transaction(client) txn_id = b"roll-bad-server" transaction._id = txn_id with pytest.raises(exceptions.InternalServerError) as exc_info: transaction._rollback() assert exc_info.value is exc assert transaction._id is None assert transaction._write_pbs == [] # Verify the called mock. firestore_api.rollback.assert_called_once_with( request={ "database": client._database_string, "transaction": txn_id }, metadata=client._rpc_metadata, )
def test__maybe_commit_failure_cannot_retry(self): from google.api_core import exceptions wrapped = self._make_one(mock.sentinel.callable_) txn_id = b"failed-but-not-retryable" transaction = _make_transaction(txn_id) transaction._id = txn_id # We won't call ``begin()``. wrapped.current_id = txn_id # We won't call ``_pre_commit()``. wrapped.retry_id = txn_id # We won't call ``_pre_commit()``. # Actually force the ``commit`` to fail. exc = exceptions.InternalServerError("Real bad thing") firestore_api = transaction._client._firestore_api firestore_api.commit.side_effect = exc with self.assertRaises(exceptions.InternalServerError) as exc_info: wrapped._maybe_commit(transaction) self.assertIs(exc_info.exception, exc) self.assertEqual(transaction._id, txn_id) self.assertEqual(wrapped.current_id, txn_id) self.assertEqual(wrapped.retry_id, txn_id) # Verify mocks. firestore_api.begin_transaction.assert_not_called() firestore_api.rollback.assert_not_called() firestore_api.commit.assert_called_once_with( transaction._client._database_string, [], transaction=txn_id, metadata=transaction._client._rpc_metadata, )
def test_failure_second_attempt(self, _sleep): from google.api_core import exceptions from google.cloud.firestore_v1beta1.gapic import firestore_client # Create a minimal fake GAPIC with a dummy result. firestore_api = mock.create_autospec(firestore_client.FirestoreClient, instance=True) # Make sure the first request fails retry-able and second # fails non-retryable. exc1 = exceptions.ServiceUnavailable('Come back next time.') exc2 = exceptions.InternalServerError('Server on fritz.') firestore_api.commit.side_effect = [exc1, exc2] # Attach the fake GAPIC to a real client. client = _make_client('peanut-butter') client._firestore_api_internal = firestore_api # Call function and check result. txn_id = b'the-journey-when-and-where-well-go' with self.assertRaises(exceptions.InternalServerError) as exc_info: self._call_fut(client, mock.sentinel.write_pbs, txn_id) self.assertIs(exc_info.exception, exc2) # Verify mocks used. _sleep.assert_called_once_with(1.0) # commit() called same way 2 times. commit_call = mock.call(client._database_string, mock.sentinel.write_pbs, transaction=txn_id, metadata=client._rpc_metadata) self.assertEqual(firestore_api.commit.mock_calls, [commit_call, commit_call])
def test__pre_commit_failure_with_rollback_failure(self): from google.api_core import exceptions exc1 = ValueError('I will not be only failure.') to_wrap = mock.Mock(side_effect=exc1, spec=[]) wrapped = self._make_one(to_wrap) txn_id = b'both-will-fail' transaction = _make_transaction(txn_id) # Actually force the ``rollback`` to fail as well. exc2 = exceptions.InternalServerError('Rollback blues.') firestore_api = transaction._client._firestore_api firestore_api.rollback.side_effect = exc2 # Try to ``_pre_commit`` with self.assertRaises(exceptions.InternalServerError) as exc_info: wrapped._pre_commit(transaction, a='b', c='zebra') self.assertIs(exc_info.exception, exc2) self.assertIsNone(transaction._id) self.assertEqual(wrapped.current_id, txn_id) self.assertEqual(wrapped.retry_id, txn_id) # Verify mocks. to_wrap.assert_called_once_with(transaction, a='b', c='zebra') firestore_api.begin_transaction.assert_called_once_with( transaction._client._database_string, options_=None, metadata=transaction._client._rpc_metadata) firestore_api.rollback.assert_called_once_with( transaction._client._database_string, txn_id, metadata=transaction._client._rpc_metadata) firestore_api.commit.assert_not_called()
def test__commit_failure(self): from google.api_core import exceptions from google.cloud.firestore_v1beta1.gapic import firestore_client # Create a minimal fake GAPIC with a dummy failure. firestore_api = mock.create_autospec(firestore_client.FirestoreClient, instance=True) exc = exceptions.InternalServerError('Fire during commit.') firestore_api.commit.side_effect = exc # Attach the fake GAPIC to a real client. client = _make_client() client._firestore_api_internal = firestore_api # Actually make a transaction with some mutations and call _commit(). transaction = self._make_one(client) txn_id = b'beep-fail-commit' transaction._id = txn_id transaction.create(client.document('up', 'down'), {'water': 1.0}) transaction.delete(client.document('up', 'left')) write_pbs = transaction._write_pbs[::] with self.assertRaises(exceptions.InternalServerError) as exc_info: transaction._commit() self.assertIs(exc_info.exception, exc) self.assertEqual(transaction._id, txn_id) self.assertEqual(transaction._write_pbs, write_pbs) # Verify the called mock. firestore_api.commit.assert_called_once_with( client._database_string, write_pbs, transaction=txn_id, metadata=client._rpc_metadata)
def test_retry(self): model = QuestionnaireState("someuser", "data", 1) self.mock_client.put = mock.Mock(side_effect=[ exceptions.InternalServerError("error"), mock.DEFAULT ]) self.ds.put(model, True) assert self.mock_client.put.call_count > 1
def test_wrap_method_with_default_retry_and_timeout(unusued_sleep): method = mock.Mock(spec=["__call__"], side_effect=[exceptions.InternalServerError(None), 42]) default_retry = retry.Retry() default_timeout = timeout.ConstantTimeout(60) wrapped_method = google.api_core.gapic_v1.method.wrap_method( method, default_retry, default_timeout) result = wrapped_method() assert result == 42 assert method.call_count == 2 method.assert_called_with(timeout=60, metadata=mock.ANY)
def test_execute_internal_server_error(self): from google.api_core import exceptions from google.cloud.spanner_dbapi.exceptions import OperationalError connection = self._make_connection(self.INSTANCE, mock.MagicMock()) cursor = self._make_one(connection) with mock.patch( "google.cloud.spanner_dbapi.parse_utils.classify_stmt", side_effect=exceptions.InternalServerError("message"), ): with self.assertRaises(OperationalError): cursor.execute(sql="sql")
async def test_wrap_method_with_default_retry_and_timeout(unused_sleep): fake_call = grpc_helpers_async.FakeUnaryUnaryCall(42) method = mock.Mock( spec=aio.UnaryUnaryMultiCallable, side_effect=[exceptions.InternalServerError(None), fake_call], ) default_retry = retry_async.AsyncRetry() default_timeout = timeout.ConstantTimeout(60) wrapped_method = gapic_v1.method_async.wrap_method(method, default_retry, default_timeout) result = await wrapped_method() assert result == 42 assert method.call_count == 2 method.assert_called_with(timeout=60, metadata=mock.ANY)
def test_wrap_method_with_overriding_retry_deadline(utcnow, unused_sleep): method = mock.Mock( spec=['__call__'], side_effect=([exceptions.InternalServerError(None)] * 4) + [42]) default_retry = retry.Retry() default_timeout = timeout.ExponentialTimeout(deadline=60) wrapped_method = google.api_core.gapic_v1.method.wrap_method( method, default_retry, default_timeout) # Overriding only the retry's deadline should also override the timeout's # deadline. result = wrapped_method(retry=default_retry.with_deadline(30)) assert result == 42 timeout_args = [call[1]['timeout'] for call in method.call_args_list] assert timeout_args == [5.0, 10.0, 20.0, 26.0, 25.0] assert utcnow.call_count == ( 1 + # First to set the deadline. 5 + # One for each min(timeout, maximum, (DEADLINE - NOW).seconds) 5)
def test_get_query_results_raises_error_if_deadline_exceeded(self, mock_api_request): # type: (MagicMock) -> None """Tests that get_query_results raises a RetryError if the deadline is exceeded. Args: mock_api_request: The MagicMock object for the api_request method. """ # Raise a transient error on every API request so that the job doesn't finish. mock_api_request.side_effect = exceptions.InternalServerError('Transient error.') client_in_a_hurry = bq.Client(self.TEST_PROJECT, max_wait_secs=1) before_get_query_results = datetime.now() with self.assertRaises(exceptions.RetryError): client_in_a_hurry.get_query_results('SELECT 5') # Test that the process timed out before the default timeout. (It should time out after 1 # second, but this leaves buffer for BQ being slow.) # Use the underlying Retry library's default timeout, because it's shorter than our # DEFAULT_TIMEOUT_SEC, and we want to fail if our timeout didn't get passed in somehow. diff = datetime.now() - before_get_query_results self.assertTrue(diff.seconds < retry._DEFAULT_DEADLINE)
def test__transactional__pre_commit_failure_with_rollback_failure(): from google.api_core import exceptions exc1 = ValueError("I will not be only failure.") to_wrap = mock.Mock(side_effect=exc1, spec=[]) wrapped = _make__transactional(to_wrap) txn_id = b"both-will-fail" transaction = _make_transaction_pb(txn_id) # Actually force the ``rollback`` to fail as well. exc2 = exceptions.InternalServerError("Rollback blues.") firestore_api = transaction._client._firestore_api firestore_api.rollback.side_effect = exc2 # Try to ``_pre_commit`` with pytest.raises(exceptions.InternalServerError) as exc_info: wrapped._pre_commit(transaction, a="b", c="zebra") assert exc_info.value is exc2 assert transaction._id is None assert wrapped.current_id == txn_id assert wrapped.retry_id == txn_id # Verify mocks. to_wrap.assert_called_once_with(transaction, a="b", c="zebra") firestore_api.begin_transaction.assert_called_once_with( request={ "database": transaction._client._database_string, "options": None }, metadata=transaction._client._rpc_metadata, ) firestore_api.rollback.assert_called_once_with( request={ "database": transaction._client._database_string, "transaction": txn_id, }, metadata=transaction._client._rpc_metadata, ) firestore_api.commit.assert_not_called()
def test__commit_failure(self): from google.api_core import exceptions from google.cloud.firestore_v1.services.firestore import ( client as firestore_client, ) # Create a minimal fake GAPIC with a dummy failure. firestore_api = mock.create_autospec(firestore_client.FirestoreClient, instance=True) exc = exceptions.InternalServerError("Fire during commit.") firestore_api.commit.side_effect = exc # Attach the fake GAPIC to a real client. client = _make_client() client._firestore_api_internal = firestore_api # Actually make a transaction with some mutations and call _commit(). transaction = self._make_one(client) txn_id = b"beep-fail-commit" transaction._id = txn_id transaction.create(client.document("up", "down"), {"water": 1.0}) transaction.delete(client.document("up", "left")) write_pbs = transaction._write_pbs[::] with self.assertRaises(exceptions.InternalServerError) as exc_info: transaction._commit() self.assertIs(exc_info.exception, exc) self.assertEqual(transaction._id, txn_id) self.assertEqual(transaction._write_pbs, write_pbs) # Verify the called mock. firestore_api.commit.assert_called_once_with( request={ "database": client._database_string, "writes": write_pbs, "transaction": txn_id, }, metadata=client._rpc_metadata, )
def test__commit_with_retry_failure_second_attempt(_sleep): from google.api_core import exceptions from google.cloud.firestore_v1.services.firestore import client as firestore_client from google.cloud.firestore_v1.transaction import _commit_with_retry # Create a minimal fake GAPIC with a dummy result. firestore_api = mock.create_autospec(firestore_client.FirestoreClient, instance=True) # Make sure the first request fails retry-able and second # fails non-retryable. exc1 = exceptions.ServiceUnavailable("Come back next time.") exc2 = exceptions.InternalServerError("Server on fritz.") firestore_api.commit.side_effect = [exc1, exc2] # Attach the fake GAPIC to a real client. client = _make_client("peanut-butter") client._firestore_api_internal = firestore_api # Call function and check result. txn_id = b"the-journey-when-and-where-well-go" with pytest.raises(exceptions.InternalServerError) as exc_info: _commit_with_retry(client, mock.sentinel.write_pbs, txn_id) assert exc_info.value is exc2 # Verify mocks used. _sleep.assert_called_once_with(1.0) # commit() called same way 2 times. commit_call = mock.call( request={ "database": client._database_string, "writes": mock.sentinel.write_pbs, "transaction": txn_id, }, metadata=client._rpc_metadata, ) assert firestore_api.commit.mock_calls == [commit_call, commit_call]
async def test_asynctransaction__commit_failure(): from google.api_core import exceptions # Create a minimal fake GAPIC with a dummy failure. firestore_api = AsyncMock() exc = exceptions.InternalServerError("Fire during commit.") firestore_api.commit.side_effect = exc # Attach the fake GAPIC to a real client. client = _make_client() client._firestore_api_internal = firestore_api # Actually make a transaction with some mutations and call _commit(). transaction = _make_async_transaction(client) txn_id = b"beep-fail-commit" transaction._id = txn_id transaction.create(client.document("up", "down"), {"water": 1.0}) transaction.delete(client.document("up", "left")) write_pbs = transaction._write_pbs[::] with pytest.raises(exceptions.InternalServerError) as exc_info: await transaction._commit() assert exc_info.value is exc assert transaction._id == txn_id assert transaction._write_pbs == write_pbs # Verify the called mock. firestore_api.commit.assert_called_once_with( request={ "database": client._database_string, "writes": write_pbs, "transaction": txn_id, }, metadata=client._rpc_metadata, )
def test_if_transient_error(): assert retry.if_transient_error(exceptions.InternalServerError('')) assert retry.if_transient_error(exceptions.TooManyRequests('')) assert not retry.if_transient_error(exceptions.InvalidArgument(''))
def test_internal(core_retry): error = core_exceptions.InternalServerError("testing") core_retry.if_transient_error.return_value = False assert _retry.is_transient_error(error) is True core_retry.if_transient_error.assert_called_once_with(error)
from verily.bigquery_wrapper.bq_base import (DEFAULT_RETRY_FOR_API_CALLS, is_job_done, validate_query_job) # Arguments to pass to retry-related tests EXCEPTION_RETRY_TEST_ARGS = ( dict( exc=exceptions.BadRequest('Extra comma before FROM clause.'), should_retry=False ), dict( exc=exceptions.BadRequest( 'The job encountered an internal error during execution. (Transient error.)'), should_retry=True ), dict( exc=exceptions.InternalServerError('Transient error.'), should_retry=True ), dict( exc=exceptions.TooManyRequests('Transient error.'), should_retry=True ), dict( exc=exceptions.BadGateway('Transient error.'), should_retry=True ), ) @ddt class BQTest(bq_shared_tests.BQSharedTests):
def test_if_transient_error(): assert retry.if_transient_error(exceptions.InternalServerError("")) assert retry.if_transient_error(exceptions.TooManyRequests("")) assert retry.if_transient_error(exceptions.ServiceUnavailable("")) assert not retry.if_transient_error(exceptions.InvalidArgument(""))
from mock import MagicMock, call from google.api_core import exceptions # pylint: disable=no-name-in-module from recidiviz.common.common_utils import ( create_generated_id, is_generated_id, retry_grpc, create_synthetic_id, get_external_id, date_intersects_with_span, date_spans_overlap_exclusive, date_spans_overlap_inclusive, ) GO_AWAY_ERROR = exceptions.InternalServerError("500 GOAWAY received") DEADLINE_EXCEEDED_ERROR = exceptions.InternalServerError("504 Deadline " "Exceeded") OTHER_ERROR = exceptions.InternalServerError("500 received") _DATE_1 = datetime.date(year=2019, month=1, day=1) _DATE_2 = datetime.date(year=2019, month=2, day=1) _DATE_3 = datetime.date(year=2019, month=3, day=1) _DATE_4 = datetime.date(year=2019, month=4, day=1) _DATE_5 = datetime.date(year=2019, month=5, day=1) _DATE_6 = datetime.date(year=2019, month=6, day=1) _DATE_7 = datetime.date(year=2019, month=7, day=1) class CommonUtilsTest(unittest.TestCase): """Tests for common_utils.py.""" def test_create_generated_id(self) -> None: