def test__transactional__maybe_commit_failure_cannot_retry():
from google.api_core import exceptions
wrapped = _make__transactional(mock.sentinel.callable_)
txn_id = b"failed-but-not-retryable"
transaction = _make_transaction_pb(txn_id)
transaction._id = txn_id # We won't call ``begin()``.
wrapped.current_id = txn_id # We won't call ``_pre_commit()``.
wrapped.retry_id = txn_id # We won't call ``_pre_commit()``.
# Actually force the ``commit`` to fail.
exc = exceptions.InternalServerError("Real bad thing")
firestore_api = transaction._client._firestore_api
firestore_api.commit.side_effect = exc
with pytest.raises(exceptions.InternalServerError) as exc_info:
wrapped._maybe_commit(transaction)
assert exc_info.value is exc
assert transaction._id == txn_id
assert wrapped.current_id == txn_id
assert wrapped.retry_id == txn_id
# Verify mocks.
firestore_api.begin_transaction.assert_not_called()
firestore_api.rollback.assert_not_called()
firestore_api.commit.assert_called_once_with(
request={
"database": transaction._client._database_string,
"writes": [],
"transaction": txn_id,
},
metadata=transaction._client._rpc_metadata,
)
async def test_wrap_method_with_overriding_retry_deadline(
utcnow, unused_sleep):
fake_call = grpc_helpers_async.FakeUnaryUnaryCall(42)
method = mock.Mock(
spec=aio.UnaryUnaryMultiCallable,
side_effect=([exceptions.InternalServerError(None)] * 4) + [fake_call],
)
default_retry = retry_async.AsyncRetry()
default_timeout = timeout.ExponentialTimeout(deadline=60)
wrapped_method = gapic_v1.method_async.wrap_method(method, default_retry,
default_timeout)
# Overriding only the retry's deadline should also override the timeout's
# deadline.
result = await wrapped_method(retry=default_retry.with_deadline(30))
assert result == 42
timeout_args = [call[1]["timeout"] for call in method.call_args_list]
assert timeout_args == [5.0, 10.0, 20.0, 26.0, 25.0]
assert utcnow.call_count == (
1 + 1 # Compute wait_for timeout in retry_async
+ 5 # First to set the deadline.
+ 5 # One for each min(timeout, maximum, (DEADLINE - NOW).seconds)
)
def test__rollback_failure(self):
from google.api_core import exceptions
from google.cloud.firestore_v1.gapic import firestore_client
# Create a minimal fake GAPIC with a dummy failure.
firestore_api = mock.create_autospec(
firestore_client.FirestoreClient, instance=True
)
exc = exceptions.InternalServerError("Fire during rollback.")
firestore_api.rollback.side_effect = exc
# Attach the fake GAPIC to a real client.
client = _make_client()
client._firestore_api_internal = firestore_api
# Actually make a transaction and roll it back.
transaction = self._make_one(client)
txn_id = b"roll-bad-server"
transaction._id = txn_id
with self.assertRaises(exceptions.InternalServerError) as exc_info:
transaction._rollback()
self.assertIs(exc_info.exception, exc)
self.assertIsNone(transaction._id)
self.assertEqual(transaction._write_pbs, [])
# Verify the called mock.
firestore_api.rollback.assert_called_once_with(
client._database_string, txn_id, metadata=client._rpc_metadata
)
async def test__rollback_failure(self):
from google.api_core import exceptions
# Create a minimal fake GAPIC with a dummy failure.
firestore_api = AsyncMock()
exc = exceptions.InternalServerError("Fire during rollback.")
firestore_api.rollback.side_effect = exc
# Attach the fake GAPIC to a real client.
client = _make_client()
client._firestore_api_internal = firestore_api
# Actually make a transaction and roll it back.
transaction = self._make_one(client)
txn_id = b"roll-bad-server"
transaction._id = txn_id
with self.assertRaises(exceptions.InternalServerError) as exc_info:
await transaction._rollback()
self.assertIs(exc_info.exception, exc)
self.assertIsNone(transaction._id)
self.assertEqual(transaction._write_pbs, [])
# Verify the called mock.
firestore_api.rollback.assert_called_once_with(
request={"database": client._database_string, "transaction": txn_id},
metadata=client._rpc_metadata,
)
async def test_failure_second_attempt(self, _sleep):
from google.api_core import exceptions
# Create a minimal fake GAPIC with a dummy result.
firestore_api = AsyncMock()
# Make sure the first request fails retry-able and second
# fails non-retryable.
exc1 = exceptions.ServiceUnavailable("Come back next time.")
exc2 = exceptions.InternalServerError("Server on fritz.")
firestore_api.commit.side_effect = [exc1, exc2]
# Attach the fake GAPIC to a real client.
client = _make_client("peanut-butter")
client._firestore_api_internal = firestore_api
# Call function and check result.
txn_id = b"the-journey-when-and-where-well-go"
with self.assertRaises(exceptions.InternalServerError) as exc_info:
await self._call_fut(client, mock.sentinel.write_pbs, txn_id)
self.assertIs(exc_info.exception, exc2)
# Verify mocks used.
_sleep.assert_called_once_with(1.0)
# commit() called same way 2 times.
commit_call = mock.call(
request={
"database": client._database_string,
"writes": mock.sentinel.write_pbs,
"transaction": txn_id,
},
metadata=client._rpc_metadata,
)
self.assertEqual(firestore_api.commit.mock_calls, [commit_call, commit_call])
def test_transaction__rollback_failure():
from google.api_core import exceptions
from google.cloud.firestore_v1.services.firestore import client as firestore_client
# Create a minimal fake GAPIC with a dummy failure.
firestore_api = mock.create_autospec(firestore_client.FirestoreClient,
instance=True)
exc = exceptions.InternalServerError("Fire during rollback.")
firestore_api.rollback.side_effect = exc
# Attach the fake GAPIC to a real client.
client = _make_client()
client._firestore_api_internal = firestore_api
# Actually make a transaction and roll it back.
transaction = _make_transaction(client)
txn_id = b"roll-bad-server"
transaction._id = txn_id
with pytest.raises(exceptions.InternalServerError) as exc_info:
transaction._rollback()
assert exc_info.value is exc
assert transaction._id is None
assert transaction._write_pbs == []
# Verify the called mock.
firestore_api.rollback.assert_called_once_with(
request={
"database": client._database_string,
"transaction": txn_id
},
metadata=client._rpc_metadata,
)
def test__maybe_commit_failure_cannot_retry(self):
from google.api_core import exceptions
wrapped = self._make_one(mock.sentinel.callable_)
txn_id = b"failed-but-not-retryable"
transaction = _make_transaction(txn_id)
transaction._id = txn_id # We won't call ``begin()``.
wrapped.current_id = txn_id # We won't call ``_pre_commit()``.
wrapped.retry_id = txn_id # We won't call ``_pre_commit()``.
# Actually force the ``commit`` to fail.
exc = exceptions.InternalServerError("Real bad thing")
firestore_api = transaction._client._firestore_api
firestore_api.commit.side_effect = exc
with self.assertRaises(exceptions.InternalServerError) as exc_info:
wrapped._maybe_commit(transaction)
self.assertIs(exc_info.exception, exc)
self.assertEqual(transaction._id, txn_id)
self.assertEqual(wrapped.current_id, txn_id)
self.assertEqual(wrapped.retry_id, txn_id)
# Verify mocks.
firestore_api.begin_transaction.assert_not_called()
firestore_api.rollback.assert_not_called()
firestore_api.commit.assert_called_once_with(
transaction._client._database_string,
[],
transaction=txn_id,
metadata=transaction._client._rpc_metadata,
)
def test_failure_second_attempt(self, _sleep):
from google.api_core import exceptions
from google.cloud.firestore_v1beta1.gapic import firestore_client
# Create a minimal fake GAPIC with a dummy result.
firestore_api = mock.create_autospec(firestore_client.FirestoreClient,
instance=True)
# Make sure the first request fails retry-able and second
# fails non-retryable.
exc1 = exceptions.ServiceUnavailable('Come back next time.')
exc2 = exceptions.InternalServerError('Server on fritz.')
firestore_api.commit.side_effect = [exc1, exc2]
# Attach the fake GAPIC to a real client.
client = _make_client('peanut-butter')
client._firestore_api_internal = firestore_api
# Call function and check result.
txn_id = b'the-journey-when-and-where-well-go'
with self.assertRaises(exceptions.InternalServerError) as exc_info:
self._call_fut(client, mock.sentinel.write_pbs, txn_id)
self.assertIs(exc_info.exception, exc2)
# Verify mocks used.
_sleep.assert_called_once_with(1.0)
# commit() called same way 2 times.
commit_call = mock.call(client._database_string,
mock.sentinel.write_pbs,
transaction=txn_id,
metadata=client._rpc_metadata)
self.assertEqual(firestore_api.commit.mock_calls,
[commit_call, commit_call])
def test__pre_commit_failure_with_rollback_failure(self):
from google.api_core import exceptions
exc1 = ValueError('I will not be only failure.')
to_wrap = mock.Mock(side_effect=exc1, spec=[])
wrapped = self._make_one(to_wrap)
txn_id = b'both-will-fail'
transaction = _make_transaction(txn_id)
# Actually force the ``rollback`` to fail as well.
exc2 = exceptions.InternalServerError('Rollback blues.')
firestore_api = transaction._client._firestore_api
firestore_api.rollback.side_effect = exc2
# Try to ``_pre_commit``
with self.assertRaises(exceptions.InternalServerError) as exc_info:
wrapped._pre_commit(transaction, a='b', c='zebra')
self.assertIs(exc_info.exception, exc2)
self.assertIsNone(transaction._id)
self.assertEqual(wrapped.current_id, txn_id)
self.assertEqual(wrapped.retry_id, txn_id)
# Verify mocks.
to_wrap.assert_called_once_with(transaction, a='b', c='zebra')
firestore_api.begin_transaction.assert_called_once_with(
transaction._client._database_string,
options_=None,
metadata=transaction._client._rpc_metadata)
firestore_api.rollback.assert_called_once_with(
transaction._client._database_string,
txn_id,
metadata=transaction._client._rpc_metadata)
firestore_api.commit.assert_not_called()
def test__commit_failure(self):
from google.api_core import exceptions
from google.cloud.firestore_v1beta1.gapic import firestore_client
# Create a minimal fake GAPIC with a dummy failure.
firestore_api = mock.create_autospec(firestore_client.FirestoreClient,
instance=True)
exc = exceptions.InternalServerError('Fire during commit.')
firestore_api.commit.side_effect = exc
# Attach the fake GAPIC to a real client.
client = _make_client()
client._firestore_api_internal = firestore_api
# Actually make a transaction with some mutations and call _commit().
transaction = self._make_one(client)
txn_id = b'beep-fail-commit'
transaction._id = txn_id
transaction.create(client.document('up', 'down'), {'water': 1.0})
transaction.delete(client.document('up', 'left'))
write_pbs = transaction._write_pbs[::]
with self.assertRaises(exceptions.InternalServerError) as exc_info:
transaction._commit()
self.assertIs(exc_info.exception, exc)
self.assertEqual(transaction._id, txn_id)
self.assertEqual(transaction._write_pbs, write_pbs)
# Verify the called mock.
firestore_api.commit.assert_called_once_with(
client._database_string,
write_pbs,
transaction=txn_id,
metadata=client._rpc_metadata)
def test_retry(self):
model = QuestionnaireState("someuser", "data", 1)
self.mock_client.put = mock.Mock(side_effect=[
exceptions.InternalServerError("error"), mock.DEFAULT
])
self.ds.put(model, True)
assert self.mock_client.put.call_count > 1
def test_wrap_method_with_default_retry_and_timeout(unusued_sleep):
method = mock.Mock(spec=["__call__"],
side_effect=[exceptions.InternalServerError(None), 42])
default_retry = retry.Retry()
default_timeout = timeout.ConstantTimeout(60)
wrapped_method = google.api_core.gapic_v1.method.wrap_method(
method, default_retry, default_timeout)
result = wrapped_method()
assert result == 42
assert method.call_count == 2
method.assert_called_with(timeout=60, metadata=mock.ANY)
def test_execute_internal_server_error(self):
from google.api_core import exceptions
from google.cloud.spanner_dbapi.exceptions import OperationalError
connection = self._make_connection(self.INSTANCE, mock.MagicMock())
cursor = self._make_one(connection)
with mock.patch(
"google.cloud.spanner_dbapi.parse_utils.classify_stmt",
side_effect=exceptions.InternalServerError("message"),
):
with self.assertRaises(OperationalError):
cursor.execute(sql="sql")
async def test_wrap_method_with_default_retry_and_timeout(unused_sleep):
fake_call = grpc_helpers_async.FakeUnaryUnaryCall(42)
method = mock.Mock(
spec=aio.UnaryUnaryMultiCallable,
side_effect=[exceptions.InternalServerError(None), fake_call],
)
default_retry = retry_async.AsyncRetry()
default_timeout = timeout.ConstantTimeout(60)
wrapped_method = gapic_v1.method_async.wrap_method(method, default_retry,
default_timeout)
result = await wrapped_method()
assert result == 42
assert method.call_count == 2
method.assert_called_with(timeout=60, metadata=mock.ANY)
def test_wrap_method_with_overriding_retry_deadline(utcnow, unused_sleep):
method = mock.Mock(
spec=['__call__'],
side_effect=([exceptions.InternalServerError(None)] * 4) + [42])
default_retry = retry.Retry()
default_timeout = timeout.ExponentialTimeout(deadline=60)
wrapped_method = google.api_core.gapic_v1.method.wrap_method(
method, default_retry, default_timeout)
# Overriding only the retry's deadline should also override the timeout's
# deadline.
result = wrapped_method(retry=default_retry.with_deadline(30))
assert result == 42
timeout_args = [call[1]['timeout'] for call in method.call_args_list]
assert timeout_args == [5.0, 10.0, 20.0, 26.0, 25.0]
assert utcnow.call_count == (
1 + # First to set the deadline.
5 + # One for each min(timeout, maximum, (DEADLINE - NOW).seconds)
5)
def test_get_query_results_raises_error_if_deadline_exceeded(self, mock_api_request):
# type: (MagicMock) -> None
"""Tests that get_query_results raises a RetryError if the deadline is exceeded.
Args:
mock_api_request: The MagicMock object for the api_request method.
"""
# Raise a transient error on every API request so that the job doesn't finish.
mock_api_request.side_effect = exceptions.InternalServerError('Transient error.')
client_in_a_hurry = bq.Client(self.TEST_PROJECT, max_wait_secs=1)
before_get_query_results = datetime.now()
with self.assertRaises(exceptions.RetryError):
client_in_a_hurry.get_query_results('SELECT 5')
# Test that the process timed out before the default timeout. (It should time out after 1
# second, but this leaves buffer for BQ being slow.)
# Use the underlying Retry library's default timeout, because it's shorter than our
# DEFAULT_TIMEOUT_SEC, and we want to fail if our timeout didn't get passed in somehow.
diff = datetime.now() - before_get_query_results
self.assertTrue(diff.seconds < retry._DEFAULT_DEADLINE)
def test__transactional__pre_commit_failure_with_rollback_failure():
from google.api_core import exceptions
exc1 = ValueError("I will not be only failure.")
to_wrap = mock.Mock(side_effect=exc1, spec=[])
wrapped = _make__transactional(to_wrap)
txn_id = b"both-will-fail"
transaction = _make_transaction_pb(txn_id)
# Actually force the ``rollback`` to fail as well.
exc2 = exceptions.InternalServerError("Rollback blues.")
firestore_api = transaction._client._firestore_api
firestore_api.rollback.side_effect = exc2
# Try to ``_pre_commit``
with pytest.raises(exceptions.InternalServerError) as exc_info:
wrapped._pre_commit(transaction, a="b", c="zebra")
assert exc_info.value is exc2
assert transaction._id is None
assert wrapped.current_id == txn_id
assert wrapped.retry_id == txn_id
# Verify mocks.
to_wrap.assert_called_once_with(transaction, a="b", c="zebra")
firestore_api.begin_transaction.assert_called_once_with(
request={
"database": transaction._client._database_string,
"options": None
},
metadata=transaction._client._rpc_metadata,
)
firestore_api.rollback.assert_called_once_with(
request={
"database": transaction._client._database_string,
"transaction": txn_id,
},
metadata=transaction._client._rpc_metadata,
)
firestore_api.commit.assert_not_called()
def test__commit_failure(self):
from google.api_core import exceptions
from google.cloud.firestore_v1.services.firestore import (
client as firestore_client, )
# Create a minimal fake GAPIC with a dummy failure.
firestore_api = mock.create_autospec(firestore_client.FirestoreClient,
instance=True)
exc = exceptions.InternalServerError("Fire during commit.")
firestore_api.commit.side_effect = exc
# Attach the fake GAPIC to a real client.
client = _make_client()
client._firestore_api_internal = firestore_api
# Actually make a transaction with some mutations and call _commit().
transaction = self._make_one(client)
txn_id = b"beep-fail-commit"
transaction._id = txn_id
transaction.create(client.document("up", "down"), {"water": 1.0})
transaction.delete(client.document("up", "left"))
write_pbs = transaction._write_pbs[::]
with self.assertRaises(exceptions.InternalServerError) as exc_info:
transaction._commit()
self.assertIs(exc_info.exception, exc)
self.assertEqual(transaction._id, txn_id)
self.assertEqual(transaction._write_pbs, write_pbs)
# Verify the called mock.
firestore_api.commit.assert_called_once_with(
request={
"database": client._database_string,
"writes": write_pbs,
"transaction": txn_id,
},
metadata=client._rpc_metadata,
)
def test__commit_with_retry_failure_second_attempt(_sleep):
from google.api_core import exceptions
from google.cloud.firestore_v1.services.firestore import client as firestore_client
from google.cloud.firestore_v1.transaction import _commit_with_retry
# Create a minimal fake GAPIC with a dummy result.
firestore_api = mock.create_autospec(firestore_client.FirestoreClient,
instance=True)
# Make sure the first request fails retry-able and second
# fails non-retryable.
exc1 = exceptions.ServiceUnavailable("Come back next time.")
exc2 = exceptions.InternalServerError("Server on fritz.")
firestore_api.commit.side_effect = [exc1, exc2]
# Attach the fake GAPIC to a real client.
client = _make_client("peanut-butter")
client._firestore_api_internal = firestore_api
# Call function and check result.
txn_id = b"the-journey-when-and-where-well-go"
with pytest.raises(exceptions.InternalServerError) as exc_info:
_commit_with_retry(client, mock.sentinel.write_pbs, txn_id)
assert exc_info.value is exc2
# Verify mocks used.
_sleep.assert_called_once_with(1.0)
# commit() called same way 2 times.
commit_call = mock.call(
request={
"database": client._database_string,
"writes": mock.sentinel.write_pbs,
"transaction": txn_id,
},
metadata=client._rpc_metadata,
)
assert firestore_api.commit.mock_calls == [commit_call, commit_call]
async def test_asynctransaction__commit_failure():
from google.api_core import exceptions
# Create a minimal fake GAPIC with a dummy failure.
firestore_api = AsyncMock()
exc = exceptions.InternalServerError("Fire during commit.")
firestore_api.commit.side_effect = exc
# Attach the fake GAPIC to a real client.
client = _make_client()
client._firestore_api_internal = firestore_api
# Actually make a transaction with some mutations and call _commit().
transaction = _make_async_transaction(client)
txn_id = b"beep-fail-commit"
transaction._id = txn_id
transaction.create(client.document("up", "down"), {"water": 1.0})
transaction.delete(client.document("up", "left"))
write_pbs = transaction._write_pbs[::]
with pytest.raises(exceptions.InternalServerError) as exc_info:
await transaction._commit()
assert exc_info.value is exc
assert transaction._id == txn_id
assert transaction._write_pbs == write_pbs
# Verify the called mock.
firestore_api.commit.assert_called_once_with(
request={
"database": client._database_string,
"writes": write_pbs,
"transaction": txn_id,
},
metadata=client._rpc_metadata,
)
def test_if_transient_error():
assert retry.if_transient_error(exceptions.InternalServerError(''))
assert retry.if_transient_error(exceptions.TooManyRequests(''))
assert not retry.if_transient_error(exceptions.InvalidArgument(''))
def test_internal(core_retry):
error = core_exceptions.InternalServerError("testing")
core_retry.if_transient_error.return_value = False
assert _retry.is_transient_error(error) is True
core_retry.if_transient_error.assert_called_once_with(error)
from verily.bigquery_wrapper.bq_base import (DEFAULT_RETRY_FOR_API_CALLS, is_job_done,
validate_query_job)
# Arguments to pass to retry-related tests
EXCEPTION_RETRY_TEST_ARGS = (
dict(
exc=exceptions.BadRequest('Extra comma before FROM clause.'),
should_retry=False
),
dict(
exc=exceptions.BadRequest(
'The job encountered an internal error during execution. (Transient error.)'),
should_retry=True
),
dict(
exc=exceptions.InternalServerError('Transient error.'),
should_retry=True
),
dict(
exc=exceptions.TooManyRequests('Transient error.'),
should_retry=True
),
dict(
exc=exceptions.BadGateway('Transient error.'),
should_retry=True
),
)
@ddt
class BQTest(bq_shared_tests.BQSharedTests):
def test_if_transient_error():
assert retry.if_transient_error(exceptions.InternalServerError(""))
assert retry.if_transient_error(exceptions.TooManyRequests(""))
assert retry.if_transient_error(exceptions.ServiceUnavailable(""))
assert not retry.if_transient_error(exceptions.InvalidArgument(""))
from mock import MagicMock, call
from google.api_core import exceptions # pylint: disable=no-name-in-module
from recidiviz.common.common_utils import (
create_generated_id,
is_generated_id,
retry_grpc,
create_synthetic_id,
get_external_id,
date_intersects_with_span,
date_spans_overlap_exclusive,
date_spans_overlap_inclusive,
)
GO_AWAY_ERROR = exceptions.InternalServerError("500 GOAWAY received")
DEADLINE_EXCEEDED_ERROR = exceptions.InternalServerError("504 Deadline "
"Exceeded")
OTHER_ERROR = exceptions.InternalServerError("500 received")
_DATE_1 = datetime.date(year=2019, month=1, day=1)
_DATE_2 = datetime.date(year=2019, month=2, day=1)
_DATE_3 = datetime.date(year=2019, month=3, day=1)
_DATE_4 = datetime.date(year=2019, month=4, day=1)
_DATE_5 = datetime.date(year=2019, month=5, day=1)
_DATE_6 = datetime.date(year=2019, month=6, day=1)
_DATE_7 = datetime.date(year=2019, month=7, day=1)
class CommonUtilsTest(unittest.TestCase):
"""Tests for common_utils.py."""
def test_create_generated_id(self) -> None:
