def _export(self, data: TypingSequence[SDKDataT]) -> ExportResultT:
# expo returns a generator that yields delay values which grow
# exponentially. Once delay is greater than max_value, the yielded
# value will remain constant.
# max_value is set to 900 (900 seconds is 15 minutes) to use the same
# value as used in the Go implementation.
max_value = 900
for delay in expo(max_value=max_value):
if delay == max_value:
return self._result.FAILURE
try:
self._client.Export(
request=self._translate_data(data),
metadata=self._headers,
timeout=self._timeout,
)
return self._result.SUCCESS
except RpcError as error:
if error.code() in [
StatusCode.CANCELLED,
StatusCode.DEADLINE_EXCEEDED,
StatusCode.PERMISSION_DENIED,
StatusCode.UNAUTHENTICATED,
StatusCode.RESOURCE_EXHAUSTED,
StatusCode.ABORTED,
StatusCode.OUT_OF_RANGE,
StatusCode.UNAVAILABLE,
StatusCode.DATA_LOSS,
]:
retry_info_bin = dict(error.trailing_metadata()).get(
"google.rpc.retryinfo-bin"
)
if retry_info_bin is not None:
retry_info = RetryInfo()
retry_info.ParseFromString(retry_info_bin)
delay = (
retry_info.retry_delay.seconds
+ retry_info.retry_delay.nanos / 1.0e9
)
logger.debug(
"Waiting %ss before retrying export of span", delay
)
sleep(delay)
continue
if error.code() == StatusCode.OK:
return self._result.SUCCESS
return self._result.FAILURE
return self._result.FAILURE
def _export(self, data: TypingSequence[SDKDataT]) -> ExportResultT:
max_value = 64
# expo returns a generator that yields delay values which grow
# exponentially. Once delay is greater than max_value, the yielded
# value will remain constant.
for delay in expo(max_value=max_value):
if delay == max_value:
return self._result.FAILURE
try:
self._client.Export(
request=self._translate_data(data),
metadata=self._headers,
timeout=self._timeout,
)
return self._result.SUCCESS
except RpcError as error:
if error.code() in [
StatusCode.CANCELLED,
StatusCode.DEADLINE_EXCEEDED,
StatusCode.RESOURCE_EXHAUSTED,
StatusCode.ABORTED,
StatusCode.OUT_OF_RANGE,
StatusCode.UNAVAILABLE,
StatusCode.DATA_LOSS,
]:
retry_info_bin = dict(error.trailing_metadata()).get(
"google.rpc.retryinfo-bin")
if retry_info_bin is not None:
retry_info = RetryInfo()
retry_info.ParseFromString(retry_info_bin)
delay = (retry_info.retry_delay.seconds +
retry_info.retry_delay.nanos / 1.0e9)
logger.warning(
"Transient error %s encountered while exporting span batch, retrying in %ss.",
error.code(),
delay,
)
sleep(delay)
continue
else:
logger.error(
"Failed to export span batch, error code: %s",
error.code(),
)
if error.code() == StatusCode.OK:
return self._result.SUCCESS
return self._result.FAILURE
return self._result.FAILURE
def Export(self, request, context):
context.set_code(StatusCode.UNAVAILABLE)
context.send_initial_metadata(
(("google.rpc.retryinfo-bin", RetryInfo().SerializeToString()), ))
context.set_trailing_metadata(((
"google.rpc.retryinfo-bin",
RetryInfo(retry_delay=Duration(seconds=4)).SerializeToString(),
), ))
return ExportLogsServiceResponse()
def _trailing_metadata(self):
from google.protobuf.duration_pb2 import Duration
from google.rpc.error_details_pb2 import RetryInfo
from grpc._common import cygrpc_metadata
if self._commit_abort_retry_nanos is None:
return cygrpc_metadata(())
retry_info = RetryInfo(
retry_delay=Duration(seconds=self._commit_abort_retry_seconds,
nanos=self._commit_abort_retry_nanos))
return cygrpc_metadata([('google.rpc.retryinfo-bin',
retry_info.SerializeToString())])
def _get_retry_delay(cause):
"""Helper for :func:`_delay_until_retry`.
:type exc: :class:`grpc.Call`
:param exc: exception for aborted transaction
:rtype: float
:returns: seconds to wait before retrying the transaction.
"""
metadata = dict(cause.trailing_metadata())
retry_info_pb = metadata.get('google.rpc.retryinfo-bin')
if retry_info_pb is not None:
retry_info = RetryInfo()
retry_info.ParseFromString(retry_info_pb)
nanos = retry_info.retry_delay.nanos
return retry_info.retry_delay.seconds + nanos / 1.0e9
async def test_other_error_details_present():
any1 = Any()
any1.Pack(RetryInfo())
any2 = Any()
any2.Pack(ErrorInfo(reason="RESET", domain="pubsublite.googleapis.com"))
status_pb = Status(code=10, details=[any1, any2])
assert is_reset_signal(Aborted("", response=make_call(status_pb)))
def _get_retry_delay(cause, attempts):
"""Helper for :func:`_delay_until_retry`.
:type exc: :class:`grpc.Call`
:param exc: exception for aborted transaction
:rtype: float
:returns: seconds to wait before retrying the transaction.
:type attempts: int
:param attempts: number of call retries
"""
metadata = dict(cause.trailing_metadata())
retry_info_pb = metadata.get("google.rpc.retryinfo-bin")
if retry_info_pb is not None:
retry_info = RetryInfo()
retry_info.ParseFromString(retry_info_pb)
nanos = retry_info.retry_delay.nanos
return retry_info.retry_delay.seconds + nanos / 1.0e9
return 2**attempts + random.random()
def test_run_in_transaction_w_callback_raises_abort_wo_metadata(self):
import datetime
from google.api_core.exceptions import Aborted
from google.protobuf.duration_pb2 import Duration
from google.rpc.error_details_pb2 import RetryInfo
from google.cloud.spanner_v1.proto.spanner_pb2 import CommitResponse
from google.cloud.spanner_v1.proto.transaction_pb2 import (
Transaction as TransactionPB,
TransactionOptions,
)
from google.cloud._helpers import UTC
from google.cloud._helpers import _datetime_to_pb_timestamp
from google.cloud.spanner_v1.transaction import Transaction
TABLE_NAME = "citizens"
COLUMNS = ["email", "first_name", "last_name", "age"]
VALUES = [
["*****@*****.**", "Phred", "Phlyntstone", 32],
["*****@*****.**", "Bharney", "Rhubble", 31],
]
TRANSACTION_ID = b"FACEDACE"
RETRY_SECONDS = 1
RETRY_NANOS = 3456
transaction_pb = TransactionPB(id=TRANSACTION_ID)
now = datetime.datetime.utcnow().replace(tzinfo=UTC)
now_pb = _datetime_to_pb_timestamp(now)
response = CommitResponse(commit_timestamp=now_pb)
retry_info = RetryInfo(
retry_delay=Duration(seconds=RETRY_SECONDS, nanos=RETRY_NANOS)
)
trailing_metadata = [
("google.rpc.retryinfo-bin", retry_info.SerializeToString())
]
gax_api = self._make_spanner_api()
gax_api.begin_transaction.return_value = transaction_pb
gax_api.commit.side_effect = [response]
database = self._make_database()
database.spanner_api = gax_api
session = self._make_one(database)
session._session_id = self.SESSION_ID
called_with = []
def unit_of_work(txn, *args, **kw):
called_with.append((txn, args, kw))
if len(called_with) < 2:
raise _make_rpc_error(Aborted, trailing_metadata)
txn.insert(TABLE_NAME, COLUMNS, VALUES)
with mock.patch("time.sleep") as sleep_mock:
session.run_in_transaction(unit_of_work)
sleep_mock.assert_called_once_with(RETRY_SECONDS + RETRY_NANOS / 1.0e9)
self.assertEqual(len(called_with), 2)
for index, (txn, args, kw) in enumerate(called_with):
self.assertIsInstance(txn, Transaction)
if index == 0:
self.assertIsNone(txn.committed)
else:
self.assertEqual(txn.committed, now)
self.assertEqual(args, ())
self.assertEqual(kw, {})
expected_options = TransactionOptions(read_write=TransactionOptions.ReadWrite())
self.assertEqual(
gax_api.begin_transaction.call_args_list,
[
mock.call(
self.SESSION_NAME,
expected_options,
metadata=[("google-cloud-resource-prefix", database.name)],
)
]
* 2,
)
gax_api.commit.assert_called_once_with(
self.SESSION_NAME,
mutations=txn._mutations,
transaction_id=TRANSACTION_ID,
metadata=[("google-cloud-resource-prefix", database.name)],
)
def test_run_in_transaction_w_abort_w_retry_metadata_deadline(self):
import datetime
from google.api_core.exceptions import Aborted
from google.protobuf.duration_pb2 import Duration
from google.rpc.error_details_pb2 import RetryInfo
from google.cloud.spanner_v1.proto.spanner_pb2 import CommitResponse
from google.cloud.spanner_v1.proto.transaction_pb2 import (
Transaction as TransactionPB,
TransactionOptions,
)
from google.cloud.spanner_v1.transaction import Transaction
from google.cloud._helpers import UTC
from google.cloud._helpers import _datetime_to_pb_timestamp
TABLE_NAME = "citizens"
COLUMNS = ["email", "first_name", "last_name", "age"]
VALUES = [
["*****@*****.**", "Phred", "Phlyntstone", 32],
["*****@*****.**", "Bharney", "Rhubble", 31],
]
TRANSACTION_ID = b"FACEDACE"
RETRY_SECONDS = 1
RETRY_NANOS = 3456
transaction_pb = TransactionPB(id=TRANSACTION_ID)
now = datetime.datetime.utcnow().replace(tzinfo=UTC)
now_pb = _datetime_to_pb_timestamp(now)
response = CommitResponse(commit_timestamp=now_pb)
retry_info = RetryInfo(
retry_delay=Duration(seconds=RETRY_SECONDS, nanos=RETRY_NANOS)
)
trailing_metadata = [
("google.rpc.retryinfo-bin", retry_info.SerializeToString())
]
aborted = _make_rpc_error(Aborted, trailing_metadata=trailing_metadata)
gax_api = self._make_spanner_api()
gax_api.begin_transaction.return_value = transaction_pb
gax_api.commit.side_effect = [aborted, response]
database = self._make_database()
database.spanner_api = gax_api
session = self._make_one(database)
session._session_id = self.SESSION_ID
called_with = []
def unit_of_work(txn, *args, **kw):
called_with.append((txn, args, kw))
txn.insert(TABLE_NAME, COLUMNS, VALUES)
# retry once w/ timeout_secs=1
def _time(_results=[1, 1.5]):
return _results.pop(0)
with mock.patch("time.time", _time):
if HAS_OPENTELEMETRY_INSTALLED:
with mock.patch("opentelemetry.util.time", _ConstantTime()):
with mock.patch("time.sleep") as sleep_mock:
with self.assertRaises(Aborted):
session.run_in_transaction(
unit_of_work, "abc", timeout_secs=1
)
else:
with mock.patch("time.sleep") as sleep_mock:
with self.assertRaises(Aborted):
session.run_in_transaction(unit_of_work, "abc", timeout_secs=1)
sleep_mock.assert_not_called()
self.assertEqual(len(called_with), 1)
txn, args, kw = called_with[0]
self.assertIsInstance(txn, Transaction)
self.assertIsNone(txn.committed)
self.assertEqual(args, ("abc",))
self.assertEqual(kw, {})
expected_options = TransactionOptions(read_write=TransactionOptions.ReadWrite())
gax_api.begin_transaction.assert_called_once_with(
self.SESSION_NAME,
expected_options,
metadata=[("google-cloud-resource-prefix", database.name)],
)
gax_api.commit.assert_called_once_with(
self.SESSION_NAME,
mutations=txn._mutations,
transaction_id=TRANSACTION_ID,
metadata=[("google-cloud-resource-prefix", database.name)],
)
def test_run_in_transaction_w_abort_w_retry_metadata(self):
import datetime
from google.api_core.exceptions import Aborted
from google.protobuf.duration_pb2 import Duration
from google.rpc.error_details_pb2 import RetryInfo
from google.cloud.spanner_v1.proto.spanner_pb2 import CommitResponse
from google.cloud.spanner_v1.proto.transaction_pb2 import (
Transaction as TransactionPB, TransactionOptions)
from google.cloud._helpers import UTC
from google.cloud._helpers import _datetime_to_pb_timestamp
from google.cloud.spanner_v1.transaction import Transaction
TABLE_NAME = 'citizens'
COLUMNS = ['email', 'first_name', 'last_name', 'age']
VALUES = [
['*****@*****.**', 'Phred', 'Phlyntstone', 32],
['*****@*****.**', 'Bharney', 'Rhubble', 31],
]
TRANSACTION_ID = b'FACEDACE'
RETRY_SECONDS = 12
RETRY_NANOS = 3456
retry_info = RetryInfo(
retry_delay=Duration(seconds=RETRY_SECONDS, nanos=RETRY_NANOS))
trailing_metadata = [
('google.rpc.retryinfo-bin', retry_info.SerializeToString()),
]
aborted = _make_rpc_error(
Aborted,
trailing_metadata=trailing_metadata,
)
transaction_pb = TransactionPB(id=TRANSACTION_ID)
now = datetime.datetime.utcnow().replace(tzinfo=UTC)
now_pb = _datetime_to_pb_timestamp(now)
response = CommitResponse(commit_timestamp=now_pb)
gax_api = self._make_spanner_api()
gax_api.begin_transaction.return_value = transaction_pb
gax_api.commit.side_effect = [aborted, response]
database = self._make_database()
database.spanner_api = gax_api
session = self._make_one(database)
session._session_id = self.SESSION_ID
called_with = []
def unit_of_work(txn, *args, **kw):
called_with.append((txn, args, kw))
txn.insert(TABLE_NAME, COLUMNS, VALUES)
with mock.patch('time.sleep') as sleep_mock:
session.run_in_transaction(unit_of_work, 'abc', some_arg='def')
sleep_mock.assert_called_once_with(RETRY_SECONDS + RETRY_NANOS / 1.0e9)
self.assertEqual(len(called_with), 2)
for index, (txn, args, kw) in enumerate(called_with):
self.assertIsInstance(txn, Transaction)
if index == 1:
self.assertEqual(txn.committed, now)
else:
self.assertIsNone(txn.committed)
self.assertEqual(args, ('abc', ))
self.assertEqual(kw, {'some_arg': 'def'})
expected_options = TransactionOptions(
read_write=TransactionOptions.ReadWrite(), )
self.assertEqual(gax_api.begin_transaction.call_args_list, [
mock.call(
self.SESSION_NAME,
expected_options,
metadata=[('google-cloud-resource-prefix', database.name)],
)
] * 2)
self.assertEqual(gax_api.commit.call_args_list, [
mock.call(
self.SESSION_NAME,
txn._mutations,
transaction_id=TRANSACTION_ID,
metadata=[('google-cloud-resource-prefix', database.name)],
)
] * 2)
async def test_wrong_error_detail():
any = Any()
any.Pack(RetryInfo())
status_pb = Status(code=10, details=[any])
assert not is_reset_signal(Aborted("", response=make_call(status_pb)))
