def _export(self, data: TypingSequence[SDKDataT]) -> ExportResultT: # expo returns a generator that yields delay values which grow # exponentially. Once delay is greater than max_value, the yielded # value will remain constant. # max_value is set to 900 (900 seconds is 15 minutes) to use the same # value as used in the Go implementation. max_value = 900 for delay in expo(max_value=max_value): if delay == max_value: return self._result.FAILURE try: self._client.Export( request=self._translate_data(data), metadata=self._headers, timeout=self._timeout, ) return self._result.SUCCESS except RpcError as error: if error.code() in [ StatusCode.CANCELLED, StatusCode.DEADLINE_EXCEEDED, StatusCode.PERMISSION_DENIED, StatusCode.UNAUTHENTICATED, StatusCode.RESOURCE_EXHAUSTED, StatusCode.ABORTED, StatusCode.OUT_OF_RANGE, StatusCode.UNAVAILABLE, StatusCode.DATA_LOSS, ]: retry_info_bin = dict(error.trailing_metadata()).get( "google.rpc.retryinfo-bin" ) if retry_info_bin is not None: retry_info = RetryInfo() retry_info.ParseFromString(retry_info_bin) delay = ( retry_info.retry_delay.seconds + retry_info.retry_delay.nanos / 1.0e9 ) logger.debug( "Waiting %ss before retrying export of span", delay ) sleep(delay) continue if error.code() == StatusCode.OK: return self._result.SUCCESS return self._result.FAILURE return self._result.FAILURE
def _export(self, data: TypingSequence[SDKDataT]) -> ExportResultT: max_value = 64 # expo returns a generator that yields delay values which grow # exponentially. Once delay is greater than max_value, the yielded # value will remain constant. for delay in expo(max_value=max_value): if delay == max_value: return self._result.FAILURE try: self._client.Export( request=self._translate_data(data), metadata=self._headers, timeout=self._timeout, ) return self._result.SUCCESS except RpcError as error: if error.code() in [ StatusCode.CANCELLED, StatusCode.DEADLINE_EXCEEDED, StatusCode.RESOURCE_EXHAUSTED, StatusCode.ABORTED, StatusCode.OUT_OF_RANGE, StatusCode.UNAVAILABLE, StatusCode.DATA_LOSS, ]: retry_info_bin = dict(error.trailing_metadata()).get( "google.rpc.retryinfo-bin") if retry_info_bin is not None: retry_info = RetryInfo() retry_info.ParseFromString(retry_info_bin) delay = (retry_info.retry_delay.seconds + retry_info.retry_delay.nanos / 1.0e9) logger.warning( "Transient error %s encountered while exporting span batch, retrying in %ss.", error.code(), delay, ) sleep(delay) continue else: logger.error( "Failed to export span batch, error code: %s", error.code(), ) if error.code() == StatusCode.OK: return self._result.SUCCESS return self._result.FAILURE return self._result.FAILURE
def Export(self, request, context): context.set_code(StatusCode.UNAVAILABLE) context.send_initial_metadata( (("google.rpc.retryinfo-bin", RetryInfo().SerializeToString()), )) context.set_trailing_metadata((( "google.rpc.retryinfo-bin", RetryInfo(retry_delay=Duration(seconds=4)).SerializeToString(), ), )) return ExportLogsServiceResponse()
def _trailing_metadata(self): from google.protobuf.duration_pb2 import Duration from google.rpc.error_details_pb2 import RetryInfo from grpc._common import cygrpc_metadata if self._commit_abort_retry_nanos is None: return cygrpc_metadata(()) retry_info = RetryInfo( retry_delay=Duration(seconds=self._commit_abort_retry_seconds, nanos=self._commit_abort_retry_nanos)) return cygrpc_metadata([('google.rpc.retryinfo-bin', retry_info.SerializeToString())])
def _get_retry_delay(cause): """Helper for :func:`_delay_until_retry`. :type exc: :class:`grpc.Call` :param exc: exception for aborted transaction :rtype: float :returns: seconds to wait before retrying the transaction. """ metadata = dict(cause.trailing_metadata()) retry_info_pb = metadata.get('google.rpc.retryinfo-bin') if retry_info_pb is not None: retry_info = RetryInfo() retry_info.ParseFromString(retry_info_pb) nanos = retry_info.retry_delay.nanos return retry_info.retry_delay.seconds + nanos / 1.0e9
async def test_other_error_details_present(): any1 = Any() any1.Pack(RetryInfo()) any2 = Any() any2.Pack(ErrorInfo(reason="RESET", domain="pubsublite.googleapis.com")) status_pb = Status(code=10, details=[any1, any2]) assert is_reset_signal(Aborted("", response=make_call(status_pb)))
def _get_retry_delay(cause, attempts): """Helper for :func:`_delay_until_retry`. :type exc: :class:`grpc.Call` :param exc: exception for aborted transaction :rtype: float :returns: seconds to wait before retrying the transaction. :type attempts: int :param attempts: number of call retries """ metadata = dict(cause.trailing_metadata()) retry_info_pb = metadata.get("google.rpc.retryinfo-bin") if retry_info_pb is not None: retry_info = RetryInfo() retry_info.ParseFromString(retry_info_pb) nanos = retry_info.retry_delay.nanos return retry_info.retry_delay.seconds + nanos / 1.0e9 return 2**attempts + random.random()
def test_run_in_transaction_w_callback_raises_abort_wo_metadata(self): import datetime from google.api_core.exceptions import Aborted from google.protobuf.duration_pb2 import Duration from google.rpc.error_details_pb2 import RetryInfo from google.cloud.spanner_v1.proto.spanner_pb2 import CommitResponse from google.cloud.spanner_v1.proto.transaction_pb2 import ( Transaction as TransactionPB, TransactionOptions, ) from google.cloud._helpers import UTC from google.cloud._helpers import _datetime_to_pb_timestamp from google.cloud.spanner_v1.transaction import Transaction TABLE_NAME = "citizens" COLUMNS = ["email", "first_name", "last_name", "age"] VALUES = [ ["*****@*****.**", "Phred", "Phlyntstone", 32], ["*****@*****.**", "Bharney", "Rhubble", 31], ] TRANSACTION_ID = b"FACEDACE" RETRY_SECONDS = 1 RETRY_NANOS = 3456 transaction_pb = TransactionPB(id=TRANSACTION_ID) now = datetime.datetime.utcnow().replace(tzinfo=UTC) now_pb = _datetime_to_pb_timestamp(now) response = CommitResponse(commit_timestamp=now_pb) retry_info = RetryInfo( retry_delay=Duration(seconds=RETRY_SECONDS, nanos=RETRY_NANOS) ) trailing_metadata = [ ("google.rpc.retryinfo-bin", retry_info.SerializeToString()) ] gax_api = self._make_spanner_api() gax_api.begin_transaction.return_value = transaction_pb gax_api.commit.side_effect = [response] database = self._make_database() database.spanner_api = gax_api session = self._make_one(database) session._session_id = self.SESSION_ID called_with = [] def unit_of_work(txn, *args, **kw): called_with.append((txn, args, kw)) if len(called_with) < 2: raise _make_rpc_error(Aborted, trailing_metadata) txn.insert(TABLE_NAME, COLUMNS, VALUES) with mock.patch("time.sleep") as sleep_mock: session.run_in_transaction(unit_of_work) sleep_mock.assert_called_once_with(RETRY_SECONDS + RETRY_NANOS / 1.0e9) self.assertEqual(len(called_with), 2) for index, (txn, args, kw) in enumerate(called_with): self.assertIsInstance(txn, Transaction) if index == 0: self.assertIsNone(txn.committed) else: self.assertEqual(txn.committed, now) self.assertEqual(args, ()) self.assertEqual(kw, {}) expected_options = TransactionOptions(read_write=TransactionOptions.ReadWrite()) self.assertEqual( gax_api.begin_transaction.call_args_list, [ mock.call( self.SESSION_NAME, expected_options, metadata=[("google-cloud-resource-prefix", database.name)], ) ] * 2, ) gax_api.commit.assert_called_once_with( self.SESSION_NAME, mutations=txn._mutations, transaction_id=TRANSACTION_ID, metadata=[("google-cloud-resource-prefix", database.name)], )
def test_run_in_transaction_w_abort_w_retry_metadata_deadline(self): import datetime from google.api_core.exceptions import Aborted from google.protobuf.duration_pb2 import Duration from google.rpc.error_details_pb2 import RetryInfo from google.cloud.spanner_v1.proto.spanner_pb2 import CommitResponse from google.cloud.spanner_v1.proto.transaction_pb2 import ( Transaction as TransactionPB, TransactionOptions, ) from google.cloud.spanner_v1.transaction import Transaction from google.cloud._helpers import UTC from google.cloud._helpers import _datetime_to_pb_timestamp TABLE_NAME = "citizens" COLUMNS = ["email", "first_name", "last_name", "age"] VALUES = [ ["*****@*****.**", "Phred", "Phlyntstone", 32], ["*****@*****.**", "Bharney", "Rhubble", 31], ] TRANSACTION_ID = b"FACEDACE" RETRY_SECONDS = 1 RETRY_NANOS = 3456 transaction_pb = TransactionPB(id=TRANSACTION_ID) now = datetime.datetime.utcnow().replace(tzinfo=UTC) now_pb = _datetime_to_pb_timestamp(now) response = CommitResponse(commit_timestamp=now_pb) retry_info = RetryInfo( retry_delay=Duration(seconds=RETRY_SECONDS, nanos=RETRY_NANOS) ) trailing_metadata = [ ("google.rpc.retryinfo-bin", retry_info.SerializeToString()) ] aborted = _make_rpc_error(Aborted, trailing_metadata=trailing_metadata) gax_api = self._make_spanner_api() gax_api.begin_transaction.return_value = transaction_pb gax_api.commit.side_effect = [aborted, response] database = self._make_database() database.spanner_api = gax_api session = self._make_one(database) session._session_id = self.SESSION_ID called_with = [] def unit_of_work(txn, *args, **kw): called_with.append((txn, args, kw)) txn.insert(TABLE_NAME, COLUMNS, VALUES) # retry once w/ timeout_secs=1 def _time(_results=[1, 1.5]): return _results.pop(0) with mock.patch("time.time", _time): if HAS_OPENTELEMETRY_INSTALLED: with mock.patch("opentelemetry.util.time", _ConstantTime()): with mock.patch("time.sleep") as sleep_mock: with self.assertRaises(Aborted): session.run_in_transaction( unit_of_work, "abc", timeout_secs=1 ) else: with mock.patch("time.sleep") as sleep_mock: with self.assertRaises(Aborted): session.run_in_transaction(unit_of_work, "abc", timeout_secs=1) sleep_mock.assert_not_called() self.assertEqual(len(called_with), 1) txn, args, kw = called_with[0] self.assertIsInstance(txn, Transaction) self.assertIsNone(txn.committed) self.assertEqual(args, ("abc",)) self.assertEqual(kw, {}) expected_options = TransactionOptions(read_write=TransactionOptions.ReadWrite()) gax_api.begin_transaction.assert_called_once_with( self.SESSION_NAME, expected_options, metadata=[("google-cloud-resource-prefix", database.name)], ) gax_api.commit.assert_called_once_with( self.SESSION_NAME, mutations=txn._mutations, transaction_id=TRANSACTION_ID, metadata=[("google-cloud-resource-prefix", database.name)], )
def test_run_in_transaction_w_abort_w_retry_metadata(self): import datetime from google.api_core.exceptions import Aborted from google.protobuf.duration_pb2 import Duration from google.rpc.error_details_pb2 import RetryInfo from google.cloud.spanner_v1.proto.spanner_pb2 import CommitResponse from google.cloud.spanner_v1.proto.transaction_pb2 import ( Transaction as TransactionPB, TransactionOptions) from google.cloud._helpers import UTC from google.cloud._helpers import _datetime_to_pb_timestamp from google.cloud.spanner_v1.transaction import Transaction TABLE_NAME = 'citizens' COLUMNS = ['email', 'first_name', 'last_name', 'age'] VALUES = [ ['*****@*****.**', 'Phred', 'Phlyntstone', 32], ['*****@*****.**', 'Bharney', 'Rhubble', 31], ] TRANSACTION_ID = b'FACEDACE' RETRY_SECONDS = 12 RETRY_NANOS = 3456 retry_info = RetryInfo( retry_delay=Duration(seconds=RETRY_SECONDS, nanos=RETRY_NANOS)) trailing_metadata = [ ('google.rpc.retryinfo-bin', retry_info.SerializeToString()), ] aborted = _make_rpc_error( Aborted, trailing_metadata=trailing_metadata, ) transaction_pb = TransactionPB(id=TRANSACTION_ID) now = datetime.datetime.utcnow().replace(tzinfo=UTC) now_pb = _datetime_to_pb_timestamp(now) response = CommitResponse(commit_timestamp=now_pb) gax_api = self._make_spanner_api() gax_api.begin_transaction.return_value = transaction_pb gax_api.commit.side_effect = [aborted, response] database = self._make_database() database.spanner_api = gax_api session = self._make_one(database) session._session_id = self.SESSION_ID called_with = [] def unit_of_work(txn, *args, **kw): called_with.append((txn, args, kw)) txn.insert(TABLE_NAME, COLUMNS, VALUES) with mock.patch('time.sleep') as sleep_mock: session.run_in_transaction(unit_of_work, 'abc', some_arg='def') sleep_mock.assert_called_once_with(RETRY_SECONDS + RETRY_NANOS / 1.0e9) self.assertEqual(len(called_with), 2) for index, (txn, args, kw) in enumerate(called_with): self.assertIsInstance(txn, Transaction) if index == 1: self.assertEqual(txn.committed, now) else: self.assertIsNone(txn.committed) self.assertEqual(args, ('abc', )) self.assertEqual(kw, {'some_arg': 'def'}) expected_options = TransactionOptions( read_write=TransactionOptions.ReadWrite(), ) self.assertEqual(gax_api.begin_transaction.call_args_list, [ mock.call( self.SESSION_NAME, expected_options, metadata=[('google-cloud-resource-prefix', database.name)], ) ] * 2) self.assertEqual(gax_api.commit.call_args_list, [ mock.call( self.SESSION_NAME, txn._mutations, transaction_id=TRANSACTION_ID, metadata=[('google-cloud-resource-prefix', database.name)], ) ] * 2)
async def test_wrong_error_detail(): any = Any() any.Pack(RetryInfo()) status_pb = Status(code=10, details=[any]) assert not is_reset_signal(Aborted("", response=make_call(status_pb)))