def test_commit_w_transaction(self):
from google.cloud.proto.datastore.v1 import datastore_pb2
from google.cloud.datastore.helpers import _new_value_pb
project = 'PROJECT'
key_pb = _make_key_pb(project)
rsp_pb = datastore_pb2.CommitResponse()
req_pb = datastore_pb2.CommitRequest()
mutation = req_pb.mutations.add()
insert = mutation.upsert
insert.key.CopyFrom(key_pb)
value_pb = _new_value_pb(insert, 'foo')
value_pb.string_value = u'Foo'
# Create mock HTTP and client with response.
http = _make_requests_session(
[_make_response(content=rsp_pb.SerializeToString())])
client = mock.Mock(_http=http,
_base_url='test.invalid',
spec=['_http', '_base_url'])
# Make request.
rq_class = datastore_pb2.CommitRequest
ds_api = self._make_one(client)
mode = rq_class.TRANSACTIONAL
result = ds_api.commit(project, mode, [mutation], transaction=b'xact')
# Check the result and verify the callers.
self.assertEqual(result, rsp_pb)
uri = _build_expected_url(client._base_url, project, 'commit')
request = _verify_protobuf_call(http, uri, rq_class())
self.assertEqual(request.transaction, b'xact')
self.assertEqual(list(request.mutations), [mutation])
self.assertEqual(request.mode, rq_class.TRANSACTIONAL)
def test_commit_w_transaction(self):
from google.cloud.proto.datastore.v1 import datastore_pb2
from google.cloud.datastore.helpers import _new_value_pb
project = 'PROJECT'
key_pb = self._make_key_pb(project)
rsp_pb = datastore_pb2.CommitResponse()
req_pb = datastore_pb2.CommitRequest()
mutation = req_pb.mutations.add()
insert = mutation.upsert
insert.key.CopyFrom(key_pb)
value_pb = _new_value_pb(insert, 'foo')
value_pb.string_value = u'Foo'
http = Http({'status': '200'}, rsp_pb.SerializeToString())
client = mock.Mock(_http=http, spec=['_http'])
conn = self._make_one(client)
uri = '/'.join([
conn.api_base_url,
conn.API_VERSION,
'projects',
project + ':commit',
])
result = conn.commit(project, req_pb, b'xact')
self.assertEqual(result, rsp_pb)
# Verify the caller.
cw = http._called_with
self._verify_protobuf_call(cw, uri, conn)
rq_class = datastore_pb2.CommitRequest
request = rq_class()
request.ParseFromString(cw['body'])
self.assertEqual(request.transaction, b'xact')
self.assertEqual(list(request.mutations), [mutation])
self.assertEqual(request.mode, rq_class.TRANSACTIONAL)
def write_mutations(
datastore,
project,
mutations,
throttler,
rpc_stats_callback=None,
throttle_delay=1):
"""A helper function to write a batch of mutations to Cloud Datastore.
If a commit fails, it will be retried upto 5 times. All mutations in the
batch will be committed again, even if the commit was partially successful.
If the retry limit is exceeded, the last exception from Cloud Datastore will
be raised.
Args:
datastore: googledatastore.connection.Datastore
project: str, project id
mutations: list of google.cloud.proto.datastore.v1.datastore_pb2.Mutation
rpc_stats_callback: a function to call with arguments `successes` and
`failures` and `throttled_secs`; this is called to record successful
and failed RPCs to Datastore and time spent waiting for throttling.
throttler: AdaptiveThrottler, to use to select requests to be throttled.
throttle_delay: float, time in seconds to sleep when throttled.
Returns a tuple of:
CommitResponse, the response from Datastore;
int, the latency of the successful RPC in milliseconds.
"""
commit_request = datastore_pb2.CommitRequest()
commit_request.mode = datastore_pb2.CommitRequest.NON_TRANSACTIONAL
commit_request.project_id = project
for mutation in mutations:
commit_request.mutations.add().CopyFrom(mutation)
@retry.with_exponential_backoff(
num_retries=5, retry_filter=retry_on_rpc_error)
def commit(request):
# Client-side throttling.
while throttler.throttle_request(time.time() * 1000):
_LOGGER.info(
"Delaying request for %ds due to previous failures", throttle_delay)
time.sleep(throttle_delay)
rpc_stats_callback(throttled_secs=throttle_delay)
try:
start_time = time.time()
response = datastore.commit(request)
end_time = time.time()
rpc_stats_callback(successes=1)
throttler.successful_request(start_time * 1000)
commit_time_ms = int((end_time - start_time) * 1000)
return response, commit_time_ms
except (RPCError, SocketError):
if rpc_stats_callback:
rpc_stats_callback(errors=1)
raise
response, commit_time_ms = commit(commit_request)
return response, commit_time_ms
def commit(self, project, mode, mutations, transaction=None):
"""Perform a ``commit`` request.
:type project: str
:param project: The project to connect to. This is
usually your project name in the cloud console.
:type mode: :class:`.gapic.datastore.v1.enums.CommitRequest.Mode`
:param mode: The type of commit to perform. Expected to be one of
``TRANSACTIONAL`` or ``NON_TRANSACTIONAL``.
:type mutations: list
:param mutations: List of :class:`.datastore_pb2.Mutation`, the
mutations to perform.
:type transaction: bytes
:param transaction: (Optional) The transaction ID returned from
:meth:`begin_transaction`. Non-transactional
commits must pass :data:`None`.
:rtype: :class:`.datastore_pb2.CommitResponse`
:returns: The returned protobuf response object.
"""
request_pb = _datastore_pb2.CommitRequest(
project_id=project,
mode=mode,
transaction=transaction,
mutations=mutations,
)
return _rpc(self.client._http, project, 'commit',
self.client._base_url,
request_pb, _datastore_pb2.CommitResponse)
def test_commit(self, mock_create_stub):
# Mock gRPC layer
grpc_stub = mock.Mock()
mock_create_stub.return_value = grpc_stub
client = datastore_client.DatastoreClient()
# Mock request
project_id = 'projectId-1969970175'
mode = enums.CommitRequest.Mode.MODE_UNSPECIFIED
mutations = []
# Mock response
index_updates = -1425228195
expected_response = datastore_pb2.CommitResponse(
index_updates=index_updates)
grpc_stub.Commit.return_value = expected_response
response = client.commit(project_id, mode, mutations)
self.assertEqual(expected_response, response)
grpc_stub.Commit.assert_called_once()
args, kwargs = grpc_stub.Commit.call_args
self.assertEqual(len(args), 2)
self.assertEqual(len(kwargs), 1)
self.assertIn('metadata', kwargs)
actual_request = args[0]
expected_request = datastore_pb2.CommitRequest(project_id=project_id,
mode=mode,
mutations=mutations)
self.assertEqual(expected_request, actual_request)
def commit(self,
project_id,
mode,
mutations,
transaction=None,
options=None):
"""
Commits a transaction, optionally creating, deleting or modifying some
entities.
Example:
>>> from google.cloud.gapic.datastore.v1 import datastore_client
>>> from google.cloud.gapic.datastore.v1 import enums
>>> client = datastore_client.DatastoreClient()
>>> project_id = ''
>>> mode = enums.CommitRequest.Mode.MODE_UNSPECIFIED
>>> mutations = []
>>> response = client.commit(project_id, mode, mutations)
Args:
project_id (string): The ID of the project against which to make the request.
mode (enum :class:`google.cloud.gapic.datastore.v1.enums.CommitRequest.Mode`): The type of commit to perform. Defaults to ``TRANSACTIONAL``.
mutations (list[:class:`google.cloud.proto.datastore.v1.datastore_pb2.Mutation`]): The mutations to perform.
When mode is ``TRANSACTIONAL``, mutations affecting a single entity are
applied in order. The following sequences of mutations affecting a single
entity are not permitted in a single ``Commit`` request:
- ``insert`` followed by ``insert``
- ``update`` followed by ``insert``
- ``upsert`` followed by ``insert``
- ``delete`` followed by ``update``
When mode is ``NON_TRANSACTIONAL``, no two mutations may affect a single
entity.
transaction (bytes): The identifier of the transaction associated with the commit. A
transaction identifier is returned by a call to
``Datastore.BeginTransaction``.
options (:class:`google.gax.CallOptions`): Overrides the default
settings for this call, e.g, timeout, retries etc.
Returns:
A :class:`google.cloud.proto.datastore.v1.datastore_pb2.CommitResponse` instance.
Raises:
:exc:`google.gax.errors.GaxError` if the RPC is aborted.
:exc:`ValueError` if the parameters are invalid.
"""
# Sanity check: We have some fields which are mutually exclusive;
# raise ValueError if more than one is sent.
oneof.check_oneof(
transaction=transaction, )
# Create the request object.
request = datastore_pb2.CommitRequest(
project_id=project_id,
mode=mode,
mutations=mutations,
transaction=transaction)
return self._commit(request, options)
def write_mutations(datastore, project, mutations, rpc_stats_callback=None):
"""A helper function to write a batch of mutations to Cloud Datastore.
If a commit fails, it will be retried upto 5 times. All mutations in the
batch will be committed again, even if the commit was partially successful.
If the retry limit is exceeded, the last exception from Cloud Datastore will
be raised.
Args:
datastore: googledatastore.connection.Datastore
project: str, project id
mutations: list of google.cloud.proto.datastore.v1.datastore_pb2.Mutation
rpc_stats_callback: a function to call with arguments `successes` and
`failures`; this is called to record successful and failed RPCs to
Datastore.
Returns a tuple of:
CommitResponse, the response from Datastore;
int, the latency of the successful RPC in milliseconds.
"""
commit_request = datastore_pb2.CommitRequest()
commit_request.mode = datastore_pb2.CommitRequest.NON_TRANSACTIONAL
commit_request.project_id = project
for mutation in mutations:
commit_request.mutations.add().CopyFrom(mutation)
@retry.with_exponential_backoff(num_retries=5,
retry_filter=retry_on_rpc_error)
def commit(request):
try:
start_time = time.time()
response = datastore.commit(request)
end_time = time.time()
rpc_stats_callback(successes=1)
commit_time_ms = int((end_time - start_time) * 1000)
return response, commit_time_ms
except (RPCError, SocketError):
if rpc_stats_callback:
rpc_stats_callback(errors=1)
raise
response, commit_time_ms = commit(commit_request)
return response, commit_time_ms
def write_mutations(datastore, project, mutations):
"""A helper function to write a batch of mutations to Cloud Datastore.
If a commit fails, it will be retried upto 5 times. All mutations in the
batch will be committed again, even if the commit was partially successful.
If the retry limit is exceeded, the last exception from Cloud Datastore will
be raised.
"""
commit_request = datastore_pb2.CommitRequest()
commit_request.mode = datastore_pb2.CommitRequest.NON_TRANSACTIONAL
commit_request.project_id = project
for mutation in mutations:
commit_request.mutations.add().CopyFrom(mutation)
@retry.with_exponential_backoff(num_retries=5,
retry_filter=retry_on_rpc_error)
def commit(req):
datastore.commit(req)
commit(commit_request)
def __init__(self, client):
self._client = client
self._commit_request = _datastore_pb2.CommitRequest()
self._partial_key_entities = []
self._status = self._INITIAL
