def test_commit_w_transaction(self): from google.cloud.proto.datastore.v1 import datastore_pb2 from google.cloud.datastore.helpers import _new_value_pb project = 'PROJECT' key_pb = _make_key_pb(project) rsp_pb = datastore_pb2.CommitResponse() req_pb = datastore_pb2.CommitRequest() mutation = req_pb.mutations.add() insert = mutation.upsert insert.key.CopyFrom(key_pb) value_pb = _new_value_pb(insert, 'foo') value_pb.string_value = u'Foo' # Create mock HTTP and client with response. http = _make_requests_session( [_make_response(content=rsp_pb.SerializeToString())]) client = mock.Mock(_http=http, _base_url='test.invalid', spec=['_http', '_base_url']) # Make request. rq_class = datastore_pb2.CommitRequest ds_api = self._make_one(client) mode = rq_class.TRANSACTIONAL result = ds_api.commit(project, mode, [mutation], transaction=b'xact') # Check the result and verify the callers. self.assertEqual(result, rsp_pb) uri = _build_expected_url(client._base_url, project, 'commit') request = _verify_protobuf_call(http, uri, rq_class()) self.assertEqual(request.transaction, b'xact') self.assertEqual(list(request.mutations), [mutation]) self.assertEqual(request.mode, rq_class.TRANSACTIONAL)
def test_commit_w_transaction(self): from google.cloud.proto.datastore.v1 import datastore_pb2 from google.cloud.datastore.helpers import _new_value_pb project = 'PROJECT' key_pb = self._make_key_pb(project) rsp_pb = datastore_pb2.CommitResponse() req_pb = datastore_pb2.CommitRequest() mutation = req_pb.mutations.add() insert = mutation.upsert insert.key.CopyFrom(key_pb) value_pb = _new_value_pb(insert, 'foo') value_pb.string_value = u'Foo' http = Http({'status': '200'}, rsp_pb.SerializeToString()) client = mock.Mock(_http=http, spec=['_http']) conn = self._make_one(client) uri = '/'.join([ conn.api_base_url, conn.API_VERSION, 'projects', project + ':commit', ]) result = conn.commit(project, req_pb, b'xact') self.assertEqual(result, rsp_pb) # Verify the caller. cw = http._called_with self._verify_protobuf_call(cw, uri, conn) rq_class = datastore_pb2.CommitRequest request = rq_class() request.ParseFromString(cw['body']) self.assertEqual(request.transaction, b'xact') self.assertEqual(list(request.mutations), [mutation]) self.assertEqual(request.mode, rq_class.TRANSACTIONAL)
def write_mutations( datastore, project, mutations, throttler, rpc_stats_callback=None, throttle_delay=1): """A helper function to write a batch of mutations to Cloud Datastore. If a commit fails, it will be retried upto 5 times. All mutations in the batch will be committed again, even if the commit was partially successful. If the retry limit is exceeded, the last exception from Cloud Datastore will be raised. Args: datastore: googledatastore.connection.Datastore project: str, project id mutations: list of google.cloud.proto.datastore.v1.datastore_pb2.Mutation rpc_stats_callback: a function to call with arguments `successes` and `failures` and `throttled_secs`; this is called to record successful and failed RPCs to Datastore and time spent waiting for throttling. throttler: AdaptiveThrottler, to use to select requests to be throttled. throttle_delay: float, time in seconds to sleep when throttled. Returns a tuple of: CommitResponse, the response from Datastore; int, the latency of the successful RPC in milliseconds. """ commit_request = datastore_pb2.CommitRequest() commit_request.mode = datastore_pb2.CommitRequest.NON_TRANSACTIONAL commit_request.project_id = project for mutation in mutations: commit_request.mutations.add().CopyFrom(mutation) @retry.with_exponential_backoff( num_retries=5, retry_filter=retry_on_rpc_error) def commit(request): # Client-side throttling. while throttler.throttle_request(time.time() * 1000): _LOGGER.info( "Delaying request for %ds due to previous failures", throttle_delay) time.sleep(throttle_delay) rpc_stats_callback(throttled_secs=throttle_delay) try: start_time = time.time() response = datastore.commit(request) end_time = time.time() rpc_stats_callback(successes=1) throttler.successful_request(start_time * 1000) commit_time_ms = int((end_time - start_time) * 1000) return response, commit_time_ms except (RPCError, SocketError): if rpc_stats_callback: rpc_stats_callback(errors=1) raise response, commit_time_ms = commit(commit_request) return response, commit_time_ms
def commit(self, project, mode, mutations, transaction=None): """Perform a ``commit`` request. :type project: str :param project: The project to connect to. This is usually your project name in the cloud console. :type mode: :class:`.gapic.datastore.v1.enums.CommitRequest.Mode` :param mode: The type of commit to perform. Expected to be one of ``TRANSACTIONAL`` or ``NON_TRANSACTIONAL``. :type mutations: list :param mutations: List of :class:`.datastore_pb2.Mutation`, the mutations to perform. :type transaction: bytes :param transaction: (Optional) The transaction ID returned from :meth:`begin_transaction`. Non-transactional commits must pass :data:`None`. :rtype: :class:`.datastore_pb2.CommitResponse` :returns: The returned protobuf response object. """ request_pb = _datastore_pb2.CommitRequest( project_id=project, mode=mode, transaction=transaction, mutations=mutations, ) return _rpc(self.client._http, project, 'commit', self.client._base_url, request_pb, _datastore_pb2.CommitResponse)
def test_commit(self, mock_create_stub): # Mock gRPC layer grpc_stub = mock.Mock() mock_create_stub.return_value = grpc_stub client = datastore_client.DatastoreClient() # Mock request project_id = 'projectId-1969970175' mode = enums.CommitRequest.Mode.MODE_UNSPECIFIED mutations = [] # Mock response index_updates = -1425228195 expected_response = datastore_pb2.CommitResponse( index_updates=index_updates) grpc_stub.Commit.return_value = expected_response response = client.commit(project_id, mode, mutations) self.assertEqual(expected_response, response) grpc_stub.Commit.assert_called_once() args, kwargs = grpc_stub.Commit.call_args self.assertEqual(len(args), 2) self.assertEqual(len(kwargs), 1) self.assertIn('metadata', kwargs) actual_request = args[0] expected_request = datastore_pb2.CommitRequest(project_id=project_id, mode=mode, mutations=mutations) self.assertEqual(expected_request, actual_request)
def commit(self, project_id, mode, mutations, transaction=None, options=None): """ Commits a transaction, optionally creating, deleting or modifying some entities. Example: >>> from google.cloud.gapic.datastore.v1 import datastore_client >>> from google.cloud.gapic.datastore.v1 import enums >>> client = datastore_client.DatastoreClient() >>> project_id = '' >>> mode = enums.CommitRequest.Mode.MODE_UNSPECIFIED >>> mutations = [] >>> response = client.commit(project_id, mode, mutations) Args: project_id (string): The ID of the project against which to make the request. mode (enum :class:`google.cloud.gapic.datastore.v1.enums.CommitRequest.Mode`): The type of commit to perform. Defaults to ``TRANSACTIONAL``. mutations (list[:class:`google.cloud.proto.datastore.v1.datastore_pb2.Mutation`]): The mutations to perform. When mode is ``TRANSACTIONAL``, mutations affecting a single entity are applied in order. The following sequences of mutations affecting a single entity are not permitted in a single ``Commit`` request: - ``insert`` followed by ``insert`` - ``update`` followed by ``insert`` - ``upsert`` followed by ``insert`` - ``delete`` followed by ``update`` When mode is ``NON_TRANSACTIONAL``, no two mutations may affect a single entity. transaction (bytes): The identifier of the transaction associated with the commit. A transaction identifier is returned by a call to ``Datastore.BeginTransaction``. options (:class:`google.gax.CallOptions`): Overrides the default settings for this call, e.g, timeout, retries etc. Returns: A :class:`google.cloud.proto.datastore.v1.datastore_pb2.CommitResponse` instance. Raises: :exc:`google.gax.errors.GaxError` if the RPC is aborted. :exc:`ValueError` if the parameters are invalid. """ # Sanity check: We have some fields which are mutually exclusive; # raise ValueError if more than one is sent. oneof.check_oneof( transaction=transaction, ) # Create the request object. request = datastore_pb2.CommitRequest( project_id=project_id, mode=mode, mutations=mutations, transaction=transaction) return self._commit(request, options)
def write_mutations(datastore, project, mutations, rpc_stats_callback=None): """A helper function to write a batch of mutations to Cloud Datastore. If a commit fails, it will be retried upto 5 times. All mutations in the batch will be committed again, even if the commit was partially successful. If the retry limit is exceeded, the last exception from Cloud Datastore will be raised. Args: datastore: googledatastore.connection.Datastore project: str, project id mutations: list of google.cloud.proto.datastore.v1.datastore_pb2.Mutation rpc_stats_callback: a function to call with arguments `successes` and `failures`; this is called to record successful and failed RPCs to Datastore. Returns a tuple of: CommitResponse, the response from Datastore; int, the latency of the successful RPC in milliseconds. """ commit_request = datastore_pb2.CommitRequest() commit_request.mode = datastore_pb2.CommitRequest.NON_TRANSACTIONAL commit_request.project_id = project for mutation in mutations: commit_request.mutations.add().CopyFrom(mutation) @retry.with_exponential_backoff(num_retries=5, retry_filter=retry_on_rpc_error) def commit(request): try: start_time = time.time() response = datastore.commit(request) end_time = time.time() rpc_stats_callback(successes=1) commit_time_ms = int((end_time - start_time) * 1000) return response, commit_time_ms except (RPCError, SocketError): if rpc_stats_callback: rpc_stats_callback(errors=1) raise response, commit_time_ms = commit(commit_request) return response, commit_time_ms
def write_mutations(datastore, project, mutations): """A helper function to write a batch of mutations to Cloud Datastore. If a commit fails, it will be retried upto 5 times. All mutations in the batch will be committed again, even if the commit was partially successful. If the retry limit is exceeded, the last exception from Cloud Datastore will be raised. """ commit_request = datastore_pb2.CommitRequest() commit_request.mode = datastore_pb2.CommitRequest.NON_TRANSACTIONAL commit_request.project_id = project for mutation in mutations: commit_request.mutations.add().CopyFrom(mutation) @retry.with_exponential_backoff(num_retries=5, retry_filter=retry_on_rpc_error) def commit(req): datastore.commit(req) commit(commit_request)
def __init__(self, client): self._client = client self._commit_request = _datastore_pb2.CommitRequest() self._partial_key_entities = [] self._status = self._INITIAL