def begin_transaction(self): request = datastore_pb.BeginTransactionRequest() request.set_app(self.project_id) response = yield self._make_request('BeginTransaction', request.Encode()) start_response = datastore_pb.Transaction(response) raise gen.Return(start_response.handle())
def _Dynamic_Transaction(self, request, response): """Handle a Transaction request. We handle transactions by accumulating Put and Delete requests on the client end, as well as recording the key and hash of Get requests. When Commit is called, Transaction is invoked, which verifies that all the entities in the precondition list still exist and their hashes match, then performs a transaction of its own to make the updates. """ begin_request = datastore_pb.BeginTransactionRequest() begin_request.set_app(os.environ['APPLICATION_ID']) begin_request.set_allow_multiple_eg(request.allow_multiple_eg()) tx = datastore_pb.Transaction() self.__call('datastore_v3', 'BeginTransaction', begin_request, tx) preconditions = request.precondition_list() if preconditions: get_request = datastore_pb.GetRequest() get_request.mutable_transaction().CopyFrom(tx) for precondition in preconditions: key = get_request.add_key() key.CopyFrom(precondition.key()) get_response = datastore_pb.GetResponse() self.__call('datastore_v3', 'Get', get_request, get_response) entities = get_response.entity_list() assert len(entities) == request.precondition_size() for precondition, entity in zip(preconditions, entities): if precondition.has_hash() != entity.has_entity(): raise apiproxy_errors.ApplicationError( datastore_pb.Error.CONCURRENT_TRANSACTION, "Transaction precondition failed.") elif entity.has_entity(): entity_hash = hashlib.sha1(entity.entity().Encode()).digest() if precondition.hash() != entity_hash: raise apiproxy_errors.ApplicationError( datastore_pb.Error.CONCURRENT_TRANSACTION, "Transaction precondition failed.") if request.has_puts(): put_request = request.puts() put_request.mutable_transaction().CopyFrom(tx) self.__call('datastore_v3', 'Put', put_request, response) if request.has_deletes(): delete_request = request.deletes() delete_request.mutable_transaction().CopyFrom(tx) self.__call('datastore_v3', 'Delete', delete_request, datastore_pb.DeleteResponse()) self.__call('datastore_v3', 'Commit', tx, datastore_pb.CommitResponse())
def _Dynamic_TransactionQuery(self, request, response): if not request.has_ancestor(): raise apiproxy_errors.ApplicationError( datastore_pb.Error.BAD_REQUEST, 'No ancestor in transactional query.') app_id = datastore_types.ResolveAppId(None) if (datastore_rpc._GetDatastoreType(app_id) != datastore_rpc.BaseConnection.HIGH_REPLICATION_DATASTORE): raise apiproxy_errors.ApplicationError( datastore_pb.Error.BAD_REQUEST, 'remote_api supports transactional queries only in the ' 'high-replication datastore.') entity_group_key = entity_pb.Reference() entity_group_key.CopyFrom(request.ancestor()) group_path = entity_group_key.mutable_path() root = entity_pb.Path_Element() root.MergeFrom(group_path.element(0)) group_path.clear_element() group_path.add_element().CopyFrom(root) eg_element = group_path.add_element() eg_element.set_type(metadata.EntityGroup.KIND_NAME) eg_element.set_id(metadata.EntityGroup.ID) begin_request = datastore_pb.BeginTransactionRequest() begin_request.set_app(app_id) tx = datastore_pb.Transaction() self.__call('datastore_v3', 'BeginTransaction', begin_request, tx) request.mutable_transaction().CopyFrom(tx) self.__call('datastore_v3', 'RunQuery', request, response.mutable_result()) get_request = datastore_pb.GetRequest() get_request.mutable_transaction().CopyFrom(tx) get_request.add_key().CopyFrom(entity_group_key) get_response = datastore_pb.GetResponse() self.__call('datastore_v3', 'Get', get_request, get_response) entity_group = get_response.entity(0) response.mutable_entity_group_key().CopyFrom(entity_group_key) if entity_group.has_entity(): response.mutable_entity_group().CopyFrom(entity_group.entity()) self.__call('datastore_v3', 'Commit', tx, datastore_pb.CommitResponse())
def _Dynamic_GetIDs(self, request, response): """Fetch unique IDs for a set of paths.""" for entity in request.entity_list(): assert entity.property_size() == 0 assert entity.raw_property_size() == 0 assert entity.entity_group().element_size() == 0 lastpart = entity.key().path().element_list()[-1] assert lastpart.id() == 0 and not lastpart.has_name() begin_request = datastore_pb.BeginTransactionRequest() begin_request.set_app(os.environ['APPLICATION_ID']) tx = datastore_pb.Transaction() self.__call('datastore_v3', 'BeginTransaction', begin_request, tx) self.__call('datastore_v3', 'Put', request, response) self.__call('datastore_v3', 'Rollback', tx, api_base_pb.VoidProto())
def begin_transaction_request(self, app_id, http_request_data): """ Handles the intial request to start a transaction. Replies with a unique identifier to handle this transaction in future requests. Args: app_id: The application ID requesting the transaction. http_request_data: The encoded request. Returns: An encoded transaction protocol buffer with a unique handler. """ global datastore_access begin_transaction_req_pb = datastore_pb.BeginTransactionRequest( http_request_data) multiple_eg = False if begin_transaction_req_pb.has_allow_multiple_eg(): multiple_eg = bool(begin_transaction_req_pb.allow_multiple_eg()) if READ_ONLY: logger.warning( 'Unable to begin transaction in read-only mode: {}'.format( begin_transaction_req_pb)) raise gen.Return(('', datastore_pb.Error.CAPABILITY_DISABLED, 'Datastore is in read-only mode.')) try: handle = yield datastore_access.setup_transaction( app_id, multiple_eg) except dbconstants.InternalError as error: raise gen.Return( ('', datastore_pb.Error.INTERNAL_ERROR, str(error))) except dbconstants.BadRequest as error: raise gen.Return(('', datastore_pb.Error.BAD_REQUEST, str(error))) except (zktransaction.ZKInternalException, dbconstants.AppScaleDBConnectionError) as error: logger.exception('Unable to begin transaction') raise gen.Return( ('', datastore_pb.Error.INTERNAL_ERROR, str(error))) transaction_pb = datastore_pb.Transaction() transaction_pb.set_app(app_id) transaction_pb.set_handle(handle) raise gen.Return((transaction_pb.Encode(), 0, ''))
def __insert_v3_entity(self, v3_entity, v3_txn): """Inserts a v3 entity. Args: v3_entity: an entity_pb.EntityProto v3_txn: a datastore_pb.Transaction or None Returns: a tuple (the number of index writes that occurred, the entity key) Raises: ApplicationError: if the entity already exists """ if not v3_txn: v3_txn = datastore_pb.Transaction() v3_begin_txn_req = datastore_pb.BeginTransactionRequest() v3_begin_txn_req.set_app(v3_entity.key().app()) self.__make_v3_call('BeginTransaction', v3_begin_txn_req, v3_txn) _, key = self.__insert_v3_entity(v3_entity, v3_txn) v3_resp = datastore_pb.CommitResponse() self.__make_v3_call('Commit', v3_txn, v3_resp) return (v3_resp.cost().index_writes(), key) if datastore_pbs.is_complete_v3_key(v3_entity.key()): v3_get_req = datastore_pb.GetRequest() v3_get_req.mutable_transaction().CopyFrom(v3_txn) v3_get_req.key_list().append(v3_entity.key()) v3_get_resp = datastore_pb.GetResponse() self.__make_v3_call('Get', v3_get_req, v3_get_resp) if v3_get_resp.entity(0).has_entity(): raise apiproxy_errors.ApplicationError( datastore_pb.Error.BAD_REQUEST, 'Entity already exists.') v3_put_req = datastore_pb.PutRequest() v3_put_req.mutable_transaction().CopyFrom(v3_txn) v3_put_req.entity_list().append(v3_entity) v3_put_resp = datastore_pb.PutResponse() self.__make_v3_call('Put', v3_put_req, v3_put_resp) return (v3_put_resp.cost().index_writes(), v3_put_resp.key(0))
def _Dynamic_Commit(self, req, resp): try: self.__service_validator.validate_commit_req(req) except cloud_datastore_validator.ValidationError as e: raise apiproxy_errors.ApplicationError(datastore_pb.Error.BAD_REQUEST, str(e)) single_use_txn = None if req.WhichOneof('transaction_selector') == 'single_use_transaction': single_use_txn = self.__begin_adhoc_txn(req) try: try: if req.transaction or single_use_txn: self.__commit(req.mutations, req.transaction or single_use_txn, resp) else: v3_txn_req = datastore_pb.BeginTransactionRequest() v3_txn_req.set_app(self.__app_id) for mutation in req.mutations: v3_txn = datastore_pb.Transaction() self.__make_v3_call('BeginTransaction', v3_txn_req, v3_txn) v1_txn = self.__service_converter._v3_to_v1_txn(v3_txn) commit_resp = googledatastore.CommitResponse() self.__commit([mutation], v1_txn, commit_resp) resp.index_updates += commit_resp.index_updates mutation_result = commit_resp.mutation_results[0] resp.mutation_results.add().CopyFrom(mutation_result) except datastore_pbs.InvalidConversionError as e: raise apiproxy_errors.ApplicationError(datastore_pb.Error.BAD_REQUEST, str(e)) except: if single_use_txn: self.__rollback_adhoc_txn(req, single_use_txn) raise
def __update_v3_entity(self, v3_entity, v3_txn): """Updates a v3 entity. Args: v3_entity: a datastore_v4_pb.Entity v3_txn: a datastore_pb.Transaction or None Returns: the number of index writes that occurred Raises: ApplicationError: if the entity does not exist """ if not v3_txn: v3_txn = datastore_pb.Transaction() v3_begin_txn_req = datastore_pb.BeginTransactionRequest() v3_begin_txn_req.set_app(v3_entity.key().app()) self.__make_v3_call('BeginTransaction', v3_begin_txn_req, v3_txn) self.__update_v3_entity(v3_entity, v3_txn) v3_resp = datastore_pb.CommitResponse() self.__make_v3_call('Commit', v3_txn, v3_resp) return v3_resp.cost().index_writes() v3_get_req = datastore_pb.GetRequest() v3_get_req.mutable_transaction().CopyFrom(v3_txn) v3_get_req.key_list().append(v3_entity.key()) v3_get_resp = datastore_pb.GetResponse() self.__make_v3_call('Get', v3_get_req, v3_get_resp) if not v3_get_resp.entity(0).has_entity(): raise apiproxy_errors.ApplicationError( datastore_v4_pb.Error.BAD_REQUEST, 'Entity does not exist.') v3_put_req = datastore_pb.PutRequest() v3_put_req.mutable_transaction().CopyFrom(v3_txn) v3_put_req.entity_list().append(v3_entity) v3_put_resp = datastore_pb.PutResponse() self.__make_v3_call('Put', v3_put_req, v3_put_resp) return v3_put_resp.cost().index_writes()
def begin_transaction_request(self, app_id, http_request_data): """ Handles the intial request to start a transaction. Replies with a unique identifier to handle this transaction in future requests. Args: app_id: The application ID requesting the transaction. http_request_data: The encoded request. Returns: An encoded transaction protocol buffer with a unique handler. """ global datastore_access begin_transaction_req_pb = datastore_pb.BeginTransactionRequest( http_request_data) multiple_eg = False if begin_transaction_req_pb.has_allow_multiple_eg(): multiple_eg = begin_transaction_req_pb.allow_multiple_eg() handle = None transaction_pb = datastore_pb.Transaction() if READ_ONLY: logger.warning( 'Unable to begin transaction in read-only mode: {}'.format( begin_transaction_req_pb)) return (transaction_pb.Encode(), datastore_pb.Error.CAPABILITY_DISABLED, 'Datastore is in read-only mode.') try: handle = datastore_access.setup_transaction(app_id, multiple_eg) except zktransaction.ZKInternalException: logger.exception('Unable to begin {}'.format(transaction_pb)) return (transaction_pb.Encode(), datastore_pb.Error.INTERNAL_ERROR, "Internal error with ZooKeeper connection.") transaction_pb.set_app(app_id) transaction_pb.set_handle(handle) return (transaction_pb.Encode(), 0, "")
try: self.__service_validator.validate_commit_req(req) except cloud_datastore_validator.ValidationError, e: raise apiproxy_errors.ApplicationError(datastore_pb.Error.BAD_REQUEST, str(e)) single_use_txn = None if req.WhichOneof('transaction_selector') == 'single_use_transaction': single_use_txn = self.__begin_adhoc_txn(req) try: try: if req.transaction or single_use_txn: self.__commit(req.mutations, req.transaction or single_use_txn, resp) else: v3_txn_req = datastore_pb.BeginTransactionRequest() v3_txn_req.set_app(self.__app_id) for mutation in req.mutations: v3_txn = datastore_pb.Transaction() self.__make_v3_call('BeginTransaction', v3_txn_req, v3_txn) v1_txn = self.__service_converter._v3_to_v1_txn(v3_txn) commit_resp = googledatastore.CommitResponse() self.__commit([mutation], v1_txn, commit_resp) resp.index_updates += commit_resp.index_updates mutation_result = commit_resp.mutation_results[0] resp.mutation_results.add().CopyFrom(mutation_result) except datastore_pbs.InvalidConversionError, e: