def begin_transaction(self):
request = datastore_pb.BeginTransactionRequest()
request.set_app(self.project_id)
response = yield self._make_request('BeginTransaction',
request.Encode())
start_response = datastore_pb.Transaction(response)
raise gen.Return(start_response.handle())
def _Dynamic_Transaction(self, request, response):
"""Handle a Transaction request.
We handle transactions by accumulating Put and Delete requests on the client
end, as well as recording the key and hash of Get requests. When Commit is
called, Transaction is invoked, which verifies that all the entities in the
precondition list still exist and their hashes match, then performs a
transaction of its own to make the updates.
"""
begin_request = datastore_pb.BeginTransactionRequest()
begin_request.set_app(os.environ['APPLICATION_ID'])
begin_request.set_allow_multiple_eg(request.allow_multiple_eg())
tx = datastore_pb.Transaction()
self.__call('datastore_v3', 'BeginTransaction', begin_request, tx)
preconditions = request.precondition_list()
if preconditions:
get_request = datastore_pb.GetRequest()
get_request.mutable_transaction().CopyFrom(tx)
for precondition in preconditions:
key = get_request.add_key()
key.CopyFrom(precondition.key())
get_response = datastore_pb.GetResponse()
self.__call('datastore_v3', 'Get', get_request, get_response)
entities = get_response.entity_list()
assert len(entities) == request.precondition_size()
for precondition, entity in zip(preconditions, entities):
if precondition.has_hash() != entity.has_entity():
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.CONCURRENT_TRANSACTION,
"Transaction precondition failed.")
elif entity.has_entity():
entity_hash = hashlib.sha1(entity.entity().Encode()).digest()
if precondition.hash() != entity_hash:
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.CONCURRENT_TRANSACTION,
"Transaction precondition failed.")
if request.has_puts():
put_request = request.puts()
put_request.mutable_transaction().CopyFrom(tx)
self.__call('datastore_v3', 'Put', put_request, response)
if request.has_deletes():
delete_request = request.deletes()
delete_request.mutable_transaction().CopyFrom(tx)
self.__call('datastore_v3', 'Delete', delete_request,
datastore_pb.DeleteResponse())
self.__call('datastore_v3', 'Commit', tx, datastore_pb.CommitResponse())
def _Dynamic_TransactionQuery(self, request, response):
if not request.has_ancestor():
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.BAD_REQUEST,
'No ancestor in transactional query.')
app_id = datastore_types.ResolveAppId(None)
if (datastore_rpc._GetDatastoreType(app_id) !=
datastore_rpc.BaseConnection.HIGH_REPLICATION_DATASTORE):
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.BAD_REQUEST,
'remote_api supports transactional queries only in the '
'high-replication datastore.')
entity_group_key = entity_pb.Reference()
entity_group_key.CopyFrom(request.ancestor())
group_path = entity_group_key.mutable_path()
root = entity_pb.Path_Element()
root.MergeFrom(group_path.element(0))
group_path.clear_element()
group_path.add_element().CopyFrom(root)
eg_element = group_path.add_element()
eg_element.set_type(metadata.EntityGroup.KIND_NAME)
eg_element.set_id(metadata.EntityGroup.ID)
begin_request = datastore_pb.BeginTransactionRequest()
begin_request.set_app(app_id)
tx = datastore_pb.Transaction()
self.__call('datastore_v3', 'BeginTransaction', begin_request, tx)
request.mutable_transaction().CopyFrom(tx)
self.__call('datastore_v3', 'RunQuery', request, response.mutable_result())
get_request = datastore_pb.GetRequest()
get_request.mutable_transaction().CopyFrom(tx)
get_request.add_key().CopyFrom(entity_group_key)
get_response = datastore_pb.GetResponse()
self.__call('datastore_v3', 'Get', get_request, get_response)
entity_group = get_response.entity(0)
response.mutable_entity_group_key().CopyFrom(entity_group_key)
if entity_group.has_entity():
response.mutable_entity_group().CopyFrom(entity_group.entity())
self.__call('datastore_v3', 'Commit', tx, datastore_pb.CommitResponse())
def _Dynamic_GetIDs(self, request, response):
"""Fetch unique IDs for a set of paths."""
for entity in request.entity_list():
assert entity.property_size() == 0
assert entity.raw_property_size() == 0
assert entity.entity_group().element_size() == 0
lastpart = entity.key().path().element_list()[-1]
assert lastpart.id() == 0 and not lastpart.has_name()
begin_request = datastore_pb.BeginTransactionRequest()
begin_request.set_app(os.environ['APPLICATION_ID'])
tx = datastore_pb.Transaction()
self.__call('datastore_v3', 'BeginTransaction', begin_request, tx)
self.__call('datastore_v3', 'Put', request, response)
self.__call('datastore_v3', 'Rollback', tx, api_base_pb.VoidProto())
def begin_transaction_request(self, app_id, http_request_data):
""" Handles the intial request to start a transaction. Replies with
a unique identifier to handle this transaction in future requests.
Args:
app_id: The application ID requesting the transaction.
http_request_data: The encoded request.
Returns:
An encoded transaction protocol buffer with a unique handler.
"""
global datastore_access
begin_transaction_req_pb = datastore_pb.BeginTransactionRequest(
http_request_data)
multiple_eg = False
if begin_transaction_req_pb.has_allow_multiple_eg():
multiple_eg = bool(begin_transaction_req_pb.allow_multiple_eg())
if READ_ONLY:
logger.warning(
'Unable to begin transaction in read-only mode: {}'.format(
begin_transaction_req_pb))
raise gen.Return(('', datastore_pb.Error.CAPABILITY_DISABLED,
'Datastore is in read-only mode.'))
try:
handle = yield datastore_access.setup_transaction(
app_id, multiple_eg)
except dbconstants.InternalError as error:
raise gen.Return(
('', datastore_pb.Error.INTERNAL_ERROR, str(error)))
except dbconstants.BadRequest as error:
raise gen.Return(('', datastore_pb.Error.BAD_REQUEST, str(error)))
except (zktransaction.ZKInternalException,
dbconstants.AppScaleDBConnectionError) as error:
logger.exception('Unable to begin transaction')
raise gen.Return(
('', datastore_pb.Error.INTERNAL_ERROR, str(error)))
transaction_pb = datastore_pb.Transaction()
transaction_pb.set_app(app_id)
transaction_pb.set_handle(handle)
raise gen.Return((transaction_pb.Encode(), 0, ''))
def __insert_v3_entity(self, v3_entity, v3_txn):
"""Inserts a v3 entity.
Args:
v3_entity: an entity_pb.EntityProto
v3_txn: a datastore_pb.Transaction or None
Returns:
a tuple (the number of index writes that occurred,
the entity key)
Raises:
ApplicationError: if the entity already exists
"""
if not v3_txn:
v3_txn = datastore_pb.Transaction()
v3_begin_txn_req = datastore_pb.BeginTransactionRequest()
v3_begin_txn_req.set_app(v3_entity.key().app())
self.__make_v3_call('BeginTransaction', v3_begin_txn_req, v3_txn)
_, key = self.__insert_v3_entity(v3_entity, v3_txn)
v3_resp = datastore_pb.CommitResponse()
self.__make_v3_call('Commit', v3_txn, v3_resp)
return (v3_resp.cost().index_writes(), key)
if datastore_pbs.is_complete_v3_key(v3_entity.key()):
v3_get_req = datastore_pb.GetRequest()
v3_get_req.mutable_transaction().CopyFrom(v3_txn)
v3_get_req.key_list().append(v3_entity.key())
v3_get_resp = datastore_pb.GetResponse()
self.__make_v3_call('Get', v3_get_req, v3_get_resp)
if v3_get_resp.entity(0).has_entity():
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.BAD_REQUEST, 'Entity already exists.')
v3_put_req = datastore_pb.PutRequest()
v3_put_req.mutable_transaction().CopyFrom(v3_txn)
v3_put_req.entity_list().append(v3_entity)
v3_put_resp = datastore_pb.PutResponse()
self.__make_v3_call('Put', v3_put_req, v3_put_resp)
return (v3_put_resp.cost().index_writes(), v3_put_resp.key(0))
def _Dynamic_Commit(self, req, resp):
try:
self.__service_validator.validate_commit_req(req)
except cloud_datastore_validator.ValidationError as e:
raise apiproxy_errors.ApplicationError(datastore_pb.Error.BAD_REQUEST,
str(e))
single_use_txn = None
if req.WhichOneof('transaction_selector') == 'single_use_transaction':
single_use_txn = self.__begin_adhoc_txn(req)
try:
try:
if req.transaction or single_use_txn:
self.__commit(req.mutations, req.transaction or single_use_txn, resp)
else:
v3_txn_req = datastore_pb.BeginTransactionRequest()
v3_txn_req.set_app(self.__app_id)
for mutation in req.mutations:
v3_txn = datastore_pb.Transaction()
self.__make_v3_call('BeginTransaction', v3_txn_req, v3_txn)
v1_txn = self.__service_converter._v3_to_v1_txn(v3_txn)
commit_resp = googledatastore.CommitResponse()
self.__commit([mutation], v1_txn, commit_resp)
resp.index_updates += commit_resp.index_updates
mutation_result = commit_resp.mutation_results[0]
resp.mutation_results.add().CopyFrom(mutation_result)
except datastore_pbs.InvalidConversionError as e:
raise apiproxy_errors.ApplicationError(datastore_pb.Error.BAD_REQUEST,
str(e))
except:
if single_use_txn:
self.__rollback_adhoc_txn(req, single_use_txn)
raise
def __update_v3_entity(self, v3_entity, v3_txn):
"""Updates a v3 entity.
Args:
v3_entity: a datastore_v4_pb.Entity
v3_txn: a datastore_pb.Transaction or None
Returns:
the number of index writes that occurred
Raises:
ApplicationError: if the entity does not exist
"""
if not v3_txn:
v3_txn = datastore_pb.Transaction()
v3_begin_txn_req = datastore_pb.BeginTransactionRequest()
v3_begin_txn_req.set_app(v3_entity.key().app())
self.__make_v3_call('BeginTransaction', v3_begin_txn_req, v3_txn)
self.__update_v3_entity(v3_entity, v3_txn)
v3_resp = datastore_pb.CommitResponse()
self.__make_v3_call('Commit', v3_txn, v3_resp)
return v3_resp.cost().index_writes()
v3_get_req = datastore_pb.GetRequest()
v3_get_req.mutable_transaction().CopyFrom(v3_txn)
v3_get_req.key_list().append(v3_entity.key())
v3_get_resp = datastore_pb.GetResponse()
self.__make_v3_call('Get', v3_get_req, v3_get_resp)
if not v3_get_resp.entity(0).has_entity():
raise apiproxy_errors.ApplicationError(
datastore_v4_pb.Error.BAD_REQUEST, 'Entity does not exist.')
v3_put_req = datastore_pb.PutRequest()
v3_put_req.mutable_transaction().CopyFrom(v3_txn)
v3_put_req.entity_list().append(v3_entity)
v3_put_resp = datastore_pb.PutResponse()
self.__make_v3_call('Put', v3_put_req, v3_put_resp)
return v3_put_resp.cost().index_writes()
def begin_transaction_request(self, app_id, http_request_data):
""" Handles the intial request to start a transaction. Replies with
a unique identifier to handle this transaction in future requests.
Args:
app_id: The application ID requesting the transaction.
http_request_data: The encoded request.
Returns:
An encoded transaction protocol buffer with a unique handler.
"""
global datastore_access
begin_transaction_req_pb = datastore_pb.BeginTransactionRequest(
http_request_data)
multiple_eg = False
if begin_transaction_req_pb.has_allow_multiple_eg():
multiple_eg = begin_transaction_req_pb.allow_multiple_eg()
handle = None
transaction_pb = datastore_pb.Transaction()
if READ_ONLY:
logger.warning(
'Unable to begin transaction in read-only mode: {}'.format(
begin_transaction_req_pb))
return (transaction_pb.Encode(),
datastore_pb.Error.CAPABILITY_DISABLED,
'Datastore is in read-only mode.')
try:
handle = datastore_access.setup_transaction(app_id, multiple_eg)
except zktransaction.ZKInternalException:
logger.exception('Unable to begin {}'.format(transaction_pb))
return (transaction_pb.Encode(), datastore_pb.Error.INTERNAL_ERROR,
"Internal error with ZooKeeper connection.")
transaction_pb.set_app(app_id)
transaction_pb.set_handle(handle)
return (transaction_pb.Encode(), 0, "")
try:
self.__service_validator.validate_commit_req(req)
except cloud_datastore_validator.ValidationError, e:
raise apiproxy_errors.ApplicationError(datastore_pb.Error.BAD_REQUEST,
str(e))
single_use_txn = None
if req.WhichOneof('transaction_selector') == 'single_use_transaction':
single_use_txn = self.__begin_adhoc_txn(req)
try:
try:
if req.transaction or single_use_txn:
self.__commit(req.mutations, req.transaction or single_use_txn, resp)
else:
v3_txn_req = datastore_pb.BeginTransactionRequest()
v3_txn_req.set_app(self.__app_id)
for mutation in req.mutations:
v3_txn = datastore_pb.Transaction()
self.__make_v3_call('BeginTransaction', v3_txn_req, v3_txn)
v1_txn = self.__service_converter._v3_to_v1_txn(v3_txn)
commit_resp = googledatastore.CommitResponse()
self.__commit([mutation], v1_txn, commit_resp)
resp.index_updates += commit_resp.index_updates
mutation_result = commit_resp.mutation_results[0]
resp.mutation_results.add().CopyFrom(mutation_result)
except datastore_pbs.InvalidConversionError, e:
