def begin_transaction(self):
request = datastore_pb.BeginTransactionRequest()
request.set_app(self.project_id)
response = yield self._make_request('BeginTransaction',
request.Encode())
start_response = datastore_pb.Transaction(response)
raise gen.Return(start_response.handle())
def rollback_transaction_request(self, app_id, http_request_data):
""" Handles the rollback phase of a transaction.
Args:
app_id: The application ID requesting the rollback.
http_request_data: The encoded request.
Returns:
An encoded protocol buffer void response.
"""
global datastore_access
if READ_ONLY:
logger.warning('Unable to rollback in read-only mode: {}'.
format(http_request_data))
return ('', datastore_pb.Error.CAPABILITY_DISABLED,
'Datastore is in read-only mode.')
try:
txn = datastore_pb.Transaction(http_request_data)
except ProtocolBufferDecodeError as error:
return '', datastore_pb.Error.BAD_REQUEST, str(error)
try:
datastore_access.rollback_transaction(app_id, txn.handle())
except dbconstants.InternalError as error:
logger.exception('Unable to rollback transaction')
return '', datastore_pb.Error.INTERNAL_ERROR, str(error)
except Exception as error:
logger.exception('Unable to rollback transaction')
return '', datastore_pb.Error.INTERNAL_ERROR, str(error)
return api_base_pb.VoidProto().Encode(), 0, ''
def __put_v1_entity(self, v1_entity, v1_txn):
"""Writes a v1 entity to the datastore in a transaction and return its key.
Args:
v1_entity: the entity to write
v1_txn: the transaction in which to write the entity.
Returns:
the key of the entity, which may have been allocated.
"""
v3_entity = entity_pb.EntityProto()
self.__entity_converter.v1_to_v3_entity(v1_entity, v3_entity)
v3_txn = datastore_pb.Transaction()
self.__service_converter.v1_to_v3_txn(v1_txn, v3_txn)
v3_put_req = datastore_pb.PutRequest()
v3_put_req.mutable_transaction().CopyFrom(v3_txn)
v3_put_req.entity_list().append(v3_entity)
v3_put_resp = datastore_pb.PutResponse()
self.__make_v3_call('Put', v3_put_req, v3_put_resp)
v3_key = v3_put_resp.key(0)
v1_key = googledatastore.Key()
self.__entity_converter.v3_to_v1_key(v3_key, v1_key)
return v1_key
def _Dynamic_BeginTransaction(self, req, resp):
try:
self.__service_validator.validate_begin_transaction_req(req)
v3_req = self.__service_converter.v4_to_v3_begin_transaction_req(
self.__app_id, req)
v3_resp = datastore_pb.Transaction()
self.__make_v3_call('BeginTransaction', v3_req, v3_resp)
except datastore_pbs.InvalidConversionError, e:
raise apiproxy_errors.ApplicationError(
datastore_v4_pb.Error.BAD_REQUEST, str(e))
def _Dynamic_Transaction(self, request, response):
"""Handle a Transaction request.
We handle transactions by accumulating Put and Delete requests on the client
end, as well as recording the key and hash of Get requests. When Commit is
called, Transaction is invoked, which verifies that all the entities in the
precondition list still exist and their hashes match, then performs a
transaction of its own to make the updates.
"""
begin_request = datastore_pb.BeginTransactionRequest()
begin_request.set_app(os.environ['APPLICATION_ID'])
begin_request.set_allow_multiple_eg(request.allow_multiple_eg())
tx = datastore_pb.Transaction()
self.__call('datastore_v3', 'BeginTransaction', begin_request, tx)
preconditions = request.precondition_list()
if preconditions:
get_request = datastore_pb.GetRequest()
get_request.mutable_transaction().CopyFrom(tx)
for precondition in preconditions:
key = get_request.add_key()
key.CopyFrom(precondition.key())
get_response = datastore_pb.GetResponse()
self.__call('datastore_v3', 'Get', get_request, get_response)
entities = get_response.entity_list()
assert len(entities) == request.precondition_size()
for precondition, entity in zip(preconditions, entities):
if precondition.has_hash() != entity.has_entity():
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.CONCURRENT_TRANSACTION,
"Transaction precondition failed.")
elif entity.has_entity():
entity_hash = hashlib.sha1(entity.entity().Encode()).digest()
if precondition.hash() != entity_hash:
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.CONCURRENT_TRANSACTION,
"Transaction precondition failed.")
if request.has_puts():
put_request = request.puts()
put_request.mutable_transaction().CopyFrom(tx)
self.__call('datastore_v3', 'Put', put_request, response)
if request.has_deletes():
delete_request = request.deletes()
delete_request.mutable_transaction().CopyFrom(tx)
self.__call('datastore_v3', 'Delete', delete_request,
datastore_pb.DeleteResponse())
self.__call('datastore_v3', 'Commit', tx, datastore_pb.CommitResponse())
def test_commit_transaction(self):
db_batch = flexmock()
zookeeper = flexmock()
zookeeper.should_receive("release_lock").and_return(True)
dd = DatastoreDistributed(db_batch, zookeeper)
commit_request = datastore_pb.Transaction()
commit_request.set_handle(123)
commit_request.set_app("aaa")
http_request = commit_request.Encode()
self.assertEquals(dd.commit_transaction("app_id", http_request),
(datastore_pb.CommitResponse().Encode(), 0, ""))
def __delete_v1_key(self, v1_key, v1_txn):
"""Deletes an entity from a v1 key in a transaction."""
v3_key = entity_pb.Reference()
self.__entity_converter.v1_to_v3_reference(v1_key, v3_key)
v3_txn = datastore_pb.Transaction()
self.__service_converter.v1_to_v3_txn(v1_txn, v3_txn)
v3_delete_req = datastore_pb.DeleteRequest()
v3_delete_req.mutable_transaction().CopyFrom(v3_txn)
v3_delete_req.add_key().CopyFrom(v3_key)
v3_delete_resp = datastore_pb.DeleteResponse()
self.__make_v3_call('Delete', v3_delete_req, v3_delete_resp)
def test_rollback_transcation(self):
db_batch = flexmock()
zookeeper = flexmock()
zookeeper.should_receive("release_lock").and_return(True)
zookeeper.should_receive("notify_failed_transaction").and_return(True)
dd = DatastoreDistributed(db_batch, zookeeper)
commit_request = datastore_pb.Transaction()
commit_request.set_handle(123)
commit_request.set_app("aaa")
http_request = commit_request.Encode()
self.assertEquals(dd.rollback_transaction("app_id", http_request),
(api_base_pb.VoidProto().Encode(), 0, ""))
def test_commit_transaction(self):
db_batch = flexmock()
db_batch.should_receive('valid_data_version').and_return(True)
zookeeper = flexmock()
zookeeper.should_receive('remove_tx_node')
dd = DatastoreDistributed(db_batch, zookeeper)
flexmock(dd).should_receive('apply_txn_changes')
commit_request = datastore_pb.Transaction()
commit_request.set_handle(123)
commit_request.set_app("aaa")
http_request = commit_request.Encode()
self.assertEquals(dd.commit_transaction("app_id", http_request),
(datastore_pb.CommitResponse().Encode(), 0, ""))
def commit_transaction_request(self, app_id, http_request_data):
""" Handles the commit phase of a transaction.
Args:
app_id: The application ID requesting the transaction commit.
http_request_data: The encoded request of datastore_pb.Transaction.
Returns:
An encoded protocol buffer commit response.
"""
global datastore_access
if READ_ONLY:
transaction_pb = datastore_pb.Transaction(http_request_data)
logger.warning('Unable to commit in read-only mode: {}'.
format(transaction_pb))
raise gen.Return(
('', datastore_pb.Error.CAPABILITY_DISABLED,
'Datastore is in read-only mode.'))
txid = datastore_pb.Transaction(http_request_data).handle()
try:
yield datastore_access.apply_txn_changes(app_id, txid)
except (dbconstants.TxTimeoutException, dbconstants.Timeout) as timeout:
raise gen.Return(('', datastore_pb.Error.TIMEOUT, str(timeout)))
except dbconstants.AppScaleDBConnectionError:
logger.exception('DB connection error during commit')
raise gen.Return(
('', datastore_pb.Error.INTERNAL_ERROR,
'Datastore connection error on Commit request.'))
except dbconstants.ConcurrentModificationException as error:
raise gen.Return(
('', datastore_pb.Error.CONCURRENT_TRANSACTION, str(error)))
except (dbconstants.TooManyGroupsException,
dbconstants.BadRequest) as error:
raise gen.Return(('', datastore_pb.Error.BAD_REQUEST, str(error)))
commitres_pb = datastore_pb.CommitResponse()
raise gen.Return((commitres_pb.Encode(), 0, ''))
def _Dynamic_Transaction(self, request, response):
"""Handle a Transaction request.
We handle transactions by accumulating Put requests on the client end, as
well as recording the key and hash of Get requests. When Commit is called,
Transaction is invoked, which verifies that all the entities in the
precondition list still exist and their hashes match, then performs a
transaction of its own to make the updates.
"""
tx = datastore_pb.Transaction()
apiproxy_stub_map.MakeSyncCall('datastore_v3', 'BeginTransaction',
api_base_pb.VoidProto(), tx)
preconditions = request.precondition_list()
if preconditions:
get_request = datastore_pb.GetRequest()
get_request.mutable_transaction().CopyFrom(tx)
for precondition in preconditions:
key = get_request.add_key()
key.CopyFrom(precondition.key())
get_response = datastore_pb.GetResponse()
apiproxy_stub_map.MakeSyncCall('datastore_v3', 'Get', get_request,
get_response)
entities = get_response.entity_list()
assert len(entities) == request.precondition_size()
for precondition, entity in zip(preconditions, entities):
if precondition.has_hash() != entity.has_entity():
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.CONCURRENT_TRANSACTION,
"Transaction precondition failed.")
elif entity.has_entity():
entity_hash = sha.new(entity.entity().Encode()).digest()
if precondition.hash() != entity_hash:
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.CONCURRENT_TRANSACTION,
"Transaction precondition failed.")
if request.has_puts():
put_request = request.puts()
put_request.mutable_transaction().CopyFrom(tx)
apiproxy_stub_map.MakeSyncCall('datastore_v3', 'Put',
put_request, response)
if request.has_deletes():
delete_request = request.deletes()
delete_request.mutable_transaction().CopyFrom(tx)
apiproxy_stub_map.MakeSyncCall('datastore_v3', 'Delete',
delete_request, api_base_pb.VoidProto())
apiproxy_stub_map.MakeSyncCall('datastore_v3', 'Commit', tx,
api_base_pb.VoidProto())
def _Dynamic_TransactionQuery(self, request, response):
if not request.has_ancestor():
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.BAD_REQUEST,
'No ancestor in transactional query.')
app_id = datastore_types.ResolveAppId(None)
if (datastore_rpc._GetDatastoreType(app_id) !=
datastore_rpc.BaseConnection.HIGH_REPLICATION_DATASTORE):
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.BAD_REQUEST,
'remote_api supports transactional queries only in the '
'high-replication datastore.')
entity_group_key = entity_pb.Reference()
entity_group_key.CopyFrom(request.ancestor())
group_path = entity_group_key.mutable_path()
root = entity_pb.Path_Element()
root.MergeFrom(group_path.element(0))
group_path.clear_element()
group_path.add_element().CopyFrom(root)
eg_element = group_path.add_element()
eg_element.set_type(metadata.EntityGroup.KIND_NAME)
eg_element.set_id(metadata.EntityGroup.ID)
begin_request = datastore_pb.BeginTransactionRequest()
begin_request.set_app(app_id)
tx = datastore_pb.Transaction()
self.__call('datastore_v3', 'BeginTransaction', begin_request, tx)
request.mutable_transaction().CopyFrom(tx)
self.__call('datastore_v3', 'RunQuery', request, response.mutable_result())
get_request = datastore_pb.GetRequest()
get_request.mutable_transaction().CopyFrom(tx)
get_request.add_key().CopyFrom(entity_group_key)
get_response = datastore_pb.GetResponse()
self.__call('datastore_v3', 'Get', get_request, get_response)
entity_group = get_response.entity(0)
response.mutable_entity_group_key().CopyFrom(entity_group_key)
if entity_group.has_entity():
response.mutable_entity_group().CopyFrom(entity_group.entity())
self.__call('datastore_v3', 'Commit', tx, datastore_pb.CommitResponse())
def _Dynamic_GetIDs(self, request, response):
"""Fetch unique IDs for a set of paths."""
for entity in request.entity_list():
assert entity.property_size() == 0
assert entity.raw_property_size() == 0
assert entity.entity_group().element_size() == 0
lastpart = entity.key().path().element_list()[-1]
assert lastpart.id() == 0 and not lastpart.has_name()
tx = datastore_pb.Transaction()
self.__call('datastore_v3', 'BeginTransaction',
api_base_pb.VoidProto(), tx)
self.__call('datastore_v3', 'Put', request, response)
self.__call('datastore_v3', 'Rollback', tx, api_base_pb.VoidProto())
def test_rollback_transcation(self):
db_batch = flexmock()
db_batch.should_receive('valid_data_version_sync').and_return(True)
zk_client = flexmock()
zk_client.should_receive('add_listener')
zookeeper = flexmock(handle=zk_client)
zookeeper.should_receive("release_lock").and_return(True)
zookeeper.should_receive("notify_failed_transaction").and_return(True)
transaction_manager = flexmock()
dd = DatastoreDistributed(db_batch, transaction_manager, zookeeper)
commit_request = datastore_pb.Transaction()
commit_request.set_handle(123)
commit_request.set_app("aaa")
self.assertEquals(
dd.rollback_transaction("app_id", commit_request.handle()), None)
def __apply_v1_mutation(self, v1_mutation, v1_txn):
"""Applies a v1 Mutation.
Args:
v1_mutation: a googledatastore.Mutation
v1_txn: an optional v1 transaction handle or None
Returns:
a tuple (googledatastore.MutationResult, number of index writes)
"""
v3_txn = None
v3_key = None
if v1_txn:
v3_txn = datastore_pb.Transaction()
self.__service_converter.v1_to_v3_txn(v1_txn, v3_txn)
if v1_mutation.op == googledatastore.Mutation.INSERT:
v3_entity = entity_pb.EntityProto()
self.__entity_converter.v1_to_v3_entity(v1_mutation.entity,
v3_entity)
index_writes, v3_key = self.__insert_v3_entity(v3_entity, v3_txn)
elif v1_mutation.op == googledatastore.Mutation.UPDATE:
v3_entity = entity_pb.EntityProto()
self.__entity_converter.v1_to_v3_entity(v1_mutation.entity,
v3_entity)
index_writes = self.__update_v3_entity(v3_entity, v3_txn)
elif v1_mutation.op == googledatastore.Mutation.UPSERT:
v3_entity = entity_pb.EntityProto()
self.__entity_converter.v1_to_v3_entity(v1_mutation.entity,
v3_entity)
index_writes, v3_key = self.__upsert_v3_entity(v3_entity, v3_txn)
elif v1_mutation.op == googledatastore.Mutation.DELETE:
v3_ref = entity_pb.Reference()
self.__entity_converter.v1_to_v3_reference(v1_mutation.key, v3_ref)
index_writes = self.__delete_v3_reference(v3_ref, v3_txn)
v1_mutation_result = googledatastore.MutationResult()
if v3_key and not datastore_pbs.is_complete_v1_key(
v1_mutation.entity.key):
self.__entity_converter.v3_to_v1_key(v3_key,
v1_mutation_result.key)
return v1_mutation_result, index_writes
def test_commit_transaction(self):
db_batch = flexmock()
db_batch.should_receive('valid_data_version_sync').and_return(True)
zk_client = flexmock()
zk_client.should_receive('add_listener')
zookeeper = flexmock(handle=zk_client)
transaction_manager = flexmock(
delete_transaction_id=lambda project, txid: None)
dd = DatastoreDistributed(db_batch, transaction_manager, zookeeper)
flexmock(dd).should_receive('apply_txn_changes').and_return(ASYNC_NONE)
commit_request = datastore_pb.Transaction()
commit_request.set_handle(123)
commit_request.set_app("aaa")
http_request = commit_request.Encode()
result = yield dd.commit_transaction("app_id", http_request)
self.assertEquals(result, (datastore_pb.CommitResponse().Encode(), 0, ""))
def _Dynamic_GetIDs(self, request, response, is_xg=False):
"""Fetch unique IDs for a set of paths."""
for entity in request.entity_list():
assert entity.property_size() == 0
assert entity.raw_property_size() == 0
assert entity.entity_group().element_size() == 0
lastpart = entity.key().path().element_list()[-1]
assert lastpart.id() == 0 and not lastpart.has_name()
begin_request = datastore_pb.BeginTransactionRequest()
begin_request.set_app(os.environ['APPLICATION_ID'])
begin_request.set_allow_multiple_eg(is_xg)
tx = datastore_pb.Transaction()
self.__call('datastore_v3', 'BeginTransaction', begin_request, tx)
self.__call('datastore_v3', 'Put', request, response)
self.__call('datastore_v3', 'Rollback', tx, api_base_pb.VoidProto())
def commit_transaction_request(self, app_id, http_request_data):
""" Handles the commit phase of a transaction.
Args:
app_id: The application ID requesting the transaction commit.
http_request_data: The encoded request of datastore_pb.Transaction.
Returns:
An encoded protocol buffer commit response.
"""
global datastore_access
if READ_ONLY:
transaction_pb = datastore_pb.Transaction(http_request_data)
logger.warning('Unable to commit in read-only mode: {}'.format(
transaction_pb))
return ('', datastore_pb.Error.CAPABILITY_DISABLED,
'Datastore is in read-only mode.')
return datastore_access.commit_transaction(app_id, http_request_data)
def begin_transaction_request(self, app_id, http_request_data):
""" Handles the intial request to start a transaction. Replies with
a unique identifier to handle this transaction in future requests.
Args:
app_id: The application ID requesting the transaction.
http_request_data: The encoded request.
Returns:
An encoded transaction protocol buffer with a unique handler.
"""
global datastore_access
begin_transaction_req_pb = datastore_pb.BeginTransactionRequest(
http_request_data)
multiple_eg = False
if begin_transaction_req_pb.has_allow_multiple_eg():
multiple_eg = bool(begin_transaction_req_pb.allow_multiple_eg())
if READ_ONLY:
logger.warning(
'Unable to begin transaction in read-only mode: {}'.format(
begin_transaction_req_pb))
raise gen.Return(('', datastore_pb.Error.CAPABILITY_DISABLED,
'Datastore is in read-only mode.'))
try:
handle = yield datastore_access.setup_transaction(
app_id, multiple_eg)
except dbconstants.InternalError as error:
raise gen.Return(
('', datastore_pb.Error.INTERNAL_ERROR, str(error)))
except dbconstants.BadRequest as error:
raise gen.Return(('', datastore_pb.Error.BAD_REQUEST, str(error)))
except (zktransaction.ZKInternalException,
dbconstants.AppScaleDBConnectionError) as error:
logger.exception('Unable to begin transaction')
raise gen.Return(
('', datastore_pb.Error.INTERNAL_ERROR, str(error)))
transaction_pb = datastore_pb.Transaction()
transaction_pb.set_app(app_id)
transaction_pb.set_handle(handle)
raise gen.Return((transaction_pb.Encode(), 0, ''))
def __insert_v3_entity(self, v3_entity, v3_txn):
"""Inserts a v3 entity.
Args:
v3_entity: an entity_pb.EntityProto
v3_txn: a datastore_pb.Transaction or None
Returns:
a tuple (the number of index writes that occurred,
the entity key)
Raises:
ApplicationError: if the entity already exists
"""
if not v3_txn:
v3_txn = datastore_pb.Transaction()
v3_begin_txn_req = datastore_pb.BeginTransactionRequest()
v3_begin_txn_req.set_app(v3_entity.key().app())
self.__make_v3_call('BeginTransaction', v3_begin_txn_req, v3_txn)
_, key = self.__insert_v3_entity(v3_entity, v3_txn)
v3_resp = datastore_pb.CommitResponse()
self.__make_v3_call('Commit', v3_txn, v3_resp)
return (v3_resp.cost().index_writes(), key)
if datastore_pbs.is_complete_v3_key(v3_entity.key()):
v3_get_req = datastore_pb.GetRequest()
v3_get_req.mutable_transaction().CopyFrom(v3_txn)
v3_get_req.key_list().append(v3_entity.key())
v3_get_resp = datastore_pb.GetResponse()
self.__make_v3_call('Get', v3_get_req, v3_get_resp)
if v3_get_resp.entity(0).has_entity():
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.BAD_REQUEST, 'Entity already exists.')
v3_put_req = datastore_pb.PutRequest()
v3_put_req.mutable_transaction().CopyFrom(v3_txn)
v3_put_req.entity_list().append(v3_entity)
v3_put_resp = datastore_pb.PutResponse()
self.__make_v3_call('Put', v3_put_req, v3_put_resp)
return (v3_put_resp.cost().index_writes(), v3_put_resp.key(0))
def _Dynamic_Commit(self, req, resp):
try:
self.__service_validator.validate_commit_req(req)
except cloud_datastore_validator.ValidationError as e:
raise apiproxy_errors.ApplicationError(datastore_pb.Error.BAD_REQUEST,
str(e))
single_use_txn = None
if req.WhichOneof('transaction_selector') == 'single_use_transaction':
single_use_txn = self.__begin_adhoc_txn(req)
try:
try:
if req.transaction or single_use_txn:
self.__commit(req.mutations, req.transaction or single_use_txn, resp)
else:
v3_txn_req = datastore_pb.BeginTransactionRequest()
v3_txn_req.set_app(self.__app_id)
for mutation in req.mutations:
v3_txn = datastore_pb.Transaction()
self.__make_v3_call('BeginTransaction', v3_txn_req, v3_txn)
v1_txn = self.__service_converter._v3_to_v1_txn(v3_txn)
commit_resp = googledatastore.CommitResponse()
self.__commit([mutation], v1_txn, commit_resp)
resp.index_updates += commit_resp.index_updates
mutation_result = commit_resp.mutation_results[0]
resp.mutation_results.add().CopyFrom(mutation_result)
except datastore_pbs.InvalidConversionError as e:
raise apiproxy_errors.ApplicationError(datastore_pb.Error.BAD_REQUEST,
str(e))
except:
if single_use_txn:
self.__rollback_adhoc_txn(req, single_use_txn)
raise
def _Dynamic_BeginTransaction(self, req, resp):
try:
self.__service_validator.validate_begin_transaction_req(req)
v3_req = self.__service_converter.v1_to_v3_begin_transaction_req(
self.__app_id, req)
except datastore_pbs.InvalidConversionError as e:
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.BAD_REQUEST, str(e))
except cloud_datastore_validator.ValidationError as e:
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.BAD_REQUEST, str(e))
v3_resp = datastore_pb.Transaction()
self.__make_v3_call('BeginTransaction', v3_req, v3_resp)
try:
v1_resp = self.__service_converter.v3_to_v1_begin_transaction_resp(
v3_resp)
except datastore_pbs.InvalidConversionError as e:
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.INTERNAL_ERROR, str(e))
resp.CopyFrom(v1_resp)
def __update_v3_entity(self, v3_entity, v3_txn):
"""Updates a v3 entity.
Args:
v3_entity: a datastore_v4_pb.Entity
v3_txn: a datastore_pb.Transaction or None
Returns:
the number of index writes that occurred
Raises:
ApplicationError: if the entity does not exist
"""
if not v3_txn:
v3_txn = datastore_pb.Transaction()
v3_begin_txn_req = datastore_pb.BeginTransactionRequest()
v3_begin_txn_req.set_app(v3_entity.key().app())
self.__make_v3_call('BeginTransaction', v3_begin_txn_req, v3_txn)
self.__update_v3_entity(v3_entity, v3_txn)
v3_resp = datastore_pb.CommitResponse()
self.__make_v3_call('Commit', v3_txn, v3_resp)
return v3_resp.cost().index_writes()
v3_get_req = datastore_pb.GetRequest()
v3_get_req.mutable_transaction().CopyFrom(v3_txn)
v3_get_req.key_list().append(v3_entity.key())
v3_get_resp = datastore_pb.GetResponse()
self.__make_v3_call('Get', v3_get_req, v3_get_resp)
if not v3_get_resp.entity(0).has_entity():
raise apiproxy_errors.ApplicationError(
datastore_v4_pb.Error.BAD_REQUEST, 'Entity does not exist.')
v3_put_req = datastore_pb.PutRequest()
v3_put_req.mutable_transaction().CopyFrom(v3_txn)
v3_put_req.entity_list().append(v3_entity)
v3_put_resp = datastore_pb.PutResponse()
self.__make_v3_call('Put', v3_put_req, v3_put_resp)
return v3_put_resp.cost().index_writes()
def begin_transaction_request(self, app_id, http_request_data):
""" Handles the intial request to start a transaction. Replies with
a unique identifier to handle this transaction in future requests.
Args:
app_id: The application ID requesting the transaction.
http_request_data: The encoded request.
Returns:
An encoded transaction protocol buffer with a unique handler.
"""
global datastore_access
begin_transaction_req_pb = datastore_pb.BeginTransactionRequest(
http_request_data)
multiple_eg = False
if begin_transaction_req_pb.has_allow_multiple_eg():
multiple_eg = begin_transaction_req_pb.allow_multiple_eg()
handle = None
transaction_pb = datastore_pb.Transaction()
if READ_ONLY:
logger.warning(
'Unable to begin transaction in read-only mode: {}'.format(
begin_transaction_req_pb))
return (transaction_pb.Encode(),
datastore_pb.Error.CAPABILITY_DISABLED,
'Datastore is in read-only mode.')
try:
handle = datastore_access.setup_transaction(app_id, multiple_eg)
except zktransaction.ZKInternalException:
logger.exception('Unable to begin {}'.format(transaction_pb))
return (transaction_pb.Encode(), datastore_pb.Error.INTERNAL_ERROR,
"Internal error with ZooKeeper connection.")
transaction_pb.set_app(app_id)
transaction_pb.set_handle(handle)
return (transaction_pb.Encode(), 0, "")
def __get_v1_entity_version(self, v1_key, v1_txn):
"""Returns the version of an entity, or _NO_VERSION if it does not exist.
Args:
v1_key: the key of the entity to lookup.
v1_txn: the transaction to use when retrieving the entity.
Returns:
the version number of the entity if it was found, or _NO_VERSION
otherwise.
"""
v3_key = entity_pb.Reference()
self.__entity_converter.v1_to_v3_reference(v1_key, v3_key)
v3_txn = datastore_pb.Transaction()
self.__service_converter.v1_to_v3_txn(v1_txn, v3_txn)
v3_get_req = datastore_pb.GetRequest()
v3_get_req.mutable_transaction().CopyFrom(v3_txn)
v3_get_req.key_list().append(v3_key)
v3_get_resp = datastore_pb.GetResponse()
self.__make_v3_call('Get', v3_get_req, v3_get_resp)
if v3_get_resp.entity(0).has_entity():
return v3_get_resp.entity(0).version()
return _NO_VERSION
self.__service_validator = cloud_datastore_validator.get_service_validator(
)
def _Dynamic_BeginTransaction(self, req, resp):
try:
self.__service_validator.validate_begin_transaction_req(req)
v3_req = self.__service_converter.v1_to_v3_begin_transaction_req(
self.__app_id, req)
except datastore_pbs.InvalidConversionError, e:
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.BAD_REQUEST, str(e))
except cloud_datastore_validator.ValidationError, e:
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.BAD_REQUEST, str(e))
v3_resp = datastore_pb.Transaction()
self.__make_v3_call('BeginTransaction', v3_req, v3_resp)
try:
v1_resp = self.__service_converter.v3_to_v1_begin_transaction_resp(
v3_resp)
except datastore_pbs.InvalidConversionError, e:
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.INTERNAL_ERROR, str(e))
resp.CopyFrom(v1_resp)
def _Dynamic_Rollback(self, req, unused_resp):
try:
self.__service_validator.validate_rollback_req(req)
v3_req = self.__service_converter.v1_rollback_req_to_v3_txn(req)
def __commit(self, v1_mutations, v1_txn, resp):
"""Commits a list of v1 mutations.
Args:
v1_mutations: the list of mutations to apply and commit
v1_txn: required v1 transaction handle in which to apply the mutations
resp: a v1 CommitResponse to update with the result of this commit
"""
mutation_keys = []
seen_keys = set()
allocated_keys = {}
conflict_cache = {}
version_cache = {}
for i, mutation in enumerate(v1_mutations):
v1_key, v1_entity = datastore_pbs.get_v1_mutation_key_and_entity(mutation)
key = datastore_types.ReferenceToKeyValue(v1_key, self._id_resolver)
if not datastore_pbs.is_complete_v1_key(v1_key):
v1_key = self.__put_v1_entity(v1_entity, v1_txn)
key = datastore_types.ReferenceToKeyValue(v1_key, self._id_resolver)
allocated_keys[key] = v1_key
elif key not in conflict_cache:
base_version = None
if mutation.HasField('base_version') and key not in seen_keys:
base_version = mutation.base_version
conflict_version = self.__apply_v1_mutation(mutation, base_version,
v1_txn, version_cache)
if conflict_version is not None:
conflict_cache[key] = conflict_version
mutation_keys.append(key)
seen_keys.add(key)
v3_txn = datastore_pb.Transaction()
self.__service_converter.v1_to_v3_txn(v1_txn, v3_txn)
v3_resp = datastore_pb.CommitResponse()
self.__make_v3_call('Commit', v3_txn, v3_resp)
resp.index_updates = v3_resp.cost().index_writes()
mutation_versions = {}
for version in v3_resp.version_list():
key = datastore_types.ReferenceToKeyValue(version.root_entity_key())
mutation_versions[key] = version.version()
for key in mutation_keys:
mutation_result = resp.mutation_results.add()
if key in allocated_keys:
mutation_result.key.CopyFrom(allocated_keys[key])
if key in conflict_cache:
mutation_result.conflict_detected = True
mutation_result.version = conflict_cache[key]
else:
mutation_result.version = mutation_versions[key]
def __apply_v4_deprecated_mutation(self, v4_deprecated_mutation, v4_txn):
"""Applies a v4 DeprecatedMutation.
Args:
v4_deprecated_mutation: a datastore_v4_pb.DeprecatedMutation
v4_txn: an optional v4 transaction handle or None
Returns:
a datastore_v4_pb.DeprecatedMutationResult
"""
index_writes = 0
v3_txn = None
if v4_txn:
v3_txn = datastore_pb.Transaction()
self.__service_converter.v4_to_v3_txn(v4_txn, v3_txn)
for v4_entity in v4_deprecated_mutation.insert_list():
v3_entity = entity_pb.EntityProto()
self.__entity_converter.v4_to_v3_entity(v4_entity, v3_entity)
index_writes += self.__insert_v3_entity(v3_entity, v3_txn)
for v4_entity in v4_deprecated_mutation.update_list():
v3_entity = entity_pb.EntityProto()
self.__entity_converter.v4_to_v3_entity(v4_entity, v3_entity)
index_writes += self.__update_v3_entity(v3_entity, v3_txn)
v3_insert_auto_req = datastore_pb.PutRequest()
if v3_txn:
v3_insert_auto_req.mutable_transaction().CopyFrom(v3_txn)
for v4_entity in v4_deprecated_mutation.insert_auto_id_list():
v3_entity = entity_pb.EntityProto()
self.__entity_converter.v4_to_v3_entity(v4_entity, v3_entity)
v3_insert_auto_req.entity_list().append(v3_entity)
v3_insert_auto_id_resp = datastore_pb.PutResponse()
self.__make_v3_call('Put', v3_insert_auto_req, v3_insert_auto_id_resp)
index_writes += v3_insert_auto_id_resp.cost().index_writes()
v3_upsert_req = datastore_pb.PutRequest()
if v3_txn:
v3_upsert_req.mutable_transaction().CopyFrom(v3_txn)
for v4_entity in v4_deprecated_mutation.upsert_list():
v3_entity = entity_pb.EntityProto()
self.__entity_converter.v4_to_v3_entity(v4_entity, v3_entity)
v3_upsert_req.entity_list().append(v3_entity)
v3_upsert_resp = datastore_pb.PutResponse()
self.__make_v3_call('Put', v3_upsert_req, v3_upsert_resp)
index_writes += v3_upsert_resp.cost().index_writes()
v3_delete_req = datastore_pb.DeleteRequest()
if v3_txn:
v3_delete_req.mutable_transaction().CopyFrom(v3_txn)
for v4_key in v4_deprecated_mutation.delete_list():
self.__entity_converter.v4_to_v3_reference(v4_key,
v3_delete_req.add_key())
v3_delete_resp = datastore_pb.DeleteResponse()
self.__make_v3_call('Delete', v3_delete_req, v3_delete_resp)
index_writes += v3_delete_resp.cost().index_writes()
v4_deprecated_mutation_result = datastore_v4_pb.DeprecatedMutationResult(
)
for v3_ref in v3_insert_auto_id_resp.key_list():
self.__entity_converter.v3_to_v4_key(
v3_ref, v4_deprecated_mutation_result.add_insert_auto_id_key())
v4_deprecated_mutation_result.set_index_updates(index_writes)
return v4_deprecated_mutation_result
def commit(self, txid):
request = datastore_pb.Transaction()
request.set_app(self.project_id)
request.set_handle(txid)
yield self._make_request('Commit', request.Encode())
