def _Dynamic_Commit(self, req, resp):
try:
self.__service_validator.validate_commit_req(req)
if req.has_transaction():
resp.mutable_deprecated_mutation_result()
resp.mutable_deprecated_mutation_result().CopyFrom(
self.__apply_v4_deprecated_mutation(
req.deprecated_mutation(), req.transaction()))
v3_req = self.__service_converter.v4_commit_req_to_v3_txn(req)
v3_resp = datastore_pb.CommitResponse()
self.__make_v3_call('Commit', v3_req, v3_resp)
total_index_updates = (
resp.mutable_deprecated_mutation_result().index_updates() +
v3_resp.cost().index_writes())
resp.mutable_deprecated_mutation_result().set_index_updates(
total_index_updates)
else:
resp.mutable_deprecated_mutation_result().CopyFrom(
self.__apply_v4_deprecated_mutation(
req.deprecated_mutation(), None))
except datastore_pbs.InvalidConversionError as e:
raise apiproxy_errors.ApplicationError(
datastore_v4_pb.Error.BAD_REQUEST, str(e))
except datastore_v4_validator.ValidationError as e:
raise apiproxy_errors.ApplicationError(
datastore_v4_pb.Error.BAD_REQUEST, str(e))
def _Dynamic_Commit(self, req, resp):
try:
self.__service_validator.validate_commit_req(req)
except cloud_datastore_validator.ValidationError as e:
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.BAD_REQUEST, str(e))
try:
txn = None
if req.transaction:
txn = req.transaction
total_index_writes = 0
for mutation in req.mutations:
mutation_result, index_writes = (self.__apply_v1_mutation(
mutation, req.transaction))
resp.mutation_results.add().CopyFrom(mutation_result)
total_index_writes += index_writes
if txn:
v3_req = self.__service_converter.v1_commit_req_to_v3_txn(req)
v3_resp = datastore_pb.CommitResponse()
self.__make_v3_call('Commit', v3_req, v3_resp)
total_index_writes += v3_resp.cost().index_writes()
resp.index_updates = total_index_writes
except datastore_pbs.InvalidConversionError as e:
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.BAD_REQUEST, str(e))
def _Dynamic_Transaction(self, request, response):
"""Handle a Transaction request.
We handle transactions by accumulating Put and Delete requests on the client
end, as well as recording the key and hash of Get requests. When Commit is
called, Transaction is invoked, which verifies that all the entities in the
precondition list still exist and their hashes match, then performs a
transaction of its own to make the updates.
"""
begin_request = datastore_pb.BeginTransactionRequest()
begin_request.set_app(os.environ['APPLICATION_ID'])
begin_request.set_allow_multiple_eg(request.allow_multiple_eg())
tx = datastore_pb.Transaction()
self.__call('datastore_v3', 'BeginTransaction', begin_request, tx)
preconditions = request.precondition_list()
if preconditions:
get_request = datastore_pb.GetRequest()
get_request.mutable_transaction().CopyFrom(tx)
for precondition in preconditions:
key = get_request.add_key()
key.CopyFrom(precondition.key())
get_response = datastore_pb.GetResponse()
self.__call('datastore_v3', 'Get', get_request, get_response)
entities = get_response.entity_list()
assert len(entities) == request.precondition_size()
for precondition, entity in zip(preconditions, entities):
if precondition.has_hash() != entity.has_entity():
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.CONCURRENT_TRANSACTION,
"Transaction precondition failed.")
elif entity.has_entity():
entity_hash = hashlib.sha1(entity.entity().Encode()).digest()
if precondition.hash() != entity_hash:
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.CONCURRENT_TRANSACTION,
"Transaction precondition failed.")
if request.has_puts():
put_request = request.puts()
put_request.mutable_transaction().CopyFrom(tx)
self.__call('datastore_v3', 'Put', put_request, response)
if request.has_deletes():
delete_request = request.deletes()
delete_request.mutable_transaction().CopyFrom(tx)
self.__call('datastore_v3', 'Delete', delete_request,
datastore_pb.DeleteResponse())
self.__call('datastore_v3', 'Commit', tx, datastore_pb.CommitResponse())
def test_commit_transaction(self):
db_batch = flexmock()
zookeeper = flexmock()
zookeeper.should_receive("release_lock").and_return(True)
dd = DatastoreDistributed(db_batch, zookeeper)
commit_request = datastore_pb.Transaction()
commit_request.set_handle(123)
commit_request.set_app("aaa")
http_request = commit_request.Encode()
self.assertEquals(dd.commit_transaction("app_id", http_request),
(datastore_pb.CommitResponse().Encode(), 0, ""))
def test_commit_transaction(self):
db_batch = flexmock()
db_batch.should_receive('valid_data_version').and_return(True)
zookeeper = flexmock()
zookeeper.should_receive('remove_tx_node')
dd = DatastoreDistributed(db_batch, zookeeper)
flexmock(dd).should_receive('apply_txn_changes')
commit_request = datastore_pb.Transaction()
commit_request.set_handle(123)
commit_request.set_app("aaa")
http_request = commit_request.Encode()
self.assertEquals(dd.commit_transaction("app_id", http_request),
(datastore_pb.CommitResponse().Encode(), 0, ""))
def _Dynamic_TransactionQuery(self, request, response):
if not request.has_ancestor():
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.BAD_REQUEST,
'No ancestor in transactional query.')
app_id = datastore_types.ResolveAppId(None)
if (datastore_rpc._GetDatastoreType(app_id) !=
datastore_rpc.BaseConnection.HIGH_REPLICATION_DATASTORE):
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.BAD_REQUEST,
'remote_api supports transactional queries only in the '
'high-replication datastore.')
entity_group_key = entity_pb.Reference()
entity_group_key.CopyFrom(request.ancestor())
group_path = entity_group_key.mutable_path()
root = entity_pb.Path_Element()
root.MergeFrom(group_path.element(0))
group_path.clear_element()
group_path.add_element().CopyFrom(root)
eg_element = group_path.add_element()
eg_element.set_type(metadata.EntityGroup.KIND_NAME)
eg_element.set_id(metadata.EntityGroup.ID)
begin_request = datastore_pb.BeginTransactionRequest()
begin_request.set_app(app_id)
tx = datastore_pb.Transaction()
self.__call('datastore_v3', 'BeginTransaction', begin_request, tx)
request.mutable_transaction().CopyFrom(tx)
self.__call('datastore_v3', 'RunQuery', request, response.mutable_result())
get_request = datastore_pb.GetRequest()
get_request.mutable_transaction().CopyFrom(tx)
get_request.add_key().CopyFrom(entity_group_key)
get_response = datastore_pb.GetResponse()
self.__call('datastore_v3', 'Get', get_request, get_response)
entity_group = get_response.entity(0)
response.mutable_entity_group_key().CopyFrom(entity_group_key)
if entity_group.has_entity():
response.mutable_entity_group().CopyFrom(entity_group.entity())
self.__call('datastore_v3', 'Commit', tx, datastore_pb.CommitResponse())
def test_commit_transaction(self):
db_batch = flexmock()
db_batch.should_receive('valid_data_version_sync').and_return(True)
zk_client = flexmock()
zk_client.should_receive('add_listener')
zookeeper = flexmock(handle=zk_client)
transaction_manager = flexmock(
delete_transaction_id=lambda project, txid: None)
dd = DatastoreDistributed(db_batch, transaction_manager, zookeeper)
flexmock(dd).should_receive('apply_txn_changes').and_return(ASYNC_NONE)
commit_request = datastore_pb.Transaction()
commit_request.set_handle(123)
commit_request.set_app("aaa")
http_request = commit_request.Encode()
result = yield dd.commit_transaction("app_id", http_request)
self.assertEquals(result, (datastore_pb.CommitResponse().Encode(), 0, ""))
def commit_transaction_request(self, app_id, http_request_data):
""" Handles the commit phase of a transaction.
Args:
app_id: The application ID requesting the transaction commit.
http_request_data: The encoded request of datastore_pb.Transaction.
Returns:
An encoded protocol buffer commit response.
"""
global datastore_access
if READ_ONLY:
commitres_pb = datastore_pb.CommitResponse()
transaction_pb = datastore_pb.Transaction(http_request_data)
logger.warning('Unable to commit in read-only mode: {}'.
format(transaction_pb))
return (commitres_pb.Encode(), datastore_pb.Error.CAPABILITY_DISABLED,
'Datastore is in read-only mode.')
return datastore_access.commit_transaction(app_id, http_request_data)
def __insert_v3_entity(self, v3_entity, v3_txn):
"""Inserts a v3 entity.
Args:
v3_entity: an entity_pb.EntityProto
v3_txn: a datastore_pb.Transaction or None
Returns:
a tuple (the number of index writes that occurred,
the entity key)
Raises:
ApplicationError: if the entity already exists
"""
if not v3_txn:
v3_txn = datastore_pb.Transaction()
v3_begin_txn_req = datastore_pb.BeginTransactionRequest()
v3_begin_txn_req.set_app(v3_entity.key().app())
self.__make_v3_call('BeginTransaction', v3_begin_txn_req, v3_txn)
_, key = self.__insert_v3_entity(v3_entity, v3_txn)
v3_resp = datastore_pb.CommitResponse()
self.__make_v3_call('Commit', v3_txn, v3_resp)
return (v3_resp.cost().index_writes(), key)
if datastore_pbs.is_complete_v3_key(v3_entity.key()):
v3_get_req = datastore_pb.GetRequest()
v3_get_req.mutable_transaction().CopyFrom(v3_txn)
v3_get_req.key_list().append(v3_entity.key())
v3_get_resp = datastore_pb.GetResponse()
self.__make_v3_call('Get', v3_get_req, v3_get_resp)
if v3_get_resp.entity(0).has_entity():
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.BAD_REQUEST, 'Entity already exists.')
v3_put_req = datastore_pb.PutRequest()
v3_put_req.mutable_transaction().CopyFrom(v3_txn)
v3_put_req.entity_list().append(v3_entity)
v3_put_resp = datastore_pb.PutResponse()
self.__make_v3_call('Put', v3_put_req, v3_put_resp)
return (v3_put_resp.cost().index_writes(), v3_put_resp.key(0))
def commit_transaction_request(self, app_id, http_request_data):
""" Handles the commit phase of a transaction.
Args:
app_id: The application ID requesting the transaction commit.
http_request_data: The encoded request of datastore_pb.Transaction.
Returns:
An encoded protocol buffer commit response.
"""
global datastore_access
if READ_ONLY:
transaction_pb = datastore_pb.Transaction(http_request_data)
logger.warning('Unable to commit in read-only mode: {}'.
format(transaction_pb))
raise gen.Return(
('', datastore_pb.Error.CAPABILITY_DISABLED,
'Datastore is in read-only mode.'))
txid = datastore_pb.Transaction(http_request_data).handle()
try:
yield datastore_access.apply_txn_changes(app_id, txid)
except (dbconstants.TxTimeoutException, dbconstants.Timeout) as timeout:
raise gen.Return(('', datastore_pb.Error.TIMEOUT, str(timeout)))
except dbconstants.AppScaleDBConnectionError:
logger.exception('DB connection error during commit')
raise gen.Return(
('', datastore_pb.Error.INTERNAL_ERROR,
'Datastore connection error on Commit request.'))
except dbconstants.ConcurrentModificationException as error:
raise gen.Return(
('', datastore_pb.Error.CONCURRENT_TRANSACTION, str(error)))
except (dbconstants.TooManyGroupsException,
dbconstants.BadRequest) as error:
raise gen.Return(('', datastore_pb.Error.BAD_REQUEST, str(error)))
commitres_pb = datastore_pb.CommitResponse()
raise gen.Return((commitres_pb.Encode(), 0, ''))
def __update_v3_entity(self, v3_entity, v3_txn):
"""Updates a v3 entity.
Args:
v3_entity: a datastore_v4_pb.Entity
v3_txn: a datastore_pb.Transaction or None
Returns:
the number of index writes that occurred
Raises:
ApplicationError: if the entity does not exist
"""
if not v3_txn:
v3_txn = datastore_pb.Transaction()
v3_begin_txn_req = datastore_pb.BeginTransactionRequest()
v3_begin_txn_req.set_app(v3_entity.key().app())
self.__make_v3_call('BeginTransaction', v3_begin_txn_req, v3_txn)
self.__update_v3_entity(v3_entity, v3_txn)
v3_resp = datastore_pb.CommitResponse()
self.__make_v3_call('Commit', v3_txn, v3_resp)
return v3_resp.cost().index_writes()
v3_get_req = datastore_pb.GetRequest()
v3_get_req.mutable_transaction().CopyFrom(v3_txn)
v3_get_req.key_list().append(v3_entity.key())
v3_get_resp = datastore_pb.GetResponse()
self.__make_v3_call('Get', v3_get_req, v3_get_resp)
if not v3_get_resp.entity(0).has_entity():
raise apiproxy_errors.ApplicationError(
datastore_v4_pb.Error.BAD_REQUEST, 'Entity does not exist.')
v3_put_req = datastore_pb.PutRequest()
v3_put_req.mutable_transaction().CopyFrom(v3_txn)
v3_put_req.entity_list().append(v3_entity)
v3_put_resp = datastore_pb.PutResponse()
self.__make_v3_call('Put', v3_put_req, v3_put_resp)
return v3_put_resp.cost().index_writes()
except cloud_datastore_validator.ValidationError, e:
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.BAD_REQUEST, str(e))
try:
txn = None
if req.transaction:
txn = req.transaction
total_index_writes = 0
for mutation in req.mutations:
mutation_result, index_writes = (self.__apply_v1_mutation(
mutation, req.transaction))
resp.mutation_results.add().CopyFrom(mutation_result)
total_index_writes += index_writes
if txn:
v3_req = self.__service_converter.v1_commit_req_to_v3_txn(req)
v3_resp = datastore_pb.CommitResponse()
self.__make_v3_call('Commit', v3_req, v3_resp)
total_index_writes += v3_resp.cost().index_writes()
resp.index_updates = total_index_writes
except datastore_pbs.InvalidConversionError, e:
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.BAD_REQUEST, str(e))
def _Dynamic_RunQuery(self, req, resp):
self.__normalize_v1_run_query_request(req)
try:
self.__service_validator.validate_run_query_req(req)
v3_req = self.__service_converter.v1_run_query_req_to_v3_query(req)
except datastore_pbs.InvalidConversionError, e:
def __commit(self, v1_mutations, v1_txn, resp):
"""Commits a list of v1 mutations.
Args:
v1_mutations: the list of mutations to apply and commit
v1_txn: required v1 transaction handle in which to apply the mutations
resp: a v1 CommitResponse to update with the result of this commit
"""
mutation_keys = []
seen_keys = set()
allocated_keys = {}
conflict_cache = {}
version_cache = {}
for i, mutation in enumerate(v1_mutations):
v1_key, v1_entity = datastore_pbs.get_v1_mutation_key_and_entity(mutation)
key = datastore_types.ReferenceToKeyValue(v1_key, self._id_resolver)
if not datastore_pbs.is_complete_v1_key(v1_key):
v1_key = self.__put_v1_entity(v1_entity, v1_txn)
key = datastore_types.ReferenceToKeyValue(v1_key, self._id_resolver)
allocated_keys[key] = v1_key
elif key not in conflict_cache:
base_version = None
if mutation.HasField('base_version') and key not in seen_keys:
base_version = mutation.base_version
conflict_version = self.__apply_v1_mutation(mutation, base_version,
v1_txn, version_cache)
if conflict_version is not None:
conflict_cache[key] = conflict_version
mutation_keys.append(key)
seen_keys.add(key)
v3_txn = datastore_pb.Transaction()
self.__service_converter.v1_to_v3_txn(v1_txn, v3_txn)
v3_resp = datastore_pb.CommitResponse()
self.__make_v3_call('Commit', v3_txn, v3_resp)
resp.index_updates = v3_resp.cost().index_writes()
mutation_versions = {}
for version in v3_resp.version_list():
key = datastore_types.ReferenceToKeyValue(version.root_entity_key())
mutation_versions[key] = version.version()
for key in mutation_keys:
mutation_result = resp.mutation_results.add()
if key in allocated_keys:
mutation_result.key.CopyFrom(allocated_keys[key])
if key in conflict_cache:
mutation_result.conflict_detected = True
mutation_result.version = conflict_cache[key]
else:
mutation_result.version = mutation_versions[key]
