def validate_commit_req(self, req):
"""Validates a normalized CommitRequest.
Args:
req: a googledatastore.CommitRequest
Raises:
ValidationError: if the request is invalid
"""
_assert_initialized(req)
if (req.mode == googledatastore.CommitRequest.MODE_UNSPECIFIED
or req.mode == googledatastore.CommitRequest.TRANSACTIONAL):
_assert_condition(req.WhichOneof('transaction_selector'),
'Transactional commit requires a transaction.')
if req.WhichOneof('transaction_selector') == 'transaction':
_assert_condition(
req.transaction, 'a transaction cannot be the empty '
'string.')
seen_base_versions = {}
for mutation in req.mutations:
v1_key, _ = datastore_pbs.get_v1_mutation_key_and_entity(
mutation)
if datastore_pbs.is_complete_v1_key(v1_key):
mutation_base_version = None
if mutation.HasField('base_version'):
mutation_base_version = mutation.base_version
key = datastore_types.ReferenceToKeyValue(
v1_key, self.__id_resolver)
if key in seen_base_versions:
_assert_condition(
seen_base_versions[key] == mutation_base_version,
'Mutations for the same entity must have the '
'same base version.')
seen_base_versions[key] = mutation_base_version
elif req.mode == googledatastore.CommitRequest.NON_TRANSACTIONAL:
_assert_condition(
not req.WhichOneof('transaction_selector'),
'Non-transactional commit cannot specify a '
'transaction.')
seen_complete_keys = set()
for mutation in req.mutations:
v1_key, _ = datastore_pbs.get_v1_mutation_key_and_entity(
mutation)
if datastore_pbs.is_complete_v1_key(v1_key):
key = datastore_types.ReferenceToKeyValue(
v1_key, self.__id_resolver)
_assert_condition(
key not in seen_complete_keys,
'A non-transactional commit may not contain '
'multiple mutations affecting the same entity.')
seen_complete_keys.add(key)
for mutation in req.mutations:
self.__validate_mutation(mutation)
def validate_commit_req(self, req):
"""Validates a normalized CommitRequest.
Args:
req: a googledatastore.CommitRequest
Raises:
ValidationError: if the request is invalid
"""
_assert_initialized(req)
if (req.mode == googledatastore.CommitRequest.MODE_UNSPECIFIED or
req.mode == googledatastore.CommitRequest.TRANSACTIONAL):
_assert_condition(req.WhichOneof('transaction_selector'),
'Transactional commit requires a transaction.')
if req.WhichOneof('transaction_selector') == 'transaction':
_assert_condition(req.transaction, 'a transaction cannot be the empty '
'string.')
seen_base_versions = {}
for mutation in req.mutations:
v1_key, _ = datastore_pbs.get_v1_mutation_key_and_entity(mutation)
if datastore_pbs.is_complete_v1_key(v1_key):
mutation_base_version = None
if mutation.HasField('base_version'):
mutation_base_version = mutation.base_version
key = datastore_types.ReferenceToKeyValue(v1_key, self.__id_resolver)
if key in seen_base_versions:
_assert_condition(seen_base_versions[key] == mutation_base_version,
'Mutations for the same entity must have the '
'same base version.')
seen_base_versions[key] = mutation_base_version
elif req.mode == googledatastore.CommitRequest.NON_TRANSACTIONAL:
_assert_condition(not req.WhichOneof('transaction_selector'),
'Non-transactional commit cannot specify a '
'transaction.')
seen_complete_keys = set()
for mutation in req.mutations:
v1_key, _ = datastore_pbs.get_v1_mutation_key_and_entity(mutation)
if datastore_pbs.is_complete_v1_key(v1_key):
key = datastore_types.ReferenceToKeyValue(v1_key, self.__id_resolver)
_assert_condition(key not in seen_complete_keys,
'A non-transactional commit may not contain '
'multiple mutations affecting the same entity.')
seen_complete_keys.add(key)
for mutation in req.mutations:
self.__validate_mutation(mutation)
def __validate_mutation(self, mutation):
if mutation.HasField('insert'):
self.__entity_validator.validate_entity(UPSERT, mutation.insert)
mutation_key = mutation.insert.key
elif mutation.HasField('update'):
self.__entity_validator.validate_entity(UPDATE, mutation.update)
mutation_key = mutation.update.key
elif mutation.HasField('upsert'):
self.__entity_validator.validate_entity(UPSERT, mutation.upsert)
mutation_key = mutation.upsert.key
elif mutation.HasField('delete'):
self.__entity_validator.validate_key(DELETE, mutation.delete)
mutation_key = mutation.delete
else:
_assert_condition(False, 'mutation lacks required op')
if mutation.WhichOneof('conflict_detection_strategy') != None:
_assert_condition(
datastore_pbs.is_complete_v1_key(mutation_key),
'conflict detection is not allowed for incomplete keys')
if mutation.HasField('base_version'):
_assert_condition(
mutation.base_version >= 0, 'Invalid base_version: %d, '
'it should be >= 0' % mutation.base_version)
def __commit(self, v1_mutations, v1_txn, resp):
"""Commits a list of v1 mutations.
Args:
v1_mutations: the list of mutations to apply and commit
v1_txn: required v1 transaction handle in which to apply the mutations
resp: a v1 CommitResponse to update with the result of this commit
"""
mutation_keys = []
seen_keys = set()
allocated_keys = {}
conflict_cache = {}
version_cache = {}
for i, mutation in enumerate(v1_mutations):
v1_key, v1_entity = datastore_pbs.get_v1_mutation_key_and_entity(mutation)
key = datastore_types.ReferenceToKeyValue(v1_key, self._id_resolver)
if not datastore_pbs.is_complete_v1_key(v1_key):
v1_key = self.__put_v1_entity(v1_entity, v1_txn)
key = datastore_types.ReferenceToKeyValue(v1_key, self._id_resolver)
allocated_keys[key] = v1_key
elif key not in conflict_cache:
base_version = None
if mutation.HasField("base_version") and key not in seen_keys:
base_version = mutation.base_version
conflict_version = self.__apply_v1_mutation(mutation, base_version, v1_txn, version_cache)
if conflict_version is not None:
conflict_cache[key] = conflict_version
mutation_keys.append(key)
seen_keys.add(key)
v3_txn = datastore_pb.Transaction()
self.__service_converter.v1_to_v3_txn(v1_txn, v3_txn)
v3_resp = datastore_pb.CommitResponse()
self.__make_v3_call("Commit", v3_txn, v3_resp)
resp.index_updates = v3_resp.cost().index_writes()
mutation_versions = {}
for version in v3_resp.version_list():
key = datastore_types.ReferenceToKeyValue(version.root_entity_key())
mutation_versions[key] = version.version()
for key in mutation_keys:
mutation_result = resp.mutation_results.add()
if key in allocated_keys:
mutation_result.key.CopyFrom(allocated_keys[key])
if key in conflict_cache:
mutation_result.conflict_detected = True
mutation_result.version = conflict_cache[key]
else:
mutation_result.version = mutation_versions[key]
def __apply_v1_mutation(self, v1_mutation, v1_txn):
"""Applies a v1 Mutation.
Args:
v1_mutation: a googledatastore.Mutation
v1_txn: an optional v1 transaction handle or None
Returns:
a tuple (googledatastore.MutationResult, number of index writes)
"""
v3_txn = None
v3_key = None
if v1_txn:
v3_txn = datastore_pb.Transaction()
self.__service_converter.v1_to_v3_txn(v1_txn, v3_txn)
if v1_mutation.HasField('insert'):
v3_entity = entity_pb.EntityProto()
v1_entity = v1_mutation.insert
self.__entity_converter.v1_to_v3_entity(v1_entity, v3_entity)
index_writes, v3_key = self.__insert_v3_entity(v3_entity, v3_txn)
elif v1_mutation.HasField('update'):
v3_entity = entity_pb.EntityProto()
self.__entity_converter.v1_to_v3_entity(v1_mutation.update,
v3_entity)
index_writes = self.__update_v3_entity(v3_entity, v3_txn)
elif v1_mutation.HasField('upsert'):
v3_entity = entity_pb.EntityProto()
v1_entity = v1_mutation.upsert
self.__entity_converter.v1_to_v3_entity(v1_entity, v3_entity)
index_writes, v3_key = self.__upsert_v3_entity(v3_entity, v3_txn)
elif v1_mutation.HasField('delete'):
v3_ref = entity_pb.Reference()
self.__entity_converter.v1_to_v3_reference(v1_mutation.delete,
v3_ref)
index_writes = self.__delete_v3_reference(v3_ref, v3_txn)
v1_mutation_result = googledatastore.MutationResult()
if v3_key and not datastore_pbs.is_complete_v1_key(v1_entity.key):
self.__entity_converter.v3_to_v1_key(v3_key, v1_mutation_result.key)
return v1_mutation_result, index_writes
def __apply_v1_mutation(self, v1_mutation, v1_txn):
"""Applies a v1 Mutation.
Args:
v1_mutation: a googledatastore.Mutation
v1_txn: an optional v1 transaction handle or None
Returns:
a tuple (googledatastore.MutationResult, number of index writes)
"""
v3_txn = None
v3_key = None
if v1_txn:
v3_txn = datastore_pb.Transaction()
self.__service_converter.v1_to_v3_txn(v1_txn, v3_txn)
if v1_mutation.op == googledatastore.Mutation.INSERT:
v3_entity = entity_pb.EntityProto()
self.__entity_converter.v1_to_v3_entity(v1_mutation.entity,
v3_entity)
index_writes, v3_key = self.__insert_v3_entity(v3_entity, v3_txn)
elif v1_mutation.op == googledatastore.Mutation.UPDATE:
v3_entity = entity_pb.EntityProto()
self.__entity_converter.v1_to_v3_entity(v1_mutation.entity,
v3_entity)
index_writes = self.__update_v3_entity(v3_entity, v3_txn)
elif v1_mutation.op == googledatastore.Mutation.UPSERT:
v3_entity = entity_pb.EntityProto()
self.__entity_converter.v1_to_v3_entity(v1_mutation.entity,
v3_entity)
index_writes, v3_key = self.__upsert_v3_entity(v3_entity, v3_txn)
elif v1_mutation.op == googledatastore.Mutation.DELETE:
v3_ref = entity_pb.Reference()
self.__entity_converter.v1_to_v3_reference(v1_mutation.key, v3_ref)
index_writes = self.__delete_v3_reference(v3_ref, v3_txn)
v1_mutation_result = googledatastore.MutationResult()
if v3_key and not datastore_pbs.is_complete_v1_key(
v1_mutation.entity.key):
self.__entity_converter.v3_to_v1_key(v3_key,
v1_mutation_result.key)
return v1_mutation_result, index_writes
def __validate_mutation(self, mutation):
if mutation.HasField('insert'):
self.__entity_validator.validate_entity(UPSERT, mutation.insert)
mutation_key = mutation.insert.key
elif mutation.HasField('update'):
self.__entity_validator.validate_entity(UPDATE, mutation.update)
mutation_key = mutation.update.key
elif mutation.HasField('upsert'):
self.__entity_validator.validate_entity(UPSERT, mutation.upsert)
mutation_key = mutation.upsert.key
elif mutation.HasField('delete'):
self.__entity_validator.validate_key(DELETE, mutation.delete)
mutation_key = mutation.delete
else:
_assert_condition(False, 'mutation lacks required op')
if mutation.WhichOneof('conflict_detection_strategy') != None:
_assert_condition(datastore_pbs.is_complete_v1_key(mutation_key),
'conflict detection is not allowed for incomplete keys')
if mutation.HasField('base_version'):
_assert_condition(mutation.base_version >= 0,
'Invalid base_version: %d, '
'it should be >= 0' % mutation.base_version)
def __commit(self, v1_mutations, v1_txn, resp):
"""Commits a list of v1 mutations.
Args:
v1_mutations: the list of mutations to apply and commit
v1_txn: required v1 transaction handle in which to apply the mutations
resp: a v1 CommitResponse to update with the result of this commit
"""
mutation_keys = []
seen_keys = set()
allocated_keys = {}
conflict_cache = {}
version_cache = {}
for i, mutation in enumerate(v1_mutations):
v1_key, v1_entity = datastore_pbs.get_v1_mutation_key_and_entity(mutation)
key = datastore_types.ReferenceToKeyValue(v1_key, self._id_resolver)
if not datastore_pbs.is_complete_v1_key(v1_key):
v1_key = self.__put_v1_entity(v1_entity, v1_txn)
key = datastore_types.ReferenceToKeyValue(v1_key, self._id_resolver)
allocated_keys[key] = v1_key
elif key not in conflict_cache:
base_version = None
if mutation.HasField('base_version') and key not in seen_keys:
base_version = mutation.base_version
conflict_version = self.__apply_v1_mutation(mutation, base_version,
v1_txn, version_cache)
if conflict_version is not None:
conflict_cache[key] = conflict_version
mutation_keys.append(key)
seen_keys.add(key)
v3_txn = datastore_pb.Transaction()
self.__service_converter.v1_to_v3_txn(v1_txn, v3_txn)
v3_resp = datastore_pb.CommitResponse()
self.__make_v3_call('Commit', v3_txn, v3_resp)
resp.index_updates = v3_resp.cost().index_writes()
mutation_versions = {}
for version in v3_resp.version_list():
key = datastore_types.ReferenceToKeyValue(version.root_entity_key())
mutation_versions[key] = version.version()
for key in mutation_keys:
mutation_result = resp.mutation_results.add()
if key in allocated_keys:
mutation_result.key.CopyFrom(allocated_keys[key])
if key in conflict_cache:
mutation_result.conflict_detected = True
mutation_result.version = conflict_cache[key]
else:
mutation_result.version = mutation_versions[key]
