def validate_commit_req(self, req): """Validates a normalized CommitRequest. Args: req: a googledatastore.CommitRequest Raises: ValidationError: if the request is invalid """ _assert_initialized(req) if (req.mode == googledatastore.CommitRequest.MODE_UNSPECIFIED or req.mode == googledatastore.CommitRequest.TRANSACTIONAL): _assert_condition(req.WhichOneof('transaction_selector'), 'Transactional commit requires a transaction.') if req.WhichOneof('transaction_selector') == 'transaction': _assert_condition( req.transaction, 'a transaction cannot be the empty ' 'string.') seen_base_versions = {} for mutation in req.mutations: v1_key, _ = datastore_pbs.get_v1_mutation_key_and_entity( mutation) if datastore_pbs.is_complete_v1_key(v1_key): mutation_base_version = None if mutation.HasField('base_version'): mutation_base_version = mutation.base_version key = datastore_types.ReferenceToKeyValue( v1_key, self.__id_resolver) if key in seen_base_versions: _assert_condition( seen_base_versions[key] == mutation_base_version, 'Mutations for the same entity must have the ' 'same base version.') seen_base_versions[key] = mutation_base_version elif req.mode == googledatastore.CommitRequest.NON_TRANSACTIONAL: _assert_condition( not req.WhichOneof('transaction_selector'), 'Non-transactional commit cannot specify a ' 'transaction.') seen_complete_keys = set() for mutation in req.mutations: v1_key, _ = datastore_pbs.get_v1_mutation_key_and_entity( mutation) if datastore_pbs.is_complete_v1_key(v1_key): key = datastore_types.ReferenceToKeyValue( v1_key, self.__id_resolver) _assert_condition( key not in seen_complete_keys, 'A non-transactional commit may not contain ' 'multiple mutations affecting the same entity.') seen_complete_keys.add(key) for mutation in req.mutations: self.__validate_mutation(mutation)
def validate_commit_req(self, req): """Validates a normalized CommitRequest. Args: req: a googledatastore.CommitRequest Raises: ValidationError: if the request is invalid """ _assert_initialized(req) if (req.mode == googledatastore.CommitRequest.MODE_UNSPECIFIED or req.mode == googledatastore.CommitRequest.TRANSACTIONAL): _assert_condition(req.WhichOneof('transaction_selector'), 'Transactional commit requires a transaction.') if req.WhichOneof('transaction_selector') == 'transaction': _assert_condition(req.transaction, 'a transaction cannot be the empty ' 'string.') seen_base_versions = {} for mutation in req.mutations: v1_key, _ = datastore_pbs.get_v1_mutation_key_and_entity(mutation) if datastore_pbs.is_complete_v1_key(v1_key): mutation_base_version = None if mutation.HasField('base_version'): mutation_base_version = mutation.base_version key = datastore_types.ReferenceToKeyValue(v1_key, self.__id_resolver) if key in seen_base_versions: _assert_condition(seen_base_versions[key] == mutation_base_version, 'Mutations for the same entity must have the ' 'same base version.') seen_base_versions[key] = mutation_base_version elif req.mode == googledatastore.CommitRequest.NON_TRANSACTIONAL: _assert_condition(not req.WhichOneof('transaction_selector'), 'Non-transactional commit cannot specify a ' 'transaction.') seen_complete_keys = set() for mutation in req.mutations: v1_key, _ = datastore_pbs.get_v1_mutation_key_and_entity(mutation) if datastore_pbs.is_complete_v1_key(v1_key): key = datastore_types.ReferenceToKeyValue(v1_key, self.__id_resolver) _assert_condition(key not in seen_complete_keys, 'A non-transactional commit may not contain ' 'multiple mutations affecting the same entity.') seen_complete_keys.add(key) for mutation in req.mutations: self.__validate_mutation(mutation)
def __validate_mutation(self, mutation): if mutation.HasField('insert'): self.__entity_validator.validate_entity(UPSERT, mutation.insert) mutation_key = mutation.insert.key elif mutation.HasField('update'): self.__entity_validator.validate_entity(UPDATE, mutation.update) mutation_key = mutation.update.key elif mutation.HasField('upsert'): self.__entity_validator.validate_entity(UPSERT, mutation.upsert) mutation_key = mutation.upsert.key elif mutation.HasField('delete'): self.__entity_validator.validate_key(DELETE, mutation.delete) mutation_key = mutation.delete else: _assert_condition(False, 'mutation lacks required op') if mutation.WhichOneof('conflict_detection_strategy') != None: _assert_condition( datastore_pbs.is_complete_v1_key(mutation_key), 'conflict detection is not allowed for incomplete keys') if mutation.HasField('base_version'): _assert_condition( mutation.base_version >= 0, 'Invalid base_version: %d, ' 'it should be >= 0' % mutation.base_version)
def __commit(self, v1_mutations, v1_txn, resp): """Commits a list of v1 mutations. Args: v1_mutations: the list of mutations to apply and commit v1_txn: required v1 transaction handle in which to apply the mutations resp: a v1 CommitResponse to update with the result of this commit """ mutation_keys = [] seen_keys = set() allocated_keys = {} conflict_cache = {} version_cache = {} for i, mutation in enumerate(v1_mutations): v1_key, v1_entity = datastore_pbs.get_v1_mutation_key_and_entity(mutation) key = datastore_types.ReferenceToKeyValue(v1_key, self._id_resolver) if not datastore_pbs.is_complete_v1_key(v1_key): v1_key = self.__put_v1_entity(v1_entity, v1_txn) key = datastore_types.ReferenceToKeyValue(v1_key, self._id_resolver) allocated_keys[key] = v1_key elif key not in conflict_cache: base_version = None if mutation.HasField("base_version") and key not in seen_keys: base_version = mutation.base_version conflict_version = self.__apply_v1_mutation(mutation, base_version, v1_txn, version_cache) if conflict_version is not None: conflict_cache[key] = conflict_version mutation_keys.append(key) seen_keys.add(key) v3_txn = datastore_pb.Transaction() self.__service_converter.v1_to_v3_txn(v1_txn, v3_txn) v3_resp = datastore_pb.CommitResponse() self.__make_v3_call("Commit", v3_txn, v3_resp) resp.index_updates = v3_resp.cost().index_writes() mutation_versions = {} for version in v3_resp.version_list(): key = datastore_types.ReferenceToKeyValue(version.root_entity_key()) mutation_versions[key] = version.version() for key in mutation_keys: mutation_result = resp.mutation_results.add() if key in allocated_keys: mutation_result.key.CopyFrom(allocated_keys[key]) if key in conflict_cache: mutation_result.conflict_detected = True mutation_result.version = conflict_cache[key] else: mutation_result.version = mutation_versions[key]
def __apply_v1_mutation(self, v1_mutation, v1_txn): """Applies a v1 Mutation. Args: v1_mutation: a googledatastore.Mutation v1_txn: an optional v1 transaction handle or None Returns: a tuple (googledatastore.MutationResult, number of index writes) """ v3_txn = None v3_key = None if v1_txn: v3_txn = datastore_pb.Transaction() self.__service_converter.v1_to_v3_txn(v1_txn, v3_txn) if v1_mutation.HasField('insert'): v3_entity = entity_pb.EntityProto() v1_entity = v1_mutation.insert self.__entity_converter.v1_to_v3_entity(v1_entity, v3_entity) index_writes, v3_key = self.__insert_v3_entity(v3_entity, v3_txn) elif v1_mutation.HasField('update'): v3_entity = entity_pb.EntityProto() self.__entity_converter.v1_to_v3_entity(v1_mutation.update, v3_entity) index_writes = self.__update_v3_entity(v3_entity, v3_txn) elif v1_mutation.HasField('upsert'): v3_entity = entity_pb.EntityProto() v1_entity = v1_mutation.upsert self.__entity_converter.v1_to_v3_entity(v1_entity, v3_entity) index_writes, v3_key = self.__upsert_v3_entity(v3_entity, v3_txn) elif v1_mutation.HasField('delete'): v3_ref = entity_pb.Reference() self.__entity_converter.v1_to_v3_reference(v1_mutation.delete, v3_ref) index_writes = self.__delete_v3_reference(v3_ref, v3_txn) v1_mutation_result = googledatastore.MutationResult() if v3_key and not datastore_pbs.is_complete_v1_key(v1_entity.key): self.__entity_converter.v3_to_v1_key(v3_key, v1_mutation_result.key) return v1_mutation_result, index_writes
def __apply_v1_mutation(self, v1_mutation, v1_txn): """Applies a v1 Mutation. Args: v1_mutation: a googledatastore.Mutation v1_txn: an optional v1 transaction handle or None Returns: a tuple (googledatastore.MutationResult, number of index writes) """ v3_txn = None v3_key = None if v1_txn: v3_txn = datastore_pb.Transaction() self.__service_converter.v1_to_v3_txn(v1_txn, v3_txn) if v1_mutation.op == googledatastore.Mutation.INSERT: v3_entity = entity_pb.EntityProto() self.__entity_converter.v1_to_v3_entity(v1_mutation.entity, v3_entity) index_writes, v3_key = self.__insert_v3_entity(v3_entity, v3_txn) elif v1_mutation.op == googledatastore.Mutation.UPDATE: v3_entity = entity_pb.EntityProto() self.__entity_converter.v1_to_v3_entity(v1_mutation.entity, v3_entity) index_writes = self.__update_v3_entity(v3_entity, v3_txn) elif v1_mutation.op == googledatastore.Mutation.UPSERT: v3_entity = entity_pb.EntityProto() self.__entity_converter.v1_to_v3_entity(v1_mutation.entity, v3_entity) index_writes, v3_key = self.__upsert_v3_entity(v3_entity, v3_txn) elif v1_mutation.op == googledatastore.Mutation.DELETE: v3_ref = entity_pb.Reference() self.__entity_converter.v1_to_v3_reference(v1_mutation.key, v3_ref) index_writes = self.__delete_v3_reference(v3_ref, v3_txn) v1_mutation_result = googledatastore.MutationResult() if v3_key and not datastore_pbs.is_complete_v1_key( v1_mutation.entity.key): self.__entity_converter.v3_to_v1_key(v3_key, v1_mutation_result.key) return v1_mutation_result, index_writes
def __validate_mutation(self, mutation): if mutation.HasField('insert'): self.__entity_validator.validate_entity(UPSERT, mutation.insert) mutation_key = mutation.insert.key elif mutation.HasField('update'): self.__entity_validator.validate_entity(UPDATE, mutation.update) mutation_key = mutation.update.key elif mutation.HasField('upsert'): self.__entity_validator.validate_entity(UPSERT, mutation.upsert) mutation_key = mutation.upsert.key elif mutation.HasField('delete'): self.__entity_validator.validate_key(DELETE, mutation.delete) mutation_key = mutation.delete else: _assert_condition(False, 'mutation lacks required op') if mutation.WhichOneof('conflict_detection_strategy') != None: _assert_condition(datastore_pbs.is_complete_v1_key(mutation_key), 'conflict detection is not allowed for incomplete keys') if mutation.HasField('base_version'): _assert_condition(mutation.base_version >= 0, 'Invalid base_version: %d, ' 'it should be >= 0' % mutation.base_version)
def __commit(self, v1_mutations, v1_txn, resp): """Commits a list of v1 mutations. Args: v1_mutations: the list of mutations to apply and commit v1_txn: required v1 transaction handle in which to apply the mutations resp: a v1 CommitResponse to update with the result of this commit """ mutation_keys = [] seen_keys = set() allocated_keys = {} conflict_cache = {} version_cache = {} for i, mutation in enumerate(v1_mutations): v1_key, v1_entity = datastore_pbs.get_v1_mutation_key_and_entity(mutation) key = datastore_types.ReferenceToKeyValue(v1_key, self._id_resolver) if not datastore_pbs.is_complete_v1_key(v1_key): v1_key = self.__put_v1_entity(v1_entity, v1_txn) key = datastore_types.ReferenceToKeyValue(v1_key, self._id_resolver) allocated_keys[key] = v1_key elif key not in conflict_cache: base_version = None if mutation.HasField('base_version') and key not in seen_keys: base_version = mutation.base_version conflict_version = self.__apply_v1_mutation(mutation, base_version, v1_txn, version_cache) if conflict_version is not None: conflict_cache[key] = conflict_version mutation_keys.append(key) seen_keys.add(key) v3_txn = datastore_pb.Transaction() self.__service_converter.v1_to_v3_txn(v1_txn, v3_txn) v3_resp = datastore_pb.CommitResponse() self.__make_v3_call('Commit', v3_txn, v3_resp) resp.index_updates = v3_resp.cost().index_writes() mutation_versions = {} for version in v3_resp.version_list(): key = datastore_types.ReferenceToKeyValue(version.root_entity_key()) mutation_versions[key] = version.version() for key in mutation_keys: mutation_result = resp.mutation_results.add() if key in allocated_keys: mutation_result.key.CopyFrom(allocated_keys[key]) if key in conflict_cache: mutation_result.conflict_detected = True mutation_result.version = conflict_cache[key] else: mutation_result.version = mutation_versions[key]