def _upsert(self, tr, entity, old_entry_future=None):
auto_id = self._auto_id(entity)
if auto_id:
# Avoid mutating the object given.
new_entity = entity_pb.EntityProto()
new_entity.CopyFrom(entity)
entity = new_entity
last_element = entity.key().path().element(-1)
last_element.set_id(self._scattered_allocator.get_id())
if old_entry_future is None:
old_entry = yield self._data_manager.get_latest(tr, entity.key())
else:
old_entry = yield old_entry_future
# If the datastore chose an ID, don't overwrite existing data.
if auto_id and old_entry.present:
self._scattered_allocator.invalidate()
raise InternalError(u'The datastore chose an existing ID')
new_version = next_entity_version(old_entry.version)
encoded_entity = entity.Encode()
yield self._data_manager.put(tr, entity.key(), new_version,
encoded_entity)
index_stats = yield self._index_manager.put_entries(
tr, old_entry, entity)
if old_entry.present:
yield self._gc.index_deleted_version(tr, old_entry)
new_entry = VersionEntry.from_key(entity.key())
new_entry._encoded_entity = encoded_entity
new_entry._decoded_entity = entity
new_entry.version = new_version
raise gen.Return((old_entry, new_entry, index_stats))
def dynamic_delete(self, project_id, delete_request, retries=5):
logger.debug(u'delete_request:\n{}'.format(delete_request))
project_id = decode_str(project_id)
tr = self._db.create_transaction()
if delete_request.has_transaction():
yield self._tx_manager.log_deletes(tr, project_id, delete_request)
deletes = [(VersionEntry.from_key(key), None, None)
for key in delete_request.key_list()]
else:
# Eliminate multiple deletes to the same key.
deletes_by_key = {
key.Encode(): key
for key in delete_request.key_list()
}
deletes = yield [
self._delete(tr, key) for key in six.itervalues(deletes_by_key)
]
old_entries = [
old_entry for old_entry, _, _ in deletes if old_entry.present
]
versionstamp_future = None
if old_entries:
versionstamp_future = tr.get_versionstamp()
try:
yield self._tornado_fdb.commit(tr, convert_exceptions=False)
except fdb.FDBError as fdb_error:
if fdb_error.code == FDBErrorCodes.NOT_COMMITTED:
pass
elif fdb_error.code == FDBErrorCodes.COMMIT_RESULT_UNKNOWN:
logger.error('Unable to determine commit result. Retrying.')
else:
raise InternalError(fdb_error.description)
retries -= 1
if retries < 0:
raise InternalError(fdb_error.description)
yield self.dynamic_delete(project_id, delete_request, retries)
return
if old_entries:
self._gc.clear_later(old_entries, versionstamp_future.wait().value)
mutations = [(old_entry, None, stats)
for old_entry, _, stats in deletes if stats is not None]
IOLoop.current().spawn_callback(self._stats_buffer.update, project_id,
mutations)
# TODO: Once the Cassandra backend is removed, populate a delete response.
for old_entry, new_version, _ in deletes:
logger.debug(u'new_version: {}'.format(new_version))
def dynamic_put(self, project_id, put_request, put_response, retries=5):
# logger.debug(u'put_request:\n{}'.format(put_request))
project_id = decode_str(project_id)
# TODO: Enforce max key length (100 elements).
# Enforce max element size (1500 bytes).
# Enforce max kind size (1500 bytes).
# Enforce key name regex (reserved names match __.*__).
if put_request.auto_id_policy() != put_request.CURRENT:
raise BadRequest(u'Sequential allocator is not implemented')
tr = self._db.create_transaction()
if put_request.has_transaction():
yield self._tx_manager.log_puts(tr, project_id, put_request)
writes = {
self._collapsible_id(entity):
(VersionEntry.from_key(entity.key()),
VersionEntry.from_key(entity.key()), None)
for entity in put_request.entity_list()
}
else:
# Eliminate multiple puts to the same key.
puts_by_key = {
self._collapsible_id(entity): entity
for entity in put_request.entity_list()
}
writes = yield {
key: self._upsert(tr, entity)
for key, entity in six.iteritems(puts_by_key)
}
old_entries = [
old_entry for old_entry, _, _ in six.itervalues(writes)
if old_entry.present
]
versionstamp_future = None
if old_entries:
versionstamp_future = tr.get_versionstamp()
try:
yield self._tornado_fdb.commit(tr, convert_exceptions=False)
except fdb.FDBError as fdb_error:
if fdb_error.code == FDBErrorCodes.NOT_COMMITTED:
pass
elif fdb_error.code == FDBErrorCodes.COMMIT_RESULT_UNKNOWN:
logger.error('Unable to determine commit result. Retrying.')
else:
raise InternalError(fdb_error.description)
retries -= 1
if retries < 0:
raise InternalError(fdb_error.description)
yield self.dynamic_put(project_id, put_request, put_response,
retries)
return
if old_entries:
self._gc.clear_later(old_entries, versionstamp_future.wait().value)
mutations = [
(old_entry, new_entry, index_stats)
for old_entry, new_entry, index_stats in six.itervalues(writes)
if index_stats is not None
]
IOLoop.current().spawn_callback(self._stats_buffer.update, project_id,
mutations)
for entity in put_request.entity_list():
write_entry = writes[self._collapsible_id(entity)][1]
put_response.add_key().CopyFrom(write_entry.key)
if write_entry.version != ABSENT_VERSION:
put_response.add_version(write_entry.version)