def start(self, fdb_clusterfile):
self._db = fdb.open(fdb_clusterfile)
self._tornado_fdb = TornadoFDB(IOLoop.current())
ds_dir = fdb.directory.create_or_open(self._db, DS_ROOT)
self._directory_cache = DirectoryCache(self._db, self._tornado_fdb,
ds_dir)
self._directory_cache.initialize()
self._data_manager = DataManager(self._tornado_fdb,
self._directory_cache)
self._tx_manager = TransactionManager(self._db, self._tornado_fdb,
self._directory_cache)
self._index_manager = IndexManager(self._db, self._tornado_fdb,
self._data_manager,
self._directory_cache)
self._index_manager.start()
self._gc = GarbageCollector(self._db, self._tornado_fdb,
self._data_manager, self._index_manager,
self._tx_manager, self._directory_cache)
self._gc.start()
self._stats_buffer = StatsBuffer(self._db, self._tornado_fdb,
self._directory_cache, self)
self._stats_buffer.start()
class FDBDatastore(object):
""" A datastore implementation that uses FoundationDB. """
def __init__(self):
self._data_manager = None
self._db = None
self._scattered_allocator = ScatteredAllocator()
self._tornado_fdb = None
self._tx_manager = None
self._gc = None
self._index_manager = None
self._stats_buffer = None
def start(self, fdb_clusterfile):
self._db = fdb.open(fdb_clusterfile)
self._tornado_fdb = TornadoFDB(IOLoop.current())
ds_dir = fdb.directory.create_or_open(self._db, DS_ROOT)
self._directory_cache = DirectoryCache(self._db, self._tornado_fdb,
ds_dir)
self._directory_cache.initialize()
self._data_manager = DataManager(self._tornado_fdb,
self._directory_cache)
self._tx_manager = TransactionManager(self._db, self._tornado_fdb,
self._directory_cache)
self._index_manager = IndexManager(self._db, self._tornado_fdb,
self._data_manager,
self._directory_cache)
self._index_manager.start()
self._gc = GarbageCollector(self._db, self._tornado_fdb,
self._data_manager, self._index_manager,
self._tx_manager, self._directory_cache)
self._gc.start()
self._stats_buffer = StatsBuffer(self._db, self._tornado_fdb,
self._directory_cache, self)
self._stats_buffer.start()
@gen.coroutine
def dynamic_put(self, project_id, put_request, put_response, retries=5):
# logger.debug(u'put_request:\n{}'.format(put_request))
project_id = decode_str(project_id)
# TODO: Enforce max key length (100 elements).
# Enforce max element size (1500 bytes).
# Enforce max kind size (1500 bytes).
# Enforce key name regex (reserved names match __.*__).
if put_request.auto_id_policy() != put_request.CURRENT:
raise BadRequest(u'Sequential allocator is not implemented')
tr = self._db.create_transaction()
if put_request.has_transaction():
yield self._tx_manager.log_puts(tr, project_id, put_request)
writes = {
self._collapsible_id(entity):
(VersionEntry.from_key(entity.key()),
VersionEntry.from_key(entity.key()), None)
for entity in put_request.entity_list()
}
else:
# Eliminate multiple puts to the same key.
puts_by_key = {
self._collapsible_id(entity): entity
for entity in put_request.entity_list()
}
writes = yield {
key: self._upsert(tr, entity)
for key, entity in six.iteritems(puts_by_key)
}
old_entries = [
old_entry for old_entry, _, _ in six.itervalues(writes)
if old_entry.present
]
versionstamp_future = None
if old_entries:
versionstamp_future = tr.get_versionstamp()
try:
yield self._tornado_fdb.commit(tr, convert_exceptions=False)
except fdb.FDBError as fdb_error:
if fdb_error.code == FDBErrorCodes.NOT_COMMITTED:
pass
elif fdb_error.code == FDBErrorCodes.COMMIT_RESULT_UNKNOWN:
logger.error('Unable to determine commit result. Retrying.')
else:
raise InternalError(fdb_error.description)
retries -= 1
if retries < 0:
raise InternalError(fdb_error.description)
yield self.dynamic_put(project_id, put_request, put_response,
retries)
return
if old_entries:
self._gc.clear_later(old_entries, versionstamp_future.wait().value)
mutations = [
(old_entry, new_entry, index_stats)
for old_entry, new_entry, index_stats in six.itervalues(writes)
if index_stats is not None
]
IOLoop.current().spawn_callback(self._stats_buffer.update, project_id,
mutations)
for entity in put_request.entity_list():
write_entry = writes[self._collapsible_id(entity)][1]
put_response.add_key().CopyFrom(write_entry.key)
if write_entry.version != ABSENT_VERSION:
put_response.add_version(write_entry.version)
#logger.debug('put_response:\n{}'.format(put_response))
@gen.coroutine
def dynamic_get(self, project_id, get_request, get_response):
logger.debug(u'get_request:\n{}'.format(get_request))
project_id = decode_str(project_id)
tr = self._db.create_transaction()
read_versionstamp = None
if get_request.has_transaction():
yield self._tx_manager.log_lookups(tr, project_id, get_request)
# Ensure the GC hasn't cleaned up an entity written after the tx start.
safe_read_stamps = yield [
self._gc.safe_read_versionstamp(tr, key)
for key in get_request.key_list()
]
safe_read_stamps = [
vs for vs in safe_read_stamps if vs is not None
]
read_versionstamp = TransactionID.decode(
get_request.transaction().handle())[1]
if any(safe_versionstamp > read_versionstamp
for safe_versionstamp in safe_read_stamps):
raise BadRequest(u'The specified transaction has expired')
futures = []
for key in get_request.key_list():
futures.append(
self._data_manager.get_latest(tr,
key,
read_versionstamp,
snapshot=True))
version_entries = yield futures
# If this read is in a transaction, logging the RPC is a mutation.
yield self._tornado_fdb.commit(tr)
for entry in version_entries:
response_entity = get_response.add_entity()
response_entity.set_version(entry.version)
if entry.has_entity:
response_entity.mutable_entity().MergeFrom(entry.decoded)
else:
response_entity.mutable_key().MergeFrom(entry.key)
logger.debug(u'fetched paths: {}'.format(
[entry.path for entry in version_entries if entry.has_entity]))
@gen.coroutine
def dynamic_delete(self, project_id, delete_request, retries=5):
logger.debug(u'delete_request:\n{}'.format(delete_request))
project_id = decode_str(project_id)
tr = self._db.create_transaction()
if delete_request.has_transaction():
yield self._tx_manager.log_deletes(tr, project_id, delete_request)
deletes = [(VersionEntry.from_key(key), None, None)
for key in delete_request.key_list()]
else:
# Eliminate multiple deletes to the same key.
deletes_by_key = {
key.Encode(): key
for key in delete_request.key_list()
}
deletes = yield [
self._delete(tr, key) for key in six.itervalues(deletes_by_key)
]
old_entries = [
old_entry for old_entry, _, _ in deletes if old_entry.present
]
versionstamp_future = None
if old_entries:
versionstamp_future = tr.get_versionstamp()
try:
yield self._tornado_fdb.commit(tr, convert_exceptions=False)
except fdb.FDBError as fdb_error:
if fdb_error.code == FDBErrorCodes.NOT_COMMITTED:
pass
elif fdb_error.code == FDBErrorCodes.COMMIT_RESULT_UNKNOWN:
logger.error('Unable to determine commit result. Retrying.')
else:
raise InternalError(fdb_error.description)
retries -= 1
if retries < 0:
raise InternalError(fdb_error.description)
yield self.dynamic_delete(project_id, delete_request, retries)
return
if old_entries:
self._gc.clear_later(old_entries, versionstamp_future.wait().value)
mutations = [(old_entry, None, stats)
for old_entry, _, stats in deletes if stats is not None]
IOLoop.current().spawn_callback(self._stats_buffer.update, project_id,
mutations)
# TODO: Once the Cassandra backend is removed, populate a delete response.
for old_entry, new_version, _ in deletes:
logger.debug(u'new_version: {}'.format(new_version))
@gen.coroutine
def _dynamic_run_query(self, query, query_result):
logger.debug(u'query: {}'.format(query))
project_id = decode_str(query.app())
tr = self._db.create_transaction()
read_versionstamp = None
if query.has_transaction():
yield self._tx_manager.log_query(tr, project_id, query)
# Ensure the GC hasn't cleaned up an entity written after the tx start.
safe_versionstamp = yield self._gc.safe_read_versionstamp(
tr, query.ancestor())
read_versionstamp = TransactionID.decode(
query.transaction().handle())[1]
if (safe_versionstamp is not None
and safe_versionstamp > read_versionstamp):
raise BadRequest(u'The specified transaction has expired')
fetch_data = self._index_manager.include_data(query)
rpc_limit, check_more_results = self._index_manager.rpc_limit(query)
iterator = yield self._index_manager.get_iterator(
tr, query, read_versionstamp)
for prop_name in query.property_name_list():
prop_name = decode_str(prop_name)
if prop_name not in iterator.prop_names:
raise BadRequest(
u'Projections on {} are not supported'.format(prop_name))
data_futures = [] if fetch_data else None
unique_keys = set()
results = []
entries_fetched = 0
skipped_results = 0
cursor = None
while True:
remainder = rpc_limit - entries_fetched
iter_offset = max(query.offset() - entries_fetched, 0)
entries, more_iterator_results = yield iterator.next_page()
entries_fetched += len(entries)
if not entries and more_iterator_results:
continue
if not entries and not more_iterator_results:
break
skipped_results += min(len(entries), iter_offset)
suitable_entries = entries[iter_offset:remainder]
if entries[:remainder]:
cursor = entries[:remainder][-1]
if not fetch_data and not query.keys_only():
results.extend(
[entry.prop_result() for entry in suitable_entries])
continue
for entry in suitable_entries:
if entry.path in unique_keys:
continue
unique_keys.add(entry.path)
if fetch_data:
data_futures.append(
self._data_manager.get_entry(tr, entry, snapshot=True))
else:
results.append(entry.key_result())
if not more_iterator_results:
break
if fetch_data:
entity_results = yield data_futures
results = [entity.encoded for entity in entity_results]
else:
results = [result.Encode() for result in results]
yield self._tornado_fdb.commit(tr)
query_result.result_list().extend(results)
# TODO: Figure out how ndb multi queries use compiled cursors.
if query.compile():
ordered_props = tuple(prop_name
for prop_name, _ in get_order_info(query)
if prop_name != KEY_PROP)
mutable_cursor = query_result.mutable_compiled_cursor()
if cursor is not None:
mutable_cursor.MergeFrom(cursor.cursor_result(ordered_props))
more_results = check_more_results and entries_fetched > rpc_limit
query_result.set_more_results(more_results)
if skipped_results:
query_result.set_skipped_results(skipped_results)
if query.keys_only():
query_result.set_keys_only(True)
logger.debug(u'{} results'.format(len(query_result.result_list())))
@gen.coroutine
def setup_transaction(self, project_id, is_xg):
project_id = decode_str(project_id)
txid = yield self._tx_manager.create(project_id)
logger.debug(u'Started new transaction: {}:{}'.format(
project_id, txid))
raise gen.Return(txid)
@gen.coroutine
def apply_txn_changes(self, project_id, txid, retries=5):
logger.debug(u'Applying {}:{}'.format(project_id, txid))
project_id = decode_str(project_id)
tr = self._db.create_transaction()
read_versionstamp = TransactionID.decode(txid)[1]
lookups, queried_groups, mutations = yield self._tx_manager.get_metadata(
tr, project_id, txid)
try:
writes = yield self._apply_mutations(tr, project_id,
queried_groups, mutations,
lookups, read_versionstamp)
finally:
yield self._tx_manager.delete(tr, project_id, txid)
versionstamp_future = None
old_entries = [
old_entry for old_entry, _, _ in writes if old_entry.present
]
if old_entries:
versionstamp_future = tr.get_versionstamp()
try:
yield self._tornado_fdb.commit(tr, convert_exceptions=False)
except fdb.FDBError as fdb_error:
if fdb_error.code != FDBErrorCodes.NOT_COMMITTED:
raise InternalError(fdb_error.description)
retries -= 1
if retries < 0:
raise InternalError(fdb_error.description)
yield self.apply_txn_changes(project_id, txid, retries)
return
if old_entries:
self._gc.clear_later(old_entries, versionstamp_future.wait().value)
mutations = [(old_entry, FDBDatastore._filter_version(new_entry),
index_stats)
for old_entry, new_entry, index_stats in writes
if index_stats is not None]
IOLoop.current().spawn_callback(self._stats_buffer.update, project_id,
mutations)
logger.debug(u'Finished applying {}:{}'.format(project_id, txid))
@gen.coroutine
def rollback_transaction(self, project_id, txid):
project_id = decode_str(project_id)
logger.debug(u'Rolling back {}:{}'.format(project_id, txid))
tr = self._db.create_transaction()
yield self._tx_manager.delete(tr, project_id, txid)
yield self._tornado_fdb.commit(tr)
@gen.coroutine
def update_composite_index(self, project_id, index):
project_id = decode_str(project_id)
yield self._index_manager.update_composite_index(project_id, index)
@gen.coroutine
def get_indexes(self, project_id):
""" Retrieves list of indexes for a project.
Args:
project_id: A string specifying a project ID.
Returns:
A list of entity_pb.CompositeIndex objects.
Raises:
BadRequest if project_id is not found.
"""
tr = self._db.create_transaction()
composite_index_manager = self._index_manager._composite_index_manager
project_indexes = yield composite_index_manager.get_definitions(
tr, project_id)
raise gen.Return([index.to_pb() for index in project_indexes])
@gen.coroutine
def add_indexes(self, project_id, indexes):
""" Adds composite index definitions to a project.
Only indexes that do not already exist will be created.
Args:
project_id: A string specifying a project ID.
indexes: An iterable containing index definitions.
"""
tr = self._db.create_transaction()
yield self._index_manager.merge(tr, project_id, indexes)
yield self._tornado_fdb.commit(tr)
@gen.coroutine
def allocate_size(self,
project_id,
namespace,
path_prefix,
size,
retries=5):
tr = self._db.create_transaction()
key = yield sequential_id_key(tr, project_id, namespace, path_prefix,
self._directory_cache)
old_max = yield old_max_id(tr, key, self._tornado_fdb)
new_max = old_max + size
# TODO: Check behavior on reaching max sequential ID.
if new_max > _MAX_SEQUENTIAL_ID:
raise BadRequest(
u'There are not enough remaining IDs to satisfy request')
tr[key] = SequentialIDsNamespace.encode_value(new_max)
try:
yield self._tornado_fdb.commit(tr)
except fdb.FDBError as fdb_error:
if fdb_error.code != FDBErrorCodes.NOT_COMMITTED:
raise InternalError(fdb_error.description)
retries -= 1
if retries < 0:
raise InternalError(fdb_error.description)
range_start, range_end = yield self.allocate_size(
project_id, namespace, path_prefix, size, retries)
raise gen.Return((range_start, range_end))
raise gen.Return((old_max + 1, new_max))
@gen.coroutine
def allocate_max(self,
project_id,
namespace,
path_prefix,
new_max,
retries=5):
tr = self._db.create_transaction()
key = yield sequential_id_key(tr, project_id, namespace, path_prefix,
self._directory_cache)
old_max = yield old_max_id(tr, key, self._tornado_fdb)
if new_max > old_max:
tr[key] = SequentialIDsNamespace.encode_value(new_max)
try:
yield self._tornado_fdb.commit(tr)
except fdb.FDBError as fdb_error:
if fdb_error.code != FDBErrorCodes.NOT_COMMITTED:
raise InternalError(fdb_error.description)
retries -= 1
if retries < 0:
raise InternalError(fdb_error.description)
range_start, range_end = yield self.allocate_max(
project_id, namespace, path_prefix, new_max, retries)
raise gen.Return((range_start, range_end))
raise gen.Return((old_max + 1, max(new_max, old_max)))
@gen.coroutine
def _upsert(self, tr, entity, old_entry_future=None):
auto_id = self._auto_id(entity)
if auto_id:
# Avoid mutating the object given.
new_entity = entity_pb.EntityProto()
new_entity.CopyFrom(entity)
entity = new_entity
last_element = entity.key().path().element(-1)
last_element.set_id(self._scattered_allocator.get_id())
if old_entry_future is None:
old_entry = yield self._data_manager.get_latest(tr, entity.key())
else:
old_entry = yield old_entry_future
# If the datastore chose an ID, don't overwrite existing data.
if auto_id and old_entry.present:
self._scattered_allocator.invalidate()
raise InternalError(u'The datastore chose an existing ID')
new_version = next_entity_version(old_entry.version)
encoded_entity = entity.Encode()
yield self._data_manager.put(tr, entity.key(), new_version,
encoded_entity)
index_stats = yield self._index_manager.put_entries(
tr, old_entry, entity)
if old_entry.present:
yield self._gc.index_deleted_version(tr, old_entry)
new_entry = VersionEntry.from_key(entity.key())
new_entry._encoded_entity = encoded_entity
new_entry._decoded_entity = entity
new_entry.version = new_version
raise gen.Return((old_entry, new_entry, index_stats))
@gen.coroutine
def _delete(self, tr, key, old_entry_future=None):
if old_entry_future is None:
old_entry = yield self._data_manager.get_latest(tr, key)
else:
old_entry = yield old_entry_future
if not old_entry.present:
raise gen.Return((old_entry, None, None))
new_version = next_entity_version(old_entry.version)
yield self._data_manager.put(tr, key, new_version, b'')
index_stats = yield self._index_manager.put_entries(tr,
old_entry,
new_entity=None)
if old_entry.present:
yield self._gc.index_deleted_version(tr, old_entry)
raise gen.Return((old_entry, new_version, index_stats))
@gen.coroutine
def _apply_mutations(self, tr, project_id, queried_groups, mutations,
lookups, read_versionstamp):
# TODO: Check if transactional tasks count as a side effect.
if not mutations:
raise gen.Return([])
group_update_futures = [
self._data_manager.last_group_versionstamp(tr, project_id,
namespace, group_path)
for namespace, group_path in queried_groups
]
# Index keys that require a full lookup rather than a versionstamp.
require_data = set()
for mutation in mutations:
key = (mutation if isinstance(mutation, entity_pb.Reference) else
mutation.key())
require_data.add(key.Encode())
# Start fetching versionstamps for lookups first to invalidate sooner.
futures = {}
for key in lookups:
encoded_key = key.Encode()
futures[encoded_key] = self._data_manager.get_latest(
tr, key, include_data=encoded_key in require_data)
group_updates = yield group_update_futures
group_updates = [vs for vs in group_updates if vs is not None]
if any(commit_vs > read_versionstamp for commit_vs in group_updates):
raise ConcurrentModificationException(
u'A queried group was modified after this transaction was started.'
)
version_entries = yield [futures[key.Encode()] for key in lookups]
if any(entry.present and entry.commit_versionstamp > read_versionstamp
for entry in version_entries):
raise ConcurrentModificationException(
u'An entity was modified after this transaction was started.')
# TODO: Check if this constraint is still needed.
self._enforce_max_groups(mutations)
# Apply mutations.
mutation_futures = []
for mutation in self._collapse_mutations(mutations):
if isinstance(mutation, entity_pb.Reference):
old_entry_future = futures.get(mutation.Encode())
mutation_futures.append(
self._delete(tr, mutation, old_entry_future))
else:
old_entry_future = futures.get(mutation.key().Encode())
mutation_futures.append(
self._upsert(tr, mutation, old_entry_future))
responses = yield mutation_futures
raise gen.Return(responses)
@staticmethod
def _collapse_mutations(mutations):
""" Selects the last mutation for each key as the one to apply. """
# TODO: For the v1 API, test if insert, update succeeds in mutation list.
mutations_by_key = {}
for mutation in mutations:
if isinstance(mutation, entity_pb.Reference):
key = mutation.Encode()
else:
key = FDBDatastore._collapsible_id(mutation)
mutations_by_key[key] = mutation
return tuple(mutation for mutation in six.itervalues(mutations_by_key))
@staticmethod
def _collapsible_id(entity):
""" The "collapsible" identity is the encoded key or the entity if
an identity will be allocated later. """
if FDBDatastore._auto_id(entity):
return id(entity)
else:
return entity.key().Encode()
@staticmethod
def _filter_version(entity):
""" Filter out any entity that is actually a delete version. """
if isinstance(entity, (int, long)):
return None
else:
return entity
@staticmethod
def _enforce_max_groups(mutations):
""" Raises an exception if too many groups were modified. """
mutated_groups = set()
for mutation in mutations:
if isinstance(mutation, entity_pb.Reference):
key = mutation
else:
key = mutation.key()
namespace = decode_str(key.name_space())
flat_group = (namespace, ) + Path.flatten(key.path())[:2]
mutated_groups.add(flat_group)
if len(mutated_groups) > 25:
raise BadRequest(
u'Too many entity groups modified in transaction')
@staticmethod
def _auto_id(entity):
""" Should perform auto identity allocation for entity. """
last_element = entity.key().path().element(-1)
auto_id = False
if not last_element.has_name():
auto_id = not (last_element.has_id() and last_element.id() != 0)
return auto_id