def allocate_max(self,
project_id,
namespace,
path_prefix,
new_max,
retries=5):
tr = self._db.create_transaction()
key = yield sequential_id_key(tr, project_id, namespace, path_prefix,
self._directory_cache)
old_max = yield old_max_id(tr, key, self._tornado_fdb)
if new_max > old_max:
tr[key] = SequentialIDsNamespace.encode_value(new_max)
try:
yield self._tornado_fdb.commit(tr)
except fdb.FDBError as fdb_error:
if fdb_error.code != FDBErrorCodes.NOT_COMMITTED:
raise InternalError(fdb_error.description)
retries -= 1
if retries < 0:
raise InternalError(fdb_error.description)
range_start, range_end = yield self.allocate_max(
project_id, namespace, path_prefix, new_max, retries)
raise gen.Return((range_start, range_end))
raise gen.Return((old_max + 1, max(new_max, old_max)))
def allocate_size(self,
project_id,
namespace,
path_prefix,
size,
retries=5):
tr = self._db.create_transaction()
key = yield sequential_id_key(tr, project_id, namespace, path_prefix,
self._directory_cache)
old_max = yield old_max_id(tr, key, self._tornado_fdb)
new_max = old_max + size
# TODO: Check behavior on reaching max sequential ID.
if new_max > _MAX_SEQUENTIAL_ID:
raise BadRequest(
u'There are not enough remaining IDs to satisfy request')
tr[key] = SequentialIDsNamespace.encode_value(new_max)
try:
yield self._tornado_fdb.commit(tr)
except fdb.FDBError as fdb_error:
if fdb_error.code != FDBErrorCodes.NOT_COMMITTED:
raise InternalError(fdb_error.description)
retries -= 1
if retries < 0:
raise InternalError(fdb_error.description)
range_start, range_end = yield self.allocate_size(
project_id, namespace, path_prefix, size, retries)
raise gen.Return((range_start, range_end))
raise gen.Return((old_max + 1, new_max))
def dynamic_delete(self, project_id, delete_request, retries=5):
logger.debug(u'delete_request:\n{}'.format(delete_request))
project_id = decode_str(project_id)
tr = self._db.create_transaction()
if delete_request.has_transaction():
yield self._tx_manager.log_deletes(tr, project_id, delete_request)
deletes = [(VersionEntry.from_key(key), None, None)
for key in delete_request.key_list()]
else:
# Eliminate multiple deletes to the same key.
deletes_by_key = {
key.Encode(): key
for key in delete_request.key_list()
}
deletes = yield [
self._delete(tr, key) for key in six.itervalues(deletes_by_key)
]
old_entries = [
old_entry for old_entry, _, _ in deletes if old_entry.present
]
versionstamp_future = None
if old_entries:
versionstamp_future = tr.get_versionstamp()
try:
yield self._tornado_fdb.commit(tr, convert_exceptions=False)
except fdb.FDBError as fdb_error:
if fdb_error.code == FDBErrorCodes.NOT_COMMITTED:
pass
elif fdb_error.code == FDBErrorCodes.COMMIT_RESULT_UNKNOWN:
logger.error('Unable to determine commit result. Retrying.')
else:
raise InternalError(fdb_error.description)
retries -= 1
if retries < 0:
raise InternalError(fdb_error.description)
yield self.dynamic_delete(project_id, delete_request, retries)
return
if old_entries:
self._gc.clear_later(old_entries, versionstamp_future.wait().value)
mutations = [(old_entry, None, stats)
for old_entry, _, stats in deletes if stats is not None]
IOLoop.current().spawn_callback(self._stats_buffer.update, project_id,
mutations)
# TODO: Once the Cassandra backend is removed, populate a delete response.
for old_entry, new_version, _ in deletes:
logger.debug(u'new_version: {}'.format(new_version))
def encode(cls, scatter_val, commit_versionstamp):
commit_version = struct.unpack('>Q', commit_versionstamp[:8])[0]
batch_order = struct.unpack('>H', commit_versionstamp[8:])[0]
if not 0 <= scatter_val <= 15:
raise InternalError(u'Invalid scatter value')
if commit_version >= 2 ** cls.COMMIT_VERSION_BITS:
raise InternalError(u'Commit version too high')
if batch_order >= 2 ** cls.BATCH_ORDER_BITS:
raise InternalError(u'Batch order too high')
return (commit_version << 12) + (batch_order << 4) + scatter_val
def type_range(self, type_name):
""" Returns a slice that encompasses all values for a property type. """
if type_name == u'NULL':
start = six.int2byte(codecs.NULL_CODE)
stop = six.int2byte(codecs.NULL_CODE + 1)
elif type_name == u'INT64':
start = six.int2byte(codecs.MIN_INT64_CODE)
stop = six.int2byte(codecs.MAX_INT64_CODE + 1)
elif type_name == u'BOOLEAN':
start = six.int2byte(codecs.FALSE_CODE)
stop = six.int2byte(codecs.TRUE_CODE + 1)
elif type_name == u'STRING':
start = six.int2byte(codecs.BYTES_CODE)
stop = six.int2byte(codecs.BYTES_CODE + 1)
elif type_name == u'DOUBLE':
start = six.int2byte(codecs.DOUBLE_CODE)
stop = six.int2byte(codecs.DOUBLE_CODE + 1)
elif type_name == u'POINT':
start = six.int2byte(codecs.POINT_CODE)
stop = six.int2byte(codecs.POINT_CODE + 1)
elif type_name == u'USER':
start = six.int2byte(codecs.USER_CODE)
stop = six.int2byte(codecs.USER_CODE + 1)
elif type_name == u'REFERENCE':
start = six.int2byte(codecs.REFERENCE_CODE)
stop = six.int2byte(codecs.REFERENCE_CODE + 1)
else:
raise InternalError(u'Unknown type name')
return slice(self.directory.rawPrefix + start,
self.directory.rawPrefix + stop)
def unpack(cls, blob, pos, reverse=False):
items = []
terminator = encode_marker(TERMINATOR, reverse)
kind_marker = encode_marker(cls.KIND_MARKER, reverse)
name_marker = encode_marker(cls.NAME_MARKER, reverse)
while pos < len(blob):
marker = blob[pos]
pos += 1
if marker == terminator:
break
if marker != kind_marker:
raise InternalError(u'Encoded path is missing kind')
kind, pos = Text.decode(blob, pos, reverse)
items.append(kind)
marker = blob[pos]
pos += 1
if marker == name_marker:
elem_name, pos = Text.decode(blob, pos, reverse)
items.append(elem_name)
else:
elem_id, pos = Int64.decode(marker, blob, pos, reverse)
items.append(elem_id)
return tuple(items), pos
def apply_path_filter(self, op, path, ancestor_path=()):
if not isinstance(path, tuple):
path = Path.flatten(path)
remaining_path = path[len(ancestor_path):] if self._ancestor else path
if not remaining_path:
raise InternalError(u'Path filter must be within ancestor')
start = Path.pack(remaining_path, omit_terminator=True)
# Since the commit versionstamp could potentially start with 0xFF, this
# selection scans up to the next possible path value.
stop = start + six.int2byte(Path.MIN_ID_MARKER)
index = -2
if op == Query_Filter.EQUAL:
self._set_start(index, start)
self._set_stop(index, stop)
self._set_stop(index + 1, b'\xFF')
return
if op == Query_Filter.GREATER_THAN_OR_EQUAL:
self._set_start(index, start)
elif op == Query_Filter.GREATER_THAN:
self._set_start(index, stop)
elif op == Query_Filter.LESS_THAN_OR_EQUAL:
self._set_stop(index, stop)
elif op == Query_Filter.LESS_THAN:
self._set_stop(index, start)
else:
raise BadRequest(u'Unexpected filter operation')
def _upsert(self, tr, entity, old_entry_future=None):
auto_id = self._auto_id(entity)
if auto_id:
# Avoid mutating the object given.
new_entity = entity_pb.EntityProto()
new_entity.CopyFrom(entity)
entity = new_entity
last_element = entity.key().path().element(-1)
last_element.set_id(self._scattered_allocator.get_id())
if old_entry_future is None:
old_entry = yield self._data_manager.get_latest(tr, entity.key())
else:
old_entry = yield old_entry_future
# If the datastore chose an ID, don't overwrite existing data.
if auto_id and old_entry.present:
self._scattered_allocator.invalidate()
raise InternalError(u'The datastore chose an existing ID')
new_version = next_entity_version(old_entry.version)
encoded_entity = entity.Encode()
yield self._data_manager.put(tr, entity.key(), new_version,
encoded_entity)
index_stats = yield self._index_manager.put_entries(
tr, old_entry, entity)
if old_entry.present:
yield self._gc.index_deleted_version(tr, old_entry)
new_entry = VersionEntry.from_key(entity.key())
new_entry._encoded_entity = encoded_entity
new_entry._decoded_entity = entity
new_entry.version = new_version
raise gen.Return((old_entry, new_entry, index_stats))
def apply_txn_changes(self, project_id, txid, retries=5):
logger.debug(u'Applying {}:{}'.format(project_id, txid))
project_id = decode_str(project_id)
tr = self._db.create_transaction()
read_versionstamp = TransactionID.decode(txid)[1]
lookups, queried_groups, mutations = yield self._tx_manager.get_metadata(
tr, project_id, txid)
try:
writes = yield self._apply_mutations(tr, project_id,
queried_groups, mutations,
lookups, read_versionstamp)
finally:
yield self._tx_manager.delete(tr, project_id, txid)
versionstamp_future = None
old_entries = [
old_entry for old_entry, _, _ in writes if old_entry.present
]
if old_entries:
versionstamp_future = tr.get_versionstamp()
try:
yield self._tornado_fdb.commit(tr, convert_exceptions=False)
except fdb.FDBError as fdb_error:
if fdb_error.code != FDBErrorCodes.NOT_COMMITTED:
raise InternalError(fdb_error.description)
retries -= 1
if retries < 0:
raise InternalError(fdb_error.description)
yield self.apply_txn_changes(project_id, txid, retries)
return
if old_entries:
self._gc.clear_later(old_entries, versionstamp_future.wait().value)
mutations = [(old_entry, FDBDatastore._filter_version(new_entry),
index_stats)
for old_entry, new_entry, index_stats in writes
if index_stats is not None]
IOLoop.current().spawn_callback(self._stats_buffer.update, project_id,
mutations)
logger.debug(u'Finished applying {}:{}'.format(project_id, txid))
def encode_bare(cls, value, byte_count):
"""
Encodes an integer without a prefix using the specified number of bytes.
"""
encoded = struct.pack('>Q', value)
if any(byte != b'\x00' for byte in encoded[:-byte_count]):
raise InternalError(u'Value exceeds maximum size')
return encoded[-byte_count:]
def clear_later(self, entries, new_versionstamp):
""" Clears deleted entities after sufficient time has passed. """
safe_time = monotonic.monotonic() + MAX_TX_DURATION
for entry in entries:
# TODO: Strip raw properties and enforce a max queue size to keep memory
# usage reasonable.
if entry.commit_versionstamp is None:
raise InternalError(
u'Deleted entry must have a commit versionstamp')
self._queue.append((safe_time, entry, new_versionstamp))
def _index_details(self, tr, project_id, index_id):
project_indexes = yield self._composite_index_manager.get_definitions(
tr, project_id)
index_def = next(
(ds_index
for ds_index in project_indexes if ds_index.id == index_id), None)
if index_def is None:
raise InternalError(u'Unable to retrieve index details')
order_info = tuple((decode_str(prop.name), prop.to_pb().direction())
for prop in index_def.properties)
raise gen.Return((index_def.kind, index_def.ancestor, order_info))
def _prop_details(self, prop_name):
prop_index = next(
(index for index, (name, direction) in enumerate(self._order_info)
if name == prop_name), None)
if prop_index is None:
raise InternalError(u'{} is not in index'.format(prop_name))
index = prop_index + 1 # Account for directory prefix.
if self._ancestor:
index += 1
return index, self._order_info[prop_index][1]
def _find_terminator(blob, pos, reverse=False):
""" Finds the position of the terminator. """
terminator = encode_marker(TERMINATOR, reverse)
escape_byte = b'\x00' if reverse else b'\xFF'
while True:
pos = blob.find(terminator, pos)
if pos < 0:
raise InternalError(u'Byte string is missing terminator')
if blob[pos + 1:pos + 2] != escape_byte:
return pos
pos += 2
def commit(self, tr, convert_exceptions=True):
tornado_future = TornadoFuture()
callback = lambda fdb_future: self._handle_fdb_result(
fdb_future, tornado_future)
commit_future = tr.commit()
commit_future.on_ready(callback)
try:
yield tornado_future
except fdb.FDBError as fdb_error:
if convert_exceptions:
raise InternalError(fdb_error.description)
else:
raise
def apply_txn_changes(self, project_id, txid, retries=5):
logger.debug(u'Applying {}:{}'.format(project_id, txid))
project_id = decode_str(project_id)
tr = self._db.create_transaction()
read_versionstamp = TransactionID.decode(txid)[1]
lookups, queried_groups, mutations = yield self._tx_manager.get_metadata(
tr, project_id, txid)
try:
old_entries = yield self._apply_mutations(tr, project_id,
queried_groups,
mutations, lookups,
read_versionstamp)
finally:
yield self._tx_manager.delete(tr, project_id, txid)
versionstamp_future = None
if old_entries:
versionstamp_future = tr.get_versionstamp()
try:
yield self._tornado_fdb.commit(tr, convert_exceptions=False)
except fdb.FDBError as fdb_error:
if fdb_error.code != FDBErrorCodes.NOT_COMMITTED:
raise InternalError(fdb_error.description)
retries -= 1
if retries < 0:
raise InternalError(fdb_error.description)
yield self.apply_txn_changes(project_id, txid, retries)
return
if old_entries:
self._gc.clear_later(old_entries, versionstamp_future.wait().value)
logger.debug(u'Finished applying {}:{}'.format(project_id, txid))
def decode_metadata(self, txid, kvs):
lookup_rpcs = defaultdict(list)
queried_groups = set()
mutation_rpcs = []
rpc_type_index = len(self._txid_prefix(txid))
current_versionstamp = None
for kv in kvs:
rpc_type = kv.key[rpc_type_index]
pos = rpc_type_index + 1
if rpc_type == self.QUERIES:
namespace, pos = Text.decode(kv.key, pos)
group_path = Path.unpack(kv.key, pos)[0]
queried_groups.add((namespace, group_path))
continue
rpc_versionstamp = kv.key[pos:pos + VERSIONSTAMP_SIZE]
if rpc_type == self.LOOKUPS:
lookup_rpcs[rpc_versionstamp].append(kv.value)
elif rpc_type in (self.PUTS, self.DELETES):
if current_versionstamp == rpc_versionstamp:
mutation_rpcs[-1].append(kv.value)
else:
current_versionstamp = rpc_versionstamp
mutation_rpcs.append([rpc_type, kv.value])
else:
raise InternalError(u'Unrecognized RPC type')
lookups = dict()
mutations = []
for chunks in six.itervalues(lookup_rpcs):
lookups.update([(key.SerializeToString(), key)
for key in self._unpack_keys(b''.join(chunks))])
for rpc_info in mutation_rpcs:
rpc_type = rpc_info[0]
blob = b''.join(rpc_info[1:])
if rpc_type == self.PUTS:
mutations.extend(self._unpack_entities(blob))
else:
mutations.extend(self._unpack_keys(blob))
return list(six.itervalues(lookups)), queried_groups, mutations
def get(self, tr, key):
current_version = yield self._tornado_fdb.get(tr, self.METADATA_KEY)
if not current_version.present():
raise InternalError(u'The FDB cluster metadata key is missing')
if current_version.value != self._metadata_version:
self._metadata_version = current_version.value
self._directory_dict.clear()
self._directory_keys.clear()
full_key = self.root_dir.get_path() + key
if full_key not in self:
# TODO: This can be made async.
# This is performed in a separate transaction so that it can be retried
# automatically and so that it's only added to the cache when the
# directory has been successfully created.
self[full_key] = fdb.directory.create_or_open(self._db, full_key)
raise gen.Return(self[full_key])
def current_metadata_version(tr, tornado_fdb):
current_version = yield tornado_fdb.get(tr, METADATA_KEY)
if not current_version.present():
raise InternalError(u'The FDB cluster metadata key is missing')
raise gen.Return(current_version.value)
def dynamic_put(self, project_id, put_request, put_response, retries=5):
# logger.debug(u'put_request:\n{}'.format(put_request))
project_id = decode_str(project_id)
# TODO: Enforce max key length (100 elements).
# Enforce max element size (1500 bytes).
# Enforce max kind size (1500 bytes).
# Enforce key name regex (reserved names match __.*__).
if put_request.auto_id_policy() != put_request.CURRENT:
raise BadRequest(u'Sequential allocator is not implemented')
tr = self._db.create_transaction()
if put_request.has_transaction():
yield self._tx_manager.log_puts(tr, project_id, put_request)
writes = {
self._collapsible_id(entity):
(VersionEntry.from_key(entity.key()),
VersionEntry.from_key(entity.key()), None)
for entity in put_request.entity_list()
}
else:
# Eliminate multiple puts to the same key.
puts_by_key = {
self._collapsible_id(entity): entity
for entity in put_request.entity_list()
}
writes = yield {
key: self._upsert(tr, entity)
for key, entity in six.iteritems(puts_by_key)
}
old_entries = [
old_entry for old_entry, _, _ in six.itervalues(writes)
if old_entry.present
]
versionstamp_future = None
if old_entries:
versionstamp_future = tr.get_versionstamp()
try:
yield self._tornado_fdb.commit(tr, convert_exceptions=False)
except fdb.FDBError as fdb_error:
if fdb_error.code == FDBErrorCodes.NOT_COMMITTED:
pass
elif fdb_error.code == FDBErrorCodes.COMMIT_RESULT_UNKNOWN:
logger.error('Unable to determine commit result. Retrying.')
else:
raise InternalError(fdb_error.description)
retries -= 1
if retries < 0:
raise InternalError(fdb_error.description)
yield self.dynamic_put(project_id, put_request, put_response,
retries)
return
if old_entries:
self._gc.clear_later(old_entries, versionstamp_future.wait().value)
mutations = [
(old_entry, new_entry, index_stats)
for old_entry, new_entry, index_stats in six.itervalues(writes)
if index_stats is not None
]
IOLoop.current().spawn_callback(self._stats_buffer.update, project_id,
mutations)
for entity in put_request.entity_list():
write_entry = writes[self._collapsible_id(entity)][1]
put_response.add_key().CopyFrom(write_entry.key)
if write_entry.version != ABSENT_VERSION:
put_response.add_version(write_entry.version)
