def allocate_max(self, project_id, namespace, path_prefix, new_max, retries=5): tr = self._db.create_transaction() key = yield sequential_id_key(tr, project_id, namespace, path_prefix, self._directory_cache) old_max = yield old_max_id(tr, key, self._tornado_fdb) if new_max > old_max: tr[key] = SequentialIDsNamespace.encode_value(new_max) try: yield self._tornado_fdb.commit(tr) except fdb.FDBError as fdb_error: if fdb_error.code != FDBErrorCodes.NOT_COMMITTED: raise InternalError(fdb_error.description) retries -= 1 if retries < 0: raise InternalError(fdb_error.description) range_start, range_end = yield self.allocate_max( project_id, namespace, path_prefix, new_max, retries) raise gen.Return((range_start, range_end)) raise gen.Return((old_max + 1, max(new_max, old_max)))
def allocate_size(self, project_id, namespace, path_prefix, size, retries=5): tr = self._db.create_transaction() key = yield sequential_id_key(tr, project_id, namespace, path_prefix, self._directory_cache) old_max = yield old_max_id(tr, key, self._tornado_fdb) new_max = old_max + size # TODO: Check behavior on reaching max sequential ID. if new_max > _MAX_SEQUENTIAL_ID: raise BadRequest( u'There are not enough remaining IDs to satisfy request') tr[key] = SequentialIDsNamespace.encode_value(new_max) try: yield self._tornado_fdb.commit(tr) except fdb.FDBError as fdb_error: if fdb_error.code != FDBErrorCodes.NOT_COMMITTED: raise InternalError(fdb_error.description) retries -= 1 if retries < 0: raise InternalError(fdb_error.description) range_start, range_end = yield self.allocate_size( project_id, namespace, path_prefix, size, retries) raise gen.Return((range_start, range_end)) raise gen.Return((old_max + 1, new_max))
def dynamic_delete(self, project_id, delete_request, retries=5): logger.debug(u'delete_request:\n{}'.format(delete_request)) project_id = decode_str(project_id) tr = self._db.create_transaction() if delete_request.has_transaction(): yield self._tx_manager.log_deletes(tr, project_id, delete_request) deletes = [(VersionEntry.from_key(key), None, None) for key in delete_request.key_list()] else: # Eliminate multiple deletes to the same key. deletes_by_key = { key.Encode(): key for key in delete_request.key_list() } deletes = yield [ self._delete(tr, key) for key in six.itervalues(deletes_by_key) ] old_entries = [ old_entry for old_entry, _, _ in deletes if old_entry.present ] versionstamp_future = None if old_entries: versionstamp_future = tr.get_versionstamp() try: yield self._tornado_fdb.commit(tr, convert_exceptions=False) except fdb.FDBError as fdb_error: if fdb_error.code == FDBErrorCodes.NOT_COMMITTED: pass elif fdb_error.code == FDBErrorCodes.COMMIT_RESULT_UNKNOWN: logger.error('Unable to determine commit result. Retrying.') else: raise InternalError(fdb_error.description) retries -= 1 if retries < 0: raise InternalError(fdb_error.description) yield self.dynamic_delete(project_id, delete_request, retries) return if old_entries: self._gc.clear_later(old_entries, versionstamp_future.wait().value) mutations = [(old_entry, None, stats) for old_entry, _, stats in deletes if stats is not None] IOLoop.current().spawn_callback(self._stats_buffer.update, project_id, mutations) # TODO: Once the Cassandra backend is removed, populate a delete response. for old_entry, new_version, _ in deletes: logger.debug(u'new_version: {}'.format(new_version))
def encode(cls, scatter_val, commit_versionstamp): commit_version = struct.unpack('>Q', commit_versionstamp[:8])[0] batch_order = struct.unpack('>H', commit_versionstamp[8:])[0] if not 0 <= scatter_val <= 15: raise InternalError(u'Invalid scatter value') if commit_version >= 2 ** cls.COMMIT_VERSION_BITS: raise InternalError(u'Commit version too high') if batch_order >= 2 ** cls.BATCH_ORDER_BITS: raise InternalError(u'Batch order too high') return (commit_version << 12) + (batch_order << 4) + scatter_val
def type_range(self, type_name): """ Returns a slice that encompasses all values for a property type. """ if type_name == u'NULL': start = six.int2byte(codecs.NULL_CODE) stop = six.int2byte(codecs.NULL_CODE + 1) elif type_name == u'INT64': start = six.int2byte(codecs.MIN_INT64_CODE) stop = six.int2byte(codecs.MAX_INT64_CODE + 1) elif type_name == u'BOOLEAN': start = six.int2byte(codecs.FALSE_CODE) stop = six.int2byte(codecs.TRUE_CODE + 1) elif type_name == u'STRING': start = six.int2byte(codecs.BYTES_CODE) stop = six.int2byte(codecs.BYTES_CODE + 1) elif type_name == u'DOUBLE': start = six.int2byte(codecs.DOUBLE_CODE) stop = six.int2byte(codecs.DOUBLE_CODE + 1) elif type_name == u'POINT': start = six.int2byte(codecs.POINT_CODE) stop = six.int2byte(codecs.POINT_CODE + 1) elif type_name == u'USER': start = six.int2byte(codecs.USER_CODE) stop = six.int2byte(codecs.USER_CODE + 1) elif type_name == u'REFERENCE': start = six.int2byte(codecs.REFERENCE_CODE) stop = six.int2byte(codecs.REFERENCE_CODE + 1) else: raise InternalError(u'Unknown type name') return slice(self.directory.rawPrefix + start, self.directory.rawPrefix + stop)
def unpack(cls, blob, pos, reverse=False): items = [] terminator = encode_marker(TERMINATOR, reverse) kind_marker = encode_marker(cls.KIND_MARKER, reverse) name_marker = encode_marker(cls.NAME_MARKER, reverse) while pos < len(blob): marker = blob[pos] pos += 1 if marker == terminator: break if marker != kind_marker: raise InternalError(u'Encoded path is missing kind') kind, pos = Text.decode(blob, pos, reverse) items.append(kind) marker = blob[pos] pos += 1 if marker == name_marker: elem_name, pos = Text.decode(blob, pos, reverse) items.append(elem_name) else: elem_id, pos = Int64.decode(marker, blob, pos, reverse) items.append(elem_id) return tuple(items), pos
def apply_path_filter(self, op, path, ancestor_path=()): if not isinstance(path, tuple): path = Path.flatten(path) remaining_path = path[len(ancestor_path):] if self._ancestor else path if not remaining_path: raise InternalError(u'Path filter must be within ancestor') start = Path.pack(remaining_path, omit_terminator=True) # Since the commit versionstamp could potentially start with 0xFF, this # selection scans up to the next possible path value. stop = start + six.int2byte(Path.MIN_ID_MARKER) index = -2 if op == Query_Filter.EQUAL: self._set_start(index, start) self._set_stop(index, stop) self._set_stop(index + 1, b'\xFF') return if op == Query_Filter.GREATER_THAN_OR_EQUAL: self._set_start(index, start) elif op == Query_Filter.GREATER_THAN: self._set_start(index, stop) elif op == Query_Filter.LESS_THAN_OR_EQUAL: self._set_stop(index, stop) elif op == Query_Filter.LESS_THAN: self._set_stop(index, start) else: raise BadRequest(u'Unexpected filter operation')
def _upsert(self, tr, entity, old_entry_future=None): auto_id = self._auto_id(entity) if auto_id: # Avoid mutating the object given. new_entity = entity_pb.EntityProto() new_entity.CopyFrom(entity) entity = new_entity last_element = entity.key().path().element(-1) last_element.set_id(self._scattered_allocator.get_id()) if old_entry_future is None: old_entry = yield self._data_manager.get_latest(tr, entity.key()) else: old_entry = yield old_entry_future # If the datastore chose an ID, don't overwrite existing data. if auto_id and old_entry.present: self._scattered_allocator.invalidate() raise InternalError(u'The datastore chose an existing ID') new_version = next_entity_version(old_entry.version) encoded_entity = entity.Encode() yield self._data_manager.put(tr, entity.key(), new_version, encoded_entity) index_stats = yield self._index_manager.put_entries( tr, old_entry, entity) if old_entry.present: yield self._gc.index_deleted_version(tr, old_entry) new_entry = VersionEntry.from_key(entity.key()) new_entry._encoded_entity = encoded_entity new_entry._decoded_entity = entity new_entry.version = new_version raise gen.Return((old_entry, new_entry, index_stats))
def apply_txn_changes(self, project_id, txid, retries=5): logger.debug(u'Applying {}:{}'.format(project_id, txid)) project_id = decode_str(project_id) tr = self._db.create_transaction() read_versionstamp = TransactionID.decode(txid)[1] lookups, queried_groups, mutations = yield self._tx_manager.get_metadata( tr, project_id, txid) try: writes = yield self._apply_mutations(tr, project_id, queried_groups, mutations, lookups, read_versionstamp) finally: yield self._tx_manager.delete(tr, project_id, txid) versionstamp_future = None old_entries = [ old_entry for old_entry, _, _ in writes if old_entry.present ] if old_entries: versionstamp_future = tr.get_versionstamp() try: yield self._tornado_fdb.commit(tr, convert_exceptions=False) except fdb.FDBError as fdb_error: if fdb_error.code != FDBErrorCodes.NOT_COMMITTED: raise InternalError(fdb_error.description) retries -= 1 if retries < 0: raise InternalError(fdb_error.description) yield self.apply_txn_changes(project_id, txid, retries) return if old_entries: self._gc.clear_later(old_entries, versionstamp_future.wait().value) mutations = [(old_entry, FDBDatastore._filter_version(new_entry), index_stats) for old_entry, new_entry, index_stats in writes if index_stats is not None] IOLoop.current().spawn_callback(self._stats_buffer.update, project_id, mutations) logger.debug(u'Finished applying {}:{}'.format(project_id, txid))
def encode_bare(cls, value, byte_count): """ Encodes an integer without a prefix using the specified number of bytes. """ encoded = struct.pack('>Q', value) if any(byte != b'\x00' for byte in encoded[:-byte_count]): raise InternalError(u'Value exceeds maximum size') return encoded[-byte_count:]
def clear_later(self, entries, new_versionstamp): """ Clears deleted entities after sufficient time has passed. """ safe_time = monotonic.monotonic() + MAX_TX_DURATION for entry in entries: # TODO: Strip raw properties and enforce a max queue size to keep memory # usage reasonable. if entry.commit_versionstamp is None: raise InternalError( u'Deleted entry must have a commit versionstamp') self._queue.append((safe_time, entry, new_versionstamp))
def _index_details(self, tr, project_id, index_id): project_indexes = yield self._composite_index_manager.get_definitions( tr, project_id) index_def = next( (ds_index for ds_index in project_indexes if ds_index.id == index_id), None) if index_def is None: raise InternalError(u'Unable to retrieve index details') order_info = tuple((decode_str(prop.name), prop.to_pb().direction()) for prop in index_def.properties) raise gen.Return((index_def.kind, index_def.ancestor, order_info))
def _prop_details(self, prop_name): prop_index = next( (index for index, (name, direction) in enumerate(self._order_info) if name == prop_name), None) if prop_index is None: raise InternalError(u'{} is not in index'.format(prop_name)) index = prop_index + 1 # Account for directory prefix. if self._ancestor: index += 1 return index, self._order_info[prop_index][1]
def _find_terminator(blob, pos, reverse=False): """ Finds the position of the terminator. """ terminator = encode_marker(TERMINATOR, reverse) escape_byte = b'\x00' if reverse else b'\xFF' while True: pos = blob.find(terminator, pos) if pos < 0: raise InternalError(u'Byte string is missing terminator') if blob[pos + 1:pos + 2] != escape_byte: return pos pos += 2
def commit(self, tr, convert_exceptions=True): tornado_future = TornadoFuture() callback = lambda fdb_future: self._handle_fdb_result( fdb_future, tornado_future) commit_future = tr.commit() commit_future.on_ready(callback) try: yield tornado_future except fdb.FDBError as fdb_error: if convert_exceptions: raise InternalError(fdb_error.description) else: raise
def apply_txn_changes(self, project_id, txid, retries=5): logger.debug(u'Applying {}:{}'.format(project_id, txid)) project_id = decode_str(project_id) tr = self._db.create_transaction() read_versionstamp = TransactionID.decode(txid)[1] lookups, queried_groups, mutations = yield self._tx_manager.get_metadata( tr, project_id, txid) try: old_entries = yield self._apply_mutations(tr, project_id, queried_groups, mutations, lookups, read_versionstamp) finally: yield self._tx_manager.delete(tr, project_id, txid) versionstamp_future = None if old_entries: versionstamp_future = tr.get_versionstamp() try: yield self._tornado_fdb.commit(tr, convert_exceptions=False) except fdb.FDBError as fdb_error: if fdb_error.code != FDBErrorCodes.NOT_COMMITTED: raise InternalError(fdb_error.description) retries -= 1 if retries < 0: raise InternalError(fdb_error.description) yield self.apply_txn_changes(project_id, txid, retries) return if old_entries: self._gc.clear_later(old_entries, versionstamp_future.wait().value) logger.debug(u'Finished applying {}:{}'.format(project_id, txid))
def decode_metadata(self, txid, kvs): lookup_rpcs = defaultdict(list) queried_groups = set() mutation_rpcs = [] rpc_type_index = len(self._txid_prefix(txid)) current_versionstamp = None for kv in kvs: rpc_type = kv.key[rpc_type_index] pos = rpc_type_index + 1 if rpc_type == self.QUERIES: namespace, pos = Text.decode(kv.key, pos) group_path = Path.unpack(kv.key, pos)[0] queried_groups.add((namespace, group_path)) continue rpc_versionstamp = kv.key[pos:pos + VERSIONSTAMP_SIZE] if rpc_type == self.LOOKUPS: lookup_rpcs[rpc_versionstamp].append(kv.value) elif rpc_type in (self.PUTS, self.DELETES): if current_versionstamp == rpc_versionstamp: mutation_rpcs[-1].append(kv.value) else: current_versionstamp = rpc_versionstamp mutation_rpcs.append([rpc_type, kv.value]) else: raise InternalError(u'Unrecognized RPC type') lookups = dict() mutations = [] for chunks in six.itervalues(lookup_rpcs): lookups.update([(key.SerializeToString(), key) for key in self._unpack_keys(b''.join(chunks))]) for rpc_info in mutation_rpcs: rpc_type = rpc_info[0] blob = b''.join(rpc_info[1:]) if rpc_type == self.PUTS: mutations.extend(self._unpack_entities(blob)) else: mutations.extend(self._unpack_keys(blob)) return list(six.itervalues(lookups)), queried_groups, mutations
def get(self, tr, key): current_version = yield self._tornado_fdb.get(tr, self.METADATA_KEY) if not current_version.present(): raise InternalError(u'The FDB cluster metadata key is missing') if current_version.value != self._metadata_version: self._metadata_version = current_version.value self._directory_dict.clear() self._directory_keys.clear() full_key = self.root_dir.get_path() + key if full_key not in self: # TODO: This can be made async. # This is performed in a separate transaction so that it can be retried # automatically and so that it's only added to the cache when the # directory has been successfully created. self[full_key] = fdb.directory.create_or_open(self._db, full_key) raise gen.Return(self[full_key])
def current_metadata_version(tr, tornado_fdb): current_version = yield tornado_fdb.get(tr, METADATA_KEY) if not current_version.present(): raise InternalError(u'The FDB cluster metadata key is missing') raise gen.Return(current_version.value)
def dynamic_put(self, project_id, put_request, put_response, retries=5): # logger.debug(u'put_request:\n{}'.format(put_request)) project_id = decode_str(project_id) # TODO: Enforce max key length (100 elements). # Enforce max element size (1500 bytes). # Enforce max kind size (1500 bytes). # Enforce key name regex (reserved names match __.*__). if put_request.auto_id_policy() != put_request.CURRENT: raise BadRequest(u'Sequential allocator is not implemented') tr = self._db.create_transaction() if put_request.has_transaction(): yield self._tx_manager.log_puts(tr, project_id, put_request) writes = { self._collapsible_id(entity): (VersionEntry.from_key(entity.key()), VersionEntry.from_key(entity.key()), None) for entity in put_request.entity_list() } else: # Eliminate multiple puts to the same key. puts_by_key = { self._collapsible_id(entity): entity for entity in put_request.entity_list() } writes = yield { key: self._upsert(tr, entity) for key, entity in six.iteritems(puts_by_key) } old_entries = [ old_entry for old_entry, _, _ in six.itervalues(writes) if old_entry.present ] versionstamp_future = None if old_entries: versionstamp_future = tr.get_versionstamp() try: yield self._tornado_fdb.commit(tr, convert_exceptions=False) except fdb.FDBError as fdb_error: if fdb_error.code == FDBErrorCodes.NOT_COMMITTED: pass elif fdb_error.code == FDBErrorCodes.COMMIT_RESULT_UNKNOWN: logger.error('Unable to determine commit result. Retrying.') else: raise InternalError(fdb_error.description) retries -= 1 if retries < 0: raise InternalError(fdb_error.description) yield self.dynamic_put(project_id, put_request, put_response, retries) return if old_entries: self._gc.clear_later(old_entries, versionstamp_future.wait().value) mutations = [ (old_entry, new_entry, index_stats) for old_entry, new_entry, index_stats in six.itervalues(writes) if index_stats is not None ] IOLoop.current().spawn_callback(self._stats_buffer.update, project_id, mutations) for entity in put_request.entity_list(): write_entry = writes[self._collapsible_id(entity)][1] put_response.add_key().CopyFrom(write_entry.key) if write_entry.version != ABSENT_VERSION: put_response.add_version(write_entry.version)