def _make_simulated_node(self) -> HexaryTrieNode: from trie.utils.nodes import ( key_starts_with, compute_extension_key, compute_leaf_key, ) actual_node = self.node key_tail = self.untraversed_tail actual_sub_segments = actual_node.sub_segments if len(key_tail) == 0: raise ValueError( "Can only raise a TraversedPartialPath when some series of nibbles was untraversed" ) if len(actual_sub_segments) == 0: if not key_starts_with(actual_node.suffix, key_tail): raise ValidationError( f"Internal traverse bug: {actual_node.suffix} does not start with {key_tail}" ) else: trimmed_suffix = Nibbles(actual_node.suffix[len(key_tail):]) return HexaryTrieNode( (), actual_node.value, trimmed_suffix, [compute_leaf_key(trimmed_suffix), actual_node.raw[1]], NodeType(NODE_TYPE_LEAF), ) elif len(actual_sub_segments) == 1: extension = actual_sub_segments[0] if not key_starts_with(extension, key_tail): raise ValidationError( f"Internal traverse bug: extension {extension} does not start with {key_tail}" ) elif len(key_tail) == len(extension): raise ValidationError( f"Internal traverse bug: {key_tail} should not equal {extension}" ) else: trimmed_extension = Nibbles(extension[len(key_tail):]) return HexaryTrieNode( (trimmed_extension, ), actual_node.value, actual_node.suffix, [compute_extension_key(trimmed_extension), actual_node.raw[1]], NodeType(NODE_TYPE_EXTENSION), ) else: raise ValidationError( f"Can only partially traverse into leaf or extension, got {actual_node}" )
def _get_proof(self, node, trie_key, proven_len=0, last_proof=tuple()): updated_proof = last_proof + (node, ) unproven_key = trie_key[proven_len:] node_type = get_node_type(node) if node_type == NODE_TYPE_BLANK: return last_proof elif node_type == NODE_TYPE_LEAF: return updated_proof elif node_type == NODE_TYPE_EXTENSION: current_key = extract_key(node) if key_starts_with(unproven_key, current_key): next_node = self.get_node(node[1]) new_proven_len = proven_len + len(current_key) return self._get_proof(next_node, trie_key, new_proven_len, updated_proof) else: return updated_proof elif node_type == NODE_TYPE_BRANCH: if not unproven_key: return updated_proof next_node = self.get_node(node[unproven_key[0]]) new_proven_len = proven_len + 1 return self._get_proof(next_node, trie_key, new_proven_len, updated_proof) else: raise Exception("Invariant: This shouldn't ever happen")
def nearest_right(self, key_input: NibblesInput) -> Nibbles: """ Find the foggy prefix that is nearest on the right to the supplied key. :raises PerfectVisibility: if there are no foggy prefixes to the right """ key = Nibbles(key_input) index = self._unexplored_prefixes.bisect(key) if index == 0: # If sorted set is empty, bisect will return 0 # But it might also return 0 if the search value is lower than the lowest existing try: return self._unexplored_prefixes[0] except IndexError as exc: raise PerfectVisibility("There are no more unexplored prefixes") from exc else: nearest_left = self._unexplored_prefixes[index - 1] # always return nearest right, unless prefix of key is unexplored if key_starts_with(key, nearest_left): return nearest_left else: try: # This can raise a IndexError if index == len(unexplored prefixes) return self._unexplored_prefixes[index] except IndexError as exc: raise FullDirectionalVisibility( f"There are no unexplored prefixes to the right of {key}" ) from exc
def _iter(self, node, key): node_type = get_node_type(node) if node_type == NODE_TYPE_BLANK: return None elif node_type == NODE_TYPE_LEAF: descend_key = extract_key(node) if descend_key > key: return descend_key return None elif node_type == NODE_TYPE_BRANCH: scan_range = range(16) if len(key): sub_node = self.trie.get_node(node[key[0]]) nibbles = self._iter(sub_node, key[1:]) if nibbles is not None: return (key[0],) + nibbles scan_range = range(key[0] + 1, 16) for i in scan_range: sub_node = self.trie.get_node(node[i]) nibbles = self._get_next(sub_node) if nibbles is not None: return (i,) + nibbles return None elif node_type == NODE_TYPE_EXTENSION: descend_key = extract_key(node) sub_node = self.trie.get_node(node[1]) sub_key = key[len(descend_key):] if key_starts_with(key, descend_key): nibbles = self._iter(sub_node, sub_key) if nibbles is not None: return descend_key + nibbles return None
def _contiguous_accounts_complete_fraction(self) -> float: """ Estimate the completed fraction of the trie that is contiguous with the current index (which rotates every 32 blocks) It will be probably be quite noticeable that it will get "stuck" when downloading a lot of storage, because we'll have to blow it up to more than a percentage to see any significant change within 32 blocks. (when the index will change again anyway) :return: a number in the range [0, 1] (+/- rounding error) estimating trie completion contiguous with the current backfill index key """ starting_index = bytes_to_nibbles(self._next_trie_root_hash) unknown_prefixes = self._account_tracker._trie_fog._unexplored_prefixes if len(unknown_prefixes) == 0: return 1 # find the nearest unknown prefix (typically, on the right) nearest_index = unknown_prefixes.bisect(starting_index) # Get the nearest unknown prefix to the left if nearest_index == 0: left_prefix = (0, ) * 64 else: left_prefix = unknown_prefixes[nearest_index - 1] if key_starts_with(starting_index, left_prefix): # The prefix of the starting index is unknown, so the index # itself is unknown. return 0 # Get the nearest unknown prefix to the right if len(unknown_prefixes) == nearest_index: right_prefix = (0xf, ) * 64 else: right_prefix = unknown_prefixes[nearest_index] # Use the space between the unknown prefixes to estimate the completed contiguous fraction # At the base, every gap in the first nibble is a full 1/16th of the state complete known_first_nibbles = right_prefix[0] - left_prefix[0] - 1 completed_fraction_base = (1 / 16) * known_first_nibbles # Underneath, you can count completed subtrees on the right, each child 1/16 of the parent right_side_completed = sum( nibble * (1 / 16)**nibble_depth for nibble_depth, nibble in enumerate(right_prefix[1:], 2)) # Do the same on the left left_side_completed = sum( (0xf - nibble) * (1 / 16)**nibble_depth for nibble_depth, nibble in enumerate(left_prefix[1:], 2)) # Add up all completed areas return left_side_completed + completed_fraction_base + right_side_completed
def _traverse_from(self, node: RawHexaryNode, trie_key) -> Tuple[RawHexaryNode, Nibbles]: """ Traverse down the trie from the given node, using the trie_key to navigate. At each node, consume a prefix from the key, and navigate to its child. Repeat with that child node and so on, until: - there is no key remaining, or - the child node is a blank node, or - the child node is a leaf node :return: (the deepest child node, the unconsumed suffix of the key) :raises MissingTraversalNode: if a node body is missing from the database """ remaining_key = trie_key while remaining_key: node_type = get_node_type(node) if node_type == NODE_TYPE_BLANK: return BLANK_NODE, ( ) # type: ignore # mypy thinks BLANK_NODE != b'' elif node_type == NODE_TYPE_LEAF: leaf_key = extract_key(node) if key_starts_with(leaf_key, remaining_key): return node, remaining_key else: # The trie key and leaf node key branch away from each other, so there # is no node at the specified key. return BLANK_NODE, ( ) # type: ignore # mypy thinks BLANK_NODE != b'' elif node_type == NODE_TYPE_EXTENSION: try: next_node_pointer, remaining_key = self._traverse_extension( node, remaining_key) except _PartialTraversal: # could only descend part-way into an extension node return node, remaining_key elif node_type == NODE_TYPE_BRANCH: next_node_pointer = node[remaining_key[0]] remaining_key = remaining_key[1:] else: raise Exception("Invariant: This shouldn't ever happen") try: node = self.get_node(next_node_pointer) except KeyError as exc: used_key = trie_key[:len(trie_key) - len(remaining_key)] raise MissingTraversalNode(exc.args[0], used_key) # navigated down the full key return node, Nibbles(())
def _get_kv_node(self, node, trie_key): current_key = extract_key(node) node_type = get_node_type(node) if node_type == NODE_TYPE_LEAF: if trie_key == current_key: return node[1] else: return BLANK_NODE elif node_type == NODE_TYPE_EXTENSION: if key_starts_with(trie_key, current_key): sub_node = self.get_node(node[1]) return self._get(sub_node, trie_key[len(current_key):]) else: return BLANK_NODE else: raise Exception("Invariant: unreachable code path")
def _delete_kv_node(self, node, trie_key): current_key = extract_key(node) if not key_starts_with(trie_key, current_key): # key not present?.... return node node_type = get_node_type(node) if node_type == NODE_TYPE_LEAF: if trie_key == current_key: return BLANK_NODE else: return node sub_node_key = trie_key[len(current_key):] sub_node = self.get_node(node[1]) new_sub_node = self._delete(sub_node, sub_node_key) encoded_new_sub_node = self._persist_node(new_sub_node) if encoded_new_sub_node == node[1]: return node if new_sub_node == BLANK_NODE: return BLANK_NODE new_sub_node_type = get_node_type(new_sub_node) if new_sub_node_type in {NODE_TYPE_LEAF, NODE_TYPE_EXTENSION}: self._prune_node(new_sub_node) new_key = current_key + decode_nibbles(new_sub_node[0]) return [encode_nibbles(new_key), new_sub_node[1]] if new_sub_node_type == NODE_TYPE_BRANCH: return [encode_nibbles(current_key), encoded_new_sub_node] raise Exception("Invariant, this code path should not be reachable")