def add(
self,
node_prefix_input: NibblesInput,
trie_node: HexaryTrieNode,
sub_segments: Sequence[NibblesInput]) -> None:
"""
Add a new cached node body for each of the sub segments supplied. Later cache
lookups will be in the form of get(node_prefix + sub_segments[0]).
:param node_prefix: the path from the root to the cached node
:param trie_node: the body to cache
:param sub_segments: all of the children of the parent which should be made indexable
"""
node_prefix = Nibbles(node_prefix_input)
# remove the cache entry for looking up node_prefix as a child
if node_prefix != ():
# If the cache entry doesn't exist, we can just ignore its absence
self._cache.pop(Nibbles(node_prefix), None)
# add cache entry for each child
for segment in sub_segments:
new_prefix = node_prefix + Nibbles(segment)
self._cache[new_prefix] = (trie_node, Nibbles(segment))
def __init__(self, nibbles_traversed: NibblesInput, node: HexaryTrieNode,
untraversed_tail: NibblesInput, *args) -> None:
super().__init__(
Nibbles(nibbles_traversed),
node,
Nibbles(untraversed_tail),
*args,
)
self._simulated_node = self._make_simulated_node()
def _make_simulated_node(self) -> HexaryTrieNode:
from trie.utils.nodes import (
key_starts_with,
compute_extension_key,
compute_leaf_key,
)
actual_node = self.node
key_tail = self.untraversed_tail
actual_sub_segments = actual_node.sub_segments
if len(key_tail) == 0:
raise ValueError(
"Can only raise a TraversedPartialPath when some series of nibbles was untraversed"
)
if len(actual_sub_segments) == 0:
if not key_starts_with(actual_node.suffix, key_tail):
raise ValidationError(
f"Internal traverse bug: {actual_node.suffix} does not start with {key_tail}"
)
else:
trimmed_suffix = Nibbles(actual_node.suffix[len(key_tail):])
return HexaryTrieNode(
(),
actual_node.value,
trimmed_suffix,
[compute_leaf_key(trimmed_suffix), actual_node.raw[1]],
NodeType(NODE_TYPE_LEAF),
)
elif len(actual_sub_segments) == 1:
extension = actual_sub_segments[0]
if not key_starts_with(extension, key_tail):
raise ValidationError(
f"Internal traverse bug: extension {extension} does not start with {key_tail}"
)
elif len(key_tail) == len(extension):
raise ValidationError(
f"Internal traverse bug: {key_tail} should not equal {extension}"
)
else:
trimmed_extension = Nibbles(extension[len(key_tail):])
return HexaryTrieNode(
(trimmed_extension, ),
actual_node.value,
actual_node.suffix,
[compute_extension_key(trimmed_extension), actual_node.raw[1]],
NodeType(NODE_TYPE_EXTENSION),
)
else:
raise ValidationError(
f"Can only partially traverse into leaf or extension, got {actual_node}"
)
def explore(
self,
old_prefix_input: NibblesInput,
foggy_sub_segments: Sequence[NibblesInput]) -> 'HexaryTrieFog':
"""
The fog lifts from the old prefix. This call returns a HexaryTrieFog that narrows
down the unexplored key prefixes. from the old prefix to the indicated children.
For example, if only the key prefix 0x12 is unexplored, then calling
explore((1, 2), ((3,), (0xe, 0xf))) would mark large swaths of 0x12 explored, leaving only
two prefixes as unknown: 0x123 and 0x12ef. To continue exploring those prefixes, navigate
to them using traverse() or traverse_from().
The sub_segments_input may be empty, which means the old prefix has been fully explored.
"""
old_prefix = Nibbles(old_prefix_input)
sub_segments = [Nibbles(segment) for segment in foggy_sub_segments]
new_fog_prefixes = self._unexplored_prefixes.copy()
try:
new_fog_prefixes.remove(old_prefix)
except KeyError:
raise ValidationError(f"Old parent {old_prefix} not found in {new_fog_prefixes!r}")
if len(set(sub_segments)) != len(sub_segments):
raise ValidationError(
f"Got duplicate sub_segments in {sub_segments} to HexaryTrieFog.explore()"
)
# Further validation that no segment is a prefix of another
all_lengths = set(len(segment) for segment in sub_segments)
if len(all_lengths) > 1:
# The known use case of exploring nodes one at a time will never arrive in this
# validation check which might be slow. Leaf nodes have no sub segments,
# extension nodes have exactly one, and branch nodes have all sub_segments
# of length 1. If a new use case hits this verification, and speed becomes an issue,
# see https://github.com/ethereum/py-trie/issues/107
for segment in sub_segments:
shorter_lengths = [length for length in all_lengths if length < len(segment)]
for check_length in shorter_lengths:
trimmed_segment = segment[:check_length]
if trimmed_segment in sub_segments:
raise ValidationError(
f"Cannot add {segment} which is a child of segment {trimmed_segment}"
)
new_fog_prefixes.update([old_prefix + segment for segment in sub_segments])
return self._new_trie_fog(new_fog_prefixes)
def delete(self, prefix: NibblesInput) -> None:
"""
Delete the cache of the parent node for the given prefix. This only deletes
this prefix's reference to the parent node, not all references to the parent node.
"""
# If the cache entry doesn't exist, we can just ignore its absence
self._cache.pop(Nibbles(prefix), None)
def __init__(self,
missing_node_hash: Hash32,
root_hash: Hash32,
requested_key: bytes,
prefix: Nibbles = None,
*args):
if not isinstance(missing_node_hash, bytes):
raise TypeError("Missing node hash must be bytes, was: %r" %
missing_node_hash)
elif not isinstance(root_hash, bytes):
raise TypeError("Root hash must be bytes, was: %r" % root_hash)
elif not isinstance(requested_key, bytes):
raise TypeError("Requested key must be bytes, was: %r" %
requested_key)
if prefix is not None:
prefix_nibbles: Optional[Nibbles] = Nibbles(prefix)
else:
prefix_nibbles = None
super().__init__(
HexBytes(missing_node_hash),
HexBytes(root_hash),
HexBytes(requested_key),
prefix_nibbles,
*args,
)
def next(self, key_bytes: Optional[bytes] = None) -> Optional[bytes]:
"""
Find the next key to the right from the given key, or None if there is
no key to the right.
.. NOTE:: To iterate the full trie, consider using keys() instead, for performance
:param key_bytes: the key to start your search from. If None, return
the first possible key.
:return: key in bytes to the right of key_bytes, or None
"""
root = self._trie.root_node
none_traversed = Nibbles(())
if key_bytes is None:
next_key = self._get_next_key(root, none_traversed)
else:
key = bytes_to_nibbles(key_bytes)
next_key = self._get_key_after(root, key, none_traversed)
if next_key is None:
return None
else:
return nibbles_to_bytes(next_key)
def test_valid_TraversedPartialPath_traversed_nibbles(valid_nibbles,
key_encoding):
some_node_key = (1, 2)
node = annotate_node([key_encoding(some_node_key), b'random-value'])
exception = TraversedPartialPath(valid_nibbles, node, some_node_key[:1])
assert exception.nibbles_traversed == valid_nibbles
assert str(Nibbles(valid_nibbles)) in repr(exception)
def nearest_unknown(self, key_input: NibblesInput = ()) -> Nibbles:
"""
Find the foggy prefix that is nearest to the supplied key.
If prefixes are exactly the same distance to the left and right,
then return the prefix on the right.
:raises PerfectVisibility: if there are no foggy prefixes remaining
"""
key = Nibbles(key_input)
index = self._unexplored_prefixes.bisect(key)
if index == 0:
# If sorted set is empty, bisect will return 0
# But it might also return 0 if the search value is lower than the lowest existing
try:
return self._unexplored_prefixes[0]
except IndexError as exc:
raise PerfectVisibility("There are no more unexplored prefixes") from exc
elif index == len(self._unexplored_prefixes):
return self._unexplored_prefixes[-1]
else:
nearest_left = self._unexplored_prefixes[index - 1]
nearest_right = self._unexplored_prefixes[index]
# is the left or right unknown prefix closer?
left_distance = self._prefix_distance(nearest_left, key)
right_distance = self._prefix_distance(key, nearest_right)
if left_distance < right_distance:
return nearest_left
else:
return nearest_right
def nearest_right(self, key_input: NibblesInput) -> Nibbles:
"""
Find the foggy prefix that is nearest on the right to the supplied key.
:raises PerfectVisibility: if there are no foggy prefixes to the right
"""
key = Nibbles(key_input)
index = self._unexplored_prefixes.bisect(key)
if index == 0:
# If sorted set is empty, bisect will return 0
# But it might also return 0 if the search value is lower than the lowest existing
try:
return self._unexplored_prefixes[0]
except IndexError as exc:
raise PerfectVisibility("There are no more unexplored prefixes") from exc
else:
nearest_left = self._unexplored_prefixes[index - 1]
# always return nearest right, unless prefix of key is unexplored
if key_starts_with(key, nearest_left):
return nearest_left
else:
try:
# This can raise a IndexError if index == len(unexplored prefixes)
return self._unexplored_prefixes[index]
except IndexError as exc:
raise FullDirectionalVisibility(
f"There are no unexplored prefixes to the right of {key}"
) from exc
def __init__(self, missing_node_hash: Hash32,
nibbles_traversed: NibblesInput, *args) -> None:
if not isinstance(missing_node_hash, bytes):
raise TypeError("Missing node hash must be bytes, was: %r" %
missing_node_hash)
super().__init__(HexBytes(missing_node_hash),
Nibbles(nibbles_traversed), *args)
def _traverse_from(self, node: RawHexaryNode,
trie_key) -> Tuple[RawHexaryNode, Nibbles]:
"""
Traverse down the trie from the given node, using the trie_key to navigate.
At each node, consume a prefix from the key, and navigate to its child. Repeat with that
child node and so on, until:
- there is no key remaining, or
- the child node is a blank node, or
- the child node is a leaf node
:return: (the deepest child node, the unconsumed suffix of the key)
:raises MissingTraversalNode: if a node body is missing from the database
"""
remaining_key = trie_key
while remaining_key:
node_type = get_node_type(node)
if node_type == NODE_TYPE_BLANK:
return BLANK_NODE, Nibbles(
()) # type: ignore # mypy thinks BLANK_NODE != b''
elif node_type == NODE_TYPE_LEAF:
return node, remaining_key
elif node_type == NODE_TYPE_EXTENSION:
try:
next_node_pointer, remaining_key = self._traverse_extension(
node, remaining_key)
except _PartialTraversal:
# could only descend part-way into an extension node
return node, remaining_key
elif node_type == NODE_TYPE_BRANCH:
next_node_pointer = node[remaining_key[0]]
remaining_key = remaining_key[1:]
else:
raise Exception("Invariant: This shouldn't ever happen")
try:
node = self.get_node(next_node_pointer)
except KeyError as exc:
used_key = trie_key[:len(trie_key) - len(remaining_key)]
raise MissingTraversalNode(exc.args[0], used_key)
# navigated down the full key
return node, Nibbles(())
def get(self, prefix: NibblesInput) -> Tuple[HexaryTrieNode, Nibbles]:
"""
Find the cached node body of the parent of the given prefix.
:return: parent node body, and the path from parent to the given prefix
:raises KeyError: if there is no cached value for the prefix
"""
return self._cache[Nibbles(prefix)]
def test_valid_TraversedPartialPath_untraversed_nibbles(
valid_nibbles, key_encoding):
# This exception means that the actual node key should have more than the untraversed amount
# So we simulate some longer key for the given node
longer_key = valid_nibbles + (0, )
node = annotate_node([key_encoding(longer_key), b'random-value'])
exception = TraversedPartialPath((), node, valid_nibbles)
assert exception.untraversed_tail == valid_nibbles
assert str(Nibbles(valid_nibbles)) in repr(exception)
def test_trie_walk_backfilling_with_traverse_from(trie_keys,
minimum_value_length,
index_nibbles):
"""
Like test_trie_walk_backfilling but using the HexaryTrie.traverse_from API
"""
node_db, trie = trie_from_keys(trie_keys, minimum_value_length, prune=True)
index_key = Nibbles(index_nibbles)
# delete all nodes
dropped_nodes = dict(node_db)
node_db.clear()
# traverse_from() cannot traverse to the root node, so resolve that manually
try:
root = trie.root_node
except MissingTraversalNode as exc:
node_db[exc.missing_node_hash] = dropped_nodes.pop(
exc.missing_node_hash)
root = trie.root_node
# Core of the test: use the fog to convince yourself that you've traversed the entire trie
fog = HexaryTrieFog()
for _ in range(100000):
# Look up the next prefix to explore
try:
nearest_key = fog.nearest_unknown(index_key)
except PerfectVisibility:
# Test Complete!
break
# Try to navigate to the prefix, catching any errors about nodes missing from the DB
try:
node = trie.traverse_from(root, nearest_key)
except MissingTraversalNode as exc:
# Node was missing, so fill in the node and try again
node_db[exc.missing_node_hash] = dropped_nodes.pop(
exc.missing_node_hash)
continue
else:
# Node was found, use the found node to "lift the fog" down to its longer prefixes
fog = fog.explore(nearest_key, node.sub_segments)
else:
assert False, "Must finish iterating the trie within ~100k runs"
# Make sure we removed all the dropped nodes to push them back to the trie db
assert len(dropped_nodes) == 0
# Make sure the fog agrees that it's completed
assert fog.is_complete
# Make sure we can walk the whole trie without any missing nodes
iterator = NodeIterator(trie)
found_keys = set(iterator.keys())
# Make sure we found all the keys
assert found_keys == set(trie_keys)
def test_nibbles_repr(nibbles_input, as_ipython):
nibbles = Nibbles(nibbles_input)
if as_ipython:
class FakePrinter:
str_buffer = ''
def text(self, new_text):
self.str_buffer += new_text
p = FakePrinter()
nibbles._repr_pretty_(p, cycle=False)
repr_string = p.str_buffer
else:
repr_string = repr(nibbles)
evaluated_repr = eval(repr_string)
assert evaluated_repr == tuple(nibbles_input)
re_cast = Nibbles(evaluated_repr)
assert re_cast == nibbles
def deserialize(cls, encoded: bytes) -> 'HexaryTrieFog':
serial_prefix = b'HexaryTrieFog:'
if not encoded.startswith(serial_prefix):
raise ValueError(f"Cannot deserialize this into HexaryTrieFog object: {encoded!r}")
else:
encoded_list = encoded[len(serial_prefix):]
prefix_list = ast.literal_eval(encoded_list.decode())
deserialized_prefixes = SortedSet(
# decode nibbles from compressed bytes value, and validate each value in range(16)
Nibbles(decode_nibbles(prefix))
for prefix in prefix_list
)
return cls._new_trie_fog(deserialized_prefixes)
def test_trie_walk_backfilling(trie_keys, index_nibbles):
"""
- Create a random trie of 3-byte keys
- Drop all node bodies from the trie
- Use fog to index into random parts of the trie
- Every time a node is missing from the DB, replace it and retry
- Repeat until full trie has been explored with the HexaryTrieFog
"""
node_db, trie = _make_trie(trie_keys)
index_key = Nibbles(index_nibbles)
# delete all nodes
dropped_nodes = dict(node_db)
node_db.clear()
# Core of the test: use the fog to convince yourself that you've traversed the entire trie
fog = HexaryTrieFog()
for _ in range(100000):
# Look up the next prefix to explore
try:
nearest_key = fog.nearest_unknown(index_key)
except PerfectVisibility:
# Test Complete!
break
# Try to navigate to the prefix, catching any errors about nodes missing from the DB
try:
node = trie.traverse(nearest_key)
except MissingTraversalNode as exc:
# Node was missing, so fill in the node and try again
node_db[exc.missing_node_hash] = dropped_nodes.pop(
exc.missing_node_hash)
continue
else:
# Node was found, use the found node to "lift the fog" down to its longer prefixes
fog = fog.explore(nearest_key, node.sub_segments)
else:
assert False, "Must finish iterating the trie within ~100k runs"
# Make sure we removed all the dropped nodes to push them back to the trie db
assert len(dropped_nodes) == 0
# Make sure the fog agrees that it's completed
assert fog.is_complete
# Make sure we can walk the whole trie without any missing nodes
iterator = NodeIterator(trie)
found_keys = set(iterator.keys())
# Make sure we found all the keys
assert found_keys == set(trie_keys)
def traverse(self, trie_key_input: NibblesInput) -> HexaryTrieNode:
"""
Find the node at the path of nibbles provided. The most trivial example is
to get the root node, using ``traverse(())``.
:param trie_key_input: the series of nibbles to traverse to arrive at the node of interest
:return: annotated node at the given path
:raises MissingTraversalNode: if a node body is missing from the database
:raises TraversedPartialPath: if trie key extends part-way down an extension or leaf node
"""
trie_key = Nibbles(trie_key_input)
node, remaining_key = self._traverse(self.root_hash, trie_key)
annotated_node = annotate_node(node)
if remaining_key:
path_to_node = trie_key[:len(trie_key) - len(remaining_key)]
raise TraversedPartialPath(path_to_node, annotated_node)
else:
return annotated_node
def traverse_from(self, parent_node: HexaryTrieNode,
trie_key_input: Nibbles) -> HexaryTrieNode:
"""
Find the node at the path of nibbles provided. You cannot navigate to the root node
this way (without already having the root node body, to supply as the argument).
The trie does *not* re-verify the path/hashes from the node prefix to the node.
:param trie_key_input: the sub-key used to traverse from the given node to the returned node
:raises MissingTraversalNode: if a node body is missing from the database
:raises TraversedPartialPath: if trie key extends part-way down an extension or leaf node
"""
trie_key = Nibbles(trie_key_input)
node, remaining_key = self._traverse_from(parent_node.raw, trie_key)
annotated_node = annotate_node(node)
if remaining_key:
path_to_node = trie_key[:len(trie_key) - len(remaining_key)]
raise TraversedPartialPath(path_to_node, annotated_node)
else:
return annotated_node
def annotate_node(node_body: RawHexaryNode) -> HexaryTrieNode:
"""
Normalize the raw node body to a HexaryTrieNode, for external consumption.
"""
node_type = get_node_type(node_body)
if node_type == NODE_TYPE_LEAF:
return HexaryTrieNode(
sub_segments=(),
value=bytes(node_body[-1]),
suffix=Nibbles(extract_key(node_body)),
raw=node_body,
node_type=NodeType(node_type),
)
elif node_type == NODE_TYPE_BRANCH:
sub_segments = tuple(
Nibbles((nibble, )) for nibble in range(16)
if bool(node_body[nibble]))
return HexaryTrieNode(
sub_segments=sub_segments,
value=bytes(node_body[-1]),
suffix=Nibbles(()),
raw=node_body,
node_type=NodeType(node_type),
)
elif node_type == NODE_TYPE_EXTENSION:
key_extension = extract_key(node_body)
return HexaryTrieNode(
sub_segments=(Nibbles(key_extension), ),
value=b'',
suffix=Nibbles(()),
raw=node_body,
node_type=NodeType(node_type),
)
elif node_type == NODE_TYPE_BLANK:
# empty trie
return HexaryTrieNode(
sub_segments=(),
value=b'',
suffix=Nibbles(()),
raw=node_body,
node_type=NodeType(node_type),
)
else:
raise NotImplementedError()
def test_valid_MissingTraversalNode_nibbles(valid_nibbles):
exception = MissingTraversalNode(b'', valid_nibbles)
assert exception.nibbles_traversed == valid_nibbles
assert str(Nibbles(valid_nibbles)) in repr(exception)
def __init__(self, nibbles_traversed: NibblesInput, node: HexaryTrieNode,
*args) -> None:
super().__init__(Nibbles(nibbles_traversed), node, *args)
def test_valid_MissingTrieNode_prefix(valid_prefix):
exception = MissingTrieNode(b'', b'', b'', valid_prefix)
assert exception.prefix == valid_prefix
if valid_prefix is not None:
assert str(Nibbles(valid_prefix)) in repr(exception)
def test_valid_TraversedPartialPath_nibbles(valid_nibbles):
exception = TraversedPartialPath(valid_nibbles, b'')
assert exception.nibbles_traversed == valid_nibbles
assert str(Nibbles(valid_nibbles)) in repr(exception)
def test_valid_nibbles(valid_nibbles):
typed_nibbles = Nibbles(valid_nibbles)
assert typed_nibbles == tuple(valid_nibbles)
def test_invalid_nibbles(invalid_nibbles, exception):
with pytest.raises(exception):
Nibbles(invalid_nibbles)