def test_iter(random):
trie, contents = make_random_trie(random)
iterator = NodeIterator(trie)
visited = []
key = iterator.next(b'')
assert key is not None
while key is not None:
visited.append(key)
key = iterator.next(key)
assert visited == sorted(contents.keys())
def test_trie_walk_backfilling_with_traverse_from(trie_keys,
minimum_value_length,
index_nibbles):
"""
Like test_trie_walk_backfilling but using the HexaryTrie.traverse_from API
"""
node_db, trie = trie_from_keys(trie_keys, minimum_value_length, prune=True)
index_key = Nibbles(index_nibbles)
# delete all nodes
dropped_nodes = dict(node_db)
node_db.clear()
# traverse_from() cannot traverse to the root node, so resolve that manually
try:
root = trie.root_node
except MissingTraversalNode as exc:
node_db[exc.missing_node_hash] = dropped_nodes.pop(
exc.missing_node_hash)
root = trie.root_node
# Core of the test: use the fog to convince yourself that you've traversed the entire trie
fog = HexaryTrieFog()
for _ in range(100000):
# Look up the next prefix to explore
try:
nearest_key = fog.nearest_unknown(index_key)
except PerfectVisibility:
# Test Complete!
break
# Try to navigate to the prefix, catching any errors about nodes missing from the DB
try:
node = trie.traverse_from(root, nearest_key)
except MissingTraversalNode as exc:
# Node was missing, so fill in the node and try again
node_db[exc.missing_node_hash] = dropped_nodes.pop(
exc.missing_node_hash)
continue
else:
# Node was found, use the found node to "lift the fog" down to its longer prefixes
fog = fog.explore(nearest_key, node.sub_segments)
else:
assert False, "Must finish iterating the trie within ~100k runs"
# Make sure we removed all the dropped nodes to push them back to the trie db
assert len(dropped_nodes) == 0
# Make sure the fog agrees that it's completed
assert fog.is_complete
# Make sure we can walk the whole trie without any missing nodes
iterator = NodeIterator(trie)
found_keys = set(iterator.keys())
# Make sure we found all the keys
assert found_keys == set(trie_keys)
def test_iter_error():
trie = HexaryTrie({})
trie[b'cat'] = b'cat'
trie[b'dog'] = b'dog'
trie[b'bird'] = b'bird'
assert is_extension_node(trie.root_node)
node_to_remove = trie.root_node[1]
trie.db.pop(node_to_remove)
iterator = NodeIterator(trie)
key = b''
with pytest.raises(KeyError):
while key is not None:
key = iterator.next(key)
def test_trie_next_prev_using_fixtures(fixture_name, fixture):
trie = HexaryTrie(db={})
for k in fixture['in']:
k = k.encode('utf-8')
trie[k] = k
iterator = NodeIterator(trie)
for point, _, nxt in fixture['tests']:
point = point.encode('utf-8')
nxt = nxt.encode('utf-8')
if nxt == b'':
nxt = None
assert nxt == iterator.next(point)
def test_trie_walk_backfilling(trie_keys, index_nibbles):
"""
- Create a random trie of 3-byte keys
- Drop all node bodies from the trie
- Use fog to index into random parts of the trie
- Every time a node is missing from the DB, replace it and retry
- Repeat until full trie has been explored with the HexaryTrieFog
"""
node_db, trie = _make_trie(trie_keys)
index_key = Nibbles(index_nibbles)
# delete all nodes
dropped_nodes = dict(node_db)
node_db.clear()
# Core of the test: use the fog to convince yourself that you've traversed the entire trie
fog = HexaryTrieFog()
for _ in range(100000):
# Look up the next prefix to explore
try:
nearest_key = fog.nearest_unknown(index_key)
except PerfectVisibility:
# Test Complete!
break
# Try to navigate to the prefix, catching any errors about nodes missing from the DB
try:
node = trie.traverse(nearest_key)
except MissingTraversalNode as exc:
# Node was missing, so fill in the node and try again
node_db[exc.missing_node_hash] = dropped_nodes.pop(
exc.missing_node_hash)
continue
else:
# Node was found, use the found node to "lift the fog" down to its longer prefixes
fog = fog.explore(nearest_key, node.sub_segments)
else:
assert False, "Must finish iterating the trie within ~100k runs"
# Make sure we removed all the dropped nodes to push them back to the trie db
assert len(dropped_nodes) == 0
# Make sure the fog agrees that it's completed
assert fog.is_complete
# Make sure we can walk the whole trie without any missing nodes
iterator = NodeIterator(trie)
found_keys = set(iterator.keys())
# Make sure we found all the keys
assert found_keys == set(trie_keys)
def _has_full_state_db(headerdb):
head = headerdb.get_canonical_head()
if head.block_number < min_block_number:
return False
# reset statistics
collected_data.clear()
raw_db = headerdb.db
state_root = head.state_root
trie = HexaryTrie(raw_db, state_root)
iterator = NodeIterator(trie)
try:
for key, value in iterator.items():
collected_data["num_accounts"] += 1
account = rlp.decode(value, sedes=Account)
if account.code_hash == EMPTY_SHA3:
pass
elif account.code_hash not in raw_db:
# missing bytecodes
logging.warning("Missing bytecode for account at 0x%s",
key.hex())
return False
else:
collected_data["num_bytecodes"] += 1
if account.storage_root != BLANK_NODE_HASH:
storage_trie = HexaryTrie(raw_db, account.storage_root)
try:
for _key in NodeIterator(storage_trie).keys():
# We don't care what the keys are, just that we can iterate through them
collected_data["num_storage_slots"] += 1
except MissingTraversalNode as exc:
logging.warning(
"Missing storage for account at 0x%s: %s",
key.hex(), exc)
return False
except MissingTraversalNode:
return False
else:
return True
def test_trie_walk_root_change_with_cached_traverse_from(
do_cache_reset,
trie_keys,
minimum_value_length,
number_explorations,
trie_changes,
index_nibbles,
index_nibbles2,
):
"""
Like test_trie_walk_root_change_with_traverse but using HexaryTrie.traverse_from
when possible.
"""
# Turn on pruning to simulate having peers lose access to old trie nodes over time
node_db, trie = trie_from_keys(trie_keys, minimum_value_length, prune=True)
number_explorations %= len(node_db)
cache = TrieFrontierCache()
# delete all nodes
missing_nodes = dict(node_db)
node_db.clear()
# First walk
index_key = tuple(index_nibbles)
fog = HexaryTrieFog()
for _ in range(number_explorations):
try:
nearest_prefix = fog.nearest_unknown(index_key)
except PerfectVisibility:
assert False, "Number explorations should be lower than database size, shouldn't finish"
try:
# Use the cache, if possible, to look up the parent node of nearest_prefix
try:
cached_node, uncached_key = cache.get(nearest_prefix)
except KeyError:
# Must navigate from the root. In this 1st walk, only the root should not be cached
assert nearest_prefix == ()
node = trie.traverse(nearest_prefix)
else:
# Only one database lookup required
node = trie.traverse_from(cached_node, uncached_key)
# Note that a TraversedPartialPath should not happen here, because no trie changes
# have happened, so we should have a perfect picture of the trie
except MissingTraversalNode as exc:
# Each missing node should only need to be retrieve (at most) once
node_db[exc.missing_node_hash] = missing_nodes.pop(
exc.missing_node_hash)
continue
else:
fog = fog.explore(nearest_prefix, node.sub_segments)
if node.sub_segments:
cache.add(nearest_prefix, node, node.sub_segments)
else:
cache.delete(nearest_prefix)
# Modify Trie mid-walk, keeping track of the expected list of final keys
expected_final_keys = set(trie_keys)
with trie.squash_changes() as trie_batch:
for change in trie_changes:
# repeat until change is complete
change_complete = False
while not change_complete:
# Catch any missing nodes during trie change, and fix them up.
# This is equivalent to Trinity's "Beam Sync".
try:
if isinstance(change, bytes):
# insert!
trie_batch[change] = change.rjust(
minimum_value_length, b'3')
expected_final_keys.add(change)
else:
key_index, new_value = change
key = trie_keys[key_index % len(trie_keys)]
if new_value is None:
del trie_batch[key]
expected_final_keys.discard(key)
else:
# update (though may be an insert, if there was a previous delete)
trie_batch[key] = new_value
expected_final_keys.add(key)
except MissingTrieNode as exc:
node_db[exc.missing_node_hash] = missing_nodes.pop(
exc.missing_node_hash)
else:
change_complete = True
# Second walk
index_key2 = tuple(index_nibbles2)
if do_cache_reset:
cache = TrieFrontierCache()
for _ in range(100000):
try:
nearest_prefix = fog.nearest_unknown(index_key2)
except PerfectVisibility:
# Complete!
break
try:
try:
cached_node, uncached_key = cache.get(nearest_prefix)
except KeyError:
node = trie.traverse(nearest_prefix)
cached_node = None
else:
node = trie.traverse_from(cached_node, uncached_key)
except MissingTraversalNode as exc:
node_hash = exc.missing_node_hash
if node_hash in missing_nodes:
# Each missing node should only need to be retrieve (at most) once
node_db[node_hash] = missing_nodes.pop(node_hash)
elif cached_node is not None:
# But, it could also be missing because of an old cached node
# Delete the bad cache and try again
cache.delete(nearest_prefix)
else:
raise AssertionError(f"Bad node hash request: {node_hash}")
continue
except TraversedPartialPath as exc:
node = exc.simulated_node
sub_segments = node.sub_segments
fog = fog.explore(nearest_prefix, sub_segments)
if sub_segments:
cache.add(nearest_prefix, node, sub_segments)
else:
cache.delete(nearest_prefix)
else:
assert False, "Must finish iterating the trie within ~100k runs"
# Final assertions
assert fog.is_complete
# We do *not* know that we have replaced all the missing_nodes, because of the trie changes
# Make sure we can walk the whole trie without any missing nodes
iterator = NodeIterator(trie)
found_keys = set(iterator.keys())
assert found_keys == expected_final_keys
def test_trie_walk_root_change_with_traverse(
trie_keys,
minimum_value_length,
number_explorations,
trie_changes,
index_nibbles,
index_nibbles2,
):
"""
Like test_trie_walk_backfilling, but:
- Halt the trie walk early
- Modify the trie according to parameter trie_changes
- Continue walking the trie using the same HexaryTrieFog, until completion
- Verify that all required database values were replaced (where only the nodes under
the NEW trie root are required)
"""
# Turn on pruning to simulate having peers lose access to old trie nodes over time
node_db, trie = trie_from_keys(trie_keys, minimum_value_length, prune=True)
number_explorations %= len(node_db)
# delete all nodes
missing_nodes = dict(node_db)
node_db.clear()
# First walk
index_key = tuple(index_nibbles)
fog = HexaryTrieFog()
for _ in range(number_explorations):
# Look up the next prefix to explore
try:
nearest_key = fog.nearest_unknown(index_key)
except PerfectVisibility:
assert False, "Number explorations should be lower than database size, shouldn't finish"
# Try to navigate to the prefix, catching any errors about nodes missing from the DB
try:
node = trie.traverse(nearest_key)
# Note that a TraversedPartialPath should not happen here, because no trie changes
# have happened, so we should have a perfect picture of the trie
except MissingTraversalNode as exc:
# Node was missing, so fill in the node and try again
node_db[exc.missing_node_hash] = missing_nodes.pop(
exc.missing_node_hash)
continue
else:
# Node was found, use the found node to "lift the fog" down to its longer prefixes
fog = fog.explore(nearest_key, node.sub_segments)
# Modify Trie mid-walk, keeping track of the expected list of final keys
expected_final_keys = set(trie_keys)
with trie.squash_changes() as trie_batch:
for change in trie_changes:
# repeat until change is complete
change_complete = False
while not change_complete:
# Catch any missing nodes during trie change, and fix them up.
# This is equivalent to Trinity's "Beam Sync".
try:
if isinstance(change, bytes):
# insert!
trie_batch[change] = change
expected_final_keys.add(change)
else:
key_index, new_value = change
key = trie_keys[key_index % len(trie_keys)]
if new_value is None:
del trie_batch[key]
expected_final_keys.discard(key)
else:
# update (though may be an insert, if there was a previous delete)
trie_batch[key] = new_value
expected_final_keys.add(key)
except MissingTrieNode as exc:
node_db[exc.missing_node_hash] = missing_nodes.pop(
exc.missing_node_hash)
else:
change_complete = True
# Second walk
index_key2 = tuple(index_nibbles2)
for _ in range(100000):
try:
nearest_key = fog.nearest_unknown(index_key2)
except PerfectVisibility:
# Complete!
break
try:
node = trie.traverse(nearest_key)
sub_segments = node.sub_segments
except MissingTraversalNode as exc:
node_db[exc.missing_node_hash] = missing_nodes.pop(
exc.missing_node_hash)
continue
except TraversedPartialPath as exc:
# You might only get part-way down a path of nibbles if your fog is based on an old trie
# Determine the new sub-segments that are accessible from this partial traversal
sub_segments = exc.simulated_node.sub_segments
# explore the fog if there were no exceptions, or if you traversed a partial path
fog = fog.explore(nearest_key, sub_segments)
else:
assert False, "Must finish iterating the trie within ~100k runs"
# Final assertions
assert fog.is_complete
# We do *not* know that we have replaced all the missing_nodes, because of the trie changes
# Make sure we can walk the whole trie without any missing nodes
iterator = NodeIterator(trie)
found_keys = set(iterator.keys())
assert found_keys == expected_final_keys