def test_hexary_trie_saving_final_root(name, updates, expected, deleted,
final_root):
db = {}
trie = HexaryTrie(db=db)
with trie.squash_changes() as memory_trie:
for key, value in updates:
if value is None:
del memory_trie[key]
else:
memory_trie[key] = value
for key in deleted:
del memory_trie[key]
# access all of the values in the trie, triggering reads for all the database keys
# that support the final state
flagged_usage_db = KeyAccessLogger(db)
flag_trie = HexaryTrie(flagged_usage_db, root_hash=trie.root_hash)
for key, val in expected.items():
assert flag_trie[key] == val
# assert that no unnecessary database values were created
unread = flagged_usage_db.unread_keys()
straggler_data = {k: (db[k], decode_node(db[k])) for k in unread}
assert len(unread) == 0, straggler_data
actual_root = trie.root_hash
assert actual_root == final_root
def get_children(self, request):
"""Return all children of the node retrieved by the given request.
:rtype: A two-tuple with one list containing the children that reference other nodes and
another containing the leaf children.
"""
node = decode_node(request.data)
return _get_children(node, request.depth)
def get_node(db, nodehash):
if len(nodehash) < 32:
# non-root nodes smaller than 32 bytes are in-lined
node_rlp = nodehash
else:
node_rlp = db.get(nodehash)
if not node_rlp:
raise Exception(f'was unable to fetch node {nodehash.hex()}')
node = decode_node(node_rlp)
return node_rlp, node
def get_node(self, node_hash):
if node_hash == BLANK_NODE:
return BLANK_NODE
elif node_hash == BLANK_NODE_HASH:
return BLANK_NODE
if len(node_hash) < 32:
encoded_node = node_hash
else:
encoded_node = self.db[node_hash]
node = decode_node(encoded_node)
return node
def test_hexary_trie_squash_all_changes(updates, deleted):
db = {}
trie = HexaryTrie(db=db)
expected = {}
root_hashes = set()
with trie.squash_changes() as memory_trie:
for _index, (key, value) in enumerate(updates):
if value is None:
del memory_trie[key]
expected.pop(key, None)
else:
memory_trie[key] = value
expected[key] = value
root_hashes.add(memory_trie.root_hash)
for _index, key in enumerate(deleted):
del memory_trie[key]
expected.pop(key, None)
root_hashes.add(memory_trie.root_hash)
final_root_hash = trie.root_hash
# access all of the values in the trie, triggering reads for all the database keys
# that support the final state
flagged_usage_db = KeyAccessLogger(db)
flag_trie = HexaryTrie(flagged_usage_db, root_hash=final_root_hash)
for key, val in expected.items():
assert flag_trie[key] == val
# assert that no unnecessary database values were created
unread = flagged_usage_db.unread_keys()
straggler_data = {k: (db[k], decode_node(db[k])) for k in unread}
assert len(unread) == 0, straggler_data
# rebuild without squashing, to compare root hash
verbose_trie = HexaryTrie({})
for key, value in updates:
if value is None:
del verbose_trie[key]
else:
verbose_trie[key] = value
for _index, key in enumerate(deleted):
del verbose_trie[key]
assert final_root_hash == verbose_trie.root_hash
def get_children(self, request):
"""Return all children of the node retrieved by the given request.
:rtype: A two-tuple with one list containing the children that reference other nodes and
another containing the leaf children.
"""
node = decode_node(request.data)
node_type = get_node_type(node)
references = []
leaves = []
if node_type == NODE_TYPE_LEAF:
leaves.append(node[1])
elif node_type == NODE_TYPE_EXTENSION:
depth = request.depth + len(node[0])
references.append((depth, node[1]))
elif node_type == NODE_TYPE_BRANCH:
depth = request.depth + 1
for item in node[:16]:
if is_blank_node(item):
continue
# In a branch, the first 16 items are either a node whose RLP-encoded
# representation is under 32 bytes or a reference to another node.
if len(item) == 2:
if get_node_type(item) != NODE_TYPE_LEAF:
raise UnexpectedNodeType(
"Expected a node of type leaf, but got %s" % item)
leaves.append(item[1])
elif len(item) == 17:
# NOTE: This can happen only if the RLP representation of all branch items fit
# in less than 32 bytes, which means the keys/values are extremely short, so
# it's probably not worth supporting it.
raise RuntimeError("If you get this, see the NOTE above")
else:
references.append((depth, item))
# The last item in a branch may contain a value.
if not is_blank_node(node[16]):
leaves.append(node[16])
return references, leaves
async def process(self, results: List[Tuple[Hash32, bytes]]) -> None:
"""Process request results.
:param results: A list of two-tuples containing the node's key and data.
"""
for node_key, data in results:
request = self.requests.get(node_key)
if request is None:
# This may happen if we resend a request for a node after waiting too long,
# and then eventually get two responses with it.
self.logger.debug2(
"No SyncRequest found for %s, maybe we got more than one response for it",
encode_hex(node_key))
return
if request.data is not None:
raise SyncRequestAlreadyProcessed(
"%s has been processed already" % request)
request.data = data
if request.is_raw:
await self.commit(request)
continue
node = decode_node(request.data)
references, leaves = _get_children(node, request.depth)
for depth, ref in references:
await self.schedule(ref, request, depth, request.leaf_callback)
if request.leaf_callback is not None:
for leaf in leaves:
await request.leaf_callback(leaf, request)
if request.dependencies == 0:
await self.commit(request)
