def test_hexary_trie_saving_final_root(name, updates, expected, deleted, final_root): db = {} trie = HexaryTrie(db=db) with trie.squash_changes() as memory_trie: for key, value in updates: if value is None: del memory_trie[key] else: memory_trie[key] = value for key in deleted: del memory_trie[key] # access all of the values in the trie, triggering reads for all the database keys # that support the final state flagged_usage_db = KeyAccessLogger(db) flag_trie = HexaryTrie(flagged_usage_db, root_hash=trie.root_hash) for key, val in expected.items(): assert flag_trie[key] == val # assert that no unnecessary database values were created unread = flagged_usage_db.unread_keys() straggler_data = {k: (db[k], decode_node(db[k])) for k in unread} assert len(unread) == 0, straggler_data actual_root = trie.root_hash assert actual_root == final_root
def get_children(self, request): """Return all children of the node retrieved by the given request. :rtype: A two-tuple with one list containing the children that reference other nodes and another containing the leaf children. """ node = decode_node(request.data) return _get_children(node, request.depth)
def get_node(db, nodehash): if len(nodehash) < 32: # non-root nodes smaller than 32 bytes are in-lined node_rlp = nodehash else: node_rlp = db.get(nodehash) if not node_rlp: raise Exception(f'was unable to fetch node {nodehash.hex()}') node = decode_node(node_rlp) return node_rlp, node
def get_node(self, node_hash): if node_hash == BLANK_NODE: return BLANK_NODE elif node_hash == BLANK_NODE_HASH: return BLANK_NODE if len(node_hash) < 32: encoded_node = node_hash else: encoded_node = self.db[node_hash] node = decode_node(encoded_node) return node
def test_hexary_trie_squash_all_changes(updates, deleted): db = {} trie = HexaryTrie(db=db) expected = {} root_hashes = set() with trie.squash_changes() as memory_trie: for _index, (key, value) in enumerate(updates): if value is None: del memory_trie[key] expected.pop(key, None) else: memory_trie[key] = value expected[key] = value root_hashes.add(memory_trie.root_hash) for _index, key in enumerate(deleted): del memory_trie[key] expected.pop(key, None) root_hashes.add(memory_trie.root_hash) final_root_hash = trie.root_hash # access all of the values in the trie, triggering reads for all the database keys # that support the final state flagged_usage_db = KeyAccessLogger(db) flag_trie = HexaryTrie(flagged_usage_db, root_hash=final_root_hash) for key, val in expected.items(): assert flag_trie[key] == val # assert that no unnecessary database values were created unread = flagged_usage_db.unread_keys() straggler_data = {k: (db[k], decode_node(db[k])) for k in unread} assert len(unread) == 0, straggler_data # rebuild without squashing, to compare root hash verbose_trie = HexaryTrie({}) for key, value in updates: if value is None: del verbose_trie[key] else: verbose_trie[key] = value for _index, key in enumerate(deleted): del verbose_trie[key] assert final_root_hash == verbose_trie.root_hash
def get_children(self, request): """Return all children of the node retrieved by the given request. :rtype: A two-tuple with one list containing the children that reference other nodes and another containing the leaf children. """ node = decode_node(request.data) node_type = get_node_type(node) references = [] leaves = [] if node_type == NODE_TYPE_LEAF: leaves.append(node[1]) elif node_type == NODE_TYPE_EXTENSION: depth = request.depth + len(node[0]) references.append((depth, node[1])) elif node_type == NODE_TYPE_BRANCH: depth = request.depth + 1 for item in node[:16]: if is_blank_node(item): continue # In a branch, the first 16 items are either a node whose RLP-encoded # representation is under 32 bytes or a reference to another node. if len(item) == 2: if get_node_type(item) != NODE_TYPE_LEAF: raise UnexpectedNodeType( "Expected a node of type leaf, but got %s" % item) leaves.append(item[1]) elif len(item) == 17: # NOTE: This can happen only if the RLP representation of all branch items fit # in less than 32 bytes, which means the keys/values are extremely short, so # it's probably not worth supporting it. raise RuntimeError("If you get this, see the NOTE above") else: references.append((depth, item)) # The last item in a branch may contain a value. if not is_blank_node(node[16]): leaves.append(node[16]) return references, leaves
async def process(self, results: List[Tuple[Hash32, bytes]]) -> None: """Process request results. :param results: A list of two-tuples containing the node's key and data. """ for node_key, data in results: request = self.requests.get(node_key) if request is None: # This may happen if we resend a request for a node after waiting too long, # and then eventually get two responses with it. self.logger.debug2( "No SyncRequest found for %s, maybe we got more than one response for it", encode_hex(node_key)) return if request.data is not None: raise SyncRequestAlreadyProcessed( "%s has been processed already" % request) request.data = data if request.is_raw: await self.commit(request) continue node = decode_node(request.data) references, leaves = _get_children(node, request.depth) for depth, ref in references: await self.schedule(ref, request, depth, request.leaf_callback) if request.leaf_callback is not None: for leaf in leaves: await request.leaf_callback(leaf, request) if request.dependencies == 0: await self.commit(request)