def iterator_for(args, db, root):
start = bytes.fromhex(args.start)
start_nibbles = bytes_to_nibbles(start)
if args.count:
iterator = traverse_until_count(db, root, start_nibbles, args.count)
else:
end = bytes.fromhex(args.end)
end_nibbles = bytes_to_nibbles(end)
iterator = traverse_until_end(db, root, start_nibbles, end_nibbles)
return iterator
def build_receipt_proof(w3, txn_hash):
receipt_trie = Trie(db={})
receipt = w3.eth.getTransactionReceipt(txn_hash)
block = w3.eth.getBlock(receipt.blockHash)
for i, tr in enumerate(block.transactions):
path = rlp.encode(i)
sibling_receipt = w3.eth.getTransactionReceipt(tr.hex())
value = get_rlp_receipt(sibling_receipt)
receipt_trie.set(path, value)
if i == receipt.transactionIndex:
rlp_txn_receipt = value # We are interested in this txn
txn_path = rlp.encode(receipt.transactionIndex)
parent_nodes = []
t = receipt_trie
parent_nodes.append(t.root_node)
node = t.root_node
nibs = nibbles.bytes_to_nibbles(txn_path)
for nib in nibs:
if len(node) == 2: # Leaf node. We are done.
break
next_node = rlp.decode(t.db[node[nib]])
parent_nodes.append(next_node)
node = next_node
rlp_parent_nodes = rlp.encode(parent_nodes)
print('Calculated hash = %s' %
HexBytes(w3.sha3(rlp.encode(t.root_node))).hex())
print('Receipts root = %s' % HexBytes(block.receiptsRoot).hex())
return rlp_txn_receipt, receipt.blockHash, txn_path, rlp_parent_nodes
def get(self, key):
validate_is_bytes(key)
trie_key = bytes_to_nibbles(key)
root_node = self.get_node(self.root_hash)
return self._get(root_node, trie_key)
def generate_proof_blob(block_dict, tx_index):
header = block_header(block_dict)
mpt = HexaryTrie(db={})
for tx_dict in block_dict["transactions"]:
key = rlp.encode(utils.parse_as_int(tx_dict['transactionIndex']))
mpt.set(key, rlp_transaction(tx_dict))
if mpt.root_hash != normalize_bytes(block_dict['transactionsRoot']):
raise ValueError(
"Tx trie root hash does not match. Calculated: {} Sent: {}".format(
mpt.root_hash.hex(),
normalize_bytes(block_dict['transactionsRoot']).hex()))
mpt_key_nibbles = bytes_to_nibbles(rlp.encode(tx_index))
mpt_path, stack_indexes, stack = generate_proof(mpt, mpt_key_nibbles)
proof_blob = rlp.encode([
1, # proof_type
header,
tx_index,
bytes(mpt_path),
bytes(stack_indexes),
stack,
])
return proof_blob
def next(self, key_bytes: Optional[bytes] = None) -> Optional[bytes]:
"""
Find the next key to the right from the given key, or None if there is
no key to the right.
.. NOTE:: To iterate the full trie, consider using keys() instead, for performance
:param key_bytes: the key to start your search from. If None, return
the first possible key.
:return: key in bytes to the right of key_bytes, or None
"""
root = self._trie.root_node
none_traversed = Nibbles(())
if key_bytes is None:
next_key = self._get_next_key(root, none_traversed)
else:
key = bytes_to_nibbles(key_bytes)
next_key = self._get_key_after(root, key, none_traversed)
if next_key is None:
return None
else:
return nibbles_to_bytes(next_key)
def delete(self, key):
validate_is_bytes(key)
trie_key = bytes_to_nibbles(key)
root_node = self.get_node(self.root_hash)
new_node = self._delete(root_node, trie_key)
self._set_root_node(new_node)
def set(self, key, value):
validate_is_bytes(key)
validate_is_bytes(value)
trie_key = bytes_to_nibbles(key)
root_node = self.get_node(self.root_hash)
new_node = self._set(root_node, trie_key, value)
self._set_root_node(new_node)
def get(self, key):
validate_is_bytes(key)
trie_key = bytes_to_nibbles(key)
try:
root_node = self.get_node(self.root_hash)
return self._get(root_node, trie_key)
except KeyError as exc:
self._raise_missing_node(exc, key)
def construct_proof_from_mpt(mpt, header, tx_index, proof_type):
mpt_key_nibbles = bytes_to_nibbles(rlp.encode(tx_index))
stack = generate_proof(mpt, mpt_key_nibbles)
proof_blob = rlp.encode([
proof_type,
header,
tx_index,
stack,
])
return proof_blob
def _contiguous_accounts_complete_fraction(self) -> float:
"""
Estimate the completed fraction of the trie that is contiguous with
the current index (which rotates every 32 blocks)
It will be probably be quite noticeable that it will get "stuck" when
downloading a lot of storage, because we'll have to blow it up to more
than a percentage to see any significant change within 32 blocks. (when
the index will change again anyway)
:return: a number in the range [0, 1] (+/- rounding error) estimating
trie completion contiguous with the current backfill index key
"""
starting_index = bytes_to_nibbles(self._next_trie_root_hash)
unknown_prefixes = self._account_tracker._trie_fog._unexplored_prefixes
if len(unknown_prefixes) == 0:
return 1
# find the nearest unknown prefix (typically, on the right)
nearest_index = unknown_prefixes.bisect(starting_index)
# Get the nearest unknown prefix to the left
if nearest_index == 0:
left_prefix = (0, ) * 64
else:
left_prefix = unknown_prefixes[nearest_index - 1]
if key_starts_with(starting_index, left_prefix):
# The prefix of the starting index is unknown, so the index
# itself is unknown.
return 0
# Get the nearest unknown prefix to the right
if len(unknown_prefixes) == nearest_index:
right_prefix = (0xf, ) * 64
else:
right_prefix = unknown_prefixes[nearest_index]
# Use the space between the unknown prefixes to estimate the completed contiguous fraction
# At the base, every gap in the first nibble is a full 1/16th of the state complete
known_first_nibbles = right_prefix[0] - left_prefix[0] - 1
completed_fraction_base = (1 / 16) * known_first_nibbles
# Underneath, you can count completed subtrees on the right, each child 1/16 of the parent
right_side_completed = sum(
nibble * (1 / 16)**nibble_depth
for nibble_depth, nibble in enumerate(right_prefix[1:], 2))
# Do the same on the left
left_side_completed = sum(
(0xf - nibble) * (1 / 16)**nibble_depth
for nibble_depth, nibble in enumerate(left_prefix[1:], 2))
# Add up all completed areas
return left_side_completed + completed_fraction_base + right_side_completed
def delete(self, key):
validate_is_bytes(key)
trie_key = bytes_to_nibbles(key)
try:
root_node = self.get_node(self.root_hash)
new_node = self._delete(root_node, trie_key)
except KeyError as exc:
self._raise_missing_node(exc, key)
self._set_root_node(new_node)
def get(self, key):
validate_is_bytes(key)
trie_key = bytes_to_nibbles(key)
root_hash = self.root_hash
try:
return self._get(root_hash, trie_key)
except MissingTraversalNode as traverse_exc:
raise MissingTrieNode(
traverse_exc.missing_node_hash,
root_hash,
key,
traverse_exc.nibbles_traversed,
) from traverse_exc
def set(self, key, value):
validate_is_bytes(key)
validate_is_bytes(value)
trie_key = bytes_to_nibbles(key)
try:
root_node = self.get_node(self.root_hash)
if value == b'':
new_node = self._delete(root_node, trie_key)
else:
new_node = self._set(root_node, trie_key, value)
except KeyError as exc:
self._raise_missing_node(exc, key)
self._set_root_node(new_node)
def verify_transaction_hash(self, block_info, tx_hash):
txns = block_info.transactions
tx_index = 0
# generate the mpt
mpt = HexaryTrie(db={})
for tx_dict in txns:
if tx_dict.hash == tx_hash:
tx_index = tx_dict.transactionIndex
key = rlp.encode(utils.parse_as_int(tx_dict['transactionIndex']))
mpt.set(key, self.rlp_transaction(tx_dict))
# verify the tx root
if mpt.root_hash != normalize_bytes(block_info.transactionsRoot):
return False
# generate the proof
mpt_key_nibbles = bytes_to_nibbles(rlp.encode(tx_index))
proof = tuple(self.generate_proof(mpt, mpt_key_nibbles))
if HexaryTrie.get_from_proof(mpt.root_hash, rlp.encode(utils.parse_as_int(tx_index)), proof) \
!= self.rlp_transaction(txns[tx_index]):
return False
return True
def generate_receipt_proof_blob(web3, block_dict, tx_index):
header = block_header(block_dict)
mpt = HexaryTrie(db={})
for tx_dict in block_dict['transactions']:
key = rlp.encode(tx_dict['transactionIndex'])
tx_receipt = web3.eth.getTransactionReceipt(tx_dict['hash'])
mpt.set(key, rlp.encode(receipt(tx_receipt)))
if mpt.root_hash != normalize_bytes(block_dict['receiptsRoot']):
raise ValueError('Receipt trie root hash does not match')
mpt_key_nibbles = bytes_to_nibbles(rlp.encode(tx_index))
mpt_path, stack_indexes, stack = generate_proof(mpt, mpt_key_nibbles)
proof_blob = rlp.encode([
2, # proof_type
header,
tx_index,
compact_encode(mpt_path),
bytes(stack_indexes),
stack,
])
return proof_blob
def test_round_trip_nibbling(value):
value_as_nibbles = bytes_to_nibbles(value)
result = nibbles_to_bytes(value_as_nibbles)
assert result == value
def _request_tracking_trie_items(
self, request_tracker: TrieNodeRequestTracker,
root_hash: Hash32) -> Iterator[Tuple[Nibbles, Nibbles, bytes]]:
"""
Walk through the supplied trie, yielding the request tracker and node
request for any missing trie nodes.
:yield: path to leaf node, a key (as nibbles), and the value found in the trie
:raise: MissingTraversalNode if a node is missing while walking the trie
"""
if self._next_trie_root_hash is None:
# We haven't started beam syncing, so don't know which root to start at
return
trie = HexaryTrie(self._db, root_hash)
starting_index = bytes_to_nibbles(root_hash)
while self.manager.is_running:
try:
path_to_node = request_tracker.next_path_to_explore(
starting_index)
except trie_exceptions.PerfectVisibility:
# This doesn't necessarily mean we are finished.
# Any active prefixes might still be hiding some significant portion of the trie
# But it's all we're able to explore for now, until more node data arrives
return
try:
cached_node, uncached_key = request_tracker.get_cached_parent(
path_to_node)
except KeyError:
cached_node = None
node_getter = partial(trie.traverse, path_to_node)
else:
node_getter = partial(trie.traverse_from, cached_node,
uncached_key)
try:
node = node_getter()
except trie_exceptions.MissingTraversalNode as exc:
# Found missing account trie node
if path_to_node == exc.nibbles_traversed:
raise
elif cached_node is None:
# The path and nibbles traversed should always match in a non-cached traversal
raise RuntimeError(
f"Unexpected: on a non-cached traversal to {path_to_node}, the"
f" exception only claimed to traverse {exc.nibbles_traversed} -- {exc}"
) from exc
else:
# We need to re-raise a version of the exception that includes the whole path
# from the root node (when using cached nodes, we only have the path from
# the parent node to the child node)
# We could always raise this re-wrapped version, but skipping it (probably?)
# improves performance.
missing_hash = exc.missing_node_hash
raise trie_exceptions.MissingTraversalNode(
missing_hash, path_to_node) from exc
except trie_exceptions.TraversedPartialPath as exc:
node = exc.simulated_node
if node.value:
full_key_nibbles = path_to_node + node.suffix
if len(node.sub_segments):
# It shouldn't be a problem to skip handling this case, because all keys are
# hashed 32 bytes.
raise NotImplementedError(
"The state backfiller doesn't handle keys of different lengths, where"
f" one key is a prefix of another. But found {node} in trie with"
f" {root_hash!r}")
yield path_to_node, full_key_nibbles, node.value
# Note that we do not mark value nodes as completed. It is up to the caller
# to do that when it is ready. For example, the storage iterator will
# immediately treat the key as completed. The account iterator will
# not treat the key as completed until all of its storage and bytecode
# are also marked as complete.
else:
# If this is just an intermediate node, then we can mark it as confirmed.
request_tracker.confirm_prefix(path_to_node, node)
def next(self, key):
key = bytes_to_nibbles(key)
nibbles = self._iter(self.trie.root_node, key)
if nibbles is None:
return None
return nibbles_to_bytes(remove_nibbles_terminator(nibbles))
