def test_get_prefix_nodes(): trie = Trie(PersistentDB(KeyValueStorageInMemory())) prefix = 'abcd' prefix_nibbles = bin_to_nibbles(to_string(prefix)) key1 = prefix + '1' key2 = prefix + '2' key3 = prefix + '3' trie.update(key1.encode(), rlp_encode(['v1'])) last_node = trie._get_last_node_for_prfx(trie.root_node, prefix_nibbles) # The last node should be a leaf since only 1 key assert trie._get_node_type(last_node) == NODE_TYPE_LEAF # The queried key is larger than prefix, results in blank node last_node_ = trie._get_last_node_for_prfx( trie.root_node, bin_to_nibbles(to_string(prefix + '5'))) assert last_node_ == BLANK_NODE trie.update(key2.encode(), rlp_encode(['v2'])) last_node = trie._get_last_node_for_prfx(trie.root_node, prefix_nibbles) # The last node should be an extension since more than 1 key assert trie._get_node_type(last_node) == NODE_TYPE_EXTENSION trie.update(key3.encode(), rlp_encode(['v3'])) last_node = trie._get_last_node_for_prfx(trie.root_node, prefix_nibbles) assert trie._get_node_type(last_node) == NODE_TYPE_EXTENSION last_node_key = without_terminator(unpack_to_nibbles(last_node[0])) # Key for the fetched prefix nodes (ignore last nibble) is same as prefix nibbles assert last_node_key[:-1] == prefix_nibbles # The extension node is correctly decoded. decoded_extension = trie._decode_to_node(last_node[1]) assert decoded_extension[1] == [b' ', rlp_encode(['v1'])] assert decoded_extension[2] == [b' ', rlp_encode(['v2'])] assert decoded_extension[3] == [b' ', rlp_encode(['v3'])]
def test_get_prefix_nodes(): trie = Trie(PersistentDB(KeyValueStorageInMemory())) prefix = 'abcd' prefix_nibbles = bin_to_nibbles(prefix) key1 = prefix + '1' key2 = prefix + '2' key3 = prefix + '3' trie.update(key1.encode(), rlp_encode(['v1'])) seen_prefix = [] last_node = trie._get_last_node_for_prfx(trie.root_node, prefix_nibbles, seen_prfx=seen_prefix) # The last node should be a leaf since only 1 key assert trie._get_node_type(last_node) == NODE_TYPE_LEAF # Seen prefix matches the prefix exactly assert seen_prefix == [] # The queried key is larger than prefix, results in blank node last_node_ = trie._get_last_node_for_prfx(trie.root_node, bin_to_nibbles(prefix + '5'), []) assert last_node_ == BLANK_NODE seen_prefix = [] trie.update(key2.encode(), rlp_encode(['v2'])) last_node = trie._get_last_node_for_prfx(trie.root_node, prefix_nibbles, seen_prfx=seen_prefix) # The last node should be an extension since more than 1 key assert trie._get_node_type(last_node) == NODE_TYPE_EXTENSION assert seen_prefix == [] seen_prefix = [] trie.update(key3.encode(), rlp_encode(['v3'])) last_node = trie._get_last_node_for_prfx(trie.root_node, prefix_nibbles, seen_prfx=seen_prefix) assert trie._get_node_type(last_node) == NODE_TYPE_EXTENSION assert seen_prefix == [] last_node_key = without_terminator(unpack_to_nibbles(last_node[0])) # Key for the fetched prefix nodes (ignore last nibble) is same as prefix nibbles assert last_node_key[:-1] == prefix_nibbles # The extension node is correctly decoded. decoded_extension = trie._decode_to_node(last_node[1]) assert decoded_extension[1] == [b' ', rlp_encode(['v1'])] assert decoded_extension[2] == [b' ', rlp_encode(['v2'])] assert decoded_extension[3] == [b' ', rlp_encode(['v3'])] # Add keys with extended prefix extended_prefix = '1' key4 = prefix + extended_prefix + '85' trie.update(key4.encode(), rlp_encode(['v11'])) key5 = prefix + extended_prefix + '96' trie.update(key5.encode(), rlp_encode(['v12'])) seen_prefix = [] new_prefix_nibbs = bin_to_nibbles(prefix + extended_prefix) last_node = trie._get_last_node_for_prfx(trie.root_node, new_prefix_nibbs, seen_prfx=seen_prefix) assert trie._get_node_type(last_node) == NODE_TYPE_BRANCH assert new_prefix_nibbs == seen_prefix assert seen_prefix == bin_to_nibbles(prefix + '1') # traverse to the next node remaining_key4_nibbs = bin_to_nibbles(key4)[len(seen_prefix):] remaining_key5_nibbs = bin_to_nibbles(key5)[len(seen_prefix):] next_nibble = remaining_key4_nibbs[0] if remaining_key4_nibbs[ 0] > remaining_key5_nibbs[0] else remaining_key5_nibbs[0] next_node = trie._decode_to_node(last_node[next_nibble]) assert trie._get_node_type(next_node) == NODE_TYPE_BRANCH # The 8th index should lead to a node with key '5', key4 ended in '85' assert trie._get_node_type(next_node[8]) == NODE_TYPE_LEAF assert without_terminator(unpack_to_nibbles( next_node[8][0])) == bin_to_nibbles('5') # The 9th index should lead to a node with key '6', key5 ended in '96' assert trie._get_node_type(next_node[9]) == NODE_TYPE_LEAF assert without_terminator(unpack_to_nibbles( next_node[9][0])) == bin_to_nibbles('6') prefix_1 = prefix + 'efgh' prefix_1_nibbles = bin_to_nibbles(prefix_1) key1 = prefix_1 + '1' key2 = prefix_1 + '2' key3 = prefix_1 + '3' trie.update(key1.encode(), rlp_encode(['v1'])) trie.update(key2.encode(), rlp_encode(['v1'])) trie.update(key3.encode(), rlp_encode(['v1'])) seen_prefix = [] last_node = trie._get_last_node_for_prfx(trie.root_node, prefix_1_nibbles, seen_prfx=seen_prefix) assert trie._get_node_type(last_node) == NODE_TYPE_EXTENSION assert len(seen_prefix) > 0 assert starts_with(prefix_1_nibbles, seen_prefix)
class PruningState(State): """ This class is used to store the committed root hash of the trie in the db. The committed root hash is only updated once a batch gets written to the ledger. It might happen that a few batches are in 3 phase commit and the node crashes. Now when the node restarts, it restores the db from the committed root hash and all entries for uncommitted batches will be ignored """ # SOME KEY THAT DOES NOT COLLIDE WITH ANY STATE VARIABLE'S NAME rootHashKey = b'\x88\xc8\x88 \x9a\xa7\x89\x1b' def __init__(self, keyValueStorage: KeyValueStorage): self._kv = keyValueStorage if self.rootHashKey in self._kv: rootHash = bytes(self._kv.get(self.rootHashKey)) else: rootHash = BLANK_ROOT self._kv.put(self.rootHashKey, BLANK_ROOT) self._trie = Trie(PersistentDB(self._kv), rootHash) @property def head(self): # The current head of the state, if the state is a merkle tree then # head is the root return self._trie.root_node @property def committedHead(self): # The committed head of the state, if the state is a merkle tree then # head is the root return self._hash_to_node(self.committedHeadHash) def get_head_by_hash(self, root_hash): # return node of a merkle tree by given hash return self._hash_to_node(root_hash) def _hash_to_node(self, node_hash): if node_hash == BLANK_ROOT: return BLANK_NODE return self._trie._decode_to_node(node_hash) def set(self, key: bytes, value: bytes): self._trie.update(key, rlp_encode([value])) def get(self, key: bytes, isCommitted: bool = True) -> Optional[bytes]: if not isCommitted: val = self._trie.get(key) else: val = self._trie._get(self.committedHead, bin_to_nibbles(to_string(key))) if val: return self.get_decoded(val) def get_for_root_hash(self, root_hash, key: bytes) -> Optional[bytes]: root = self._hash_to_node(root_hash) val = self._trie._get(root, bin_to_nibbles(to_string(key))) if val: return self.get_decoded(val) def get_all_leaves_for_root_hash(self, root_hash): node = self._hash_to_node(root_hash) leaves = self._trie.to_dict(node) return leaves def remove(self, key: bytes): self._trie.delete(key) def commit(self, rootHash=None, rootNode=None): if rootNode: rootHash = self._trie._encode_node(rootNode) elif rootHash and isHex(rootHash): if isinstance(rootHash, str): rootHash = rootHash.encode() rootHash = unhexlify(rootHash) elif rootHash: rootHash = rootHash else: rootHash = self.headHash self._kv.put(self.rootHashKey, rootHash) def revertToHead(self, headHash=None): head = self._hash_to_node(headHash) self._trie.replace_root_hash(self._trie.root_node, head) # Proofs are always generated over committed state def generate_state_proof(self, key: bytes, root=None, serialize=False, get_value=False): return self._trie.generate_state_proof(key, root, serialize, get_value=get_value) def generate_state_proof_for_keys_with_prefix(self, key_prfx, root=None, serialize=False, get_value=False): return self._trie.generate_state_proof_for_keys_with_prefix( key_prfx, root, serialize, get_value=get_value) @staticmethod def verify_state_proof(root, key, value, proof_nodes, serialized=False): encoded_key, encoded_value = PruningState.encode_kv_for_verification( key, value) return Trie.verify_spv_proof(root, encoded_key, encoded_value, proof_nodes, serialized) @staticmethod def verify_state_proof_multi(root, key_values, proof_nodes, serialized=False): encoded_key_values = dict( PruningState.encode_kv_for_verification(k, v) for k, v in key_values.items()) return Trie.verify_spv_proof_multi(root, encoded_key_values, proof_nodes, serialized) @staticmethod def encode_kv_for_verification(key, value): encoded_key = key.encode() if isinstance(key, str) else key encoded_value = rlp_encode([value]) if value is not None else b'' return encoded_key, encoded_value @property def as_dict(self): d = self._trie.to_dict() return {k: self.get_decoded(v) for k, v in d.items()} @property def headHash(self): """ The hash of the current head of the state, if the state is a merkle tree then hash of the root :return: """ return self._trie.root_hash @property def committedHeadHash(self): return self._kv.get(self.rootHashKey) @property def closed(self): return not self._kv or self._kv.closed @property def isEmpty(self): return self._kv and self.committedHeadHash == BLANK_ROOT def close(self): if self._kv: self._kv.close() self._kv = None @staticmethod def get_decoded(encoded): return rlp_decode(encoded)[0]
class PruningState(State): """ This class is used to store the committed root hash of the trie in the db. The committed root hash is only updated once a batch gets written to the ledger. It might happen that a few batches are in 3 phase commit and the node crashes. Now when the node restarts, it restores the db from the committed root hash and all entries for uncommitted batches will be ignored """ # SOME KEY THAT DOES NOT COLLIDE WITH ANY STATE VARIABLE'S NAME rootHashKey = b'\x88\xc8\x88 \x9a\xa7\x89\x1b' def __init__(self, keyValueStorage: KeyValueStorage): self._kv = keyValueStorage if self.rootHashKey in self._kv: rootHash = bytes(self._kv.get(self.rootHashKey)) else: rootHash = BLANK_ROOT self._kv.put(self.rootHashKey, BLANK_ROOT) self._trie = Trie( PersistentDB(self._kv), rootHash) @property def head(self): # The current head of the state, if the state is a merkle tree then # head is the root return self._trie.root_node @property def committedHead(self): # The committed head of the state, if the state is a merkle tree then # head is the root return self._hash_to_node(self.committedHeadHash) def get_head_by_hash(self, root_hash): # return node of a merkle tree by given hash return self._hash_to_node(root_hash) def _hash_to_node(self, node_hash): if node_hash == BLANK_ROOT: return BLANK_NODE return self._trie._decode_to_node(node_hash) def set(self, key: bytes, value: bytes): self._trie.update(key, rlp_encode([value])) def get(self, key: bytes, isCommitted: bool = True) -> Optional[bytes]: if not isCommitted: val = self._trie.get(key) else: val = self._trie._get(self.committedHead, bin_to_nibbles(to_string(key))) if val: return self.get_decoded(val) def get_for_root_hash(self, root_hash, key: bytes) -> Optional[bytes]: root = self._hash_to_node(root_hash) val = self._trie._get(root, bin_to_nibbles(to_string(key))) if val: return self.get_decoded(val) def get_all_leaves_for_root_hash(self, root_hash): node = self._hash_to_node(root_hash) leaves = self._trie.to_dict(node) return leaves def remove(self, key: bytes): self._trie.delete(key) def commit(self, rootHash=None, rootNode=None): if rootNode: rootHash = self._trie._encode_node(rootNode) elif rootHash and isHex(rootHash): if isinstance(rootHash, str): rootHash = rootHash.encode() rootHash = unhexlify(rootHash) elif rootHash: rootHash = rootHash else: rootHash = self.headHash self._kv.put(self.rootHashKey, rootHash) def revertToHead(self, headHash=None): head = self._hash_to_node(headHash) self._trie.replace_root_hash(self._trie.root_node, head) # Proofs are always generated over committed state def generate_state_proof(self, key: bytes, root=None, serialize=False, get_value=False): return self._trie.generate_state_proof(key, root, serialize, get_value=get_value) def generate_state_proof_for_keys_with_prefix(self, key_prfx, root=None, serialize=False, get_value=False): return self._trie.generate_state_proof_for_keys_with_prefix(key_prfx, root, serialize, get_value=get_value) @staticmethod def verify_state_proof(root, key, value, proof_nodes, serialized=False): encoded_key, encoded_value = PruningState.encode_kv_for_verification(key, value) return Trie.verify_spv_proof(root, encoded_key, encoded_value, proof_nodes, serialized) @staticmethod def verify_state_proof_multi(root, key_values, proof_nodes, serialized=False): encoded_key_values = dict(PruningState.encode_kv_for_verification(k, v) for k, v in key_values.items()) return Trie.verify_spv_proof_multi(root, encoded_key_values, proof_nodes, serialized) @staticmethod def encode_kv_for_verification(key, value): encoded_key = key.encode() if isinstance(key, str) else key encoded_value = rlp_encode([value]) if value is not None else b'' return encoded_key, encoded_value @property def as_dict(self): d = self._trie.to_dict() return {k: self.get_decoded(v) for k, v in d.items()} @property def headHash(self): """ The hash of the current head of the state, if the state is a merkle tree then hash of the root :return: """ return self._trie.root_hash @property def committedHeadHash(self): return self._kv.get(self.rootHashKey) @property def isEmpty(self): return self.committedHeadHash == BLANK_ROOT def close(self): if self._kv: self._kv.close() self._kv = None @staticmethod def get_decoded(encoded): return rlp_decode(encoded)[0]
def test_get_values_at_roots_in_memory(): # Update key with different values but preserve root after each update # Check values of keys with different previous roots and check that they # are correct trie = Trie(PersistentDB(KeyValueStorageInMemory())) trie.update('k1'.encode(), rlp_encode(['v1'])) # print state.root_hash.encode('hex') # print state.root_node val = trie.get('k1') print(rlp_decode(val)) oldroot1 = trie.root_node old_root1_hash = trie.root_hash assert trie._decode_to_node(old_root1_hash) == oldroot1 trie.update('k1'.encode(), rlp_encode(['v1a'])) val = trie.get('k1') assert rlp_decode(val) == [b'v1a', ] # Already saved roots help in getting previous values oldval = trie.get_at(oldroot1, 'k1') assert rlp_decode(oldval) == [b'v1', ] oldroot1a = trie.root_node trie.update('k1'.encode(), rlp_encode([b'v1b'])) val = trie.get('k1') assert rlp_decode(val) == [b'v1b'] oldval = trie.get_at(oldroot1a, 'k1') assert rlp_decode(oldval) == [b'v1a', ] oldval = trie.get_at(oldroot1, 'k1') assert rlp_decode(oldval) == [b'v1', ] oldroot1b = trie.root_node trie.update('k1'.encode(), rlp_encode([b'v1c'])) val = trie.get('k1') assert rlp_decode(val) == [b'v1c', ] oldval = trie.get_at(oldroot1b, 'k1') assert rlp_decode(oldval) == [b'v1b', ] oldval = trie.get_at(oldroot1a, 'k1') assert rlp_decode(oldval) == [b'v1a', ] oldval = trie.get_at(oldroot1, 'k1') assert rlp_decode(oldval) == [b'v1', ] oldroot1c = trie.root_node trie.delete('k1'.encode()) assert trie.get('k1') == BLANK_NODE oldval = trie.get_at(oldroot1c, 'k1') assert rlp_decode(oldval) == [b'v1c', ] oldval = trie.get_at(oldroot1b, 'k1') assert rlp_decode(oldval) == [b'v1b', ] oldval = trie.get_at(oldroot1a, 'k1') assert rlp_decode(oldval) == [b'v1a', ] oldval = trie.get_at(oldroot1, 'k1') assert rlp_decode(oldval) == [b'v1', ] trie.root_node = oldroot1c val = trie.get('k1') assert rlp_decode(val) == [b'v1c', ] trie.root_node = oldroot1 val = trie.get('k1') assert rlp_decode(val) == [b'v1', ]
class PruningState(State): # This class is used to store the # committed root hash of the trie in the db. # The committed root hash is only updated once a batch gets written to the # ledger. It might happen that a few batches are in 3 phase commit and the # node crashes. Now when the node restarts, it restores the db from the # committed root hash and all entries for uncommitted batches will be # ignored # some key that does not collide with any state variable's name rootHashKey = b'\x88\xc8\x88 \x9a\xa7\x89\x1b' def __init__(self, keyValueStorage: KeyValueStorage): self._kv = keyValueStorage if self.rootHashKey in self._kv: rootHash = bytes(self._kv.get(self.rootHashKey)) else: rootHash = BLANK_ROOT self._kv.put(self.rootHashKey, BLANK_ROOT) self._trie = Trie(PersistentDB(self._kv), rootHash) @property def head(self): # The current head of the state, if the state is a merkle tree then # head is the root return self._trie.root_node @property def committedHead(self): # The committed head of the state, if the state is a merkle tree then # head is the root if self.committedHeadHash == BLANK_ROOT: return BLANK_NODE else: return self._trie._decode_to_node(self.committedHeadHash) def set(self, key: bytes, value: bytes): self._trie.update(key, rlp_encode([value])) def get(self, key: bytes, isCommitted: bool = True): if not isCommitted: val = self._trie.get(key) else: val = self._trie._get(self.committedHead, bin_to_nibbles(to_string(key))) if val: return rlp_decode(val)[0] def remove(self, key: bytes): self._trie.delete(key) def commit(self, rootHash=None, rootNode=None): if rootNode: rootHash = self._trie._encode_node(rootNode) elif rootHash and isHex(rootHash): if isinstance(rootHash, str): rootHash = rootHash.encode() rootHash = unhexlify(rootHash) elif rootHash: rootHash = rootHash else: rootHash = self.headHash self._kv.put(self.rootHashKey, rootHash) def revertToHead(self, headHash=None): if headHash != BLANK_ROOT: head = self._trie._decode_to_node(headHash) else: head = BLANK_NODE self._trie.replace_root_hash(self._trie.root_node, head) @property def as_dict(self): d = self._trie.to_dict() return {k: rlp_decode(v)[0] for k, v in d.items()} @property def headHash(self): """ The hash of the current head of the state, if the state is a merkle tree then hash of the root :return: """ return self._trie.root_hash @property def committedHeadHash(self): return self._kv.get(self.rootHashKey) @property def isEmpty(self): return self.committedHeadHash == BLANK_ROOT def close(self): if self._kv: self._kv.close() self._kv = None
def test_get_prefix_nodes(): trie = Trie(PersistentDB(KeyValueStorageInMemory())) prefix = 'abcd' prefix_nibbles = bin_to_nibbles(prefix) key1 = prefix + '1' key2 = prefix + '2' key3 = prefix + '3' trie.update(key1.encode(), rlp_encode(['v1'])) seen_prefix = [] last_node = trie._get_last_node_for_prfx(trie.root_node, prefix_nibbles, seen_prfx=seen_prefix) # The last node should be a leaf since only 1 key assert trie._get_node_type(last_node) == NODE_TYPE_LEAF # Seen prefix matches the prefix exactly assert seen_prefix == [] # The queried key is larger than prefix, results in blank node last_node_ = trie._get_last_node_for_prfx(trie.root_node, bin_to_nibbles(prefix + '5'), []) assert last_node_ == BLANK_NODE seen_prefix = [] trie.update(key2.encode(), rlp_encode(['v2'])) last_node = trie._get_last_node_for_prfx(trie.root_node, prefix_nibbles, seen_prfx=seen_prefix) # The last node should be an extension since more than 1 key assert trie._get_node_type(last_node) == NODE_TYPE_EXTENSION assert seen_prefix == [] seen_prefix = [] trie.update(key3.encode(), rlp_encode(['v3'])) last_node = trie._get_last_node_for_prfx(trie.root_node, prefix_nibbles, seen_prfx=seen_prefix) assert trie._get_node_type(last_node) == NODE_TYPE_EXTENSION assert seen_prefix == [] last_node_key = without_terminator(unpack_to_nibbles(last_node[0])) # Key for the fetched prefix nodes (ignore last nibble) is same as prefix nibbles assert last_node_key[:-1] == prefix_nibbles # The extension node is correctly decoded. decoded_extension = trie._decode_to_node(last_node[1]) assert decoded_extension[1] == [b' ', rlp_encode(['v1'])] assert decoded_extension[2] == [b' ', rlp_encode(['v2'])] assert decoded_extension[3] == [b' ', rlp_encode(['v3'])] # Add keys with extended prefix extended_prefix = '1' key4 = prefix + extended_prefix + '85' trie.update(key4.encode(), rlp_encode(['v11'])) key5 = prefix + extended_prefix + '96' trie.update(key5.encode(), rlp_encode(['v12'])) seen_prefix = [] new_prefix_nibbs = bin_to_nibbles(prefix + extended_prefix) last_node = trie._get_last_node_for_prfx(trie.root_node, new_prefix_nibbs, seen_prfx=seen_prefix) assert trie._get_node_type(last_node) == NODE_TYPE_BRANCH assert new_prefix_nibbs == seen_prefix assert seen_prefix == bin_to_nibbles(prefix + '1') # traverse to the next node remaining_key4_nibbs = bin_to_nibbles(key4)[len(seen_prefix):] remaining_key5_nibbs = bin_to_nibbles(key5)[len(seen_prefix):] next_nibble = remaining_key4_nibbs[0] if remaining_key4_nibbs[0] > remaining_key5_nibbs[0] else remaining_key5_nibbs[0] next_node = trie._decode_to_node(last_node[next_nibble]) assert trie._get_node_type(next_node) == NODE_TYPE_BRANCH # The 8th index should lead to a node with key '5', key4 ended in '85' assert trie._get_node_type(next_node[8]) == NODE_TYPE_LEAF assert without_terminator(unpack_to_nibbles(next_node[8][0])) == bin_to_nibbles('5') # The 9th index should lead to a node with key '6', key5 ended in '96' assert trie._get_node_type(next_node[9]) == NODE_TYPE_LEAF assert without_terminator(unpack_to_nibbles(next_node[9][0])) == bin_to_nibbles('6') prefix_1 = prefix + 'efgh' prefix_1_nibbles = bin_to_nibbles(prefix_1) key1 = prefix_1 + '1' key2 = prefix_1 + '2' key3 = prefix_1 + '3' trie.update(key1.encode(), rlp_encode(['v1'])) trie.update(key2.encode(), rlp_encode(['v1'])) trie.update(key3.encode(), rlp_encode(['v1'])) seen_prefix = [] last_node = trie._get_last_node_for_prfx(trie.root_node, prefix_1_nibbles, seen_prfx=seen_prefix) assert trie._get_node_type(last_node) == NODE_TYPE_EXTENSION assert len(seen_prefix) > 0 assert starts_with(prefix_1_nibbles, seen_prefix)