def test_proof_specific_root(): node_trie = Trie(PersistentDB(KeyValueStorageInMemory())) client_trie = Trie(PersistentDB(KeyValueStorageInMemory())) kvs = OrderedDict({'k1': 'v1', 'k2': 'v2', 'x3': 'v3', 'x4': 'v5', 'x5': 'v7', 'y99': 'v6'}) size = len(kvs) # Only add some keys old_keys = set() i = 0 for k, v in kvs.items(): node_trie.update(k.encode(), rlp_encode([v])) old_keys.add(k) i += 1 if i >= size // 2: break # Record the root root_hash_0 = node_trie.root_hash root_node_0 = deepcopy(node_trie.root_node) # Add remaining keys new_keys = set() i = 0 for k, v in reversed(kvs.items()): node_trie.update(k.encode(), rlp_encode([v])) new_keys.add(k) i += 1 if i >= size // 2: break # Record new roots root_hash_1 = node_trie.root_hash root_node_1 = deepcopy(node_trie.root_node) # Check each root present for k, v in kvs.items(): assert node_trie.get(k.encode()) == rlp_encode([v]) # Old and new roots should be different assert root_hash_0 != root_hash_1 assert root_node_0 != root_node_1 # Generate and verify proof for both old (if key was present) and new roots for k, v in kvs.items(): k, v = k.encode(), rlp_encode([v]) if k in old_keys: old_root_proof = node_trie.generate_state_proof(k, root=root_node_0) assert client_trie.verify_spv_proof(root_hash_0, k, v, old_root_proof) new_root_proof = node_trie.generate_state_proof(k, root=root_node_1) assert client_trie.verify_spv_proof(root_hash_1, k, v, new_root_proof)
def test_proof_specific_root(): node_trie = Trie(PersistentDB(KeyValueStorageInMemory())) client_trie = Trie(PersistentDB(KeyValueStorageInMemory())) kvs = OrderedDict({'k1': 'v1', 'k2': 'v2', 'x3': 'v3', 'x4': 'v5', 'x5': 'v7', 'y99': 'v6'}) size = len(kvs) # Only add some keys old_keys = set() i = 0 for k, v in kvs.items(): node_trie.update(k.encode(), rlp_encode([v])) old_keys.add(k) i += 1 if i >= size // 2: break # Record the root root_hash_0 = node_trie.root_hash root_node_0 = deepcopy(node_trie.root_node) # Add remaining keys new_keys = set() i = 0 for k, v in reversed(kvs.items()): node_trie.update(k.encode(), rlp_encode([v])) new_keys.add(k) i += 1 if i >= size // 2: break # Record new roots root_hash_1 = node_trie.root_hash root_node_1 = deepcopy(node_trie.root_node) # Check each root present for k, v in kvs.items(): assert node_trie.get(k.encode()) == rlp_encode([v]) # Old and new roots should be different assert root_hash_0 != root_hash_1 assert root_node_0 != root_node_1 # Generate and verify proof for both old (if key was present) and new roots for k, v in kvs.items(): k, v = k.encode(), rlp_encode([v]) if k in old_keys: old_root_proof = node_trie.generate_state_proof(k, root=root_node_0) assert client_trie.verify_spv_proof(root_hash_0, k, v, old_root_proof) new_root_proof = node_trie.generate_state_proof(k, root=root_node_1) assert client_trie.verify_spv_proof(root_hash_1, k, v, new_root_proof)
class PruningState(State): """ This class is used to store the committed root hash of the trie in the db. The committed root hash is only updated once a batch gets written to the ledger. It might happen that a few batches are in 3 phase commit and the node crashes. Now when the node restarts, it restores the db from the committed root hash and all entries for uncommitted batches will be ignored """ # SOME KEY THAT DOES NOT COLLIDE WITH ANY STATE VARIABLE'S NAME rootHashKey = b'\x88\xc8\x88 \x9a\xa7\x89\x1b' def __init__(self, keyValueStorage: KeyValueStorage): self._kv = keyValueStorage if self.rootHashKey in self._kv: rootHash = bytes(self._kv.get(self.rootHashKey)) else: rootHash = BLANK_ROOT self._kv.put(self.rootHashKey, BLANK_ROOT) self._trie = Trie(PersistentDB(self._kv), rootHash) @property def head(self): # The current head of the state, if the state is a merkle tree then # head is the root return self._trie.root_node @property def committedHead(self): # The committed head of the state, if the state is a merkle tree then # head is the root return self._hash_to_node(self.committedHeadHash) def get_head_by_hash(self, root_hash): # return node of a merkle tree by given hash return self._hash_to_node(root_hash) def _hash_to_node(self, node_hash): if node_hash == BLANK_ROOT: return BLANK_NODE return self._trie._decode_to_node(node_hash) def set(self, key: bytes, value: bytes): self._trie.update(key, rlp_encode([value])) def get(self, key: bytes, isCommitted: bool = True) -> Optional[bytes]: if not isCommitted: val = self._trie.get(key) else: val = self._trie._get(self.committedHead, bin_to_nibbles(to_string(key))) if val: return self.get_decoded(val) def get_for_root_hash(self, root_hash, key: bytes) -> Optional[bytes]: root = self._hash_to_node(root_hash) val = self._trie._get(root, bin_to_nibbles(to_string(key))) if val: return self.get_decoded(val) def get_all_leaves_for_root_hash(self, root_hash): node = self._hash_to_node(root_hash) leaves = self._trie.to_dict(node) return leaves def remove(self, key: bytes): self._trie.delete(key) def commit(self, rootHash=None, rootNode=None): if rootNode: rootHash = self._trie._encode_node(rootNode) elif rootHash and isHex(rootHash): if isinstance(rootHash, str): rootHash = rootHash.encode() rootHash = unhexlify(rootHash) elif rootHash: rootHash = rootHash else: rootHash = self.headHash self._kv.put(self.rootHashKey, rootHash) def revertToHead(self, headHash=None): head = self._hash_to_node(headHash) self._trie.replace_root_hash(self._trie.root_node, head) # Proofs are always generated over committed state def generate_state_proof(self, key: bytes, root=None, serialize=False, get_value=False): return self._trie.generate_state_proof(key, root, serialize, get_value=get_value) def generate_state_proof_for_keys_with_prefix(self, key_prfx, root=None, serialize=False, get_value=False): return self._trie.generate_state_proof_for_keys_with_prefix( key_prfx, root, serialize, get_value=get_value) @staticmethod def verify_state_proof(root, key, value, proof_nodes, serialized=False): encoded_key, encoded_value = PruningState.encode_kv_for_verification( key, value) return Trie.verify_spv_proof(root, encoded_key, encoded_value, proof_nodes, serialized) @staticmethod def verify_state_proof_multi(root, key_values, proof_nodes, serialized=False): encoded_key_values = dict( PruningState.encode_kv_for_verification(k, v) for k, v in key_values.items()) return Trie.verify_spv_proof_multi(root, encoded_key_values, proof_nodes, serialized) @staticmethod def encode_kv_for_verification(key, value): encoded_key = key.encode() if isinstance(key, str) else key encoded_value = rlp_encode([value]) if value is not None else b'' return encoded_key, encoded_value @property def as_dict(self): d = self._trie.to_dict() return {k: self.get_decoded(v) for k, v in d.items()} @property def headHash(self): """ The hash of the current head of the state, if the state is a merkle tree then hash of the root :return: """ return self._trie.root_hash @property def committedHeadHash(self): return self._kv.get(self.rootHashKey) @property def closed(self): return not self._kv or self._kv.closed @property def isEmpty(self): return self._kv and self.committedHeadHash == BLANK_ROOT def close(self): if self._kv: self._kv.close() self._kv = None @staticmethod def get_decoded(encoded): return rlp_decode(encoded)[0]
class PruningState(State): """ This class is used to store the committed root hash of the trie in the db. The committed root hash is only updated once a batch gets written to the ledger. It might happen that a few batches are in 3 phase commit and the node crashes. Now when the node restarts, it restores the db from the committed root hash and all entries for uncommitted batches will be ignored """ # SOME KEY THAT DOES NOT COLLIDE WITH ANY STATE VARIABLE'S NAME rootHashKey = b'\x88\xc8\x88 \x9a\xa7\x89\x1b' def __init__(self, keyValueStorage: KeyValueStorage): self._kv = keyValueStorage if self.rootHashKey in self._kv: rootHash = bytes(self._kv.get(self.rootHashKey)) else: rootHash = BLANK_ROOT self._kv.put(self.rootHashKey, BLANK_ROOT) self._trie = Trie( PersistentDB(self._kv), rootHash) @property def head(self): # The current head of the state, if the state is a merkle tree then # head is the root return self._trie.root_node @property def committedHead(self): # The committed head of the state, if the state is a merkle tree then # head is the root return self._hash_to_node(self.committedHeadHash) def get_head_by_hash(self, root_hash): # return node of a merkle tree by given hash return self._hash_to_node(root_hash) def _hash_to_node(self, node_hash): if node_hash == BLANK_ROOT: return BLANK_NODE return self._trie._decode_to_node(node_hash) def set(self, key: bytes, value: bytes): self._trie.update(key, rlp_encode([value])) def get(self, key: bytes, isCommitted: bool = True) -> Optional[bytes]: if not isCommitted: val = self._trie.get(key) else: val = self._trie._get(self.committedHead, bin_to_nibbles(to_string(key))) if val: return self.get_decoded(val) def get_for_root_hash(self, root_hash, key: bytes) -> Optional[bytes]: root = self._hash_to_node(root_hash) val = self._trie._get(root, bin_to_nibbles(to_string(key))) if val: return self.get_decoded(val) def get_all_leaves_for_root_hash(self, root_hash): node = self._hash_to_node(root_hash) leaves = self._trie.to_dict(node) return leaves def remove(self, key: bytes): self._trie.delete(key) def commit(self, rootHash=None, rootNode=None): if rootNode: rootHash = self._trie._encode_node(rootNode) elif rootHash and isHex(rootHash): if isinstance(rootHash, str): rootHash = rootHash.encode() rootHash = unhexlify(rootHash) elif rootHash: rootHash = rootHash else: rootHash = self.headHash self._kv.put(self.rootHashKey, rootHash) def revertToHead(self, headHash=None): head = self._hash_to_node(headHash) self._trie.replace_root_hash(self._trie.root_node, head) # Proofs are always generated over committed state def generate_state_proof(self, key: bytes, root=None, serialize=False, get_value=False): return self._trie.generate_state_proof(key, root, serialize, get_value=get_value) def generate_state_proof_for_keys_with_prefix(self, key_prfx, root=None, serialize=False, get_value=False): return self._trie.generate_state_proof_for_keys_with_prefix(key_prfx, root, serialize, get_value=get_value) @staticmethod def verify_state_proof(root, key, value, proof_nodes, serialized=False): encoded_key, encoded_value = PruningState.encode_kv_for_verification(key, value) return Trie.verify_spv_proof(root, encoded_key, encoded_value, proof_nodes, serialized) @staticmethod def verify_state_proof_multi(root, key_values, proof_nodes, serialized=False): encoded_key_values = dict(PruningState.encode_kv_for_verification(k, v) for k, v in key_values.items()) return Trie.verify_spv_proof_multi(root, encoded_key_values, proof_nodes, serialized) @staticmethod def encode_kv_for_verification(key, value): encoded_key = key.encode() if isinstance(key, str) else key encoded_value = rlp_encode([value]) if value is not None else b'' return encoded_key, encoded_value @property def as_dict(self): d = self._trie.to_dict() return {k: self.get_decoded(v) for k, v in d.items()} @property def headHash(self): """ The hash of the current head of the state, if the state is a merkle tree then hash of the root :return: """ return self._trie.root_hash @property def committedHeadHash(self): return self._kv.get(self.rootHashKey) @property def isEmpty(self): return self.committedHeadHash == BLANK_ROOT def close(self): if self._kv: self._kv.close() self._kv = None @staticmethod def get_decoded(encoded): return rlp_decode(encoded)[0]
def test_get_values_at_roots_in_memory(): # Update key with different values but preserve root after each update # Check values of keys with different previous roots and check that they # are correct trie = Trie(PersistentDB(KeyValueStorageInMemory())) trie.update('k1'.encode(), rlp_encode(['v1'])) # print state.root_hash.encode('hex') # print state.root_node val = trie.get('k1') print(rlp_decode(val)) oldroot1 = trie.root_node old_root1_hash = trie.root_hash assert trie._decode_to_node(old_root1_hash) == oldroot1 trie.update('k1'.encode(), rlp_encode(['v1a'])) val = trie.get('k1') assert rlp_decode(val) == [b'v1a', ] # Already saved roots help in getting previous values oldval = trie.get_at(oldroot1, 'k1') assert rlp_decode(oldval) == [b'v1', ] oldroot1a = trie.root_node trie.update('k1'.encode(), rlp_encode([b'v1b'])) val = trie.get('k1') assert rlp_decode(val) == [b'v1b'] oldval = trie.get_at(oldroot1a, 'k1') assert rlp_decode(oldval) == [b'v1a', ] oldval = trie.get_at(oldroot1, 'k1') assert rlp_decode(oldval) == [b'v1', ] oldroot1b = trie.root_node trie.update('k1'.encode(), rlp_encode([b'v1c'])) val = trie.get('k1') assert rlp_decode(val) == [b'v1c', ] oldval = trie.get_at(oldroot1b, 'k1') assert rlp_decode(oldval) == [b'v1b', ] oldval = trie.get_at(oldroot1a, 'k1') assert rlp_decode(oldval) == [b'v1a', ] oldval = trie.get_at(oldroot1, 'k1') assert rlp_decode(oldval) == [b'v1', ] oldroot1c = trie.root_node trie.delete('k1'.encode()) assert trie.get('k1') == BLANK_NODE oldval = trie.get_at(oldroot1c, 'k1') assert rlp_decode(oldval) == [b'v1c', ] oldval = trie.get_at(oldroot1b, 'k1') assert rlp_decode(oldval) == [b'v1b', ] oldval = trie.get_at(oldroot1a, 'k1') assert rlp_decode(oldval) == [b'v1a', ] oldval = trie.get_at(oldroot1, 'k1') assert rlp_decode(oldval) == [b'v1', ] trie.root_node = oldroot1c val = trie.get('k1') assert rlp_decode(val) == [b'v1c', ] trie.root_node = oldroot1 val = trie.get('k1') assert rlp_decode(val) == [b'v1', ]
class PruningState(State): # This class is used to store the # committed root hash of the trie in the db. # The committed root hash is only updated once a batch gets written to the # ledger. It might happen that a few batches are in 3 phase commit and the # node crashes. Now when the node restarts, it restores the db from the # committed root hash and all entries for uncommitted batches will be # ignored # some key that does not collide with any state variable's name rootHashKey = b'\x88\xc8\x88 \x9a\xa7\x89\x1b' def __init__(self, keyValueStorage: KeyValueStorage): self._kv = keyValueStorage if self.rootHashKey in self._kv: rootHash = bytes(self._kv.get(self.rootHashKey)) else: rootHash = BLANK_ROOT self._kv.put(self.rootHashKey, BLANK_ROOT) self._trie = Trie(PersistentDB(self._kv), rootHash) @property def head(self): # The current head of the state, if the state is a merkle tree then # head is the root return self._trie.root_node @property def committedHead(self): # The committed head of the state, if the state is a merkle tree then # head is the root if self.committedHeadHash == BLANK_ROOT: return BLANK_NODE else: return self._trie._decode_to_node(self.committedHeadHash) def set(self, key: bytes, value: bytes): self._trie.update(key, rlp_encode([value])) def get(self, key: bytes, isCommitted: bool = True): if not isCommitted: val = self._trie.get(key) else: val = self._trie._get(self.committedHead, bin_to_nibbles(to_string(key))) if val: return rlp_decode(val)[0] def remove(self, key: bytes): self._trie.delete(key) def commit(self, rootHash=None, rootNode=None): if rootNode: rootHash = self._trie._encode_node(rootNode) elif rootHash and isHex(rootHash): if isinstance(rootHash, str): rootHash = rootHash.encode() rootHash = unhexlify(rootHash) elif rootHash: rootHash = rootHash else: rootHash = self.headHash self._kv.put(self.rootHashKey, rootHash) def revertToHead(self, headHash=None): if headHash != BLANK_ROOT: head = self._trie._decode_to_node(headHash) else: head = BLANK_NODE self._trie.replace_root_hash(self._trie.root_node, head) @property def as_dict(self): d = self._trie.to_dict() return {k: rlp_decode(v)[0] for k, v in d.items()} @property def headHash(self): """ The hash of the current head of the state, if the state is a merkle tree then hash of the root :return: """ return self._trie.root_hash @property def committedHeadHash(self): return self._kv.get(self.rootHashKey) @property def isEmpty(self): return self.committedHeadHash == BLANK_ROOT def close(self): if self._kv: self._kv.close() self._kv = None