def test_proof_specific_root():
node_trie = Trie(PersistentDB(KeyValueStorageInMemory()))
client_trie = Trie(PersistentDB(KeyValueStorageInMemory()))
kvs = OrderedDict({'k1': 'v1', 'k2': 'v2', 'x3': 'v3', 'x4': 'v5',
'x5': 'v7', 'y99': 'v6'})
size = len(kvs)
# Only add some keys
old_keys = set()
i = 0
for k, v in kvs.items():
node_trie.update(k.encode(), rlp_encode([v]))
old_keys.add(k)
i += 1
if i >= size // 2:
break
# Record the root
root_hash_0 = node_trie.root_hash
root_node_0 = deepcopy(node_trie.root_node)
# Add remaining keys
new_keys = set()
i = 0
for k, v in reversed(kvs.items()):
node_trie.update(k.encode(), rlp_encode([v]))
new_keys.add(k)
i += 1
if i >= size // 2:
break
# Record new roots
root_hash_1 = node_trie.root_hash
root_node_1 = deepcopy(node_trie.root_node)
# Check each root present
for k, v in kvs.items():
assert node_trie.get(k.encode()) == rlp_encode([v])
# Old and new roots should be different
assert root_hash_0 != root_hash_1
assert root_node_0 != root_node_1
# Generate and verify proof for both old (if key was present) and new roots
for k, v in kvs.items():
k, v = k.encode(), rlp_encode([v])
if k in old_keys:
old_root_proof = node_trie.generate_state_proof(k, root=root_node_0)
assert client_trie.verify_spv_proof(root_hash_0, k, v, old_root_proof)
new_root_proof = node_trie.generate_state_proof(k, root=root_node_1)
assert client_trie.verify_spv_proof(root_hash_1, k, v, new_root_proof)
def test_proof_specific_root():
node_trie = Trie(PersistentDB(KeyValueStorageInMemory()))
client_trie = Trie(PersistentDB(KeyValueStorageInMemory()))
kvs = OrderedDict({'k1': 'v1', 'k2': 'v2', 'x3': 'v3', 'x4': 'v5',
'x5': 'v7', 'y99': 'v6'})
size = len(kvs)
# Only add some keys
old_keys = set()
i = 0
for k, v in kvs.items():
node_trie.update(k.encode(), rlp_encode([v]))
old_keys.add(k)
i += 1
if i >= size // 2:
break
# Record the root
root_hash_0 = node_trie.root_hash
root_node_0 = deepcopy(node_trie.root_node)
# Add remaining keys
new_keys = set()
i = 0
for k, v in reversed(kvs.items()):
node_trie.update(k.encode(), rlp_encode([v]))
new_keys.add(k)
i += 1
if i >= size // 2:
break
# Record new roots
root_hash_1 = node_trie.root_hash
root_node_1 = deepcopy(node_trie.root_node)
# Check each root present
for k, v in kvs.items():
assert node_trie.get(k.encode()) == rlp_encode([v])
# Old and new roots should be different
assert root_hash_0 != root_hash_1
assert root_node_0 != root_node_1
# Generate and verify proof for both old (if key was present) and new roots
for k, v in kvs.items():
k, v = k.encode(), rlp_encode([v])
if k in old_keys:
old_root_proof = node_trie.generate_state_proof(k, root=root_node_0)
assert client_trie.verify_spv_proof(root_hash_0, k, v, old_root_proof)
new_root_proof = node_trie.generate_state_proof(k, root=root_node_1)
assert client_trie.verify_spv_proof(root_hash_1, k, v, new_root_proof)
def test_proof_serialize_deserialize():
node_trie = Trie(PersistentDB(KeyValueStorageInMemory()))
client_trie = Trie(PersistentDB(KeyValueStorageInMemory()))
keys = {
k.encode(): [
rlp_encode([v]),
]
for k, v in [('k1', 'v1'), ('k2', 'v2'), ('k35', 'v55'), ('k70',
'v99')]
}
for k, v in keys.items():
node_trie.update(k, v[0])
for k in keys:
keys[k].append(node_trie.generate_state_proof(k, serialize=True))
for k in keys:
prf = keys[k][1]
assert isinstance(prf, bytes)
assert client_trie.verify_spv_proof(node_trie.root_hash,
k,
keys[k][0],
prf,
serialized=True)
def test_verify_proof_random_data():
"""
Add some key value pairs in trie. Generate and verify proof for them.
:return:
"""
num_keys = 100
test_data = gen_test_data(num_keys)
partitions = 4
partition_size = num_keys // partitions
keys = [
list(list(test_data.keys())[i:i + partition_size])
for i in range(0, len(test_data), partition_size)
]
node_trie = Trie(PersistentDB(KeyValueStorageInMemory()))
client_trie = Trie(PersistentDB(KeyValueStorageInMemory()))
root_hashes = []
proofs = []
for i in range(0, partitions):
for k in keys[i]:
node_trie.update(k, test_data[k])
root_hashes.append(node_trie.root_hash)
proofs.append({k: node_trie.generate_state_proof(k) for k in keys[i]})
assert all([
client_trie.verify_spv_proof(root_hashes[i], k, test_data[k],
proofs[i][k]) for k in keys[i]
])
# Pick any keys from any partition and verify the already generated proof
for _ in range(400):
p = randint(0, partitions - 1)
key = choice(keys[p])
assert client_trie.verify_spv_proof(root_hashes[p], key,
test_data[key], proofs[p][key])
# Pick any key randomly, generate new proof corresponding to current root
# and verify proof
all_keys = [k for i in keys for k in i]
root_hash = node_trie.root_hash
for _ in range(400):
key = choice(all_keys)
proof = node_trie.generate_state_proof(key)
assert client_trie.verify_spv_proof(root_hash, key, test_data[key],
proof)
def test_verify_proof_random_data():
"""
Add some key value pairs in trie. Generate and verify proof for them.
:return:
"""
num_keys = 100
test_data = gen_test_data(num_keys)
partitions = 4
partition_size = num_keys // partitions
keys = [list(list(test_data.keys())[i:i + partition_size])
for i in range(0, len(test_data), partition_size)]
node_trie = Trie(PersistentDB(KeyValueStorageInMemory()))
client_trie = Trie(PersistentDB(KeyValueStorageInMemory()))
root_hashes = []
proofs = []
for i in range(0, partitions):
for k in keys[i]:
node_trie.update(k, test_data[k])
root_hashes.append(node_trie.root_hash)
proofs.append({k: node_trie.generate_state_proof(k) for k in keys[i]})
assert all([client_trie.verify_spv_proof(root_hashes[i],
k, test_data[k],
proofs[i][k]) for k in keys[i]])
# Pick any keys from any partition and verify the already generated proof
for _ in range(400):
p = randint(0, partitions - 1)
key = choice(keys[p])
assert client_trie.verify_spv_proof(root_hashes[p], key,
test_data[key], proofs[p][key])
# Pick any key randomly, generate new proof corresponding to current root
# and verify proof
all_keys = [k for i in keys for k in i]
root_hash = node_trie.root_hash
for _ in range(400):
key = choice(all_keys)
proof = node_trie.generate_state_proof(key)
assert client_trie.verify_spv_proof(root_hash, key,
test_data[key], proof)
def test_proof_serialize_deserialize():
node_trie = Trie(PersistentDB(KeyValueStorageInMemory()))
client_trie = Trie(PersistentDB(KeyValueStorageInMemory()))
keys = {k.encode(): [rlp_encode([v]), ] for k, v in
[('k1', 'v1'), ('k2', 'v2'), ('k35', 'v55'), ('k70', 'v99')]}
for k, v in keys.items():
node_trie.update(k, v[0])
for k in keys:
keys[k].append(node_trie.generate_state_proof(k, serialize=True))
for k in keys:
prf = keys[k][1]
assert isinstance(prf, bytes)
assert client_trie.verify_spv_proof(node_trie.root_hash, k, keys[k][0],
prf, serialized=True)
def test_verify_proof_generated_using_helper():
node_trie = Trie(PersistentDB(KeyValueStorageInMemory()))
client_trie = Trie(PersistentDB(KeyValueStorageInMemory()))
node_trie.update('k1'.encode(), rlp_encode(['v1']))
node_trie.update('k2'.encode(), rlp_encode(['v2']))
root_hash_0 = node_trie.root_hash
p0 = node_trie.generate_state_proof('k2'.encode())
assert client_trie.verify_spv_proof(root_hash_0, 'k2'.encode(),
rlp_encode(['v2']), p0)
node_trie.update('k3'.encode(), rlp_encode(['v3']))
node_trie.update('k4'.encode(), rlp_encode(['v4']))
node_trie.update('x1'.encode(), rlp_encode(['y1']))
node_trie.update('x2'.encode(), rlp_encode(['y2']))
root_hash_1 = node_trie.root_hash
# Generate 1 proof and then verify that proof
p1 = node_trie.generate_state_proof('k1'.encode())
assert client_trie.verify_spv_proof(root_hash_1, 'k1'.encode(),
rlp_encode(['v1']), p1)
p2 = node_trie.generate_state_proof('x2'.encode())
assert client_trie.verify_spv_proof(root_hash_1, 'x2'.encode(),
rlp_encode(['y2']), p2)
# Generate more than 1 proof and then verify all proofs
p3 = node_trie.generate_state_proof('k3'.encode())
p4 = node_trie.generate_state_proof('x1'.encode())
assert client_trie.verify_spv_proof(root_hash_1, 'k3'.encode(),
rlp_encode(['v3']), p3)
assert client_trie.verify_spv_proof(root_hash_1, 'x1'.encode(),
rlp_encode(['y1']), p4)
# Proof is correct but value is different
assert not client_trie.verify_spv_proof(root_hash_1, 'x1'.encode(),
rlp_encode(['y99']), p4)
# Verify same proof again
assert client_trie.verify_spv_proof(root_hash_1, 'k3'.encode(),
rlp_encode(['v3']), p3)
assert client_trie.verify_spv_proof(root_hash_0, 'k2'.encode(),
rlp_encode(['v2']), p0)
# Proof generated using non-existent key fails verification
p5 = node_trie.generate_state_proof('x909'.encode())
assert not client_trie.verify_spv_proof(root_hash_1, 'x909'.encode(),
rlp_encode(['y909']), p5)
def test_verify_proof_generated_using_helper():
node_trie = Trie(PersistentDB(KeyValueStorageInMemory()))
client_trie = Trie(PersistentDB(KeyValueStorageInMemory()))
node_trie.update('k1'.encode(), rlp_encode(['v1']))
node_trie.update('k2'.encode(), rlp_encode(['v2']))
root_hash_0 = node_trie.root_hash
p0 = node_trie.generate_state_proof('k2'.encode())
assert client_trie.verify_spv_proof(root_hash_0, 'k2'.encode(),
rlp_encode(['v2']), p0)
node_trie.update('k3'.encode(), rlp_encode(['v3']))
node_trie.update('k4'.encode(), rlp_encode(['v4']))
node_trie.update('x1'.encode(), rlp_encode(['y1']))
node_trie.update('x2'.encode(), rlp_encode(['y2']))
root_hash_1 = node_trie.root_hash
# Generate 1 proof and then verify that proof
p1 = node_trie.generate_state_proof('k1'.encode())
assert client_trie.verify_spv_proof(root_hash_1, 'k1'.encode(),
rlp_encode(['v1']), p1)
p2 = node_trie.generate_state_proof('x2'.encode())
assert client_trie.verify_spv_proof(root_hash_1, 'x2'.encode(),
rlp_encode(['y2']), p2)
# Generate more than 1 proof and then verify all proofs
p3 = node_trie.generate_state_proof('k3'.encode())
p4 = node_trie.generate_state_proof('x1'.encode())
assert client_trie.verify_spv_proof(root_hash_1, 'k3'.encode(),
rlp_encode(['v3']), p3)
assert client_trie.verify_spv_proof(root_hash_1, 'x1'.encode(),
rlp_encode(['y1']), p4)
# Proof is correct but value is different
assert not client_trie.verify_spv_proof(root_hash_1, 'x1'.encode(),
rlp_encode(['y99']), p4)
# Verify same proof again
assert client_trie.verify_spv_proof(root_hash_1, 'k3'.encode(),
rlp_encode(['v3']), p3)
assert client_trie.verify_spv_proof(root_hash_0, 'k2'.encode(),
rlp_encode(['v2']), p0)
# Proof generated using non-existent key fails verification
p5 = node_trie.generate_state_proof('x909'.encode())
assert not client_trie.verify_spv_proof(root_hash_1, 'x909'.encode(),
rlp_encode(['y909']), p5)
def test_proof_specific_root():
node_trie = Trie(PersistentDB(KeyValueStorageInMemory()))
client_trie = Trie(PersistentDB(KeyValueStorageInMemory()))
node_trie.update('k1'.encode(), rlp_encode(['v1']))
node_trie.update('k2'.encode(), rlp_encode(['v2']))
node_trie.update('x3'.encode(), rlp_encode(['v3']))
root_hash_0 = node_trie.root_hash
root_node_0 = node_trie.root_node
node_trie.update('x4'.encode(), rlp_encode(['v5']))
node_trie.update('y99'.encode(), rlp_encode(['v6']))
node_trie.update('x5'.encode(), rlp_encode(['v7']))
# root_hash_1 = node_trie.root_hash
# root_node_1 = node_trie.root_node
k, v = 'k1'.encode(), rlp_encode(['v1'])
old_root_proof = node_trie.generate_state_proof(k, root=root_node_0)
assert client_trie.verify_spv_proof(root_hash_0, k, v, old_root_proof)
class PruningState(State):
"""
This class is used to store the
committed root hash of the trie in the db.
The committed root hash is only updated once a batch gets written to the
ledger. It might happen that a few batches are in 3 phase commit and the
node crashes. Now when the node restarts, it restores the db from the
committed root hash and all entries for uncommitted batches will be
ignored
"""
# SOME KEY THAT DOES NOT COLLIDE WITH ANY STATE VARIABLE'S NAME
rootHashKey = b'\x88\xc8\x88 \x9a\xa7\x89\x1b'
def __init__(self, keyValueStorage: KeyValueStorage):
self._kv = keyValueStorage
if self.rootHashKey in self._kv:
rootHash = bytes(self._kv.get(self.rootHashKey))
else:
rootHash = BLANK_ROOT
self._kv.put(self.rootHashKey, BLANK_ROOT)
self._trie = Trie(PersistentDB(self._kv), rootHash)
@property
def head(self):
# The current head of the state, if the state is a merkle tree then
# head is the root
return self._trie.root_node
@property
def committedHead(self):
# The committed head of the state, if the state is a merkle tree then
# head is the root
return self._hash_to_node(self.committedHeadHash)
def get_head_by_hash(self, root_hash):
# return node of a merkle tree by given hash
return self._hash_to_node(root_hash)
def _hash_to_node(self, node_hash):
if node_hash == BLANK_ROOT:
return BLANK_NODE
return self._trie._decode_to_node(node_hash)
def set(self, key: bytes, value: bytes):
self._trie.update(key, rlp_encode([value]))
def get(self, key: bytes, isCommitted: bool = True) -> Optional[bytes]:
if not isCommitted:
val = self._trie.get(key)
else:
val = self._trie._get(self.committedHead,
bin_to_nibbles(to_string(key)))
if val:
return self.get_decoded(val)
def get_for_root_hash(self, root_hash, key: bytes) -> Optional[bytes]:
root = self._hash_to_node(root_hash)
val = self._trie._get(root, bin_to_nibbles(to_string(key)))
if val:
return self.get_decoded(val)
def get_all_leaves_for_root_hash(self, root_hash):
node = self._hash_to_node(root_hash)
leaves = self._trie.to_dict(node)
return leaves
def remove(self, key: bytes):
self._trie.delete(key)
def commit(self, rootHash=None, rootNode=None):
if rootNode:
rootHash = self._trie._encode_node(rootNode)
elif rootHash and isHex(rootHash):
if isinstance(rootHash, str):
rootHash = rootHash.encode()
rootHash = unhexlify(rootHash)
elif rootHash:
rootHash = rootHash
else:
rootHash = self.headHash
self._kv.put(self.rootHashKey, rootHash)
def revertToHead(self, headHash=None):
head = self._hash_to_node(headHash)
self._trie.replace_root_hash(self._trie.root_node, head)
# Proofs are always generated over committed state
def generate_state_proof(self,
key: bytes,
root=None,
serialize=False,
get_value=False):
return self._trie.generate_state_proof(key,
root,
serialize,
get_value=get_value)
def generate_state_proof_for_keys_with_prefix(self,
key_prfx,
root=None,
serialize=False,
get_value=False):
return self._trie.generate_state_proof_for_keys_with_prefix(
key_prfx, root, serialize, get_value=get_value)
@staticmethod
def verify_state_proof(root, key, value, proof_nodes, serialized=False):
encoded_key, encoded_value = PruningState.encode_kv_for_verification(
key, value)
return Trie.verify_spv_proof(root, encoded_key, encoded_value,
proof_nodes, serialized)
@staticmethod
def verify_state_proof_multi(root,
key_values,
proof_nodes,
serialized=False):
encoded_key_values = dict(
PruningState.encode_kv_for_verification(k, v)
for k, v in key_values.items())
return Trie.verify_spv_proof_multi(root, encoded_key_values,
proof_nodes, serialized)
@staticmethod
def encode_kv_for_verification(key, value):
encoded_key = key.encode() if isinstance(key, str) else key
encoded_value = rlp_encode([value]) if value is not None else b''
return encoded_key, encoded_value
@property
def as_dict(self):
d = self._trie.to_dict()
return {k: self.get_decoded(v) for k, v in d.items()}
@property
def headHash(self):
"""
The hash of the current head of the state, if the state is a merkle
tree then hash of the root
:return:
"""
return self._trie.root_hash
@property
def committedHeadHash(self):
return self._kv.get(self.rootHashKey)
@property
def closed(self):
return not self._kv or self._kv.closed
@property
def isEmpty(self):
return self._kv and self.committedHeadHash == BLANK_ROOT
def close(self):
if self._kv:
self._kv.close()
self._kv = None
@staticmethod
def get_decoded(encoded):
return rlp_decode(encoded)[0]
class PruningState(State):
"""
This class is used to store the
committed root hash of the trie in the db.
The committed root hash is only updated once a batch gets written to the
ledger. It might happen that a few batches are in 3 phase commit and the
node crashes. Now when the node restarts, it restores the db from the
committed root hash and all entries for uncommitted batches will be
ignored
"""
# SOME KEY THAT DOES NOT COLLIDE WITH ANY STATE VARIABLE'S NAME
rootHashKey = b'\x88\xc8\x88 \x9a\xa7\x89\x1b'
def __init__(self, keyValueStorage: KeyValueStorage):
self._kv = keyValueStorage
if self.rootHashKey in self._kv:
rootHash = bytes(self._kv.get(self.rootHashKey))
else:
rootHash = BLANK_ROOT
self._kv.put(self.rootHashKey, BLANK_ROOT)
self._trie = Trie(
PersistentDB(self._kv),
rootHash)
@property
def head(self):
# The current head of the state, if the state is a merkle tree then
# head is the root
return self._trie.root_node
@property
def committedHead(self):
# The committed head of the state, if the state is a merkle tree then
# head is the root
return self._hash_to_node(self.committedHeadHash)
def get_head_by_hash(self, root_hash):
# return node of a merkle tree by given hash
return self._hash_to_node(root_hash)
def _hash_to_node(self, node_hash):
if node_hash == BLANK_ROOT:
return BLANK_NODE
return self._trie._decode_to_node(node_hash)
def set(self, key: bytes, value: bytes):
self._trie.update(key, rlp_encode([value]))
def get(self, key: bytes, isCommitted: bool = True) -> Optional[bytes]:
if not isCommitted:
val = self._trie.get(key)
else:
val = self._trie._get(self.committedHead,
bin_to_nibbles(to_string(key)))
if val:
return self.get_decoded(val)
def get_for_root_hash(self, root_hash, key: bytes) -> Optional[bytes]:
root = self._hash_to_node(root_hash)
val = self._trie._get(root,
bin_to_nibbles(to_string(key)))
if val:
return self.get_decoded(val)
def get_all_leaves_for_root_hash(self, root_hash):
node = self._hash_to_node(root_hash)
leaves = self._trie.to_dict(node)
return leaves
def remove(self, key: bytes):
self._trie.delete(key)
def commit(self, rootHash=None, rootNode=None):
if rootNode:
rootHash = self._trie._encode_node(rootNode)
elif rootHash and isHex(rootHash):
if isinstance(rootHash, str):
rootHash = rootHash.encode()
rootHash = unhexlify(rootHash)
elif rootHash:
rootHash = rootHash
else:
rootHash = self.headHash
self._kv.put(self.rootHashKey, rootHash)
def revertToHead(self, headHash=None):
head = self._hash_to_node(headHash)
self._trie.replace_root_hash(self._trie.root_node, head)
# Proofs are always generated over committed state
def generate_state_proof(self, key: bytes, root=None, serialize=False, get_value=False):
return self._trie.generate_state_proof(key, root, serialize, get_value=get_value)
def generate_state_proof_for_keys_with_prefix(self, key_prfx, root=None,
serialize=False, get_value=False):
return self._trie.generate_state_proof_for_keys_with_prefix(key_prfx, root,
serialize, get_value=get_value)
@staticmethod
def verify_state_proof(root, key, value, proof_nodes, serialized=False):
encoded_key, encoded_value = PruningState.encode_kv_for_verification(key, value)
return Trie.verify_spv_proof(root, encoded_key, encoded_value,
proof_nodes, serialized)
@staticmethod
def verify_state_proof_multi(root, key_values, proof_nodes, serialized=False):
encoded_key_values = dict(PruningState.encode_kv_for_verification(k, v) for k, v in key_values.items())
return Trie.verify_spv_proof_multi(root, encoded_key_values, proof_nodes, serialized)
@staticmethod
def encode_kv_for_verification(key, value):
encoded_key = key.encode() if isinstance(key, str) else key
encoded_value = rlp_encode([value]) if value is not None else b''
return encoded_key, encoded_value
@property
def as_dict(self):
d = self._trie.to_dict()
return {k: self.get_decoded(v) for k, v in d.items()}
@property
def headHash(self):
"""
The hash of the current head of the state, if the state is a merkle
tree then hash of the root
:return:
"""
return self._trie.root_hash
@property
def committedHeadHash(self):
return self._kv.get(self.rootHashKey)
@property
def isEmpty(self):
return self.committedHeadHash == BLANK_ROOT
def close(self):
if self._kv:
self._kv.close()
self._kv = None
@staticmethod
def get_decoded(encoded):
return rlp_decode(encoded)[0]