class StateListRequest(Handler): def __init__(self, database, block_store): self._tree = MerkleDatabase(database) self._block_store = block_store def handle(self, identity, message_content): helper = _ClientHelper( message_content, client_pb2.ClientStateListRequest, client_pb2.ClientStateListResponse, validator_pb2.Message.CLIENT_STATE_LIST_RESPONSE, tree=self._tree, block_store=self._block_store) helper.set_root() if helper.has_response(): return helper.result # Fetch leaves and encode as protobuf leaves = [ client_pb2.Leaf(address=a, data=v) for a, v in self._tree.leaves( helper.request.address or '').items() ] if leaves: helper.set_response(helper.status.OK, head_id=helper.head_id, leaves=leaves) else: helper.set_response(helper.status.NO_RESOURCE, head_id=helper.head_id) return helper.result
class ClientStateListRequestHandler(object): def __init__(self, database): self._tree = MerkleDatabase(database) def handle(self, message, responder): error = False status = None request = client_pb2.ClientStateListRequest() try: request.ParseFromString(message.content) self._tree.set_merkle_root(request.merkle_root) except KeyError as e: status = client_pb2.ClientStateListResponse.NORESOURCE LOGGER.info(e) error = True except DecodeError: LOGGER.info( "Expected protobuf of class %s failed to " "deserialize", request) error = True if error: response = client_pb2.ClientStateListResponse( status=status or client_pb2.ClientStateListResponse.ERROR) else: prefix = request.prefix leaves = self._tree.leaves(prefix) if len(leaves) == 0: status = client_pb2.ClientStateListResponse.NORESOURCE response = client_pb2.ClientStateListResponse(status=status) else: status = client_pb2.ClientStateListResponse.OK entries = [Entry(address=a, data=v) for a, v in leaves.items()] response = client_pb2.ClientStateListResponse(status=status, entries=entries) responder.send( validator_pb2.Message( sender=message.sender, message_type=validator_pb2.Message.CLIENT_STATE_LIST_RESPONSE, correlation_id=message.correlation_id, content=response.SerializeToString()))
class StateListRequestHandler(Handler): def __init__(self, database): self._tree = MerkleDatabase(database) def handle(self, identity, message_content): request = client_pb2.ClientStateListRequest() resp_proto = client_pb2.ClientStateListResponse status = resp_proto.OK try: request.ParseFromString(message_content) self._tree.set_merkle_root(request.merkle_root) except KeyError as e: status = resp_proto.NORESOURCE LOGGER.debug(e) except DecodeError: status = resp_proto.ERROR LOGGER.info( "Expected protobuf of class %s failed to " "deserialize", request) if status != resp_proto.OK: response = resp_proto(status=status) else: prefix = request.prefix leaves = self._tree.leaves(prefix) if len(leaves) == 0: status = resp_proto.NORESOURCE response = resp_proto(status=status) else: entries = [Entry(address=a, data=v) for a, v in leaves.items()] response = resp_proto(status=status, entries=entries) return HandlerResult( status=HandlerStatus.RETURN, message_out=response, message_type=validator_pb2.Message.CLIENT_STATE_LIST_RESPONSE)
class StateListRequestHandler(Handler): def __init__(self, database): self._tree = MerkleDatabase(database) def handle(self, identity, message_content): request = client_pb2.ClientStateListRequest() resp_proto = client_pb2.ClientStateListResponse status = resp_proto.OK try: request.ParseFromString(message_content) self._tree.set_merkle_root(request.merkle_root) except KeyError as e: status = resp_proto.NORESOURCE LOGGER.debug(e) except DecodeError: status = resp_proto.ERROR LOGGER.info("Expected protobuf of class %s failed to " "deserialize", request) if status != resp_proto.OK: response = resp_proto(status=status) else: prefix = request.prefix leaves = self._tree.leaves(prefix) if len(leaves) == 0: status = resp_proto.NORESOURCE response = resp_proto(status=status) else: entries = [Entry(address=a, data=v) for a, v in leaves.items()] response = resp_proto(status=status, entries=entries) return HandlerResult( status=HandlerStatus.RETURN, message_out=response, message_type=validator_pb2.Message.CLIENT_STATE_LIST_RESPONSE)
class TestSawtoothMerkleTrie(unittest.TestCase): def setUp(self): self.dir = tempfile.mkdtemp() self.file = os.path.join(self.dir, 'merkle.lmdb') self.lmdb = NativeLmdbDatabase( self.file, indexes=MerkleDatabase.create_index_configuration(), _size=120 * 1024 * 1024) self.trie = MerkleDatabase(self.lmdb) def tearDown(self): self.trie.close() shutil.rmtree(self.dir) def test_merkle_trie_root_advance(self): value = {'name': 'foo', 'value': 1} orig_root = self.get_merkle_root() new_root = self.set('foo', value) self.assert_root(orig_root) self.assert_no_key('foo') self.set_merkle_root(new_root) self.assert_root(new_root) self.assert_value_at_address('foo', value) def test_merkle_trie_delete(self): value = {'name': 'bar', 'value': 1} new_root = self.set('bar', value) self.set_merkle_root(new_root) self.assert_root(new_root) self.assert_value_at_address('bar', value) # deleting an invalid key should raise an error with self.assertRaises(KeyError): self.delete('barf') del_root = self.delete('bar') # del_root hasn't been set yet, so address should still have value self.assert_root(new_root) self.assert_value_at_address('bar', value) self.set_merkle_root(del_root) self.assert_root(del_root) self.assert_no_key('bar') def test_merkle_trie_update(self): init_root = self.get_merkle_root() values = {} key_hashes = { key: _hash(key) for key in (_random_string(10) for _ in range(1000)) } for key, hashed in key_hashes.items(): value = {key: _random_string(512)} new_root = self.set(hashed, value, ishash=True) values[hashed] = value self.set_merkle_root(new_root) self.assert_not_root(init_root) for address, value in values.items(): self.assert_value_at_address(address, value, ishash=True) set_items = { hashed: { key: 5.0 } for key, hashed in random.sample(key_hashes.items(), 50) } values.update(set_items) delete_items = { hashed for hashed in random.sample(list(key_hashes.values()), 50) } # make sure there are no sets and deletes of the same key delete_items = delete_items - set_items.keys() for addr in delete_items: del values[addr] virtual_root = self.update(set_items, delete_items, virtual=True) # virtual root shouldn't match actual contents of tree with self.assertRaises(KeyError): self.set_merkle_root(virtual_root) actual_root = self.update(set_items, delete_items, virtual=False) # the virtual root should be the same as the actual root self.assertEqual(virtual_root, actual_root) # neither should be the root yet self.assert_not_root(virtual_root, actual_root) self.set_merkle_root(actual_root) self.assert_root(actual_root) for address, value in values.items(): self.assert_value_at_address(address, value, ishash=True) for address in delete_items: with self.assertRaises(KeyError): self.get(address, ishash=True) def test_merkle_trie_leaf_iteration(self): new_root = self.update( { "010101": { "my_data": 1 }, "010202": { "my_data": 2 }, "010303": { "my_data": 3 } }, [], virtual=False) # iterate over the empty trie iterator = iter(self.trie) with self.assertRaises(StopIteration): next(iterator) self.set_merkle_root(new_root) # Test complete trie iteration self.assertEqual([("010101", { "my_data": 1 }), ("010202", { "my_data": 2 }), ("010303", { "my_data": 3 })], [entry for entry in iter(self.trie)]) # Test prefixed iteration self.assertEqual([("010202", { "my_data": 2 })], [entry for entry in self.trie.leaves('0102')]) # assertions def assert_value_at_address(self, address, value, ishash=False): self.assertEqual(self.get(address, ishash), value, 'Wrong value') def assert_no_key(self, key): with self.assertRaises(KeyError): self.get(key) def assert_root(self, expected): self.assertEqual(expected, self.get_merkle_root(), 'Wrong root') def assert_not_root(self, *not_roots): root = self.get_merkle_root() for not_root in not_roots: self.assertNotEqual(root, not_root, 'Wrong root') # trie accessors # For convenience, assume keys are not hashed # unless otherwise indicated. def set(self, key, val, ishash=False): key_ = key if ishash else _hash(key) return self.trie.set(key_, val) def get(self, key, ishash=False): key_ = key if ishash else _hash(key) return self.trie.get(key_) def delete(self, key, ishash=False): key_ = key if ishash else _hash(key) return self.trie.delete(key_) def set_merkle_root(self, root): self.trie.set_merkle_root(root) def get_merkle_root(self): return self.trie.get_merkle_root() def update(self, set_items, delete_items=None, virtual=True): return self.trie.update(set_items, delete_items, virtual=virtual)
class TestSawtoothMerkleTrie(unittest.TestCase): def setUp(self): self.dir = tempfile.mkdtemp() self.file = os.path.join(self.dir, 'merkle.lmdb') self.lmdb = NativeLmdbDatabase( self.file, indexes=MerkleDatabase.create_index_configuration(), _size=120 * 1024 * 1024) self.trie = MerkleDatabase(self.lmdb) def tearDown(self): self.trie.close() shutil.rmtree(self.dir) def test_merkle_trie_root_advance(self): value = {'name': 'foo', 'value': 1} orig_root = self.get_merkle_root() new_root = self.set('foo', value) self.assert_root(orig_root) self.assert_no_key('foo') self.set_merkle_root(new_root) self.assert_root(new_root) self.assert_value_at_address('foo', value) def test_merkle_trie_delete(self): value = {'name': 'bar', 'value': 1} new_root = self.set('bar', value) self.set_merkle_root(new_root) self.assert_root(new_root) self.assert_value_at_address('bar', value) # deleting an invalid key should raise an error with self.assertRaises(KeyError): self.delete('barf') del_root = self.delete('bar') # del_root hasn't been set yet, so address should still have value self.assert_root(new_root) self.assert_value_at_address('bar', value) self.set_merkle_root(del_root) self.assert_root(del_root) self.assert_no_key('bar') def test_merkle_trie_update(self): init_root = self.get_merkle_root() values = {} key_hashes = { key: _hash(key) for key in (_random_string(10) for _ in range(1000)) } for key, hashed in key_hashes.items(): value = {key: _random_string(512)} new_root = self.set(hashed, value, ishash=True) values[hashed] = value self.set_merkle_root(new_root) self.assert_not_root(init_root) for address, value in values.items(): self.assert_value_at_address( address, value, ishash=True) set_items = { hashed: { key: 5.0 } for key, hashed in random.sample(key_hashes.items(), 50) } values.update(set_items) delete_items = { hashed for hashed in random.sample(list(key_hashes.values()), 50) } # make sure there are no sets and deletes of the same key delete_items = delete_items - set_items.keys() for addr in delete_items: del values[addr] virtual_root = self.update(set_items, delete_items, virtual=True) # virtual root shouldn't match actual contents of tree with self.assertRaises(KeyError): self.set_merkle_root(virtual_root) actual_root = self.update(set_items, delete_items, virtual=False) # the virtual root should be the same as the actual root self.assertEqual(virtual_root, actual_root) # neither should be the root yet self.assert_not_root( virtual_root, actual_root) self.set_merkle_root(actual_root) self.assert_root(actual_root) for address, value in values.items(): self.assert_value_at_address( address, value, ishash=True) for address in delete_items: with self.assertRaises(KeyError): self.get(address, ishash=True) def test_merkle_trie_leaf_iteration(self): new_root = self.update({ "010101": {"my_data": 1}, "010202": {"my_data": 2}, "010303": {"my_data": 3} }, [], virtual=False) # iterate over the empty trie iterator = iter(self.trie) with self.assertRaises(StopIteration): next(iterator) self.set_merkle_root(new_root) # Test complete trie iteration self.assertEqual( [("010101", {"my_data": 1}), ("010202", {"my_data": 2}), ("010303", {"my_data": 3})], [entry for entry in iter(self.trie)]) # Test prefixed iteration self.assertEqual([("010202", {"my_data": 2})], [entry for entry in self.trie.leaves('0102')]) # assertions def assert_value_at_address(self, address, value, ishash=False): self.assertEqual( self.get(address, ishash), value, 'Wrong value') def assert_no_key(self, key): with self.assertRaises(KeyError): self.get(key) def assert_root(self, expected): self.assertEqual( expected, self.get_merkle_root(), 'Wrong root') def assert_not_root(self, *not_roots): root = self.get_merkle_root() for not_root in not_roots: self.assertNotEqual( root, not_root, 'Wrong root') # trie accessors # For convenience, assume keys are not hashed # unless otherwise indicated. def set(self, key, val, ishash=False): key_ = key if ishash else _hash(key) return self.trie.set(key_, val) def get(self, key, ishash=False): key_ = key if ishash else _hash(key) return self.trie.get(key_) def delete(self, key, ishash=False): key_ = key if ishash else _hash(key) return self.trie.delete(key_) def set_merkle_root(self, root): self.trie.set_merkle_root(root) def get_merkle_root(self): return self.trie.get_merkle_root() def update(self, set_items, delete_items=None, virtual=True): return self.trie.update(set_items, delete_items, virtual=virtual)