class StateListRequest(Handler):
def __init__(self, database, block_store):
self._tree = MerkleDatabase(database)
self._block_store = block_store
def handle(self, identity, message_content):
helper = _ClientHelper(
message_content,
client_pb2.ClientStateListRequest,
client_pb2.ClientStateListResponse,
validator_pb2.Message.CLIENT_STATE_LIST_RESPONSE,
tree=self._tree,
block_store=self._block_store)
helper.set_root()
if helper.has_response():
return helper.result
# Fetch leaves and encode as protobuf
leaves = [
client_pb2.Leaf(address=a, data=v) for a, v in self._tree.leaves(
helper.request.address or '').items()
]
if leaves:
helper.set_response(helper.status.OK,
head_id=helper.head_id,
leaves=leaves)
else:
helper.set_response(helper.status.NO_RESOURCE,
head_id=helper.head_id)
return helper.result
class ClientStateListRequestHandler(object):
def __init__(self, database):
self._tree = MerkleDatabase(database)
def handle(self, message, responder):
error = False
status = None
request = client_pb2.ClientStateListRequest()
try:
request.ParseFromString(message.content)
self._tree.set_merkle_root(request.merkle_root)
except KeyError as e:
status = client_pb2.ClientStateListResponse.NORESOURCE
LOGGER.info(e)
error = True
except DecodeError:
LOGGER.info(
"Expected protobuf of class %s failed to "
"deserialize", request)
error = True
if error:
response = client_pb2.ClientStateListResponse(
status=status or client_pb2.ClientStateListResponse.ERROR)
else:
prefix = request.prefix
leaves = self._tree.leaves(prefix)
if len(leaves) == 0:
status = client_pb2.ClientStateListResponse.NORESOURCE
response = client_pb2.ClientStateListResponse(status=status)
else:
status = client_pb2.ClientStateListResponse.OK
entries = [Entry(address=a, data=v) for a, v in leaves.items()]
response = client_pb2.ClientStateListResponse(status=status,
entries=entries)
responder.send(
validator_pb2.Message(
sender=message.sender,
message_type=validator_pb2.Message.CLIENT_STATE_LIST_RESPONSE,
correlation_id=message.correlation_id,
content=response.SerializeToString()))
class StateListRequestHandler(Handler):
def __init__(self, database):
self._tree = MerkleDatabase(database)
def handle(self, identity, message_content):
request = client_pb2.ClientStateListRequest()
resp_proto = client_pb2.ClientStateListResponse
status = resp_proto.OK
try:
request.ParseFromString(message_content)
self._tree.set_merkle_root(request.merkle_root)
except KeyError as e:
status = resp_proto.NORESOURCE
LOGGER.debug(e)
except DecodeError:
status = resp_proto.ERROR
LOGGER.info(
"Expected protobuf of class %s failed to "
"deserialize", request)
if status != resp_proto.OK:
response = resp_proto(status=status)
else:
prefix = request.prefix
leaves = self._tree.leaves(prefix)
if len(leaves) == 0:
status = resp_proto.NORESOURCE
response = resp_proto(status=status)
else:
entries = [Entry(address=a, data=v) for a, v in leaves.items()]
response = resp_proto(status=status, entries=entries)
return HandlerResult(
status=HandlerStatus.RETURN,
message_out=response,
message_type=validator_pb2.Message.CLIENT_STATE_LIST_RESPONSE)
class StateListRequestHandler(Handler):
def __init__(self, database):
self._tree = MerkleDatabase(database)
def handle(self, identity, message_content):
request = client_pb2.ClientStateListRequest()
resp_proto = client_pb2.ClientStateListResponse
status = resp_proto.OK
try:
request.ParseFromString(message_content)
self._tree.set_merkle_root(request.merkle_root)
except KeyError as e:
status = resp_proto.NORESOURCE
LOGGER.debug(e)
except DecodeError:
status = resp_proto.ERROR
LOGGER.info("Expected protobuf of class %s failed to "
"deserialize", request)
if status != resp_proto.OK:
response = resp_proto(status=status)
else:
prefix = request.prefix
leaves = self._tree.leaves(prefix)
if len(leaves) == 0:
status = resp_proto.NORESOURCE
response = resp_proto(status=status)
else:
entries = [Entry(address=a, data=v) for a, v in leaves.items()]
response = resp_proto(status=status, entries=entries)
return HandlerResult(
status=HandlerStatus.RETURN,
message_out=response,
message_type=validator_pb2.Message.CLIENT_STATE_LIST_RESPONSE)
class TestSawtoothMerkleTrie(unittest.TestCase):
def setUp(self):
self.dir = tempfile.mkdtemp()
self.file = os.path.join(self.dir, 'merkle.lmdb')
self.lmdb = NativeLmdbDatabase(
self.file,
indexes=MerkleDatabase.create_index_configuration(),
_size=120 * 1024 * 1024)
self.trie = MerkleDatabase(self.lmdb)
def tearDown(self):
self.trie.close()
shutil.rmtree(self.dir)
def test_merkle_trie_root_advance(self):
value = {'name': 'foo', 'value': 1}
orig_root = self.get_merkle_root()
new_root = self.set('foo', value)
self.assert_root(orig_root)
self.assert_no_key('foo')
self.set_merkle_root(new_root)
self.assert_root(new_root)
self.assert_value_at_address('foo', value)
def test_merkle_trie_delete(self):
value = {'name': 'bar', 'value': 1}
new_root = self.set('bar', value)
self.set_merkle_root(new_root)
self.assert_root(new_root)
self.assert_value_at_address('bar', value)
# deleting an invalid key should raise an error
with self.assertRaises(KeyError):
self.delete('barf')
del_root = self.delete('bar')
# del_root hasn't been set yet, so address should still have value
self.assert_root(new_root)
self.assert_value_at_address('bar', value)
self.set_merkle_root(del_root)
self.assert_root(del_root)
self.assert_no_key('bar')
def test_merkle_trie_update(self):
init_root = self.get_merkle_root()
values = {}
key_hashes = {
key: _hash(key)
for key in (_random_string(10) for _ in range(1000))
}
for key, hashed in key_hashes.items():
value = {key: _random_string(512)}
new_root = self.set(hashed, value, ishash=True)
values[hashed] = value
self.set_merkle_root(new_root)
self.assert_not_root(init_root)
for address, value in values.items():
self.assert_value_at_address(address, value, ishash=True)
set_items = {
hashed: {
key: 5.0
}
for key, hashed in random.sample(key_hashes.items(), 50)
}
values.update(set_items)
delete_items = {
hashed
for hashed in random.sample(list(key_hashes.values()), 50)
}
# make sure there are no sets and deletes of the same key
delete_items = delete_items - set_items.keys()
for addr in delete_items:
del values[addr]
virtual_root = self.update(set_items, delete_items, virtual=True)
# virtual root shouldn't match actual contents of tree
with self.assertRaises(KeyError):
self.set_merkle_root(virtual_root)
actual_root = self.update(set_items, delete_items, virtual=False)
# the virtual root should be the same as the actual root
self.assertEqual(virtual_root, actual_root)
# neither should be the root yet
self.assert_not_root(virtual_root, actual_root)
self.set_merkle_root(actual_root)
self.assert_root(actual_root)
for address, value in values.items():
self.assert_value_at_address(address, value, ishash=True)
for address in delete_items:
with self.assertRaises(KeyError):
self.get(address, ishash=True)
def test_merkle_trie_leaf_iteration(self):
new_root = self.update(
{
"010101": {
"my_data": 1
},
"010202": {
"my_data": 2
},
"010303": {
"my_data": 3
}
}, [],
virtual=False)
# iterate over the empty trie
iterator = iter(self.trie)
with self.assertRaises(StopIteration):
next(iterator)
self.set_merkle_root(new_root)
# Test complete trie iteration
self.assertEqual([("010101", {
"my_data": 1
}), ("010202", {
"my_data": 2
}), ("010303", {
"my_data": 3
})], [entry for entry in iter(self.trie)])
# Test prefixed iteration
self.assertEqual([("010202", {
"my_data": 2
})], [entry for entry in self.trie.leaves('0102')])
# assertions
def assert_value_at_address(self, address, value, ishash=False):
self.assertEqual(self.get(address, ishash), value, 'Wrong value')
def assert_no_key(self, key):
with self.assertRaises(KeyError):
self.get(key)
def assert_root(self, expected):
self.assertEqual(expected, self.get_merkle_root(), 'Wrong root')
def assert_not_root(self, *not_roots):
root = self.get_merkle_root()
for not_root in not_roots:
self.assertNotEqual(root, not_root, 'Wrong root')
# trie accessors
# For convenience, assume keys are not hashed
# unless otherwise indicated.
def set(self, key, val, ishash=False):
key_ = key if ishash else _hash(key)
return self.trie.set(key_, val)
def get(self, key, ishash=False):
key_ = key if ishash else _hash(key)
return self.trie.get(key_)
def delete(self, key, ishash=False):
key_ = key if ishash else _hash(key)
return self.trie.delete(key_)
def set_merkle_root(self, root):
self.trie.set_merkle_root(root)
def get_merkle_root(self):
return self.trie.get_merkle_root()
def update(self, set_items, delete_items=None, virtual=True):
return self.trie.update(set_items, delete_items, virtual=virtual)
class TestSawtoothMerkleTrie(unittest.TestCase):
def setUp(self):
self.dir = tempfile.mkdtemp()
self.file = os.path.join(self.dir, 'merkle.lmdb')
self.lmdb = NativeLmdbDatabase(
self.file,
indexes=MerkleDatabase.create_index_configuration(),
_size=120 * 1024 * 1024)
self.trie = MerkleDatabase(self.lmdb)
def tearDown(self):
self.trie.close()
shutil.rmtree(self.dir)
def test_merkle_trie_root_advance(self):
value = {'name': 'foo', 'value': 1}
orig_root = self.get_merkle_root()
new_root = self.set('foo', value)
self.assert_root(orig_root)
self.assert_no_key('foo')
self.set_merkle_root(new_root)
self.assert_root(new_root)
self.assert_value_at_address('foo', value)
def test_merkle_trie_delete(self):
value = {'name': 'bar', 'value': 1}
new_root = self.set('bar', value)
self.set_merkle_root(new_root)
self.assert_root(new_root)
self.assert_value_at_address('bar', value)
# deleting an invalid key should raise an error
with self.assertRaises(KeyError):
self.delete('barf')
del_root = self.delete('bar')
# del_root hasn't been set yet, so address should still have value
self.assert_root(new_root)
self.assert_value_at_address('bar', value)
self.set_merkle_root(del_root)
self.assert_root(del_root)
self.assert_no_key('bar')
def test_merkle_trie_update(self):
init_root = self.get_merkle_root()
values = {}
key_hashes = {
key: _hash(key)
for key in (_random_string(10) for _ in range(1000))
}
for key, hashed in key_hashes.items():
value = {key: _random_string(512)}
new_root = self.set(hashed, value, ishash=True)
values[hashed] = value
self.set_merkle_root(new_root)
self.assert_not_root(init_root)
for address, value in values.items():
self.assert_value_at_address(
address, value, ishash=True)
set_items = {
hashed: {
key: 5.0
}
for key, hashed in random.sample(key_hashes.items(), 50)
}
values.update(set_items)
delete_items = {
hashed
for hashed in random.sample(list(key_hashes.values()), 50)
}
# make sure there are no sets and deletes of the same key
delete_items = delete_items - set_items.keys()
for addr in delete_items:
del values[addr]
virtual_root = self.update(set_items, delete_items, virtual=True)
# virtual root shouldn't match actual contents of tree
with self.assertRaises(KeyError):
self.set_merkle_root(virtual_root)
actual_root = self.update(set_items, delete_items, virtual=False)
# the virtual root should be the same as the actual root
self.assertEqual(virtual_root, actual_root)
# neither should be the root yet
self.assert_not_root(
virtual_root,
actual_root)
self.set_merkle_root(actual_root)
self.assert_root(actual_root)
for address, value in values.items():
self.assert_value_at_address(
address, value, ishash=True)
for address in delete_items:
with self.assertRaises(KeyError):
self.get(address, ishash=True)
def test_merkle_trie_leaf_iteration(self):
new_root = self.update({
"010101": {"my_data": 1},
"010202": {"my_data": 2},
"010303": {"my_data": 3}
}, [], virtual=False)
# iterate over the empty trie
iterator = iter(self.trie)
with self.assertRaises(StopIteration):
next(iterator)
self.set_merkle_root(new_root)
# Test complete trie iteration
self.assertEqual(
[("010101", {"my_data": 1}),
("010202", {"my_data": 2}),
("010303", {"my_data": 3})],
[entry for entry in iter(self.trie)])
# Test prefixed iteration
self.assertEqual([("010202", {"my_data": 2})],
[entry for entry in self.trie.leaves('0102')])
# assertions
def assert_value_at_address(self, address, value, ishash=False):
self.assertEqual(
self.get(address, ishash),
value,
'Wrong value')
def assert_no_key(self, key):
with self.assertRaises(KeyError):
self.get(key)
def assert_root(self, expected):
self.assertEqual(
expected,
self.get_merkle_root(),
'Wrong root')
def assert_not_root(self, *not_roots):
root = self.get_merkle_root()
for not_root in not_roots:
self.assertNotEqual(
root,
not_root,
'Wrong root')
# trie accessors
# For convenience, assume keys are not hashed
# unless otherwise indicated.
def set(self, key, val, ishash=False):
key_ = key if ishash else _hash(key)
return self.trie.set(key_, val)
def get(self, key, ishash=False):
key_ = key if ishash else _hash(key)
return self.trie.get(key_)
def delete(self, key, ishash=False):
key_ = key if ishash else _hash(key)
return self.trie.delete(key_)
def set_merkle_root(self, root):
self.trie.set_merkle_root(root)
def get_merkle_root(self):
return self.trie.get_merkle_root()
def update(self, set_items, delete_items=None, virtual=True):
return self.trie.update(set_items, delete_items, virtual=virtual)