def run(self):
# start once and works all time
#LOGGER.debug('_ContextReader: run \n')
while True:
context_state_addresslist_tuple = self._addresses.get(block=True)
if context_state_addresslist_tuple is _SHUTDOWN_SENTINEL:
break
c_id, state_hash, address_list = context_state_addresslist_tuple
#LOGGER.debug('_ContextReader: run state_hash=%s\n',state_hash)
tree = MerkleDatabase(self._database, state_hash)
"""
# for testing only
# check state for testing
try:
tree._get_by_addr("449095bc5d9deba00a635d8db93c9deeb043416204f494b9f07862e9445559f0185109")
LOGGER.debug('_ContextReader: ADDRESS YES \n')
except :
LOGGER.debug('_ContextReader: ADDRESS NO \n')
"""
return_values = []
for address in address_list:
value = None
try:
value = tree.get(address)
except KeyError:
pass
return_values.append((address, value))
self._inflated_addresses.put((c_id, return_values))
class StateGetRequest(Handler):
def __init__(self, database, block_store):
self._tree = MerkleDatabase(database)
self._block_store = block_store
def handle(self, identity, message_content):
helper = _ClientHelper(message_content,
client_pb2.ClientStateGetRequest,
client_pb2.ClientStateGetResponse,
validator_pb2.Message.CLIENT_STATE_GET_RESPONSE,
tree=self._tree,
block_store=self._block_store)
helper.set_root()
if helper.has_response():
return helper.result
# Fetch leaf value
address = helper.request.address
try:
value = self._tree.get(address)
except KeyError:
LOGGER.debug('Unable to find entry at address %s', address)
helper.set_response(helper.status.NO_RESOURCE)
except ValueError as e:
LOGGER.debug('Address %s is a nonleaf', address)
LOGGER.debug(e)
helper.set_response(helper.status.MISSING_ADDRESS)
if not helper.has_response():
helper.set_response(helper.status.OK,
head_id=helper.head_id,
value=value)
return helper.result
def get(self, context_id, address_list):
"""Get the values associated with list of addresses, for a specific
context referenced by context_id.
Args:
context_id (str): the return value of create_context, referencing
a particular context.
address_list (list): a list of address strs
Returns:
values_list (list): a list of (address, value) tuples
Raises:
AuthorizationException: Raised when an address in address_list is
not authorized either by not being in the inputs for the
txn associated with this context, or it is under a namespace
but the characters that are under the namespace are not valid
address characters.
"""
if context_id not in self._contexts:
return []
for add in address_list:
if not self.address_is_valid(address=add):
raise AuthorizationException(address=add)
context = self._contexts[context_id]
addresses_in_ctx = [add for add in address_list if add in context]
addresses_not_in_ctx = list(set(address_list) - set(addresses_in_ctx))
values = context.get(addresses_in_ctx)
values_list = list(zip(addresses_in_ctx, values))
if addresses_not_in_ctx:
# Validate the addresses that won't be validated by a direct get on
# the context.
for address in addresses_not_in_ctx:
context.validate_read(address)
address_values, reads = self._find_address_values_in_chain(
base_contexts=[context_id],
addresses_to_find=addresses_not_in_ctx)
values_list.extend(address_values)
if reads:
tree = MerkleDatabase(self._database, context.merkle_root)
add_values = []
for add in reads:
value = None
try:
value = tree.get(add)
except KeyError:
# The address is not in the radix tree/merkle tree
pass
add_values.append((add, value))
values_list.extend(add_values)
values_list.sort(key=lambda x: address_list.index(x[0]))
return values_list
def get(self, context_id, address_list):
"""Get the values associated with list of addresses, for a specific
context referenced by context_id.
Args:
context_id (str): the return value of create_context, referencing
a particular context.
address_list (list): a list of address strs
Returns:
values_list (list): a list of (address, value) tuples
Raises:
AuthorizationException: Raised when an address in address_list is
not authorized either by not being in the inputs for the
txn associated with this context, or it is under a namespace
but the characters that are under the namespace are not valid
address characters.
"""
if context_id not in self._contexts:
return []
for add in address_list:
if not self.address_is_valid(address=add):
raise AuthorizationException(address=add)
context = self._contexts[context_id]
addresses_in_ctx = [add for add in address_list if add in context]
addresses_not_in_ctx = list(set(address_list) - set(addresses_in_ctx))
values = context.get(addresses_in_ctx)
values_list = list(zip(addresses_in_ctx, values))
if addresses_not_in_ctx:
# Validate the addresses that won't be validated by a direct get on
# the context.
for address in addresses_not_in_ctx:
context.validate_read(address)
address_values, reads = self._find_address_values_in_chain(
base_contexts=[context_id],
addresses_to_find=addresses_not_in_ctx)
values_list.extend(address_values)
if reads:
tree = MerkleDatabase(self._database, context.merkle_root)
add_values = []
for add in reads:
value = None
try:
value = tree.get(add)
except KeyError:
# The address is not in the radix tree/merkle tree
pass
add_values.append((add, value))
values_list.extend(add_values)
values_list.sort(key=lambda x: address_list.index(x[0]))
return values_list
class TestSawtoothMerkleTrie:
def __init__(self):
self.dir = '/tmp/sawtooth' # tempfile.mkdtemp()
self.file = os.path.join(self.dir, 'merkle.lmdb')
self.lmdb = lmdb_nolock_database.LMDBNoLockDatabase(self.file, 'n')
self.trie = MerkleDatabase(self.lmdb)
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
self.trie.close()
# assertions
def assert_value_at_address(self, address, value, ishash=False):
assert self.get(address, ishash) == value
def assert_no_key(self, key):
with pytest.raises(KeyError):
self.get(key)
def assert_root(self, expected):
assert expected == self.get_merkle_root()
def assert_not_root(self, *not_roots):
root = self.get_merkle_root()
for not_root in not_roots:
assert root != not_root
# trie accessors
# For convenience, assume keys are not hashed
# unless otherwise indicated.
def set(self, key, val, ishash=False):
key_ = key if ishash else _hash(key)
return self.trie.set(key_, val)
def get(self, key, ishash=False):
key_ = key if ishash else _hash(key)
return self.trie.get(key_)
def delete(self, key, ishash=False):
key_ = key if ishash else _hash(key)
return self.trie.delete(key_)
def set_merkle_root(self, root):
self.trie.set_merkle_root(root)
def get_merkle_root(self):
return self.trie.get_merkle_root()
def update(self, set_items, delete_items=None, virtual=True):
return self.trie.update(set_items, delete_items, virtual=virtual)
def run(self):
while True:
context_state_addresslist_tuple = self._addresses.get(block=True)
c_id, state_hash, address_list = context_state_addresslist_tuple
tree = MerkleDatabase(self._database, state_hash)
return_values = []
for address in address_list:
value = None
try:
value = tree.get(address)
except KeyError:
pass
return_values.append((address, value))
self._inflated_addresses.put((c_id, return_values))
def run(self):
while True:
context_state_addresslist_tuple = self._addresses.get(block=True)
c_id, state_hash, address_list = context_state_addresslist_tuple
tree = MerkleDatabase(self._database, state_hash)
return_values = []
for address in address_list:
value = None
try:
value = tree.get(address)
except KeyError:
pass
return_values.append((address, value))
self._inflated_addresses.put((c_id, return_values))
class ClientStateGetRequestHandler(object):
def __init__(self, database):
self._tree = MerkleDatabase(database)
def handle(self, message, responder):
error = False
status = None
request = client_pb2.ClientStateGetRequest()
try:
request.ParseFromString(message.content)
self._tree.set_merkle_root(request.merkle_root)
except KeyError as e:
status = client_pb2.ClientStateGetResponse.NORESOURCE
LOGGER.info(e)
error = True
except DecodeError:
LOGGER.info(
"Expected protobuf of class %s failed to "
"deserialize", request)
error = True
if error:
response = client_pb2.ClientStateGetResponse(
status=status or client_pb2.ClientStateGetResponse.ERROR)
else:
address = request.address
try:
value = self._tree.get(address)
status = client_pb2.ClientStateGetResponse.OK
except KeyError:
status = client_pb2.ClientStateGetResponse.NORESOURCE
LOGGER.debug("No entry at state address %s", address)
error = True
except ValueError:
status = client_pb2.ClientStateGetResponse.NONLEAF
LOGGER.debug("Node at state address %s is a nonleaf", address)
error = True
response = client_pb2.ClientStateGetResponse(status=status)
if not error:
response.value = value
responder.send(
validator_pb2.Message(
sender=message.sender,
message_type=validator_pb2.Message.CLIENT_STATE_GET_RESPONSE,
correlation_id=message.correlation_id,
content=response.SerializeToString()))
class StateGetRequestHandler(Handler):
def __init__(self, database):
self._tree = MerkleDatabase(database)
def handle(self, identity, message_content):
request = client_pb2.ClientStateGetRequest()
resp_proto = client_pb2.ClientStateGetResponse
status = resp_proto.OK
try:
request.ParseFromString(message_content)
self._tree.set_merkle_root(request.merkle_root)
except KeyError as e:
status = resp_proto.NORESOURCE
LOGGER.debug(e)
except DecodeError:
status = resp_proto.ERROR
LOGGER.info(
"Expected protobuf of class %s failed to "
"deserialize", request)
if status != resp_proto.OK:
response = resp_proto(status=status)
else:
address = request.address
try:
value = self._tree.get(address)
except KeyError:
status = resp_proto.NORESOURCE
LOGGER.debug("No entry at state address %s", address)
except ValueError:
status = resp_proto.NONLEAF
LOGGER.debug("Node at state address %s is a nonleaf", address)
response = resp_proto(status=status)
if status == resp_proto.OK:
response.value = value
return HandlerResult(
status=HandlerStatus.RETURN,
message_out=response,
message_type=validator_pb2.Message.CLIENT_STATE_GET_RESPONSE)
class StateGetRequestHandler(Handler):
def __init__(self, database):
self._tree = MerkleDatabase(database)
def handle(self, identity, message_content):
request = client_pb2.ClientStateGetRequest()
resp_proto = client_pb2.ClientStateGetResponse
status = resp_proto.OK
try:
request.ParseFromString(message_content)
self._tree.set_merkle_root(request.merkle_root)
except KeyError as e:
status = resp_proto.NORESOURCE
LOGGER.debug(e)
except DecodeError:
status = resp_proto.ERROR
LOGGER.info("Expected protobuf of class %s failed to "
"deserialize", request)
if status != resp_proto.OK:
response = resp_proto(status=status)
else:
address = request.address
try:
value = self._tree.get(address)
except KeyError:
status = resp_proto.NORESOURCE
LOGGER.debug("No entry at state address %s", address)
except ValueError:
status = resp_proto.NONLEAF
LOGGER.debug("Node at state address %s is a nonleaf", address)
response = resp_proto(status=status)
if status == resp_proto.OK:
response.value = value
return HandlerResult(
status=HandlerStatus.RETURN,
message_out=response,
message_type=validator_pb2.Message.CLIENT_STATE_GET_RESPONSE)
class TestSawtoothMerkleTrie(unittest.TestCase):
def setUp(self):
self.dir = tempfile.mkdtemp()
self.file = os.path.join(self.dir, 'merkle.lmdb')
self.lmdb = NativeLmdbDatabase(
self.file,
indexes=MerkleDatabase.create_index_configuration(),
_size=120 * 1024 * 1024)
self.trie = MerkleDatabase(self.lmdb)
def tearDown(self):
self.trie.close()
shutil.rmtree(self.dir)
def test_merkle_trie_root_advance(self):
value = {'name': 'foo', 'value': 1}
orig_root = self.get_merkle_root()
new_root = self.set('foo', value)
self.assert_root(orig_root)
self.assert_no_key('foo')
self.set_merkle_root(new_root)
self.assert_root(new_root)
self.assert_value_at_address('foo', value)
def test_merkle_trie_delete(self):
value = {'name': 'bar', 'value': 1}
new_root = self.set('bar', value)
self.set_merkle_root(new_root)
self.assert_root(new_root)
self.assert_value_at_address('bar', value)
# deleting an invalid key should raise an error
with self.assertRaises(KeyError):
self.delete('barf')
del_root = self.delete('bar')
# del_root hasn't been set yet, so address should still have value
self.assert_root(new_root)
self.assert_value_at_address('bar', value)
self.set_merkle_root(del_root)
self.assert_root(del_root)
self.assert_no_key('bar')
def test_merkle_trie_update(self):
init_root = self.get_merkle_root()
values = {}
key_hashes = {
key: _hash(key)
for key in (_random_string(10) for _ in range(1000))
}
for key, hashed in key_hashes.items():
value = {key: _random_string(512)}
new_root = self.set(hashed, value, ishash=True)
values[hashed] = value
self.set_merkle_root(new_root)
self.assert_not_root(init_root)
for address, value in values.items():
self.assert_value_at_address(address, value, ishash=True)
set_items = {
hashed: {
key: 5.0
}
for key, hashed in random.sample(key_hashes.items(), 50)
}
values.update(set_items)
delete_items = {
hashed
for hashed in random.sample(list(key_hashes.values()), 50)
}
# make sure there are no sets and deletes of the same key
delete_items = delete_items - set_items.keys()
for addr in delete_items:
del values[addr]
virtual_root = self.update(set_items, delete_items, virtual=True)
# virtual root shouldn't match actual contents of tree
with self.assertRaises(KeyError):
self.set_merkle_root(virtual_root)
actual_root = self.update(set_items, delete_items, virtual=False)
# the virtual root should be the same as the actual root
self.assertEqual(virtual_root, actual_root)
# neither should be the root yet
self.assert_not_root(virtual_root, actual_root)
self.set_merkle_root(actual_root)
self.assert_root(actual_root)
for address, value in values.items():
self.assert_value_at_address(address, value, ishash=True)
for address in delete_items:
with self.assertRaises(KeyError):
self.get(address, ishash=True)
def test_merkle_trie_leaf_iteration(self):
new_root = self.update(
{
"010101": {
"my_data": 1
},
"010202": {
"my_data": 2
},
"010303": {
"my_data": 3
}
}, [],
virtual=False)
# iterate over the empty trie
iterator = iter(self.trie)
with self.assertRaises(StopIteration):
next(iterator)
self.set_merkle_root(new_root)
# Test complete trie iteration
self.assertEqual([("010101", {
"my_data": 1
}), ("010202", {
"my_data": 2
}), ("010303", {
"my_data": 3
})], [entry for entry in iter(self.trie)])
# Test prefixed iteration
self.assertEqual([("010202", {
"my_data": 2
})], [entry for entry in self.trie.leaves('0102')])
# assertions
def assert_value_at_address(self, address, value, ishash=False):
self.assertEqual(self.get(address, ishash), value, 'Wrong value')
def assert_no_key(self, key):
with self.assertRaises(KeyError):
self.get(key)
def assert_root(self, expected):
self.assertEqual(expected, self.get_merkle_root(), 'Wrong root')
def assert_not_root(self, *not_roots):
root = self.get_merkle_root()
for not_root in not_roots:
self.assertNotEqual(root, not_root, 'Wrong root')
# trie accessors
# For convenience, assume keys are not hashed
# unless otherwise indicated.
def set(self, key, val, ishash=False):
key_ = key if ishash else _hash(key)
return self.trie.set(key_, val)
def get(self, key, ishash=False):
key_ = key if ishash else _hash(key)
return self.trie.get(key_)
def delete(self, key, ishash=False):
key_ = key if ishash else _hash(key)
return self.trie.delete(key_)
def set_merkle_root(self, root):
self.trie.set_merkle_root(root)
def get_merkle_root(self):
return self.trie.get_merkle_root()
def update(self, set_items, delete_items=None, virtual=True):
return self.trie.update(set_items, delete_items, virtual=virtual)
class TestSawtoothMerkleTrie(unittest.TestCase):
def setUp(self):
self.dir = tempfile.mkdtemp()
self.file = os.path.join(self.dir, 'merkle.lmdb')
self.lmdb = lmdb_nolock_database.LMDBNoLockDatabase(self.file, 'n')
self.trie = MerkleDatabase(self.lmdb)
def tearDown(self):
self.trie.close()
def test_merkle_trie_root_advance(self):
value = {'name': 'foo', 'value': 1}
orig_root = self.get_merkle_root()
new_root = self.set('foo', value)
self.assert_root(orig_root)
self.assert_no_key('foo')
self.set_merkle_root(new_root)
self.assert_root(new_root)
self.assert_value_at_address('foo', value)
def test_merkle_trie_delete(self):
value = {'name': 'bar', 'value': 1}
new_root = self.set('bar', value)
self.set_merkle_root(new_root)
self.assert_root(new_root)
self.assert_value_at_address('bar', value)
# deleting an invalid key should raise an error
with self.assertRaises(KeyError):
self.delete('barf')
del_root = self.delete('bar')
# del_root hasn't been set yet, so address should still have value
self.assert_root(new_root)
self.assert_value_at_address('bar', value)
self.set_merkle_root(del_root)
self.assert_root(del_root)
self.assert_no_key('bar')
def test_merkle_trie_update(self):
init_root = self.get_merkle_root()
values = {}
key_hashes = {
key: _hash(key)
for key in (_random_string(10) for _ in range(1000))
}
for key, hashed in key_hashes.items():
value = {key: _random_string(512)}
new_root = self.set(hashed, value, ishash=True)
values[hashed] = value
self.set_merkle_root(new_root)
self.assert_not_root(init_root)
for address, value in values.items():
self.assert_value_at_address(address, value, ishash=True)
set_items = {
hashed: {
key: 5.0
}
for key, hashed in random.sample(key_hashes.items(), 50)
}
values.update(set_items)
delete_items = {
hashed
for hashed in random.sample(list(key_hashes.values()), 50)
}
# make sure there are no sets and deletes of the same key
delete_items = delete_items - set_items.keys()
for addr in delete_items:
del values[addr]
virtual_root = self.update(set_items, delete_items, virtual=True)
# virtual root shouldn't match actual contents of tree
with self.assertRaises(KeyError):
self.set_merkle_root(virtual_root)
actual_root = self.update(set_items, delete_items, virtual=False)
# the virtual root should be the same as the actual root
self.assertEqual(virtual_root, actual_root)
# neither should be the root yet
self.assert_not_root(virtual_root, actual_root)
self.set_merkle_root(actual_root)
self.assert_root(actual_root)
for address, value in values.items():
self.assert_value_at_address(address, value, ishash=True)
for address in delete_items:
with self.assertRaises(KeyError):
self.get(address, ishash=True)
# assertions
def assert_value_at_address(self, address, value, ishash=False):
self.assertEqual(self.get(address, ishash), value, 'Wrong value')
def assert_no_key(self, key):
with self.assertRaises(KeyError):
self.get(key)
def assert_root(self, expected):
self.assertEqual(expected, self.get_merkle_root(), 'Wrong root')
def assert_not_root(self, *not_roots):
root = self.get_merkle_root()
for not_root in not_roots:
self.assertNotEqual(root, not_root, 'Wrong root')
# trie accessors
# For convenience, assume keys are not hashed
# unless otherwise indicated.
def set(self, key, val, ishash=False):
key_ = key if ishash else _hash(key)
return self.trie.set(key_, val)
def get(self, key, ishash=False):
key_ = key if ishash else _hash(key)
return self.trie.get(key_)
def delete(self, key, ishash=False):
key_ = key if ishash else _hash(key)
return self.trie.delete(key_)
def set_merkle_root(self, root):
self.trie.set_merkle_root(root)
def get_merkle_root(self):
return self.trie.get_merkle_root()
def update(self, set_items, delete_items=None, virtual=True):
return self.trie.update(set_items, delete_items, virtual=virtual)
class TestSawtoothMerkleTrie(unittest.TestCase):
def setUp(self):
self.dir = tempfile.mkdtemp()
self.file = os.path.join(self.dir, 'merkle.lmdb')
self.lmdb = lmdb_nolock_database.LMDBNoLockDatabase(
self.file,
'n')
self.trie = MerkleDatabase(self.lmdb)
def tearDown(self):
self.trie.close()
def test_merkle_trie_root_advance(self):
value = {'name': 'foo', 'value': 1}
orig_root = self.get_merkle_root()
new_root = self.set('foo', value)
self.assert_root(orig_root)
self.assert_no_key('foo')
self.set_merkle_root(new_root)
self.assert_root(new_root)
self.assert_value_at_address('foo', value)
def test_merkle_trie_delete(self):
value = {'name': 'bar', 'value': 1}
new_root = self.set('bar', value)
self.set_merkle_root(new_root)
self.assert_root(new_root)
self.assert_value_at_address('bar', value)
# deleting an invalid key should raise an error
with self.assertRaises(KeyError):
self.delete('barf')
del_root = self.delete('bar')
# del_root hasn't been set yet, so address should still have value
self.assert_root(new_root)
self.assert_value_at_address('bar', value)
self.set_merkle_root(del_root)
self.assert_root(del_root)
self.assert_no_key('bar')
def test_merkle_trie_update(self):
init_root = self.get_merkle_root()
values = {}
key_hashes = {
key: _hash(key)
for key in (_random_string(10) for _ in range(1000))
}
for key, hashed in key_hashes.items():
value = {key: _random_string(512)}
new_root = self.set(hashed, value, ishash=True)
values[hashed] = value
self.set_merkle_root(new_root)
self.assert_not_root(init_root)
for address, value in values.items():
self.assert_value_at_address(
address, value, ishash=True)
set_items = {
hashed: {
key: 5.0
}
for key, hashed in random.sample(key_hashes.items(), 50)
}
values.update(set_items)
delete_items = {
hashed
for hashed in random.sample(list(key_hashes.values()), 50)
}
# make sure there are no sets and deletes of the same key
delete_items = delete_items - set_items.keys()
for addr in delete_items:
del values[addr]
virtual_root = self.update(set_items, delete_items, virtual=True)
# virtual root shouldn't match actual contents of tree
with self.assertRaises(KeyError):
self.set_merkle_root(virtual_root)
actual_root = self.update(set_items, delete_items, virtual=False)
# the virtual root should be the same as the actual root
self.assertEqual(virtual_root, actual_root)
# neither should be the root yet
self.assert_not_root(
virtual_root,
actual_root)
self.set_merkle_root(actual_root)
self.assert_root(actual_root)
for address, value in values.items():
self.assert_value_at_address(
address, value, ishash=True)
for address in delete_items:
with self.assertRaises(KeyError):
self.get(address, ishash=True)
# assertions
def assert_value_at_address(self, address, value, ishash=False):
self.assertEqual(
self.get(address, ishash),
value,
'Wrong value')
def assert_no_key(self, key):
with self.assertRaises(KeyError):
self.get(key)
def assert_root(self, expected):
self.assertEqual(
expected,
self.get_merkle_root(),
'Wrong root')
def assert_not_root(self, *not_roots):
root = self.get_merkle_root()
for not_root in not_roots:
self.assertNotEqual(
root,
not_root,
'Wrong root')
# trie accessors
# For convenience, assume keys are not hashed
# unless otherwise indicated.
def set(self, key, val, ishash=False):
key_ = key if ishash else _hash(key)
return self.trie.set(key_, val)
def get(self, key, ishash=False):
key_ = key if ishash else _hash(key)
return self.trie.get(key_)
def delete(self, key, ishash=False):
key_ = key if ishash else _hash(key)
return self.trie.delete(key_)
def set_merkle_root(self, root):
self.trie.set_merkle_root(root)
def get_merkle_root(self):
return self.trie.get_merkle_root()
def update(self, set_items, delete_items=None, virtual=True):
return self.trie.update(set_items, delete_items, virtual=virtual)
class TestSawtoothMerkleTrie(unittest.TestCase):
def setUp(self):
self.lmdb = lmdb_nolock_database.LMDBNoLockDatabase(
"/home/vagrant/merkle.lmdb",
'n')
self.trie = MerkleDatabase(self.lmdb)
def tearDown(self):
self.trie.close()
def test_merkle_trie_root_advance(self):
value = {"name": "foo", "value": 1}
orig_root = self.trie.get_merkle_root()
new_root = self.trie.set(MerkleDatabase.hash("foo"),
value)
with self.assertRaises(KeyError):
self.trie.get(MerkleDatabase.hash("foo"))
self.trie.set_merkle_root(new_root)
self.assertEqual(self.trie.get(MerkleDatabase.hash("foo")),
value)
def test_merkle_trie_delete(self):
value = {"name": "bar", "value": 1}
new_root = self.trie.set(MerkleDatabase.hash("bar"), value)
self.trie.set_merkle_root(new_root)
self.assertEqual(self.trie.get(MerkleDatabase.hash("bar")),
value)
del_root = self.trie.delete(MerkleDatabase.hash("bar"))
self.trie.set_merkle_root(del_root)
with self.assertRaises(KeyError):
self.trie.get(MerkleDatabase.hash("bar"))
def test_merkle_trie_update(self):
value = ''.join(random.choice(string.ascii_lowercase)
for _ in range(512))
keys = []
for i in range(1000):
key = ''.join(random.choice(string.ascii_lowercase)
for _ in range(10))
keys.append(key)
hash = MerkleDatabase.hash(key)
new_root = self.trie.set(hash, {key: value})
self.trie.set_merkle_root(new_root)
set_items = {}
for key in random.sample(keys, 50):
hash = MerkleDatabase.hash(key)
thing = {key: 5.0}
set_items[hash] = thing
update_root = self.trie.update(set_items)
self.trie.set_merkle_root(update_root)
for address in set_items:
self.assertEqual(self.trie.get(address),
set_items[address])
class TestSawtoothMerkleTrie(unittest.TestCase):
def setUp(self):
self.dir = tempfile.mkdtemp()
self.file = os.path.join(self.dir, 'merkle.lmdb')
self.lmdb = NativeLmdbDatabase(
self.file,
indexes=MerkleDatabase.create_index_configuration(),
_size=120 * 1024 * 1024)
self.trie = MerkleDatabase(self.lmdb)
def tearDown(self):
self.trie.close()
shutil.rmtree(self.dir)
def test_merkle_trie_root_advance(self):
value = {'name': 'foo', 'value': 1}
orig_root = self.get_merkle_root()
new_root = self.set('foo', value)
self.assert_root(orig_root)
self.assert_no_key('foo')
self.set_merkle_root(new_root)
self.assert_root(new_root)
self.assert_value_at_address('foo', value)
def test_merkle_trie_delete(self):
value = {'name': 'bar', 'value': 1}
new_root = self.set('bar', value)
self.set_merkle_root(new_root)
self.assert_root(new_root)
self.assert_value_at_address('bar', value)
# deleting an invalid key should raise an error
with self.assertRaises(KeyError):
self.delete('barf')
del_root = self.delete('bar')
# del_root hasn't been set yet, so address should still have value
self.assert_root(new_root)
self.assert_value_at_address('bar', value)
self.set_merkle_root(del_root)
self.assert_root(del_root)
self.assert_no_key('bar')
def test_merkle_trie_update(self):
init_root = self.get_merkle_root()
values = {}
key_hashes = {
key: _hash(key)
for key in (_random_string(10) for _ in range(1000))
}
for key, hashed in key_hashes.items():
value = {key: _random_string(512)}
new_root = self.set(hashed, value, ishash=True)
values[hashed] = value
self.set_merkle_root(new_root)
self.assert_not_root(init_root)
for address, value in values.items():
self.assert_value_at_address(
address, value, ishash=True)
set_items = {
hashed: {
key: 5.0
}
for key, hashed in random.sample(key_hashes.items(), 50)
}
values.update(set_items)
delete_items = {
hashed
for hashed in random.sample(list(key_hashes.values()), 50)
}
# make sure there are no sets and deletes of the same key
delete_items = delete_items - set_items.keys()
for addr in delete_items:
del values[addr]
virtual_root = self.update(set_items, delete_items, virtual=True)
# virtual root shouldn't match actual contents of tree
with self.assertRaises(KeyError):
self.set_merkle_root(virtual_root)
actual_root = self.update(set_items, delete_items, virtual=False)
# the virtual root should be the same as the actual root
self.assertEqual(virtual_root, actual_root)
# neither should be the root yet
self.assert_not_root(
virtual_root,
actual_root)
self.set_merkle_root(actual_root)
self.assert_root(actual_root)
for address, value in values.items():
self.assert_value_at_address(
address, value, ishash=True)
for address in delete_items:
with self.assertRaises(KeyError):
self.get(address, ishash=True)
def test_merkle_trie_leaf_iteration(self):
new_root = self.update({
"010101": {"my_data": 1},
"010202": {"my_data": 2},
"010303": {"my_data": 3}
}, [], virtual=False)
# iterate over the empty trie
iterator = iter(self.trie)
with self.assertRaises(StopIteration):
next(iterator)
self.set_merkle_root(new_root)
# Test complete trie iteration
self.assertEqual(
[("010101", {"my_data": 1}),
("010202", {"my_data": 2}),
("010303", {"my_data": 3})],
[entry for entry in iter(self.trie)])
# Test prefixed iteration
self.assertEqual([("010202", {"my_data": 2})],
[entry for entry in self.trie.leaves('0102')])
# assertions
def assert_value_at_address(self, address, value, ishash=False):
self.assertEqual(
self.get(address, ishash),
value,
'Wrong value')
def assert_no_key(self, key):
with self.assertRaises(KeyError):
self.get(key)
def assert_root(self, expected):
self.assertEqual(
expected,
self.get_merkle_root(),
'Wrong root')
def assert_not_root(self, *not_roots):
root = self.get_merkle_root()
for not_root in not_roots:
self.assertNotEqual(
root,
not_root,
'Wrong root')
# trie accessors
# For convenience, assume keys are not hashed
# unless otherwise indicated.
def set(self, key, val, ishash=False):
key_ = key if ishash else _hash(key)
return self.trie.set(key_, val)
def get(self, key, ishash=False):
key_ = key if ishash else _hash(key)
return self.trie.get(key_)
def delete(self, key, ishash=False):
key_ = key if ishash else _hash(key)
return self.trie.delete(key_)
def set_merkle_root(self, root):
self.trie.set_merkle_root(root)
def get_merkle_root(self):
return self.trie.get_merkle_root()
def update(self, set_items, delete_items=None, virtual=True):
return self.trie.update(set_items, delete_items, virtual=virtual)
