def enter_any(self, prev_state):
assert self.parent.interpreter.queue_size > 0, "Entered consensus state, but interpreter queue is empty!"
# Merkle-ize transaction queue and create signed merkle hash
all_tx = self.parent.interpreter.queue_binary
self.log.info(
"Delegate got tx from interpreter queue: {}".format(all_tx))
self.merkle = MerkleTree(all_tx)
self.merkle_hash = self.merkle.hash_of_nodes()
self.log.info("Delegate got merkle hash {}".format(self.merkle_hash))
self.signature = ED25519Wallet.sign(self.parent.signing_key,
self.merkle_hash)
# Create merkle signature message and publish it
merkle_sig = MerkleSignature.create(sig_hex=self.signature,
timestamp='now',
sender=self.parent.verifying_key)
self.log.info("Broadcasting signature {}".format(self.signature))
self.parent.composer.send_pub_msg(
filter=Constants.ZmqFilters.DelegateDelegate, message=merkle_sig)
# Now that we've computed/composed the merkle tree hash, validate all our pending signatures
for sig in [
s for s in self.parent.pending_sigs if self.validate_sig(s)
]:
self.signatures.append(sig)
self.check_majority()
def test_verify_tree_from_hex_str_invalid(self):
raw_leaves = [secrets.token_bytes(16) for _ in range(16)]
random_hash = MerkleTree.hash(secrets.token_bytes(16))
m = MerkleTree(raw_leaves)
self.assertFalse(MerkleTree.verify_tree_from_hex_str(m.leaves_as_concat_hex_str, random_hash))
def test_from_leaves_hex_str(self):
leaves = [1, 2, 3, 4]
m_from_init = MerkleTree(leaves)
m_from_hex_str = MerkleTree.from_leaves_hex_str(m_from_init.leaves_as_concat_hex_str)
self.assertEqual(m_from_hex_str.root, m_from_init.root)
def test_leaves_as_hex(self):
leaves = [1, 2, 3, 4]
hashed_leaves = [MerkleTree.hash(leaf).hex() for leaf in leaves]
m = MerkleTree(leaves)
for actual, expected in zip(m.leaves_as_hex, hashed_leaves):
self.assertEqual(actual, expected)
def test_leaves_as_hex_str(self):
leaves = [1, 2, 3, 4]
hashed_leaves = [MerkleTree.hash(leaf).hex() for leaf in leaves]
hex_leaves = ''.join(hashed_leaves)
m = MerkleTree(leaves)
self.assertEqual(hex_leaves, m.leaves_as_concat_hex_str)
def test_verify_tree_from_hex_str(self):
raw_leaves = [secrets.token_bytes(16) for _ in range(16)]
m = MerkleTree(raw_leaves)
self.assertTrue(
MerkleTree.verify_tree_from_hex_str(m.leaves_as_concat_hex_str,
m.root_as_hex))
def test_hash_leaves_false(self):
leaves = [1, 2, 3, 4]
prehashed_leaves = list(map(MerkleTree.hash, leaves))
merk_from_leaves = MerkleTree(leaves=leaves)
merk_from_hashes = MerkleTree(leaves=prehashed_leaves, hash_leaves=False)
for node1, node2 in zip(merk_from_hashes.nodes, merk_from_leaves.nodes):
self.assertEqual(node1, node2)
def test_verify_tree(self):
raw_leaves = [secrets.token_bytes(16) for _ in range(16)]
m = MerkleTree(raw_leaves)
tree_nodes = m.nodes[len(m.nodes) // 2:]
tree_hash = m.hash_of_nodes()
self.assertTrue(MerkleTree.verify_tree(tree_nodes, tree_hash))
def test_verify_tree_invalid(self):
raw_leaves = [secrets.token_bytes(16) for _ in range(16)]
m = MerkleTree(raw_leaves)
# Set the last leaf to a random value obviously different then whatever was used to build the tree
# this should cause validation to fail
bad_leaves = m.leaves
bad_leaves[-1] = secrets.token_bytes(16)
self.assertFalse(MerkleTree.verify_tree(bad_leaves, m.root))
def new_block_procedure(self, block, txs):
self.log.critical(
"\n***\nDONE COLLECTING BLOCK DATA FROM NODES. Storing new block.\n***\n"
)
hash_of_nodes = MerkleTree.hash_nodes(block.nodes)
tree = b"".join(block.nodes).hex()
signatures = "".join(
[merk_sig.signature for merk_sig in block.signatures])
# Store the block + transaction data
block_num = -1
with DB() as db:
tables = db.tables
q = insert(tables.blocks).values(hash=hash_of_nodes,
tree=tree,
signatures=signatures)
q_result = db.execute(q)
block_num = q_result.lastrowid
for key, value in txs.items():
tx = {'key': key, 'value': value}
qq = insert(tables.transactions).values(tx)
db.execute(qq)
assert block_num > 0, "Block num must be greater than 0! Was it not set in the DB() context session?"
# Notify delegates of new block
self.log.info(
"Masternode sending NewBlockNotification to delegates with new block hash {} and block num {}"
.format(hash_of_nodes, block_num))
notif = NewBlockNotification.create(new_block_hash=hash_of_nodes.hex(),
new_block_num=block_num)
self.parent.composer.send_pub_msg(
filter=Constants.ZmqFilters.MasternodeDelegate, message=notif)
def enter_from_interpret(self):
assert self.parent.interpreter.queue_size > 0, "Entered consensus state, but interpreter queue is empty!"
assert self.parent.interpreter.queue_size == BLOCK_SIZE, \
"Consensus state entered with {} transactions in queue, but the BLOCK_SIZE is {}!"\
.format(self.parent.interpreter.queue_size, BLOCK_SIZE)
# Merkle-ize transaction queue and create signed merkle hash
all_tx = self.parent.interpreter.queue_binary
self.merkle = MerkleTree.from_raw_transactions(all_tx)
self.signature = wallet.sign(self.parent.signing_key, self.merkle.root)
self.log.info("Delegate entering consensus state with merkle hash {}, and latest block hash {}"
.format(self.merkle.root_as_hex, self.parent.current_hash))
self.log.spam("Delegate got merkle leaves {}".format(self.merkle.leaves_as_hex))
# Create merkle signature message and publish it
merkle_sig = MerkleSignature.create(sig_hex=self.signature, timestamp='now',
sender=self.parent.verifying_key)
self.log.debugv("Broadcasting signature {}".format(self.signature))
self.parent.composer.send_pub_msg(filter=DELEGATE_DELEGATE_FILTER, message=merkle_sig)
# Now that we've computed/composed the merkle tree hash, validate all our pending signatures
for sig in [s for s in self.parent.pending_sigs if self.validate_sig(s)]:
self.signatures.append(sig)
# Add our own signature
self.signatures.append(merkle_sig)
self.check_majority()
def enter_from_interpret(self):
assert self.parent.interpreter.queue_size > 0, "Entered consensus state, but interpreter queue is empty!"
# Merkle-ize transaction queue and create signed merkle hash
all_tx = self.parent.interpreter.queue_binary
self.log.debugv(
"Delegate got tx from interpreter queue: {}".format(all_tx))
self.merkle = MerkleTree.from_raw_transactions(all_tx)
self.log.debugv("Delegate got merkle hash {}".format(
self.merkle.root_as_hex))
self.signature = wallet.sign(self.parent.signing_key, self.merkle.root)
# Create merkle signature message and publish it
merkle_sig = MerkleSignature.create(sig_hex=self.signature,
timestamp='now',
sender=self.parent.verifying_key)
self.log.debugv("Broadcasting signature {}".format(self.signature))
self.parent.composer.send_pub_msg(filter=delegate_delegate,
message=merkle_sig)
# Now that we've computed/composed the merkle tree hash, validate all our pending signatures
for sig in [
s for s in self.parent.pending_sigs if self.validate_sig(s)
]:
self.signatures.append(sig)
self.check_majority()
def test_data_for_hash_doesnt_exist(self):
leaves = [b'1', b'2', b'3']
hashed_leaves = list(map(Hasher.hash, leaves))
m = MerkleTree.from_raw_transactions(leaves)
self.assertRaises(AssertionError, m.data_for_hash, '0' * 64)
def test_tx_request_replies_with_correct_data(self):
"""
Tests that a delegate who receives a TransactionRequest in Consensus state replies with the correct data
"""
mock_sm = MagicMock()
state = DelegateConsensusState(mock_sm)
# Build a merkle tree and attach it to the state
tx_objects = [build_test_transaction() for _ in range(5)]
tx_blobs = [tx.serialize() for tx in tx_objects]
tx_hashes = [Hasher.hash(blob) for blob in tx_blobs]
merkle_tree = MerkleTree.from_raw_transactions(tx_blobs)
state.merkle = merkle_tree
# Build a TransactionRequest for a few of these transactions
requested_hashes = tx_hashes[1:4]
expected_blobs = tx_blobs[1:4]
request = TransactionRequest.create(
transaction_hashes=requested_hashes)
# Finally, ensure the reply is what we expect it to be
reply = state.handle_tx_request(request)
self.assertEquals(reply.raw_transactions, expected_blobs)
def validate_block_data(cls, block_data: dict):
"""
Validates the block_data dictionary. 'block_data' should be a strict subset of the 'block' dictionary, keys for all
columns in the block table EXCEPT 'number' and 'hash'. If any validation fails, an exception is raised.
For a block_data dictionary to be valid, it must:
- Have a key for each block data column specified in BLOCK_DATA_COLS (at top of blocks.py)
- BlockContender successfully validates with the Merkle root (meaning all signatures in the BlockContender
can be verified using the Merkle root as the message)
- Merkle leaves contained in BlockContender (block_contender.nodes) match Merkle leaves in block_data dict
- Merkle root is correct root if a Merkle tree is built from Merkle leaves
- Masternode signature is valid (signature is valid using Merkle root as message and masternode_vk as vk)
:param block_data: The dictionary containing a key for each column in BLOCK_DATA_COLS
(ie 'merkle_root', 'prev_block_hash', .. ect)
:raises: An BlockStorageValidationException (or subclass) if any validation fails
"""
# Check block_data has all the necessary keys
expected_keys = set(BLOCK_DATA_COLS.keys())
actual_keys = set(block_data.keys())
missing_keys = expected_keys - actual_keys
extra_keys = actual_keys - expected_keys
# Check for missing keys
if len(missing_keys) > 0:
raise BlockStorageValidationException("block_data keys {} missing key(s) {}".format(actual_keys, missing_keys))
# Check for extra (unrecognized) keys
if len(extra_keys) > 0:
raise BlockStorageValidationException("block_data keys {} has unrecognized keys {}".format(actual_keys, extra_keys))
# Validate Merkle Tree
tree = MerkleTree.from_leaves_hex_str(block_data['merkle_leaves'])
if tree.root_as_hex != block_data['merkle_root']:
raise InvalidMerkleTreeException("Merkle Tree could not be validated for block_data {}".format(block_data))
# Validate BlockContender nodes match merkle leaves
block_leaves = block_data['block_contender'].merkle_leaves
if len(block_leaves) != len(tree.leaves):
raise InvalidBlockContenderException("Number of Merkle leaves on BlockContender {} does not match number of"
" leaves in MerkleTree {}".format(len(block_leaves), len(tree.leaves)))
for block_leaf, merkle_leaf in zip(block_leaves, tree.leaves_as_hex):
if block_leaf != merkle_leaf:
raise InvalidBlockContenderException("BlockContender leaves do not match Merkle leaves\nblock leaves = "
"{}\nmerkle leaves = {}".format(block_leaves, tree.leaves_as_hex))
# Validate MerkleSignatures inside BlockContender match Merkle leaves from raw transactions
bc = block_data['block_contender']
if not bc.validate_signatures():
raise InvalidBlockContenderException("BlockContender signatures could not be validated! BC = {}".format(bc))
# TODO validate MerkleSignatures are infact signed by valid delegates
# this is tricky b/c we would need to know who the delegates were at the time of the block, not necessarily the
# current delegates
# Validate Masternode Signature
if not is_valid_hex(block_data['masternode_vk'], length=64):
raise InvalidBlockSignatureException("Invalid verifying key for field masternode_vk: {}"
.format(block_data['masternode_vk']))
if not wallet.verify(block_data['masternode_vk'], bytes.fromhex(block_data['merkle_root']), block_data['masternode_signature']):
raise InvalidBlockSignatureException("Could not validate Masternode's signature on block data")
def validate_block_contender(self, block: BlockContender) -> bool:
"""
Helper method to validate a block contender. For a block contender to be valid it must:
1) Have a provable merkle tree, ie. all nodes must be hash of (left child + right child)
2) Be signed by at least 2/3 of the top 32 delegates
:param block_contender: The BlockContender to validate
:return: True if the BlockContender is valid, false otherwise
"""
def _validate_sigs(signatures, msg) -> bool:
for sig in signatures:
self.log.info("mn verifying signature: {}".format(sig))
if not sig.verify(msg, sig.sender):
self.log.error(
"!!!! Oh no why couldnt we verify sig {}???".format(
sig))
return False
return True
# Development sanity checks (these should be removed in production)
assert len(
block.nodes
) >= 1, "Masternode got block contender with no nodes! {}".format(
block)
assert len(block.signatures) >= Constants.Testnet.Majority, \
"Received a block contender with only {} signatures (which is less than a majority of {}"\
.format(len(block.signatures), Constants.Testnet.Majority)
# TODO -- ensure that this block contender's previous block is this Masternode's current block...
# Prove Merkle Tree
hash_of_nodes = MerkleTree.hash_nodes(block.nodes)
tx_hashes = block.nodes[len(block.nodes) // 2:]
if not MerkleTree.verify_tree(tx_hashes, hash_of_nodes):
self.log.error(
"\n\n\n\nCOULD NOT VERIFY MERKLE TREE FOR BLOCK CONTENDER {}\n\n\n"
.format(block))
return False
# Validate signatures
if not _validate_sigs(block.signatures, hash_of_nodes):
self.log.error(
"MN COULD NOT VALIDATE SIGNATURES FOR CONTENDER {}".format(
block))
return False
return True
def test_data_for_hash(self):
leaves = [b'1', b'2', b'3']
hashed_leaves = list(map(Hasher.hash, leaves))
m = MerkleTree.from_raw_transactions(leaves)
for i in range(len(leaves)):
self.assertEquals(m.data_for_hash(hashed_leaves[i]), leaves[i])
def test_creation(self):
"""
Tests that a created block data reply has the expected properties
"""
tx_binary = b'some random binary'
bdr = BlockDataReply.create(tx_binary)
self.assertEqual(tx_binary, bdr.raw_tx)
self.assertEqual(bdr.tx_hash, MerkleTree.hash(tx_binary))
def build_test_contender(tree: MerkleTree=None):
"""
Method to build a 'test' block contender. Used exclusively in unit tests.
"""
if not tree:
nodes = [1, 2, 3, 4]
tree = MerkleTree(leaves=nodes)
sigs = [build_test_merkle_sig(msg=tree.root) for _ in range(8)]
return BlockContender.create(signatures=sigs, merkle_leaves=tree.leaves_as_hex)
def _validate_sigs(self, block: BlockContender) -> bool:
signatures = block.signatures
msg = MerkleTree.hash_nodes(block.merkle_leaves)
for sig in signatures:
# TODO -- ensure that the sender belongs to the top delegate pool
self.log.debug("mn verifying signature: {}".format(sig))
if not sig.verify(msg, sig.sender):
self.log.error("Masternode could not verify signature!!! Sig={}".format(sig))
return False
return True
def validate_signatures(self):
"""
Validates the signatures in the block contender. Returns true if all signatures are valid, and false otherwise
:return: True if the signatures are valid; False otherwise
"""
tree = MerkleTree.from_hex_leaves(self.merkle_leaves)
for sig in self.signatures:
if not sig.verify(tree.root):
return False
return True
def test_make_merkle_works(self):
leaves = [1, 2, 3, 4]
test_merkle = [
None, None, None,
MerkleTree.hash(1),
MerkleTree.hash(2),
MerkleTree.hash(3),
MerkleTree.hash(4)
]
test_merkle[2] = MerkleTree.hash(test_merkle[-2] + test_merkle[-1])
test_merkle[1] = MerkleTree.hash(test_merkle[3] + test_merkle[4])
test_merkle[0] = MerkleTree.hash(test_merkle[1] + test_merkle[2])
m = MerkleTree(leaves)
self.assertEqual(test_merkle, m.nodes)
def test_make_merkle_works(self):
# create a merkle tree that should be
leaves = [1, 2, 3, 4]
test_merkle = [None,
None,
None,
MerkleTree.hash(bytes(1)),
MerkleTree.hash(bytes(2)),
MerkleTree.hash(bytes(3)),
MerkleTree.hash(bytes(4))]
test_merkle[2] = MerkleTree.hash(test_merkle[-2] + test_merkle[-1])
test_merkle[1] = MerkleTree.hash(test_merkle[3] + test_merkle[4])
test_merkle[0] = MerkleTree.hash(test_merkle[1] + test_merkle[2])
m = MerkleTree(leaves)
self.assertEqual(test_merkle, m.nodes)
def test_root_as_hex(self):
leaves = [1, 2, 3, 4]
test_merkle = [
None, None, None,
MerkleTree.hash(1),
MerkleTree.hash(2),
MerkleTree.hash(3),
MerkleTree.hash(4)
]
test_merkle[2] = MerkleTree.hash(test_merkle[-2] + test_merkle[-1])
test_merkle[1] = MerkleTree.hash(test_merkle[3] + test_merkle[4])
test_merkle[0] = MerkleTree.hash(test_merkle[1] + test_merkle[2])
m = MerkleTree(leaves)
self.assertEqual(m.root, test_merkle[0])
self.assertEqual(m.root_as_hex, test_merkle[0].hex())
def test_eq(self):
nodes = [secrets.token_bytes(8) for _ in range(4)]
tree = MerkleTree.from_raw_transactions(nodes)
msg = tree.root
sig1, sk1, vk1 = self._create_merkle_sig(msg)
sig2, sk2, vk2 = self._create_merkle_sig(msg)
sig3, sk3, vk3 = self._create_merkle_sig(msg)
sig4, sk4, vk4 = self._create_merkle_sig(msg)
signatures = [sig1, sig2, sig3, sig4]
bc1 = BlockContender.create(signatures, merkle_leaves=tree.leaves_as_hex, prev_block_hash='A' * 64)
bc2 = BlockContender.create(signatures, merkle_leaves=tree.leaves_as_hex, prev_block_hash='A' * 64)
self.assertEquals(bc1, bc2)
def build_test_contender(tree: MerkleTree = None, prev_block_hash=''):
"""
Method to build a 'test' block contender. Used exclusively in unit tests.
"""
from cilantro.storage.blocks import BlockStorageDriver
from cilantro.constants.nodes import BLOCK_SIZE
if not tree:
nodes = [str(i).encode() for i in range(BLOCK_SIZE)]
tree = MerkleTree(leaves=nodes)
if not prev_block_hash:
prev_block_hash = BlockStorageDriver.get_latest_block_hash()
sigs = [build_test_merkle_sig(msg=tree.root) for _ in range(8)]
return BlockContender.create(signatures=sigs,
merkle_leaves=tree.leaves_as_hex,
prev_block_hash=prev_block_hash)
def test_validate_signatures(self):
nodes = [secrets.token_bytes(8) for _ in range(4)]
tree = MerkleTree.from_raw_transactions(nodes)
msg = tree.root
sig1, sk1, vk1 = self._create_merkle_sig(msg)
sig2, sk2, vk2 = self._create_merkle_sig(msg)
sig3, sk3, vk3 = self._create_merkle_sig(msg)
sig4, sk4, vk4 = self._create_merkle_sig(msg)
signatures = [sig1, sig2, sig3, sig4]
bc = BlockContender.create(signatures,
merkle_leaves=tree.leaves_as_hex)
is_valid = bc.validate_signatures()
self.assertTrue(is_valid)
def test_validate_signatures_invalid(self):
nodes = [secrets.token_bytes(8) for _ in range(4)]
tree = MerkleTree.from_raw_transactions(nodes)
msg = tree.root
bad_msg = b'lol this is def not a merkle root'
sig1, sk1, vk1 = self._create_merkle_sig(msg)
sig2, sk2, vk2 = self._create_merkle_sig(msg)
sig3, sk3, vk3 = self._create_merkle_sig(msg)
sig4, sk4, vk4 = self._create_merkle_sig(bad_msg)
signatures = [sig1, sig2, sig3, sig4]
bc = BlockContender.create(signatures, merkle_leaves=tree.leaves_as_hex, prev_block_hash="A" * 64)
is_valid = bc.validate_signatures()
self.assertFalse(is_valid)
class Block:
def __init__(self, txs, last_block):
self.merkle_tree = MerkleTree(txs)
self.last_block = last_block
self.hash = self.merkle_tree.parent()
def encode(self):
return pickle.dumps([self.merkle_tree.raw_leaves, self.last_block])
@classmethod
def decode(cls, b):
block = pickle.loads(b)
return Block(block[0], block[1])
def __eq__(self, other):
if self.hash != other.hash:
return False
elif self.merkle_tree.nodes != other.merkle_tree.merkle_leaves:
return False
elif self.last_block != other.last_block:
return False
return True
async def _interpret_next_subtree(self, num_of_batches = 1):
self.log.debug("Starting to make a new sub-block {} for block {}"
.format(self.sub_block_num, self.block_num))
# get next batch of txns ?? still need to decide whether to unpack a bag or check for end of txn batch
while(num_of_batches > 0):
txn = self.pending_txs.popleft()
if txn == end_of_batch:
num_of_batches = num_of_batches - 1
else:
self.interpreter.interpret(txn) # this is a blocking call. either async or threads??
if not self._interpret: # do we need abort??
self.interpreter.flush(update_state=False)
return;
# Merkle-ize transaction queue and create signed merkle hash
all_tx = self.interpreter.queue_binary
self.merkle = MerkleTree.from_raw_transactions(all_tx)
self.signature = wallet.sign(self.signing_key, self.merkle.root)
# Create merkle signature message and publish it
merkle_sig = MerkleSignature.create(sig_hex=self.signature,
timestamp='now',
sender=self.verifying_key)
self.send_signature(merkle_sig) # send signature to block manager
