def enter_any(self, prev_state): assert self.parent.interpreter.queue_size > 0, "Entered consensus state, but interpreter queue is empty!" # Merkle-ize transaction queue and create signed merkle hash all_tx = self.parent.interpreter.queue_binary self.log.info( "Delegate got tx from interpreter queue: {}".format(all_tx)) self.merkle = MerkleTree(all_tx) self.merkle_hash = self.merkle.hash_of_nodes() self.log.info("Delegate got merkle hash {}".format(self.merkle_hash)) self.signature = ED25519Wallet.sign(self.parent.signing_key, self.merkle_hash) # Create merkle signature message and publish it merkle_sig = MerkleSignature.create(sig_hex=self.signature, timestamp='now', sender=self.parent.verifying_key) self.log.info("Broadcasting signature {}".format(self.signature)) self.parent.composer.send_pub_msg( filter=Constants.ZmqFilters.DelegateDelegate, message=merkle_sig) # Now that we've computed/composed the merkle tree hash, validate all our pending signatures for sig in [ s for s in self.parent.pending_sigs if self.validate_sig(s) ]: self.signatures.append(sig) self.check_majority()
def test_verify_tree_from_hex_str_invalid(self): raw_leaves = [secrets.token_bytes(16) for _ in range(16)] random_hash = MerkleTree.hash(secrets.token_bytes(16)) m = MerkleTree(raw_leaves) self.assertFalse(MerkleTree.verify_tree_from_hex_str(m.leaves_as_concat_hex_str, random_hash))
def test_from_leaves_hex_str(self): leaves = [1, 2, 3, 4] m_from_init = MerkleTree(leaves) m_from_hex_str = MerkleTree.from_leaves_hex_str(m_from_init.leaves_as_concat_hex_str) self.assertEqual(m_from_hex_str.root, m_from_init.root)
def test_leaves_as_hex(self): leaves = [1, 2, 3, 4] hashed_leaves = [MerkleTree.hash(leaf).hex() for leaf in leaves] m = MerkleTree(leaves) for actual, expected in zip(m.leaves_as_hex, hashed_leaves): self.assertEqual(actual, expected)
def test_leaves_as_hex_str(self): leaves = [1, 2, 3, 4] hashed_leaves = [MerkleTree.hash(leaf).hex() for leaf in leaves] hex_leaves = ''.join(hashed_leaves) m = MerkleTree(leaves) self.assertEqual(hex_leaves, m.leaves_as_concat_hex_str)
def test_verify_tree_from_hex_str(self): raw_leaves = [secrets.token_bytes(16) for _ in range(16)] m = MerkleTree(raw_leaves) self.assertTrue( MerkleTree.verify_tree_from_hex_str(m.leaves_as_concat_hex_str, m.root_as_hex))
def test_hash_leaves_false(self): leaves = [1, 2, 3, 4] prehashed_leaves = list(map(MerkleTree.hash, leaves)) merk_from_leaves = MerkleTree(leaves=leaves) merk_from_hashes = MerkleTree(leaves=prehashed_leaves, hash_leaves=False) for node1, node2 in zip(merk_from_hashes.nodes, merk_from_leaves.nodes): self.assertEqual(node1, node2)
def test_verify_tree(self): raw_leaves = [secrets.token_bytes(16) for _ in range(16)] m = MerkleTree(raw_leaves) tree_nodes = m.nodes[len(m.nodes) // 2:] tree_hash = m.hash_of_nodes() self.assertTrue(MerkleTree.verify_tree(tree_nodes, tree_hash))
def test_verify_tree_invalid(self): raw_leaves = [secrets.token_bytes(16) for _ in range(16)] m = MerkleTree(raw_leaves) # Set the last leaf to a random value obviously different then whatever was used to build the tree # this should cause validation to fail bad_leaves = m.leaves bad_leaves[-1] = secrets.token_bytes(16) self.assertFalse(MerkleTree.verify_tree(bad_leaves, m.root))
def new_block_procedure(self, block, txs): self.log.critical( "\n***\nDONE COLLECTING BLOCK DATA FROM NODES. Storing new block.\n***\n" ) hash_of_nodes = MerkleTree.hash_nodes(block.nodes) tree = b"".join(block.nodes).hex() signatures = "".join( [merk_sig.signature for merk_sig in block.signatures]) # Store the block + transaction data block_num = -1 with DB() as db: tables = db.tables q = insert(tables.blocks).values(hash=hash_of_nodes, tree=tree, signatures=signatures) q_result = db.execute(q) block_num = q_result.lastrowid for key, value in txs.items(): tx = {'key': key, 'value': value} qq = insert(tables.transactions).values(tx) db.execute(qq) assert block_num > 0, "Block num must be greater than 0! Was it not set in the DB() context session?" # Notify delegates of new block self.log.info( "Masternode sending NewBlockNotification to delegates with new block hash {} and block num {}" .format(hash_of_nodes, block_num)) notif = NewBlockNotification.create(new_block_hash=hash_of_nodes.hex(), new_block_num=block_num) self.parent.composer.send_pub_msg( filter=Constants.ZmqFilters.MasternodeDelegate, message=notif)
def enter_from_interpret(self): assert self.parent.interpreter.queue_size > 0, "Entered consensus state, but interpreter queue is empty!" assert self.parent.interpreter.queue_size == BLOCK_SIZE, \ "Consensus state entered with {} transactions in queue, but the BLOCK_SIZE is {}!"\ .format(self.parent.interpreter.queue_size, BLOCK_SIZE) # Merkle-ize transaction queue and create signed merkle hash all_tx = self.parent.interpreter.queue_binary self.merkle = MerkleTree.from_raw_transactions(all_tx) self.signature = wallet.sign(self.parent.signing_key, self.merkle.root) self.log.info("Delegate entering consensus state with merkle hash {}, and latest block hash {}" .format(self.merkle.root_as_hex, self.parent.current_hash)) self.log.spam("Delegate got merkle leaves {}".format(self.merkle.leaves_as_hex)) # Create merkle signature message and publish it merkle_sig = MerkleSignature.create(sig_hex=self.signature, timestamp='now', sender=self.parent.verifying_key) self.log.debugv("Broadcasting signature {}".format(self.signature)) self.parent.composer.send_pub_msg(filter=DELEGATE_DELEGATE_FILTER, message=merkle_sig) # Now that we've computed/composed the merkle tree hash, validate all our pending signatures for sig in [s for s in self.parent.pending_sigs if self.validate_sig(s)]: self.signatures.append(sig) # Add our own signature self.signatures.append(merkle_sig) self.check_majority()
def enter_from_interpret(self): assert self.parent.interpreter.queue_size > 0, "Entered consensus state, but interpreter queue is empty!" # Merkle-ize transaction queue and create signed merkle hash all_tx = self.parent.interpreter.queue_binary self.log.debugv( "Delegate got tx from interpreter queue: {}".format(all_tx)) self.merkle = MerkleTree.from_raw_transactions(all_tx) self.log.debugv("Delegate got merkle hash {}".format( self.merkle.root_as_hex)) self.signature = wallet.sign(self.parent.signing_key, self.merkle.root) # Create merkle signature message and publish it merkle_sig = MerkleSignature.create(sig_hex=self.signature, timestamp='now', sender=self.parent.verifying_key) self.log.debugv("Broadcasting signature {}".format(self.signature)) self.parent.composer.send_pub_msg(filter=delegate_delegate, message=merkle_sig) # Now that we've computed/composed the merkle tree hash, validate all our pending signatures for sig in [ s for s in self.parent.pending_sigs if self.validate_sig(s) ]: self.signatures.append(sig) self.check_majority()
def test_data_for_hash_doesnt_exist(self): leaves = [b'1', b'2', b'3'] hashed_leaves = list(map(Hasher.hash, leaves)) m = MerkleTree.from_raw_transactions(leaves) self.assertRaises(AssertionError, m.data_for_hash, '0' * 64)
def test_tx_request_replies_with_correct_data(self): """ Tests that a delegate who receives a TransactionRequest in Consensus state replies with the correct data """ mock_sm = MagicMock() state = DelegateConsensusState(mock_sm) # Build a merkle tree and attach it to the state tx_objects = [build_test_transaction() for _ in range(5)] tx_blobs = [tx.serialize() for tx in tx_objects] tx_hashes = [Hasher.hash(blob) for blob in tx_blobs] merkle_tree = MerkleTree.from_raw_transactions(tx_blobs) state.merkle = merkle_tree # Build a TransactionRequest for a few of these transactions requested_hashes = tx_hashes[1:4] expected_blobs = tx_blobs[1:4] request = TransactionRequest.create( transaction_hashes=requested_hashes) # Finally, ensure the reply is what we expect it to be reply = state.handle_tx_request(request) self.assertEquals(reply.raw_transactions, expected_blobs)
def validate_block_data(cls, block_data: dict): """ Validates the block_data dictionary. 'block_data' should be a strict subset of the 'block' dictionary, keys for all columns in the block table EXCEPT 'number' and 'hash'. If any validation fails, an exception is raised. For a block_data dictionary to be valid, it must: - Have a key for each block data column specified in BLOCK_DATA_COLS (at top of blocks.py) - BlockContender successfully validates with the Merkle root (meaning all signatures in the BlockContender can be verified using the Merkle root as the message) - Merkle leaves contained in BlockContender (block_contender.nodes) match Merkle leaves in block_data dict - Merkle root is correct root if a Merkle tree is built from Merkle leaves - Masternode signature is valid (signature is valid using Merkle root as message and masternode_vk as vk) :param block_data: The dictionary containing a key for each column in BLOCK_DATA_COLS (ie 'merkle_root', 'prev_block_hash', .. ect) :raises: An BlockStorageValidationException (or subclass) if any validation fails """ # Check block_data has all the necessary keys expected_keys = set(BLOCK_DATA_COLS.keys()) actual_keys = set(block_data.keys()) missing_keys = expected_keys - actual_keys extra_keys = actual_keys - expected_keys # Check for missing keys if len(missing_keys) > 0: raise BlockStorageValidationException("block_data keys {} missing key(s) {}".format(actual_keys, missing_keys)) # Check for extra (unrecognized) keys if len(extra_keys) > 0: raise BlockStorageValidationException("block_data keys {} has unrecognized keys {}".format(actual_keys, extra_keys)) # Validate Merkle Tree tree = MerkleTree.from_leaves_hex_str(block_data['merkle_leaves']) if tree.root_as_hex != block_data['merkle_root']: raise InvalidMerkleTreeException("Merkle Tree could not be validated for block_data {}".format(block_data)) # Validate BlockContender nodes match merkle leaves block_leaves = block_data['block_contender'].merkle_leaves if len(block_leaves) != len(tree.leaves): raise InvalidBlockContenderException("Number of Merkle leaves on BlockContender {} does not match number of" " leaves in MerkleTree {}".format(len(block_leaves), len(tree.leaves))) for block_leaf, merkle_leaf in zip(block_leaves, tree.leaves_as_hex): if block_leaf != merkle_leaf: raise InvalidBlockContenderException("BlockContender leaves do not match Merkle leaves\nblock leaves = " "{}\nmerkle leaves = {}".format(block_leaves, tree.leaves_as_hex)) # Validate MerkleSignatures inside BlockContender match Merkle leaves from raw transactions bc = block_data['block_contender'] if not bc.validate_signatures(): raise InvalidBlockContenderException("BlockContender signatures could not be validated! BC = {}".format(bc)) # TODO validate MerkleSignatures are infact signed by valid delegates # this is tricky b/c we would need to know who the delegates were at the time of the block, not necessarily the # current delegates # Validate Masternode Signature if not is_valid_hex(block_data['masternode_vk'], length=64): raise InvalidBlockSignatureException("Invalid verifying key for field masternode_vk: {}" .format(block_data['masternode_vk'])) if not wallet.verify(block_data['masternode_vk'], bytes.fromhex(block_data['merkle_root']), block_data['masternode_signature']): raise InvalidBlockSignatureException("Could not validate Masternode's signature on block data")
def validate_block_contender(self, block: BlockContender) -> bool: """ Helper method to validate a block contender. For a block contender to be valid it must: 1) Have a provable merkle tree, ie. all nodes must be hash of (left child + right child) 2) Be signed by at least 2/3 of the top 32 delegates :param block_contender: The BlockContender to validate :return: True if the BlockContender is valid, false otherwise """ def _validate_sigs(signatures, msg) -> bool: for sig in signatures: self.log.info("mn verifying signature: {}".format(sig)) if not sig.verify(msg, sig.sender): self.log.error( "!!!! Oh no why couldnt we verify sig {}???".format( sig)) return False return True # Development sanity checks (these should be removed in production) assert len( block.nodes ) >= 1, "Masternode got block contender with no nodes! {}".format( block) assert len(block.signatures) >= Constants.Testnet.Majority, \ "Received a block contender with only {} signatures (which is less than a majority of {}"\ .format(len(block.signatures), Constants.Testnet.Majority) # TODO -- ensure that this block contender's previous block is this Masternode's current block... # Prove Merkle Tree hash_of_nodes = MerkleTree.hash_nodes(block.nodes) tx_hashes = block.nodes[len(block.nodes) // 2:] if not MerkleTree.verify_tree(tx_hashes, hash_of_nodes): self.log.error( "\n\n\n\nCOULD NOT VERIFY MERKLE TREE FOR BLOCK CONTENDER {}\n\n\n" .format(block)) return False # Validate signatures if not _validate_sigs(block.signatures, hash_of_nodes): self.log.error( "MN COULD NOT VALIDATE SIGNATURES FOR CONTENDER {}".format( block)) return False return True
def test_data_for_hash(self): leaves = [b'1', b'2', b'3'] hashed_leaves = list(map(Hasher.hash, leaves)) m = MerkleTree.from_raw_transactions(leaves) for i in range(len(leaves)): self.assertEquals(m.data_for_hash(hashed_leaves[i]), leaves[i])
def test_creation(self): """ Tests that a created block data reply has the expected properties """ tx_binary = b'some random binary' bdr = BlockDataReply.create(tx_binary) self.assertEqual(tx_binary, bdr.raw_tx) self.assertEqual(bdr.tx_hash, MerkleTree.hash(tx_binary))
def build_test_contender(tree: MerkleTree=None): """ Method to build a 'test' block contender. Used exclusively in unit tests. """ if not tree: nodes = [1, 2, 3, 4] tree = MerkleTree(leaves=nodes) sigs = [build_test_merkle_sig(msg=tree.root) for _ in range(8)] return BlockContender.create(signatures=sigs, merkle_leaves=tree.leaves_as_hex)
def _validate_sigs(self, block: BlockContender) -> bool: signatures = block.signatures msg = MerkleTree.hash_nodes(block.merkle_leaves) for sig in signatures: # TODO -- ensure that the sender belongs to the top delegate pool self.log.debug("mn verifying signature: {}".format(sig)) if not sig.verify(msg, sig.sender): self.log.error("Masternode could not verify signature!!! Sig={}".format(sig)) return False return True
def validate_signatures(self): """ Validates the signatures in the block contender. Returns true if all signatures are valid, and false otherwise :return: True if the signatures are valid; False otherwise """ tree = MerkleTree.from_hex_leaves(self.merkle_leaves) for sig in self.signatures: if not sig.verify(tree.root): return False return True
def test_make_merkle_works(self): leaves = [1, 2, 3, 4] test_merkle = [ None, None, None, MerkleTree.hash(1), MerkleTree.hash(2), MerkleTree.hash(3), MerkleTree.hash(4) ] test_merkle[2] = MerkleTree.hash(test_merkle[-2] + test_merkle[-1]) test_merkle[1] = MerkleTree.hash(test_merkle[3] + test_merkle[4]) test_merkle[0] = MerkleTree.hash(test_merkle[1] + test_merkle[2]) m = MerkleTree(leaves) self.assertEqual(test_merkle, m.nodes)
def test_make_merkle_works(self): # create a merkle tree that should be leaves = [1, 2, 3, 4] test_merkle = [None, None, None, MerkleTree.hash(bytes(1)), MerkleTree.hash(bytes(2)), MerkleTree.hash(bytes(3)), MerkleTree.hash(bytes(4))] test_merkle[2] = MerkleTree.hash(test_merkle[-2] + test_merkle[-1]) test_merkle[1] = MerkleTree.hash(test_merkle[3] + test_merkle[4]) test_merkle[0] = MerkleTree.hash(test_merkle[1] + test_merkle[2]) m = MerkleTree(leaves) self.assertEqual(test_merkle, m.nodes)
def test_root_as_hex(self): leaves = [1, 2, 3, 4] test_merkle = [ None, None, None, MerkleTree.hash(1), MerkleTree.hash(2), MerkleTree.hash(3), MerkleTree.hash(4) ] test_merkle[2] = MerkleTree.hash(test_merkle[-2] + test_merkle[-1]) test_merkle[1] = MerkleTree.hash(test_merkle[3] + test_merkle[4]) test_merkle[0] = MerkleTree.hash(test_merkle[1] + test_merkle[2]) m = MerkleTree(leaves) self.assertEqual(m.root, test_merkle[0]) self.assertEqual(m.root_as_hex, test_merkle[0].hex())
def test_eq(self): nodes = [secrets.token_bytes(8) for _ in range(4)] tree = MerkleTree.from_raw_transactions(nodes) msg = tree.root sig1, sk1, vk1 = self._create_merkle_sig(msg) sig2, sk2, vk2 = self._create_merkle_sig(msg) sig3, sk3, vk3 = self._create_merkle_sig(msg) sig4, sk4, vk4 = self._create_merkle_sig(msg) signatures = [sig1, sig2, sig3, sig4] bc1 = BlockContender.create(signatures, merkle_leaves=tree.leaves_as_hex, prev_block_hash='A' * 64) bc2 = BlockContender.create(signatures, merkle_leaves=tree.leaves_as_hex, prev_block_hash='A' * 64) self.assertEquals(bc1, bc2)
def build_test_contender(tree: MerkleTree = None, prev_block_hash=''): """ Method to build a 'test' block contender. Used exclusively in unit tests. """ from cilantro.storage.blocks import BlockStorageDriver from cilantro.constants.nodes import BLOCK_SIZE if not tree: nodes = [str(i).encode() for i in range(BLOCK_SIZE)] tree = MerkleTree(leaves=nodes) if not prev_block_hash: prev_block_hash = BlockStorageDriver.get_latest_block_hash() sigs = [build_test_merkle_sig(msg=tree.root) for _ in range(8)] return BlockContender.create(signatures=sigs, merkle_leaves=tree.leaves_as_hex, prev_block_hash=prev_block_hash)
def test_validate_signatures(self): nodes = [secrets.token_bytes(8) for _ in range(4)] tree = MerkleTree.from_raw_transactions(nodes) msg = tree.root sig1, sk1, vk1 = self._create_merkle_sig(msg) sig2, sk2, vk2 = self._create_merkle_sig(msg) sig3, sk3, vk3 = self._create_merkle_sig(msg) sig4, sk4, vk4 = self._create_merkle_sig(msg) signatures = [sig1, sig2, sig3, sig4] bc = BlockContender.create(signatures, merkle_leaves=tree.leaves_as_hex) is_valid = bc.validate_signatures() self.assertTrue(is_valid)
def test_validate_signatures_invalid(self): nodes = [secrets.token_bytes(8) for _ in range(4)] tree = MerkleTree.from_raw_transactions(nodes) msg = tree.root bad_msg = b'lol this is def not a merkle root' sig1, sk1, vk1 = self._create_merkle_sig(msg) sig2, sk2, vk2 = self._create_merkle_sig(msg) sig3, sk3, vk3 = self._create_merkle_sig(msg) sig4, sk4, vk4 = self._create_merkle_sig(bad_msg) signatures = [sig1, sig2, sig3, sig4] bc = BlockContender.create(signatures, merkle_leaves=tree.leaves_as_hex, prev_block_hash="A" * 64) is_valid = bc.validate_signatures() self.assertFalse(is_valid)
class Block: def __init__(self, txs, last_block): self.merkle_tree = MerkleTree(txs) self.last_block = last_block self.hash = self.merkle_tree.parent() def encode(self): return pickle.dumps([self.merkle_tree.raw_leaves, self.last_block]) @classmethod def decode(cls, b): block = pickle.loads(b) return Block(block[0], block[1]) def __eq__(self, other): if self.hash != other.hash: return False elif self.merkle_tree.nodes != other.merkle_tree.merkle_leaves: return False elif self.last_block != other.last_block: return False return True
async def _interpret_next_subtree(self, num_of_batches = 1): self.log.debug("Starting to make a new sub-block {} for block {}" .format(self.sub_block_num, self.block_num)) # get next batch of txns ?? still need to decide whether to unpack a bag or check for end of txn batch while(num_of_batches > 0): txn = self.pending_txs.popleft() if txn == end_of_batch: num_of_batches = num_of_batches - 1 else: self.interpreter.interpret(txn) # this is a blocking call. either async or threads?? if not self._interpret: # do we need abort?? self.interpreter.flush(update_state=False) return; # Merkle-ize transaction queue and create signed merkle hash all_tx = self.interpreter.queue_binary self.merkle = MerkleTree.from_raw_transactions(all_tx) self.signature = wallet.sign(self.signing_key, self.merkle.root) # Create merkle signature message and publish it merkle_sig = MerkleSignature.create(sig_hex=self.signature, timestamp='now', sender=self.verifying_key) self.send_signature(merkle_sig) # send signature to block manager