def _build_valid_block_data(self, num_transactions=4) -> dict:
"""
Utility method to build a dictionary with all the params needed to invoke store_block
:param num_transactions:
:return:
"""
mn_sk = Constants.Testnet.Masternodes[0]['sk']
mn_vk = ED25519Wallet.get_vk(mn_sk)
timestamp = 9000
raw_transactions = [build_test_transaction().serialize() for _ in range(num_transactions)]
tree = MerkleTree(raw_transactions)
merkle_leaves = tree.leaves_as_concat_hex_str
merkle_root = tree.root_as_hex
bc = build_test_contender(tree=tree)
prev_block_hash = '0' * 64
mn_sig = ED25519Wallet.sign(mn_sk, tree.root)
return {
'prev_block_hash': prev_block_hash,
'block_contender': bc,
'merkle_leaves': merkle_leaves,
'merkle_root': merkle_root,
'masternode_signature': mn_sig,
'masternode_vk': mn_vk,
'timestamp': timestamp
}
def test_valid_sender(self):
# Confirm no error when correct public key is used
msg = b'this is a pretend merkle tree hash'
sk, vk = ED25519Wallet.new()
signature = ED25519Wallet.sign(sk, msg)
timestamp = 'now'
MerkleSignature.create(sig_hex=signature,
timestamp=timestamp,
sender=vk) # no error thrown
def _convenience_build_standard_transaction(self):
"""These transactions get POSTed directly to masternodes by TAU wallet software"""
STU = (ED25519Wallet.new())
DAVIS = (ED25519Wallet.new())
DENTON = (ED25519Wallet.new())
FALCON = (ED25519Wallet.new())
KNOWN_ADRS = (STU, DAVIS, DENTON, FALCON)
amount = 10
tx = StandardTransactionBuilder.create_tx(STU[0], STU[1], DAVIS[1], amount)
return tx
def test_deserialization_valid_json(self):
# Test valid json throws no errors
msg = b'this is a pretend merkle tree hash'
sk, vk = ED25519Wallet.new()
signature = ED25519Wallet.sign(sk, msg)
d = {
MerkleSignature.SIG: signature,
MerkleSignature.TS: 'now',
MerkleSignature.SENDER: vk
}
binary = json.dumps(d).encode()
MerkleSignature.from_bytes(binary)
def test_verify_wrong_ms(self):
# Test merkle tree validation returns false for incorrect verifying (public) key
msg = b'this is a pretend merkle tree hash'
timestamp = 'now'
sk, vk = ED25519Wallet.new()
signature = ED25519Wallet.sign(sk, msg)
ms = MerkleSignature.create(sig_hex=signature,
timestamp=timestamp,
sender=vk)
sk1, vk1 = ED25519Wallet.new()
self.assertFalse(ms.verify(msg, vk1))
def test_verify_valid_ms(self):
"""
Tests that MerkleSignature.verify(...) returns true given a proper msg and vk
"""
# Test merkle tree verify() validates correct verifying (public) keys
msg = b'this is a pretend merkle tree hash'
timestamp = 'now'
sk, vk = ED25519Wallet.new()
signature = ED25519Wallet.sign(sk, msg)
ms = MerkleSignature.create(sig_hex=signature,
timestamp=timestamp,
sender=vk)
self.assertTrue(ms.verify(msg))
def _create_merkle_sig(self, msg: bytes):
"""
Helper method to create a MerkleSignature and wallet keys
:return: A tuple container (MerkleSignature, signing_key, verifying_key)
"""
assert type(msg) == bytes, "Must pass in bytes"
sk, vk = ED25519Wallet.new()
signature = ED25519Wallet.sign(sk, msg)
ms = MerkleSignature.create(sig_hex=signature,
timestamp=TIMESTAMP,
sender=vk)
return ms, sk, vk
def test_invalid_timestamp(self):
"""
Test that if the timestamp field is not formatted as expected an error will be thrown
"""
msg = b'this is a pretend merkle tree hash'
sk, vk = ED25519Wallet.new()
signature = ED25519Wallet.sign(sk, msg)
timestamp = 99
self.assertRaises(
TypeError,
MerkleSignature.create(sig_hex=signature,
timestamp=timestamp,
sender=vk))
def test_valid_creation(self):
"""
Tests that a MerkleSignature created with some argument has the expected properties
"""
msg = b'this is a pretend merkle tree hash'
timestamp = 'now'
sk, vk = ED25519Wallet.new()
signature = ED25519Wallet.sign(sk, msg)
ms = MerkleSignature.create(sig_hex=signature,
timestamp=timestamp,
sender=vk)
self.assertEqual(ms.signature, signature)
self.assertEqual(ms.timestamp, timestamp)
self.assertEqual(ms.sender, vk)
def test_invalid_sender_bad_hash(self):
# Test an error is thrown when created with a sender of not valid hash
msg = b'this is a pretend merkle tree hash'
sk, vk = ED25519Wallet.new()
signature = ED25519Wallet.sign(sk, msg)
timestamp = 'now'
vk_bad_hash = ''.join(
'Z' for _ in range(64)) # verifying (public) key with bad hash
self.assertRaises(
Exception,
MerkleSignature.create(sig_hex=signature,
timestamp=timestamp,
sender=vk_bad_hash,
validate=False))
def test_invalid_sender_wrong_sender(self):
"""
Tests that an error is raised during creation if an invalid sender field is passed in. A sender should be a
64 character hex string verifying key.
"""
# Test an error is thrown when MerkleSignature created with a sender that is not the correct public key
msg = b'this is a pretend merkle tree hash'
sk, vk = ED25519Wallet.new()
signature = ED25519Wallet.sign(sk, msg)
timestamp = 'now'
vk_bad = ED25519Wallet.new()[1] # different verifying (public) key
bad_public_key = MerkleSignature.create(sig_hex=signature,
timestamp=timestamp,
sender=vk_bad)
self.assertRaises(Exception, bad_public_key)
def enter_any(self, prev_state):
assert self.parent.interpreter.queue_size > 0, "Entered consensus state, but interpreter queue is empty!"
# Merkle-ize transaction queue and create signed merkle hash
all_tx = self.parent.interpreter.queue_binary
self.log.info(
"Delegate got tx from interpreter queue: {}".format(all_tx))
self.merkle = MerkleTree(all_tx)
self.merkle_hash = self.merkle.hash_of_nodes()
self.log.info("Delegate got merkle hash {}".format(self.merkle_hash))
self.signature = ED25519Wallet.sign(self.parent.signing_key,
self.merkle_hash)
# Create merkle signature message and publish it
merkle_sig = MerkleSignature.create(sig_hex=self.signature,
timestamp='now',
sender=self.parent.verifying_key)
self.log.info("Broadcasting signature {}".format(self.signature))
self.parent.composer.send_pub_msg(
filter=Constants.ZmqFilters.DelegateDelegate, message=merkle_sig)
# Now that we've computed/composed the merkle tree hash, validate all our pending signatures
for sig in [
s for s in self.parent.pending_sigs if self.validate_sig(s)
]:
self.signatures.append(sig)
self.check_majority()
def random_envelope():
sk, vk = ED25519Wallet.new()
tx = StandardTransactionBuilder.random_tx()
sender = 'me'
return Envelope.create_from_message(message=tx,
signing_key=sk,
sender_id=sender)
def build_test_merkle_sig(msg: bytes = b'some default payload',
sk=None,
vk=None) -> MerkleSignature:
"""
Builds a 'test' merkle signature. Used exclusively for unit tests
:return:
"""
import time
if not sk:
sk, vk = ED25519Wallet.new()
signature = ED25519Wallet.sign(sk, msg)
return MerkleSignature.create(sig_hex=signature,
timestamp=str(time.time()),
sender=vk)
def test_invalid_signature_bad_length(self):
# Test signature incorrect length (but valid hex)
sk, vk = ED25519Wallet.new()
timestamp = 'now'
sig = ''.join(('A' for _ in range(100)))
wrong_len = MerkleSignature.create(sig_hex=sig,
timestamp=timestamp,
sender=vk,
validate=False)
self.assertRaises(Exception, wrong_len.validate)
def __init__(self, *args, **kwargs):
self.log = get_logger("BaseNode")
self.log.info("-- BaseNode Initiating --")
self.port = int(os.getenv('PORT', 31337))
self.host = os.getenv('HOST_IP', '127.0.0.1')
self.loop = asyncio.get_event_loop()
self.wallet = ED25519Wallet()
self.router = Router(statemachine=self)
self.reactor = ReactorInterface(self.router, self.loop, self.wallet.s)
self.composer = Composer(self.reactor, self.wallet.s)
def test_validate_catches_bad_sig(self):
# test that validate is called by default and throws an exception with bad sig
sk, vk = ED25519Wallet.new()
sig = ''.join(('X' for _ in range(128)))
timestamp = 'now'
self.assertRaises(Exception,
MerkleSignature.create,
sig_hex=sig,
timestamp=timestamp,
sender=vk)
def gather_consensus(self):
self.log.debug("Starting consesnsus, with peers: {}"
.format(['{}:{}'.format(d['url'], d['port']) for d in self.delegates]))
# if str(self.port)[-1] == '0':
# self.log.debug("Exiting consensus as I am not the chosen one")
# return
# Get all tx
tx = self.queue.dequeue_all()
# Merkleize them and sign
self.merkle = MerkleTree(tx)
self.signature = ED25519Wallet.sign(self.signing_key, self.merkle.hash_of_nodes())
self.signature = self.signature.encode()
self.log.debug('signature is {}'.format(self.signature))
# Time to gather from others
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
def get_message(connection):
# DEBUG
# if connection[-1] != '0':
# self.log.debug("Ignoring poke attempt, as he was not the chosen one")
# return
# END DEBUG
context = zmq.Context()
request_socket = context.socket(socket_type=zmq.REQ)
request_socket.connect(connection)
self.log.warning("Poking url: {}".format(connection))
poke = Poke.create()
poke_msg = Message.create(Poke, poke.serialize())
request_socket.send(poke_msg.serialize())
self.log.warning("Waiting for poke reply...")
msg = request_socket.recv()
self.log.critical("Got reply from the poked delegate: {}".format(msg))
connections = ['{}:{}'.format(d['url'], d['port']) for d in self.delegates]
tasks = [loop.run_in_executor(None, get_message, connection) for connection in connections]
loop.run_until_complete(asyncio.wait(tasks))
self.log.critical("\nClosing event loop!\n")
loop.close()
def test_serialization(self):
"""
Tests that a created block data reply successfully serializes and deserializes. The deserialized object should
have the same properties as the original one before it was serialized.
"""
msg = b'this is a pretend merkle tree hash'
sk, vk = ED25519Wallet.new()
signature = ED25519Wallet.sign(sk, msg)
timestamp = 'now'
valid_merkle_sig = MerkleSignature.create(sig_hex=signature,
timestamp=timestamp,
sender=vk)
valid_merkle_sig_serialized = valid_merkle_sig.serialize()
clone = MerkleSignature.from_bytes(valid_merkle_sig_serialized)
self.assertEqual(valid_merkle_sig.signature, clone.signature)
self.assertEqual(valid_merkle_sig.timestamp, clone.timestamp)
self.assertEqual(valid_merkle_sig.sender, clone.sender)
def test_create_with_envelope(self):
"""
Tests creating a message with an envelope produces an object with the expected properties
"""
sk, vk = ED25519Wallet.new()
tx = StandardTransactionBuilder.random_tx()
sender = 'me'
env = Envelope.create_from_message(message=tx, signing_key=sk)
cmd = ReactorCommand.create_cmd('some_cls', 'some_func', envelope=env)
self.assertTrue(ReactorCommand.envelope, env)
def test_deserialization_bad_sk_hex(self):
# Test valid json but signature (private key) not proper hex
msg = b'this is a pretend merkle tree hash'
sk, vk = ED25519Wallet.new()
sig = ''.join(('Z' for _ in range(128)))
d = {
MerkleSignature.SIG: sig,
MerkleSignature.TS: 'now',
MerkleSignature.SENDER: vk
}
binary = json.dumps(d).encode()
self.assertRaises(Exception, MerkleSignature.from_bytes, binary)
def print_stuff():
pprint(connection_list)
print('\n===MERKLE TREE HASHED===')
h = hashlib.sha3_256()
[h.update(mm) for mm in m.nodes]
print(h.digest().hex())
print('\n===ENTIRE MERKLE TREE===')
[print(mm.hex()) for mm in m.nodes]
print('\n===SIGNATURE OF MERKLE HASH===')
[print(ED25519Wallet.sign(k[0], h.digest())) for k in delegates]
def run_mgmt():
from cilantro.logger import get_logger
from cilantro import Constants
from cilantro.db import DB, DB_NAME
from cilantro.utils.test import MPComposer
from cilantro.protocol.wallets import ED25519Wallet
import os, time, asyncio
log = get_logger(__name__)
sk = Constants.Testnet.Masternodes[0]['sk']
vk = Constants.Protocol.Wallets.get_vk(sk)
s,v = ED25519Wallet.new()
mpc = MPComposer(name='mgmt', sk=s)
mpc.add_sub(filter='a', vk=vk)
def test_invalid_signature_nonhex(self):
"""
Tests that an error is raised during validation if an invalid signature is passed (nonhex, or length != 128)
"""
msg = b'this is a pretend merkle tree hash'
timestamp = 'now'
sk, vk = ED25519Wallet.new()
# Test nonhex signature (but valid length)
sig = ''.join(('X' for _ in range(128)))
nonhex = MerkleSignature.create(sig_hex=sig,
timestamp=timestamp,
sender=vk,
validate=False)
self.assertRaises(Exception, nonhex.validate)
def run_mgmt():
from cilantro.logger import get_logger
from cilantro import Constants
from cilantro.db import DB, DB_NAME
from cilantro.utils.test import MPComposer
from cilantro.protocol.wallets import ED25519Wallet
import os, time, asyncio
log = get_logger("MANAGEMENT NODE")
sk = Constants.Testnet.Masternode.Sk
vk = Constants.Protocol.Wallets.get_vk(sk)
s, v = ED25519Wallet.new()
mpc = MPComposer(name='mgmt', sk=s)
log.critical("trying to look at vk: {}".format(vk))
mpc.add_sub(filter='a', url='tcp://{}:33333'.format(vk))
def create_std_tx(sender: tuple, recipient: tuple, amount: float):
"""
Utility method to create signed transaction
:param sender: A tuple containing the (signing_key, verifying_key) of the sender
:param recipient: A tuple containing the (signing_key, verifying_key) of the recipient
:param amount: The amount to send
:return:
"""
# TestNetTransaction.TX, sender, to, amount
tx = {
'payload': ('t', sender[1], recipient[1], str(amount)),
'metadata': {}
}
tx["metadata"]["proof"] = SHA3POW.find(
JSONSerializer.serialize(tx["payload"]))[0]
# tx["metadata"]["signature"] = ED25519Wallet.sign(sender[0], json.dumps(tx["payload"]).encode())
tx["metadata"]["signature"] = ED25519Wallet.sign(
sender[0], JSONSerializer.serialize(tx['payload']))
return tx
def verify_signature(future):
signature, verifying_key = future.result()
if ED25519Wallet.verify(verifying_key, self.merkle, signature):
self.signatures.append(future.result())
def _validate_block_data(cls, block_data: dict):
"""
Validates the block_data dictionary. 'block_data' should be a strict subset of the 'block' dictionary, keys for all
columns in the block table EXCEPT 'number' and 'hash'. If any validation fails, an exception is raised.
For a block_data dictionary to be valid, it must:
- Have a key for each block data column specified in BLOCK_DATA_COLS (at top of blocks.py)
- BlockContender successfully validates with the Merkle root (meaning all signatures in the BlockContender
can be verified using the Merkle root as the message)
- Merkle leaves contained in BlockContender (block_contender.nodes) match Merkle leaves in block_data dict
- Merkle root is correct root if a Merkle tree is built from Merkle leaves
- Masternode signature is valid (signature is valid using Merkle root as message and masternode_vk as vk)
:param block_data: The dictionary containing a key for each column in BLOCK_DATA_COLS
(ie 'merkle_root', 'prev_block_hash', .. ect)
:raises: An BlockStorageValidationException (or subclass) if any validation fails
"""
# Check block_data has all the necessary keys
expected_keys = set(BLOCK_DATA_COLS.keys())
actual_keys = set(block_data.keys())
missing_keys = expected_keys - actual_keys
extra_keys = actual_keys - expected_keys
# Check for missing keys
if len(missing_keys) > 0:
raise BlockStorageValidationException(
"block_data keys {} missing key(s) {}".format(
actual_keys, missing_keys))
# Check for extra (unrecognized) keys
if len(extra_keys) > 0:
raise BlockStorageValidationException(
"block_data keys {} has unrecognized keys {}".format(
actual_keys, extra_keys))
# Validate Merkle Tree
tree = MerkleTree.from_leaves_hex_str(block_data['merkle_leaves'])
if tree.root_as_hex != block_data['merkle_root']:
raise InvalidMerkleTreeException(
"Merkle Tree could not be validated for block_data {}".format(
block_data))
# Validate BlockContender nodes match merkle leaves
block_leaves = block_data['block_contender'].merkle_leaves
if len(block_leaves) != len(tree.leaves):
raise InvalidBlockContenderException(
"Number of Merkle leaves on BlockContender {} does not match number of"
" leaves in MerkleTree {}".format(len(block_leaves),
len(tree.leaves)))
for block_leaf, merkle_leaf in zip(block_leaves, tree.leaves_as_hex):
if block_leaf != merkle_leaf:
raise InvalidBlockContenderException(
"BlockContender leaves do not match Merkle leaves\nblock leaves = "
"{}\nmerkle leaves = {}".format(block_leaves,
tree.leaves_as_hex))
# Validate MerkleSignatures inside BlockContender match Merkle leaves from raw transactions
bc = block_data['block_contender']
if not bc.validate_signatures():
raise InvalidBlockContenderException(
"BlockContender signatures could not be validated! BC = {}".
format(bc))
# TODO validate MerkleSignatures are infact signed by valid delegates
# this is tricky b/c we would need to know who the delegates were at the time of the block, not necessarily the
# current delegates
# Validate Masternode Signature
if not is_valid_hex(block_data['masternode_vk'], length=64):
raise InvalidBlockSignatureException(
"Invalid verifying key for field masternode_vk: {}".format(
block_data['masternode_vk']))
if not ED25519Wallet.verify(block_data['masternode_vk'],
bytes.fromhex(block_data['merkle_root']),
block_data['masternode_signature']):
raise InvalidBlockSignatureException(
"Could not validate Masternode's signature on block data")
def store_block(cls,
block_contender: BlockContender,
raw_transactions: List[bytes],
publisher_sk: str,
timestamp: int = 0):
"""
Persist a new block to the blockchain, along with the raw transactions associated with the block. An exception
will be raised if an error occurs either validating the new block data, or storing the block. Thus, it is
recommended that this method is wrapped in a try block.
:param block_contender: A BlockContender instance
:param raw_transactions: A list of ordered raw transactions contained in the block
:param publisher_sk: The signing key of the publisher (a Masternode) who is publishing the block
:param timestamp: The time the block was published, in unix epoch time. If 0, time.time() is used
:return: None
:raises: An assertion error if invalid args are passed into this function, or a BlockStorageValidationException
if validation fails on the attempted block
TODO -- think really hard and make sure that this is 'collision proof' (extremely unlikely, but still possible)
- could there be a hash collision in the Merkle tree nodes?
- hash collision in block hash space?
- hash collision in transaction space?
"""
assert isinstance(
block_contender, BlockContender
), "Expected block_contender arg to be BlockContender instance"
assert is_valid_hex(
publisher_sk,
64), "Invalid signing key {}. Expected 64 char hex str".format(
publisher_sk)
if not timestamp:
timestamp = int(time.time())
tree = MerkleTree.from_raw_transactions(raw_transactions)
publisher_vk = ED25519Wallet.get_vk(publisher_sk)
publisher_sig = ED25519Wallet.sign(publisher_sk, tree.root)
# Build and validate block_data
block_data = {
'block_contender': block_contender,
'timestamp': timestamp,
'merkle_root': tree.root_as_hex,
'merkle_leaves': tree.leaves_as_concat_hex_str,
'prev_block_hash': cls._get_latest_block_hash(),
'masternode_signature': publisher_sig,
'masternode_vk': publisher_vk,
}
cls._validate_block_data(block_data)
# Compute block hash
block_hash = cls._compute_block_hash(block_data)
# Encode block data for serialization and finally persist the data
log.info(
"Attempting to persist new block with hash {}".format(block_hash))
block_data = cls._encode_block(block_data)
with DB() as db:
# Store block
res = db.tables.blocks.insert([{
'hash': block_hash,
**block_data
}]).run(db.ex)
if res:
log.info(
"Successfully inserted new block with number {} and hash {}"
.format(res['last_row_id'], block_hash))
else:
log.error(
"Error inserting block! Got None/False result back from insert query. Result={}"
.format(res))
return
# Store raw transactions
log.info(
"Attempting to store {} raw transactions associated with block hash {}"
.format(len(raw_transactions), block_hash))
tx_rows = [{
'hash': Hasher.hash(raw_tx),
'data': encode_tx(raw_tx),
'block_hash': block_hash
} for raw_tx in raw_transactions]
res = db.tables.transactions.insert(tx_rows).run(db.ex)
if res:
log.info("Successfully inserted {} transactions".format(
res['row_count']))
else:
log.error(
"Error inserting raw transactions! Got None from insert query. Result={}"
.format(res))
from cilantro.protocol.proofs import POW
from cilantro.protocol.interpreters import TestNetInterpreter
from cilantro.protocol.wallets import ED25519Wallet
'''
Steps:
1. create a ton of wallets
2. create a ton of random transaction for those wallets
3. interpret those transaction and verify them
4. put the db output stack in a list
5. figure it out from there.
'''
import random
import redis
NUM_WALLETS = 100
wallets = [ED25519Wallet.new() for x in range(NUM_WALLETS)]
txs = []
interpreter = TestNetInterpreter(proof_system=POW)
setup_queries = {}
def generate_random_std_transaction():
from_wallet, to_wallet = random.sample(wallets, 2)
amount = str(random.randint(1, 1000))
tx = TestNetTransaction.standard_tx(from_wallet[1], to_wallet[1], amount)
transaction_builder = TestNetTransaction(ED25519Wallet, POW)
