def test_compute_block_hash(self):
# NOTE -- this implicitly tests Hasher.hash_iterable
bd = self._build_valid_block_data()
# Correct order of keys should be:
# ['block_contender',
# 'masternode_signature',
# 'masternode_vk',
# 'merkle_leaves',
# 'merkle_root',
# 'prev_block_hash',
# 'timestamp']
binary_data = bd['block_contender'].serialize()
binary_data += bd['masternode_signature'].encode()
binary_data += bd['masternode_vk'].encode()
binary_data += bd['merkle_leaves'].encode()
binary_data += bd['merkle_root'].encode()
binary_data += bd['prev_block_hash'].encode()
binary_data += int_to_bytes(bd['timestamp'])
expected_hash = Hasher.hash(binary_data)
actual_hash = BlockStorageDriver._compute_block_hash(bd)
self.assertEqual(expected_hash, actual_hash)
def test_store_block_inserts_transactions(self):
num_txs = 4
with DB() as db:
initial_txs = len(db.tables.transactions.select().run(db.ex))
mn_sk = Constants.Testnet.Masternodes[0]['sk']
timestamp = random.randint(0, pow(2, 32))
raw_transactions = [build_test_transaction().serialize() for _ in range(num_txs)]
tree = MerkleTree(raw_transactions)
bc = build_test_contender(tree=tree)
BlockStorageDriver.store_block(block_contender=bc, raw_transactions=raw_transactions, publisher_sk=mn_sk, timestamp=timestamp)
block_hash = BlockStorageDriver._get_latest_block_hash()
with DB() as db:
transactions = db.tables.transactions
all_tx_query = transactions.select().run(db.ex)
# Ensure the correct number of transactions was inserted
self.assertEquals(len(all_tx_query) - initial_txs, num_txs)
# Ensure the transactions were correctly inserted
for raw_tx in raw_transactions:
tx_hash = Hasher.hash(raw_tx)
rows = transactions.select().where(transactions.hash == tx_hash).run(db.ex)
self.assertTrue(rows, "Expected there to be a row for inserted tx {}".format(raw_tx))
tx_row = rows[0]
self.assertEquals(tx_row['hash'], tx_hash, "Expected fetched tx to have hash equal to its hashed data")
self.assertEquals(tx_row['data'], encode_tx(raw_tx), "Expected tx data col to equal encoded raw tx")
self.assertEquals(tx_row['block_hash'], block_hash, "Expected inserted tx to reference last block")
def hash(o):
# If 'o' is a str, it is assumed to be a hex string, thus we cast it to bytes using bytes.fromhex(...)
# instead of the default str.encode(..) that the Hasher module uses
if type(o) is str:
o = bytes.fromhex(o)
return Hasher.hash(o, algorithm=Hasher.Alg.SHA3_256, return_bytes=True)
def handle_tx_request(self, reply: TransactionReply):
self.log.debug("Masternode got block data reply: {}".format(reply))
for tx in reply.raw_transactions:
tx_hash = Hasher.hash(tx)
if tx_hash in self.tx_hashes:
self.retrieved_txs[tx_hash] = tx
else:
self.log.error(
"Received block data reply with tx hash {} that is not in tx_hashes"
)
return
if len(self.retrieved_txs) == len(self.tx_hashes):
self.log.debug(
"Done collecting block data. Transitioning back to NewBlockState."
)
self.parent.transition(MNNewBlockState,
success=True,
retrieved_txs=self._get_ordered_raw_txs(),
pending_blocks=self.pending_blocks)
return
else:
self.log.debug(
"Still {} transactions yet to request until we can build the block"
.format(len(self.tx_hashes) - len(self.retrieved_txs)))
def node_create(cls, user_id: str, contract_code: str, block_hash: str):
contract_id = Hasher.hash(user_id + contract_code + block_hash)
data = {
cls.USER_ID: user_id,
cls.CONTRACT_CODE: contract_code,
cls.CONTRACT_ID: contract_id
}
return cls.from_data(data)
def reply_uuid(request_uuid: int):
"""
Returns the associated reply UUID for some request UUID. This is simply the SHA3 256 hash of the request's UUID.
:param request_uuid: The request UUID to generate a reply UUID for
:return: An int, denoting the reply UUID associated with the passed in request UUID
"""
int_binary = Hasher.hash(request_uuid, algorithm=Hasher.Alg.SHAKE_128, digest_len=UUID_SIZE//8, return_bytes=True)
rep_uuid = int.from_bytes(int_binary, byteorder='little') # capnp int fields encoded in little endian
# This assertion is for debugging only. Remove for production.
assert rep_uuid <= max_uuid, "OH NO! Got reply uuid greater than MaxUUID! LOGIC ERROR!!!"
return rep_uuid
def test_get_raw_transaction(self):
mn_sk = Constants.Testnet.Masternodes[0]['sk']
timestamp = random.randint(0, pow(2, 32))
raw_transactions = [build_test_transaction().serialize() for _ in range(4)]
tree = MerkleTree(raw_transactions)
bc = build_test_contender(tree=tree)
BlockStorageDriver.store_block(block_contender=bc, raw_transactions=raw_transactions, publisher_sk=mn_sk, timestamp=timestamp)
# Ensure all these transactions are retrievable
for raw_tx in raw_transactions:
retrieved_tx = BlockStorageDriver.get_raw_transaction(Hasher.hash(raw_tx))
self.assertEquals(raw_tx, retrieved_tx)
def _compute_block_hash(cls, block_data: dict) -> str:
"""
Computes the block's hash as a function of the block data. The process for computing the block hash follows:
- Binarize all block_data values
- Lexicographically sort block_data keys
- Concatenate binarized block_data values in this lexicographical order
- SHA3 hash this concatenated binary
:param block_data: The dictionary of containing a key for each column in BLOCK_DATA_COLS
(ie 'merkle_root', 'prev_block_hash', .. ect)
:return: The block's hash, as a 64 character hex string
"""
ordered_values = [block_data[key] for key in sorted(block_data.keys())]
return Hasher.hash_iterable(ordered_values)
def verify_tree(nodes: list, tree_hash: bytes):
"""
Attempts to verify merkle tree represented implicitly by the list 'nodes'. The tree is valid if it maintains
the invariant that the value of each non-leaf node is the hash of its left child's value concatenated with
its right child's value.
:param nodes: The nodes in the tree, represented implicitly as a list
:param tree_hash: The expected hash of the merkle tree formed from nodes (the 'hash of a merkle tree' is the
value returned by the .hash_of_nodes method on this class)
:return: True if the tree is valid; False otherwise
"""
nodes = MerkleTree.merklize(nodes, hash_leaves=False)
h = Hasher.hash_iterable(nodes,
algorithm=Hasher.Alg.SHA3_256,
return_bytes=True)
return h == tree_hash
def validate_matches_request(self, request: TransactionRequest) -> bool:
"""
Validates that this TransactionReply contains transactions, whose hashes match the hashes in TransactionRequest.
Returns True if this class contains every transaction for each transaction hash in the TransactionRequest
(none missing, and no extras). Otherwise, returns false
:param request:
:return: True if this object has a transaction for every hash in TransactionRequest. False otherwise.
"""
if len(self._data.transactions) != len(request._data.transactions):
return False
req_hashes = request.tx_hashes
for i, self_t in enumerate(self._data.transactions):
request_t = req_hashes[i]
if Hasher.hash(self_t) != request_t:
return False
return True
def __repr__(self):
"""
Printing the full capnp struct (which is the default MessageBase __repr__ behvaior) is way to verbose for
the logs. Here we just slim this guy down a little to make the logs easier to read
TODO -- the hashing bit should not be done in production as this wastes computational cycles
"""
msg_type = str(MessageBase.registry[self.meta.type])
msg_hash = Hasher.hash(data=self.message_binary, digest_len=3) # compressed representation of the message
seal_vk = self.seal.verifying_key
uuid = self.meta.uuid
repr = "\nEnvelope from sender {}".format(seal_vk)
repr += "\n\tuuid: {}".format(uuid)
repr += "\n\tmessage type: {}".format(msg_type)
repr += "\n\tmessage hash: {}".format(msg_hash)
return repr
def test_validate_matches_request_no_match(self):
"""
Tests that a created block data reply has the expected properties
"""
sk = wallet.new()[0]
code_strs = ['some random binary', 'some deterministic binary', 'While True: self.eatAss()']
contracts = [
ContractTransactionBuilder.create_contract_tx(sk, code_str) \
for code_str in code_strs
]
tx_binaries = [c.serialize() for c in contracts]
tx_hashes = [Hasher.hash(cs+b'bbb') for cs in tx_binaries]
bdr_req = TransactionRequest.create(tx_hashes)
bdr_rep = TransactionReply.create(tx_binaries)
assert not bdr_rep.validate_matches_request(bdr_req)
def store_block(cls,
block_contender: BlockContender,
raw_transactions: List[bytes],
publisher_sk: str,
timestamp: int = 0):
"""
Persist a new block to the blockchain, along with the raw transactions associated with the block. An exception
will be raised if an error occurs either validating the new block data, or storing the block. Thus, it is
recommended that this method is wrapped in a try block.
:param block_contender: A BlockContender instance
:param raw_transactions: A list of ordered raw transactions contained in the block
:param publisher_sk: The signing key of the publisher (a Masternode) who is publishing the block
:param timestamp: The time the block was published, in unix epoch time. If 0, time.time() is used
:return: None
:raises: An assertion error if invalid args are passed into this function, or a BlockStorageValidationException
if validation fails on the attempted block
TODO -- think really hard and make sure that this is 'collision proof' (extremely unlikely, but still possible)
- could there be a hash collision in the Merkle tree nodes?
- hash collision in block hash space?
- hash collision in transaction space?
"""
assert isinstance(
block_contender, BlockContender
), "Expected block_contender arg to be BlockContender instance"
assert is_valid_hex(
publisher_sk,
64), "Invalid signing key {}. Expected 64 char hex str".format(
publisher_sk)
if not timestamp:
timestamp = int(time.time())
tree = MerkleTree.from_raw_transactions(raw_transactions)
publisher_vk = ED25519Wallet.get_vk(publisher_sk)
publisher_sig = ED25519Wallet.sign(publisher_sk, tree.root)
# Build and validate block_data
block_data = {
'block_contender': block_contender,
'timestamp': timestamp,
'merkle_root': tree.root_as_hex,
'merkle_leaves': tree.leaves_as_concat_hex_str,
'prev_block_hash': cls._get_latest_block_hash(),
'masternode_signature': publisher_sig,
'masternode_vk': publisher_vk,
}
cls._validate_block_data(block_data)
# Compute block hash
block_hash = cls._compute_block_hash(block_data)
# Encode block data for serialization and finally persist the data
log.info(
"Attempting to persist new block with hash {}".format(block_hash))
block_data = cls._encode_block(block_data)
with DB() as db:
# Store block
res = db.tables.blocks.insert([{
'hash': block_hash,
**block_data
}]).run(db.ex)
if res:
log.info(
"Successfully inserted new block with number {} and hash {}"
.format(res['last_row_id'], block_hash))
else:
log.error(
"Error inserting block! Got None/False result back from insert query. Result={}"
.format(res))
return
# Store raw transactions
log.info(
"Attempting to store {} raw transactions associated with block hash {}"
.format(len(raw_transactions), block_hash))
tx_rows = [{
'hash': Hasher.hash(raw_tx),
'data': encode_tx(raw_tx),
'block_hash': block_hash
} for raw_tx in raw_transactions]
res = db.tables.transactions.insert(tx_rows).run(db.ex)
if res:
log.info("Successfully inserted {} transactions".format(
res['row_count']))
else:
log.error(
"Error inserting raw transactions! Got None from insert query. Result={}"
.format(res))
def hash_nodes(nodes: list):
log.warning("HASH LEAVES API SHOULD BE DEPRECATED")
return Hasher.hash_iterable(nodes,
algorithm=Hasher.Alg.SHA3_256,
return_bytes=True)
def hash_nodes(nodes: list):
return Hasher.hash_iterable(nodes,
algorithm=Hasher.Alg.SHA3_256,
return_bytes=True)
def hash(o):
return Hasher.hash(o, algorithm=Hasher.Alg.SHA3_256, return_bytes=True)