def _get_raw_transactions(cls, hashes: str or list, is_block_hashes=False):
"""
Helper method to query raw transactions, either from a list of hashes or a single hash. If is_block_hashes=True,
then the hashes are assumed to refer to block hashes, in which case all the transactions belonging to that
particular block is returned.
:param hashes: A single hash (as a string), or a list of hashes
:param is_block_hashes: If true, 'hashes' is assumed to refer to block hashes, and all transactions belonging
to the specified block hash(es) will be returned. If False, 'hashes' are assumed to be transaction hashes.
:return: A list of raw transactions, each as bytes, or None if no block with the given hash can be found
"""
assert isinstance(hashes, str) or isinstance(
hashes, list), "Expected hashes to be a str or list"
if isinstance(hashes, str):
hashes = [hashes]
for h in hashes:
assert is_valid_hex(
h, length=64
), "Expected hashes to be 64 char hex str, not {}".format(h)
with DB() as db:
transactions = db.tables.transactions
comparison_col = transactions.block_hash if is_block_hashes else transactions.hash
where_clause = or_(*[(comparison_col == h) for h in hashes])
rows = transactions.select().where(where_clause).run(db.ex)
if not rows:
return None
else:
return [decode_tx(row['data']) for row in rows]
def get_block(cls, number: int = 0, hash: str = '') -> dict or None:
"""
Retrieves a block by its hash, or autoincrement number. Returns a dictionary with a key for each column in the
blocks table. Returns None if no block with the specified hash/number is found.
:param number: The number of the block to fetch. The genesis block is number 1, and the first 'real' block
is number 2, and so on.
:param hash: The hash of the block to lookup. Must be valid 64 char hex string
:return: A dictionary, containing a key for each column in the blocks table.
"""
assert bool(number > 0) ^ bool(
hash), "Either 'number' XOR 'hash' arg must be given"
with DB() as db:
blocks = db.tables.blocks
if number > 0:
block = blocks.select().where(blocks.number == number).run(
db.ex)
return cls._decode_block(block[0]) if block else None
elif hash:
assert is_valid_hex(
hash, length=64), "Invalid block hash {}".format(hash)
block = blocks.select().where(blocks.hash == hash).run(db.ex)
return cls._decode_block(block[0]) if block else None
def get_child_block_hashes(cls, parent_hash: str, limit=0) -> List[str] or None:
"""
Retrieve a list of child block hashes from a given a parent block. In other words, this method gets the hashes
for all blocks created "after" the specified block with hash 'parent_hash'.
:param parent_hash: The hash of the parent block
:param limit: If specified,
:return: A list of hashes for the blocks that descend the parent block. These will be sorted by their order
in the block chain, such that the first element is the block immediately after parent_hash, and the last element
is the latest block in the block chain. Returns None if parent_hash is already the latest block, or if no
block with 'parent_hash' can be found.
"""
assert is_valid_hex(parent_hash, 64), "parent_hash {} is not valid 64 char hex str".format(parent_hash)
assert limit >= 0, "Limit must be >= 0 (not {})".format(limit)
# TODO implement functionality for limits
if limit:
raise NotImplementedError("Limiting logic not implemented")
# TODO optimize this once we get more functionality in EasyDB ...
# we would like to select all rows where the number >= the number associated with 'parent_hash', ordered
# ascending by number. For now, we must do this in 2 queries
with DB() as db:
blocks = db.tables.blocks
# Get block index number associated with 'parent_hash'
row = blocks.select().where(blocks.hash == parent_hash).run(db.ex)
if not row:
log.warning("No block with hash {} could be found!".format(parent_hash))
return None
parent_number = row[0]['number']
rows = blocks.select('hash').where(blocks.number > parent_number).order_by('number', desc=False).run(db.ex)
if not rows:
return None
return [row['hash'] for row in rows]
def validate_block_data(cls, block_data: dict):
"""
Validates the block_data dictionary. 'block_data' should be a strict subset of the 'block' dictionary, keys for all
columns in the block table EXCEPT 'number' and 'hash'. If any validation fails, an exception is raised.
For a block_data dictionary to be valid, it must:
- Have a key for each block data column specified in BLOCK_DATA_COLS (at top of blocks.py)
- BlockContender successfully validates with the Merkle root (meaning all signatures in the BlockContender
can be verified using the Merkle root as the message)
- Merkle leaves contained in BlockContender (block_contender.nodes) match Merkle leaves in block_data dict
- Merkle root is correct root if a Merkle tree is built from Merkle leaves
- Masternode signature is valid (signature is valid using Merkle root as message and masternode_vk as vk)
:param block_data: The dictionary containing a key for each column in BLOCK_DATA_COLS
(ie 'merkle_root', 'prev_block_hash', .. ect)
:raises: An BlockStorageValidationException (or subclass) if any validation fails
"""
# Check block_data has all the necessary keys
expected_keys = set(BLOCK_DATA_COLS.keys())
actual_keys = set(block_data.keys())
missing_keys = expected_keys - actual_keys
extra_keys = actual_keys - expected_keys
# Check for missing keys
if len(missing_keys) > 0:
raise BlockStorageValidationException("block_data keys {} missing key(s) {}".format(actual_keys, missing_keys))
# Check for extra (unrecognized) keys
if len(extra_keys) > 0:
raise BlockStorageValidationException("block_data keys {} has unrecognized keys {}".format(actual_keys, extra_keys))
# Validate Merkle Tree
tree = MerkleTree.from_leaves_hex_str(block_data['merkle_leaves'])
if tree.root_as_hex != block_data['merkle_root']:
raise InvalidMerkleTreeException("Merkle Tree could not be validated for block_data {}".format(block_data))
# Validate BlockContender nodes match merkle leaves
block_leaves = block_data['block_contender'].merkle_leaves
if len(block_leaves) != len(tree.leaves):
raise InvalidBlockContenderException("Number of Merkle leaves on BlockContender {} does not match number of"
" leaves in MerkleTree {}".format(len(block_leaves), len(tree.leaves)))
for block_leaf, merkle_leaf in zip(block_leaves, tree.leaves_as_hex):
if block_leaf != merkle_leaf:
raise InvalidBlockContenderException("BlockContender leaves do not match Merkle leaves\nblock leaves = "
"{}\nmerkle leaves = {}".format(block_leaves, tree.leaves_as_hex))
# Validate MerkleSignatures inside BlockContender match Merkle leaves from raw transactions
bc = block_data['block_contender']
if not bc.validate_signatures():
raise InvalidBlockContenderException("BlockContender signatures could not be validated! BC = {}".format(bc))
# TODO validate MerkleSignatures are infact signed by valid delegates
# this is tricky b/c we would need to know who the delegates were at the time of the block, not necessarily the
# current delegates
# Validate Masternode Signature
if not is_valid_hex(block_data['masternode_vk'], length=64):
raise InvalidBlockSignatureException("Invalid verifying key for field masternode_vk: {}"
.format(block_data['masternode_vk']))
if not wallet.verify(block_data['masternode_vk'], bytes.fromhex(block_data['merkle_root']), block_data['masternode_signature']):
raise InvalidBlockSignatureException("Could not validate Masternode's signature on block data")
def validate(self):
# Validate field types and existence
assert type(self._data) == dict, "BlockContender's _data must be a dict"
assert BlockContender.SIGS in self._data, "signature field missing from data {}".format(self._data)
assert BlockContender.LEAVES in self._data, "leaves field missing from data {}".format(self._data)
# Ensure merkle leaves are valid hex
for leaf in self.merkle_leaves:
assert is_valid_hex(leaf, length=64), "Invalid Merkle leaf {} ... expected 64 char hex string".format(leaf)
# Attempt to deserialize signatures by reading property (will raise exception if can't)
self.signatures
def get_latest_block_hash(cls) -> str:
"""
Looks into the DB, and returns the latest block's hash. If the latest block_hash is for whatever reason invalid,
(ie. not valid 64 char hex string), then this method will raise an assertion.
:return: A string, representing the latest (most recent) block's hash
:raises: An assertion if the latest block hash is not vaild 64 character hex. If this happens, something was
seriously messed up in the block storage process.
# TODO this can perhaps be made more efficient by memoizing the new block_hash each time we store it
"""
with DB() as db:
row = db.tables.blocks.select('hash').order_by('number', desc=True).limit(1).run(db.ex)[0]
last_hash = row['hash']
assert is_valid_hex(last_hash, length=64), "Latest block hash is invalid 64 char hex! Got {}".format(last_hash)
return last_hash
def get_raw_transactions_from_block(
cls, block_hash: str) -> List[bytes] or None:
"""
Retrieves a list of raw transactions associated with a particular block. Returns None if no block with
the given hash can be found.
:param block_hash:
:return: A list of raw transactions each as bytes, or None if no block with the given hash can be found
"""
assert is_valid_hex(
block_hash, length=64
), "Expected block_hash to be 64 char hex str, not {}".format(
block_hash)
with DB() as db:
transactions = db.tables.transactions
rows = transactions.select().where(
transactions.block_hash == block_hash).run(db.ex)
if not rows:
return None
else:
return [decode_tx(row['data']) for row in rows]
def get_raw_transaction(cls, tx_hash: str) -> bytes or None:
"""
Retrieves a single raw transaction from its hash. Returns None if no transaction for that hash can be found
:param tx_hash: The hash of the raw transaction to lookup. Should be a 64 character hex string
:return: The raw transactions as bytes, or None if no transaction with that hash can be found
"""
assert is_valid_hex(
tx_hash, length=64
), "Expected tx_hash to be 64 char hex str, not {}".format(tx_hash)
with DB() as db:
transactions = db.tables.transactions
rows = transactions.select().where(
transactions.hash == tx_hash).run(db.ex)
if not rows:
return None
else:
assert len(
rows
) == 1, "Multiple Transactions found with has {}! BIG TIME DEVELOPMENT ERROR!!!".format(
tx_hash)
return decode_tx(rows[0]['data'])
def store_block(cls,
block_contender: BlockContender,
raw_transactions: List[bytes],
publisher_sk: str,
timestamp: int = 0):
"""
Persist a new block to the blockchain, along with the raw transactions associated with the block. An exception
will be raised if an error occurs either validating the new block data, or storing the block. Thus, it is
recommended that this method is wrapped in a try block.
:param block_contender: A BlockContender instance
:param raw_transactions: A list of ordered raw transactions contained in the block
:param publisher_sk: The signing key of the publisher (a Masternode) who is publishing the block
:param timestamp: The time the block was published, in unix epoch time. If 0, time.time() is used
:return: None
:raises: An assertion error if invalid args are passed into this function, or a BlockStorageValidationException
if validation fails on the attempted block
TODO -- think really hard and make sure that this is 'collision proof' (extremely unlikely, but still possible)
- could there be a hash collision in the Merkle tree nodes?
- hash collision in block hash space?
- hash collision in transaction space?
"""
assert isinstance(
block_contender, BlockContender
), "Expected block_contender arg to be BlockContender instance"
assert is_valid_hex(
publisher_sk,
64), "Invalid signing key {}. Expected 64 char hex str".format(
publisher_sk)
if not timestamp:
timestamp = int(time.time())
tree = MerkleTree.from_raw_transactions(raw_transactions)
publisher_vk = ED25519Wallet.get_vk(publisher_sk)
publisher_sig = ED25519Wallet.sign(publisher_sk, tree.root)
# Build and validate block_data
block_data = {
'block_contender': block_contender,
'timestamp': timestamp,
'merkle_root': tree.root_as_hex,
'merkle_leaves': tree.leaves_as_concat_hex_str,
'prev_block_hash': cls._get_latest_block_hash(),
'masternode_signature': publisher_sig,
'masternode_vk': publisher_vk,
}
cls._validate_block_data(block_data)
# Compute block hash
block_hash = cls._compute_block_hash(block_data)
# Encode block data for serialization and finally persist the data
log.info(
"Attempting to persist new block with hash {}".format(block_hash))
block_data = cls._encode_block(block_data)
with DB() as db:
# Store block
res = db.tables.blocks.insert([{
'hash': block_hash,
**block_data
}]).run(db.ex)
if res:
log.info(
"Successfully inserted new block with number {} and hash {}"
.format(res['last_row_id'], block_hash))
else:
log.error(
"Error inserting block! Got None/False result back from insert query. Result={}"
.format(res))
return
# Store raw transactions
log.info(
"Attempting to store {} raw transactions associated with block hash {}"
.format(len(raw_transactions), block_hash))
tx_rows = [{
'hash': Hasher.hash(raw_tx),
'data': encode_tx(raw_tx),
'block_hash': block_hash
} for raw_tx in raw_transactions]
res = db.tables.transactions.insert(tx_rows).run(db.ex)
if res:
log.info("Successfully inserted {} transactions".format(
res['row_count']))
else:
log.error(
"Error inserting raw transactions! Got None from insert query. Result={}"
.format(res))
