def __init__(self, state):
self.state = state
self.tx_pool = TransactionPool() # TODO: Move to some pool manager
self.last_block = Block.from_json(GenesisBlock().to_json())
self.current_difficulty = StringToUInt256(str(config.dev.genesis_difficulty))
self.trigger_miner = False
def create_block(self,
last_block,
mining_nonce,
tx_pool: TransactionPool,
miner_address) -> Optional[Block]:
# TODO: Persistence will move to rocksdb
# FIXME: Difference between this and create block?????????????
dummy_block = Block.create(block_number=last_block.block_number + 1,
prevblock_headerhash=last_block.headerhash,
transactions=[],
miner_address=miner_address)
dummy_block.set_nonces(mining_nonce, 0)
t_pool2 = tx_pool.transactions
addresses_set = set()
for tx_set in t_pool2:
tx = tx_set[1]
tx.set_effected_address(addresses_set)
addresses_state = dict()
for address in addresses_set:
addresses_state[address] = self.state.get_address(address)
block_size = dummy_block.size
block_size_limit = self.state.get_block_size_limit(last_block)
transactions = []
for tx_set in t_pool2:
tx = tx_set[1]
# Skip Transactions for later, which doesn't fit into block
if block_size + tx.size + config.dev.tx_extra_overhead > block_size_limit:
continue
addr_from_pk_state = addresses_state[tx.addr_from]
addr_from_pk = Transaction.get_slave(tx)
if addr_from_pk:
addr_from_pk_state = addresses_state[addr_from_pk]
if not tx.validate_extended(addresses_state[tx.addr_from], addr_from_pk_state):
logger.warning('Txn validation failed for tx in tx_pool')
tx_pool.remove_tx_from_pool(tx)
continue
tx.apply_on_state(addresses_state)
tx._data.nonce = addr_from_pk_state.nonce
block_size += tx.size + config.dev.tx_extra_overhead
transactions.append(tx)
block = Block.create(block_number=last_block.block_number + 1,
prevblock_headerhash=last_block.headerhash,
transactions=transactions,
miner_address=miner_address)
return block
def __init__(self, state):
self._state = state
self.tx_pool = TransactionPool(None)
self._last_block = Block.deserialize(GenesisBlock().serialize())
self.current_difficulty = StringToUInt256(
str(config.user.genesis_difficulty))
self.trigger_miner = False
self.lock = threading.RLock()
def create_block(self, last_block, mining_nonce, tx_pool: TransactionPool,
miner_address) -> Optional[Block]:
dummy_block = Block.create(block_number=last_block.block_number + 1,
prev_headerhash=last_block.headerhash,
prev_timestamp=last_block.timestamp,
transactions=[],
miner_address=miner_address)
dummy_block.set_nonces(mining_nonce, 0)
t_pool2 = tx_pool.transactions
addresses_set = set()
for tx_set in t_pool2:
tx = tx_set[1].transaction
tx.set_affected_address(addresses_set)
addresses_state = dict()
for address in addresses_set:
addresses_state[address] = self._chain_manager.get_address_state(
address)
block_size = dummy_block.size
block_size_limit = self._chain_manager.get_block_size_limit(last_block)
transactions = []
for tx_set in t_pool2:
tx = tx_set[1].transaction
# Skip Transactions for later, which doesn't fit into block
if block_size + tx.size + config.dev.tx_extra_overhead > block_size_limit:
continue
addr_from_pk_state = addresses_state[tx.addr_from]
addr_from_pk = Transaction.get_slave(tx)
if addr_from_pk:
addr_from_pk_state = addresses_state[addr_from_pk]
if not tx.validate_extended(addresses_state[tx.addr_from],
addr_from_pk_state):
logger.warning('Txn validation failed for tx in tx_pool')
tx_pool.remove_tx_from_pool(tx)
continue
tx.apply_state_changes(addresses_state)
tx._data.nonce = addr_from_pk_state.nonce
block_size += tx.size + config.dev.tx_extra_overhead
transactions.append(tx)
block = Block.create(block_number=last_block.block_number + 1,
prev_headerhash=last_block.headerhash,
prev_timestamp=last_block.timestamp,
transactions=transactions,
miner_address=miner_address)
return block
def __init__(self, state):
self.state = state
self.tx_pool = TransactionPool() # TODO: Move to some pool manager
self.last_block = Block.from_json(GenesisBlock().to_json())
self.current_difficulty = StringToUInt256(str(config.dev.genesis_difficulty))
self._difficulty_tracker = DifficultyTracker()
self.trigger_miner = False
def setUp(self):
self.txpool = TransactionPool(None)
self.tx_3907 = make_tx(name='Mock TX 3907', txhash=b'h3907', ots_key=3907)
self.tx_4095 = make_tx(name='Mock TX 4095', txhash=b'h4095', ots_key=4095)
self.tx_4096 = make_tx(name='Mock TX 4096', txhash=b'h4096', ots_key=4096)
self.tx_4097 = make_tx(name='Mock TX 4097', txhash=b'h4097', ots_key=4097)
self.tx_4098 = make_tx(name='Mock TX 4098', txhash=b'h4098', ots_key=4098)
self.tx_4099 = make_tx(name='Mock TX 4099', txhash=b'h4099', ots_key=4099)
self.tx_4100 = make_tx(name='Mock TX 4100', txhash=b'h4100', ots_key=4100)
self.tx_4200 = make_tx(name='Mock TX 4200', txhash=b'h4200', ots_key=4200)
# To remove the tx from the pool we have to add it first!
self.txpool.add_tx_to_pool(self.tx_4095, 5)
self.txpool.add_tx_to_pool(self.tx_4096, 5)
self.txpool.add_tx_to_pool(self.tx_4097, 5)
self.txpool.add_tx_to_pool(self.tx_4098, 5)
self.txpool.add_tx_to_pool(self.tx_4099, 5)
self.txpool.add_tx_to_pool(self.tx_4100, 5)
self.txpool.add_tx_to_pool(self.tx_4200, 5)
def setUp(self):
self.m_mining_qaddress = alice.qaddress
self.m_mining_address = parse_qaddress(self.m_mining_qaddress)
self.alice_address_state = Mock(
autospec=OptimizedAddressState,
name='mock alice OptimizedAddressState')
self.chain_manager = Mock(spec=ChainManager)
self.chain_manager.get_block_size_limit.return_value = 500
self.chain_manager.get_address_state.return_value = self.alice_address_state
self.chain_manager.get_config_by_block_number.return_value = config.dev
self.parent_block = Block()
self.parent_difficulty = StringToUInt256(
'0') # tuple (0,0,0,0,0...) length 32
self.m_pre_block_logic = Mock(spec=POW.pre_block_logic, name='hello')
mining_thread_count = 1
self.miner = Miner(self.chain_manager, self.m_pre_block_logic,
self.m_mining_address, mining_thread_count)
self.txpool = TransactionPool(None)
def replacement_set_affected_address(addresses_set):
return addresses_set.add(alice.address)
self.m_tx_args = {
"addr_from": alice.address,
"addrs_to": [bob.address],
"amounts": [10],
"fee": 1,
"PK": alice.pk,
"master_addr": None,
"size": 150,
"validate_extended.return_value": True,
"set_affected_address": replacement_set_affected_address
}
class ChainManager:
def __init__(self, state):
self.state = state
self.tx_pool = TransactionPool(None)
self.last_block = Block.from_json(GenesisBlock().to_json())
self.current_difficulty = StringToUInt256(
str(config.dev.genesis_difficulty))
self.trigger_miner = False
@property
def height(self):
return self.last_block.block_number
def set_broadcast_tx(self, broadcast_tx):
self.tx_pool.set_broadcast_tx(broadcast_tx)
def get_last_block(self) -> Block:
return self.last_block
def get_cumulative_difficulty(self):
last_block_metadata = self.state.get_block_metadata(
self.last_block.headerhash)
return last_block_metadata.cumulative_difficulty
def load(self, genesis_block):
height = self.state.get_mainchain_height()
if height == -1:
self.state.put_block(genesis_block, None)
block_number_mapping = qrl_pb2.BlockNumberMapping(
headerhash=genesis_block.headerhash,
prev_headerhash=genesis_block.prev_headerhash)
self.state.put_block_number_mapping(genesis_block.block_number,
block_number_mapping, None)
parent_difficulty = StringToUInt256(
str(config.dev.genesis_difficulty))
self.current_difficulty, _ = DifficultyTracker.get(
measurement=config.dev.mining_setpoint_blocktime,
parent_difficulty=parent_difficulty)
block_metadata = BlockMetadata.create()
block_metadata.set_orphan(False)
block_metadata.set_block_difficulty(self.current_difficulty)
block_metadata.set_cumulative_difficulty(self.current_difficulty)
self.state.put_block_metadata(genesis_block.headerhash,
block_metadata, None)
addresses_state = dict()
for genesis_balance in GenesisBlock().genesis_balance:
bytes_addr = genesis_balance.address
addresses_state[bytes_addr] = AddressState.get_default(
bytes_addr)
addresses_state[
bytes_addr]._data.balance = genesis_balance.balance
for tx_idx in range(1, len(genesis_block.transactions)):
tx = Transaction.from_pbdata(
genesis_block.transactions[tx_idx])
for addr in tx.addrs_to:
addresses_state[addr] = AddressState.get_default(addr)
coinbase_tx = Transaction.from_pbdata(
genesis_block.transactions[0])
if not isinstance(coinbase_tx, CoinBase):
return False
addresses_state[coinbase_tx.addr_to] = AddressState.get_default(
coinbase_tx.addr_to)
if not coinbase_tx.validate_extended():
return False
coinbase_tx.apply_state_changes(addresses_state)
for tx_idx in range(1, len(genesis_block.transactions)):
tx = Transaction.from_pbdata(
genesis_block.transactions[tx_idx])
tx.apply_state_changes(addresses_state)
self.state.put_addresses_state(addresses_state)
self.state.update_tx_metadata(genesis_block, None)
self.state.update_mainchain_height(0, None)
else:
self.last_block = self.get_block_by_number(height)
self.current_difficulty = self.state.get_block_metadata(
self.last_block.headerhash).block_difficulty
def _try_orphan_add_block(self, block, batch):
prev_block_metadata = self.state.get_block_metadata(
block.prev_headerhash)
if prev_block_metadata is None or prev_block_metadata.is_orphan:
self.state.put_block(block, batch)
self.add_block_metadata(block.headerhash, block.timestamp,
block.prev_headerhash, batch)
return True
return False
def _try_branch_add_block(self, block, batch=None) -> bool:
address_set = self.state.prepare_address_list(
block) # Prepare list for current block
if self.last_block.headerhash == block.prev_headerhash:
address_txn = self.state.get_state_mainchain(address_set)
else:
address_txn, rollback_headerhash, hash_path = self.state.get_state(
block.prev_headerhash, address_set)
if block.apply_state_changes(address_txn):
self.state.put_block(block, None)
self.add_block_metadata(block.headerhash, block.timestamp,
block.prev_headerhash, None)
last_block_metadata = self.state.get_block_metadata(
self.last_block.headerhash)
new_block_metadata = self.state.get_block_metadata(
block.headerhash)
last_block_difficulty = int(
UInt256ToString(last_block_metadata.cumulative_difficulty))
new_block_difficulty = int(
UInt256ToString(new_block_metadata.cumulative_difficulty))
if new_block_difficulty > last_block_difficulty:
if self.last_block.headerhash != block.prev_headerhash:
self.rollback(rollback_headerhash, hash_path,
block.block_number)
self.state.put_addresses_state(address_txn)
self.last_block = block
self._update_mainchain(block, batch)
self.tx_pool.remove_tx_in_block_from_pool(block)
self.tx_pool.check_stale_txn(block.block_number)
self.state.update_mainchain_height(block.block_number, batch)
self.state.update_tx_metadata(block, batch)
self.trigger_miner = True
return True
return False
def remove_block_from_mainchain(self, block: Block,
latest_block_number: int, batch):
addresses_set = self.state.prepare_address_list(block)
addresses_state = self.state.get_state_mainchain(addresses_set)
for tx_idx in range(len(block.transactions) - 1, 0, -1):
tx = Transaction.from_pbdata(block.transactions[tx_idx])
tx.revert_state_changes(addresses_state, self.state)
self.tx_pool.add_tx_from_block_to_pool(block, latest_block_number)
self.state.update_mainchain_height(block.block_number - 1, batch)
self.state.rollback_tx_metadata(block, batch)
self.state.remove_blocknumber_mapping(block.block_number, batch)
self.state.put_addresses_state(addresses_state, batch)
def rollback(self, rollback_headerhash, hash_path, latest_block_number):
while self.last_block.headerhash != rollback_headerhash:
self.remove_block_from_mainchain(self.last_block,
latest_block_number, None)
self.last_block = self.state.get_block(
self.last_block.prev_headerhash)
for header_hash in hash_path[-1::-1]:
block = self.state.get_block(header_hash)
address_set = self.state.prepare_address_list(
block) # Prepare list for current block
addresses_state = self.state.get_state_mainchain(address_set)
for tx_idx in range(0, len(block.transactions)):
tx = Transaction.from_pbdata(block.transactions[tx_idx])
tx.apply_state_changes(addresses_state)
self.state.put_addresses_state(addresses_state)
self.last_block = block
self._update_mainchain(block, None)
self.tx_pool.remove_tx_in_block_from_pool(block)
self.state.update_mainchain_height(block.block_number, None)
self.state.update_tx_metadata(block, None)
self.trigger_miner = True
def _add_block(self, block, batch=None):
self.trigger_miner = False
block_size_limit = self.state.get_block_size_limit(block)
if block_size_limit and block.size > block_size_limit:
logger.info('Block Size greater than threshold limit %s > %s',
block.size, block_size_limit)
return False
if self._try_orphan_add_block(block, batch):
return True
if self._try_branch_add_block(block, batch):
return True
return False
def add_block(self, block: Block) -> bool:
if block.block_number < self.height - config.dev.reorg_limit:
logger.debug('Skipping block #%s as beyond re-org limit',
block.block_number)
return False
batch = self.state.get_batch()
if self._add_block(block, batch=batch):
self.state.write_batch(batch)
self.update_child_metadata(
block.headerhash
) # TODO: Not needed to execute when an orphan block is added
logger.info('Added Block #%s %s', block.block_number,
bin2hstr(block.headerhash))
return True
return False
def update_child_metadata(self, headerhash):
block_metadata = self.state.get_block_metadata(headerhash)
childs = list(block_metadata.child_headerhashes)
while childs:
child_headerhash = childs.pop(0)
block = self.state.get_block(child_headerhash)
if not block:
continue
if not self._add_block(block):
self._prune([block.headerhash], None)
continue
block_metadata = self.state.get_block_metadata(child_headerhash)
childs += block_metadata.child_headerhashes
def _prune(self, childs, batch):
while childs:
child_headerhash = childs.pop(0)
block_metadata = self.state.get_block_metadata(child_headerhash)
childs += block_metadata.child_headerhashes
self.state.delete(bin2hstr(child_headerhash).encode(), batch)
self.state.delete(
b'metadata_' + bin2hstr(child_headerhash).encode(), batch)
def add_block_metadata(self, headerhash, block_timestamp,
parent_headerhash, batch):
block_metadata = self.state.get_block_metadata(headerhash)
if not block_metadata:
block_metadata = BlockMetadata.create()
parent_metadata = self.state.get_block_metadata(parent_headerhash)
block_difficulty = (0, ) * 32 # 32 bytes to represent 256 bit of 0
block_cumulative_difficulty = (
0, ) * 32 # 32 bytes to represent 256 bit of 0
if not parent_metadata:
parent_metadata = BlockMetadata.create()
else:
parent_block = self.state.get_block(parent_headerhash)
if parent_block:
parent_block_difficulty = parent_metadata.block_difficulty
parent_cumulative_difficulty = parent_metadata.cumulative_difficulty
if not parent_metadata.is_orphan:
block_metadata.update_last_headerhashes(
parent_metadata.last_N_headerhashes, parent_headerhash)
measurement = self.state.get_measurement(
block_timestamp, parent_headerhash, parent_metadata)
block_difficulty, _ = DifficultyTracker.get(
measurement=measurement,
parent_difficulty=parent_block_difficulty)
block_cumulative_difficulty = StringToUInt256(
str(
int(UInt256ToString(block_difficulty)) +
int(UInt256ToString(parent_cumulative_difficulty)))
)
block_metadata.set_orphan(parent_metadata.is_orphan)
block_metadata.set_block_difficulty(block_difficulty)
block_metadata.set_cumulative_difficulty(block_cumulative_difficulty)
parent_metadata.add_child_headerhash(headerhash)
self.state.put_block_metadata(parent_headerhash, parent_metadata,
batch)
self.state.put_block_metadata(headerhash, block_metadata, batch)
# Call once to populate the cache
self.state.get_block_datapoint(headerhash)
def _update_mainchain(self, block, batch):
block_number_mapping = None
while block_number_mapping is None or block.headerhash != block_number_mapping.headerhash:
block_number_mapping = qrl_pb2.BlockNumberMapping(
headerhash=block.headerhash,
prev_headerhash=block.prev_headerhash)
self.state.put_block_number_mapping(block.block_number,
block_number_mapping, batch)
block = self.state.get_block(block.prev_headerhash)
block_number_mapping = self.state.get_block_number_mapping(
block.block_number)
def get_block_by_number(self, block_number) -> Optional[Block]:
return self.state.get_block_by_number(block_number)
def get_state(self, headerhash):
return self.state.get_state(headerhash, set())
def get_address(self, address):
return self.state.get_address_state(address)
def get_transaction(self, transaction_hash) -> list:
for tx_set in self.tx_pool.transactions:
tx = tx_set[1].transaction
if tx.txhash == transaction_hash:
return [tx, None]
return self.state.get_tx_metadata(transaction_hash)
def get_headerhashes(self, start_blocknumber):
start_blocknumber = max(0, start_blocknumber)
end_blocknumber = min(self.last_block.block_number,
start_blocknumber + 2 * config.dev.reorg_limit)
total_expected_headerhash = end_blocknumber - start_blocknumber + 1
node_header_hash = qrl_pb2.NodeHeaderHash()
node_header_hash.block_number = start_blocknumber
block = self.state.get_block_by_number(end_blocknumber)
block_headerhash = block.headerhash
node_header_hash.headerhashes.append(block_headerhash)
end_blocknumber -= 1
while end_blocknumber >= start_blocknumber:
block_metadata = self.state.get_block_metadata(block_headerhash)
for headerhash in block_metadata.last_N_headerhashes[-1::-1]:
node_header_hash.headerhashes.append(headerhash)
end_blocknumber -= len(block_metadata.last_N_headerhashes)
if len(block_metadata.last_N_headerhashes) == 0:
break
block_headerhash = block_metadata.last_N_headerhashes[0]
node_header_hash.headerhashes[:] = node_header_hash.headerhashes[
-1::-1]
del node_header_hash.headerhashes[:len(node_header_hash.headerhashes) -
total_expected_headerhash]
return node_header_hash
class ChainManager:
def __init__(self, state):
self._state = state
self.tx_pool = TransactionPool(None)
self._last_block = Block.deserialize(GenesisBlock().serialize())
self.current_difficulty = StringToUInt256(
str(config.user.genesis_difficulty))
self.trigger_miner = False
self.lock = threading.RLock()
@property
def height(self):
with self.lock:
if not self._last_block:
return -1
return self._last_block.block_number
@property
def last_block(self) -> Block:
with self.lock:
return self._last_block
@property
def total_coin_supply(self):
with self.lock:
return self._state.total_coin_supply
def get_block_datapoint(self, headerhash):
with self.lock:
return self._state.get_block_datapoint(headerhash)
def get_cumulative_difficulty(self):
with self.lock:
last_block_metadata = self._state.get_block_metadata(
self._last_block.headerhash)
return last_block_metadata.cumulative_difficulty
def get_block_by_number(self, block_number) -> Optional[Block]:
with self.lock:
return self._state.get_block_by_number(block_number)
def get_block(self, header_hash: bytes) -> Optional[Block]:
with self.lock:
return self._state.get_block(header_hash)
def get_address_balance(self, address: bytes) -> int:
with self.lock:
return self._state.get_address_balance(address)
def get_address_is_used(self, address: bytes) -> bool:
with self.lock:
return self._state.get_address_is_used(address)
def get_address_state(self, address: bytes) -> AddressState:
with self.lock:
return self._state.get_address_state(address)
def get_all_address_state(self):
with self.lock:
return self._state.get_all_address_state()
def get_tx_metadata(self, transaction_hash) -> list:
with self.lock:
return self._state.get_tx_metadata(transaction_hash)
def get_last_transactions(self):
with self.lock:
return self._state.get_last_txs()
def get_unconfirmed_transaction(self, transaction_hash) -> list:
with self.lock:
for tx_set in self.tx_pool.transactions:
tx = tx_set[1].transaction
if tx.txhash == transaction_hash:
return [tx, tx_set[1].timestamp]
return []
def get_block_metadata(self,
header_hash: bytes) -> Optional[BlockMetadata]:
with self.lock:
return self._state.get_block_metadata(header_hash)
def get_blockheader_and_metadata(self, block_number=0) -> Tuple:
with self.lock:
block_number = block_number or self.height # if both are non-zero, then block_number takes priority
result = (None, None)
block = self.get_block_by_number(block_number)
if block:
blockheader = block.blockheader
blockmetadata = self.get_block_metadata(blockheader.headerhash)
result = (blockheader, blockmetadata)
return result
def get_block_to_mine(self, miner, wallet_address) -> list:
with miner.lock: # Trying to acquire miner.lock to make sure pre_block_logic is not running
with self.lock:
last_block = self.last_block
last_block_metadata = self.get_block_metadata(
last_block.headerhash)
return miner.get_block_to_mine(
wallet_address, self.tx_pool, last_block,
last_block_metadata.block_difficulty)
def get_measurement(self, block_timestamp, parent_headerhash,
parent_metadata: BlockMetadata):
with self.lock:
return self._state.get_measurement(block_timestamp,
parent_headerhash,
parent_metadata)
def get_block_size_limit(self, block: Block):
with self.lock:
return self._state.get_block_size_limit(block)
def get_block_is_duplicate(self, block: Block) -> bool:
with self.lock:
return self._state.get_block(block.headerhash) is not None
def validate_mining_nonce(self,
blockheader: BlockHeader,
enable_logging=True):
with self.lock:
parent_metadata = self.get_block_metadata(
blockheader.prev_headerhash)
parent_block = self._state.get_block(blockheader.prev_headerhash)
measurement = self.get_measurement(blockheader.timestamp,
blockheader.prev_headerhash,
parent_metadata)
diff, target = DifficultyTracker.get(
measurement=measurement,
parent_difficulty=parent_metadata.block_difficulty)
if enable_logging:
logger.debug('-----------------START--------------------')
logger.debug('Validate #%s',
blockheader.block_number)
logger.debug('block.timestamp %s',
blockheader.timestamp)
logger.debug('parent_block.timestamp %s',
parent_block.timestamp)
logger.debug('parent_block.difficulty %s',
UInt256ToString(parent_metadata.block_difficulty))
logger.debug('diff %s',
UInt256ToString(diff))
logger.debug('target %s', bin2hstr(target))
logger.debug('-------------------END--------------------')
if not PoWValidator().verify_input(blockheader.mining_blob,
target):
if enable_logging:
logger.warning("PoW verification failed")
qn = Qryptonight()
tmp_hash = qn.hash(blockheader.mining_blob)
logger.warning("{}".format(bin2hstr(tmp_hash)))
logger.debug('%s', blockheader.to_json())
return False
return True
def get_headerhashes(self, start_blocknumber):
with self.lock:
start_blocknumber = max(0, start_blocknumber)
end_blocknumber = min(
self._last_block.block_number,
start_blocknumber + 2 * config.dev.reorg_limit)
total_expected_headerhash = end_blocknumber - start_blocknumber + 1
node_header_hash = qrl_pb2.NodeHeaderHash()
node_header_hash.block_number = start_blocknumber
block = self._state.get_block_by_number(end_blocknumber)
block_headerhash = block.headerhash
node_header_hash.headerhashes.append(block_headerhash)
end_blocknumber -= 1
while end_blocknumber >= start_blocknumber:
block_metadata = self._state.get_block_metadata(
block_headerhash)
for headerhash in block_metadata.last_N_headerhashes[-1::-1]:
node_header_hash.headerhashes.append(headerhash)
end_blocknumber -= len(block_metadata.last_N_headerhashes)
if len(block_metadata.last_N_headerhashes) == 0:
break
block_headerhash = block_metadata.last_N_headerhashes[0]
node_header_hash.headerhashes[:] = node_header_hash.headerhashes[
-1::-1]
del node_header_hash.headerhashes[:len(node_header_hash.
headerhashes) -
total_expected_headerhash]
return node_header_hash
def set_broadcast_tx(self, broadcast_tx):
with self.lock:
self.tx_pool.set_broadcast_tx(broadcast_tx)
def load(self, genesis_block):
# load() has the following tasks:
# Write Genesis Block into State immediately
# Register block_number <-> blockhash mapping
# Calculate difficulty Metadata for Genesis Block
# Generate AddressStates from Genesis Block balances
# Apply Genesis Block's transactions to the state
# Detect if we are forked from genesis block and if so initiate recovery.
height = self._state.get_mainchain_height()
if height == -1:
self._state.put_block(genesis_block, None)
block_number_mapping = qrl_pb2.BlockNumberMapping(
headerhash=genesis_block.headerhash,
prev_headerhash=genesis_block.prev_headerhash)
self._state.put_block_number_mapping(genesis_block.block_number,
block_number_mapping, None)
parent_difficulty = StringToUInt256(
str(config.user.genesis_difficulty))
self.current_difficulty, _ = DifficultyTracker.get(
measurement=config.dev.mining_setpoint_blocktime,
parent_difficulty=parent_difficulty)
block_metadata = BlockMetadata.create()
block_metadata.set_block_difficulty(self.current_difficulty)
block_metadata.set_cumulative_difficulty(self.current_difficulty)
self._state.put_block_metadata(genesis_block.headerhash,
block_metadata, None)
addresses_state = dict()
for genesis_balance in GenesisBlock().genesis_balance:
bytes_addr = genesis_balance.address
addresses_state[bytes_addr] = AddressState.get_default(
bytes_addr)
addresses_state[
bytes_addr]._data.balance = genesis_balance.balance
for tx_idx in range(1, len(genesis_block.transactions)):
tx = Transaction.from_pbdata(
genesis_block.transactions[tx_idx])
for addr in tx.addrs_to:
addresses_state[addr] = AddressState.get_default(addr)
coinbase_tx = Transaction.from_pbdata(
genesis_block.transactions[0])
if not isinstance(coinbase_tx, CoinBase):
return False
addresses_state[coinbase_tx.addr_to] = AddressState.get_default(
coinbase_tx.addr_to)
if not coinbase_tx.validate_extended(genesis_block.block_number):
return False
coinbase_tx.apply_state_changes(addresses_state)
for tx_idx in range(1, len(genesis_block.transactions)):
tx = Transaction.from_pbdata(
genesis_block.transactions[tx_idx])
tx.apply_state_changes(addresses_state)
self._state.put_addresses_state(addresses_state)
self._state.update_tx_metadata(genesis_block, None)
self._state.update_mainchain_height(0, None)
else:
self._last_block = self.get_block_by_number(height)
self.current_difficulty = self._state.get_block_metadata(
self._last_block.headerhash).block_difficulty
fork_state = self._state.get_fork_state()
if fork_state:
block = self._state.get_block(fork_state.initiator_headerhash)
self._fork_recovery(block, fork_state)
def _apply_block(self, block: Block, batch) -> bool:
address_set = self._state.prepare_address_list(
block) # Prepare list for current block
addresses_state = self._state.get_state_mainchain(address_set)
if not block.apply_state_changes(addresses_state):
return False
self._state.put_addresses_state(addresses_state, batch)
return True
def _update_chainstate(self, block: Block, batch):
self._last_block = block
self._update_block_number_mapping(block, batch)
self.tx_pool.remove_tx_in_block_from_pool(block)
self._state.update_mainchain_height(block.block_number, batch)
self._state.update_tx_metadata(block, batch)
def _try_branch_add_block(self,
block,
batch,
check_stale=True) -> (bool, bool):
"""
This function returns list of bool types. The first bool represent
if the block has been added successfully and the second bool
represent the fork_flag, which becomes true when a block triggered
into fork recovery.
:param block:
:param batch:
:return: [Added successfully, fork_flag]
"""
if self._last_block.headerhash == block.prev_headerhash:
if not self._apply_block(block, batch):
return False, False
self._state.put_block(block, batch)
last_block_metadata = self._state.get_block_metadata(
self._last_block.headerhash)
if last_block_metadata is None:
logger.warning("Could not find log metadata for %s",
bin2hstr(self._last_block.headerhash))
return False, False
last_block_difficulty = int(
UInt256ToString(last_block_metadata.cumulative_difficulty))
new_block_metadata = self._add_block_metadata(block.headerhash,
block.timestamp,
block.prev_headerhash,
batch)
new_block_difficulty = int(
UInt256ToString(new_block_metadata.cumulative_difficulty))
if new_block_difficulty > last_block_difficulty:
if self._last_block.headerhash != block.prev_headerhash:
fork_state = qrlstateinfo_pb2.ForkState(
initiator_headerhash=block.headerhash)
self._state.put_fork_state(fork_state, batch)
self._state.write_batch(batch)
return self._fork_recovery(block, fork_state), True
self._update_chainstate(block, batch)
if check_stale:
self.tx_pool.check_stale_txn(self._state, block.block_number)
self.trigger_miner = True
return True, False
def _remove_block_from_mainchain(self, block: Block,
latest_block_number: int, batch):
addresses_set = self._state.prepare_address_list(block)
addresses_state = self._state.get_state_mainchain(addresses_set)
for tx_idx in range(len(block.transactions) - 1, -1, -1):
tx = Transaction.from_pbdata(block.transactions[tx_idx])
tx.revert_state_changes(addresses_state, self)
self.tx_pool.add_tx_from_block_to_pool(block, latest_block_number)
self._state.update_mainchain_height(block.block_number - 1, batch)
self._state.rollback_tx_metadata(block, batch)
self._state.remove_blocknumber_mapping(block.block_number, batch)
self._state.put_addresses_state(addresses_state, batch)
def _get_fork_point(self, block: Block):
tmp_block = block
hash_path = []
while True:
if not block:
raise Exception('[get_state] No Block Found %s, Initiator %s',
block.headerhash, tmp_block.headerhash)
mainchain_block = self.get_block_by_number(block.block_number)
if mainchain_block and mainchain_block.headerhash == block.headerhash:
break
if block.block_number == 0:
raise Exception(
'[get_state] Alternate chain genesis is different, Initiator %s',
tmp_block.headerhash)
hash_path.append(block.headerhash)
block = self._state.get_block(block.prev_headerhash)
return block.headerhash, hash_path
def _rollback(self,
forked_header_hash: bytes,
fork_state: qrlstateinfo_pb2.ForkState = None):
"""
Rollback from last block to the block just before the forked_header_hash
:param forked_header_hash:
:param fork_state:
:return:
"""
hash_path = []
while self._last_block.headerhash != forked_header_hash:
block = self._state.get_block(self._last_block.headerhash)
mainchain_block = self._state.get_block_by_number(
block.block_number)
if block is None:
logger.warning(
"self.state.get_block(self.last_block.headerhash) returned None"
)
if mainchain_block is None:
logger.warning(
"self.get_block_by_number(block.block_number) returned None"
)
if block.headerhash != mainchain_block.headerhash:
break
hash_path.append(self._last_block.headerhash)
batch = self._state.batch
self._remove_block_from_mainchain(self._last_block,
block.block_number, batch)
if fork_state:
fork_state.old_mainchain_hash_path.extend(
[self._last_block.headerhash])
self._state.put_fork_state(fork_state, batch)
self._state.write_batch(batch)
self._last_block = self._state.get_block(
self._last_block.prev_headerhash)
return hash_path
def add_chain(self, hash_path: list,
fork_state: qrlstateinfo_pb2.ForkState) -> bool:
"""
Add series of blocks whose headerhash mentioned into hash_path
:param hash_path:
:param fork_state:
:param batch:
:return:
"""
with self.lock:
start = 0
try:
start = hash_path.index(self._last_block.headerhash) + 1
except ValueError:
# Following condition can only be true if the fork recovery was interrupted last time
if self._last_block.headerhash in fork_state.old_mainchain_hash_path:
return False
for i in range(start, len(hash_path)):
header_hash = hash_path[i]
block = self._state.get_block(header_hash)
batch = self._state.batch
if not self._apply_block(block, batch):
return False
self._update_chainstate(block, batch)
logger.debug('Apply block #%d - [batch %d | %s]',
block.block_number, i, hash_path[i])
self._state.write_batch(batch)
self._state.delete_fork_state()
return True
def _fork_recovery(self, block: Block,
fork_state: qrlstateinfo_pb2.ForkState) -> bool:
logger.info("Triggered Fork Recovery")
# This condition only becomes true, when fork recovery was interrupted
if fork_state.fork_point_headerhash:
logger.info("Recovering from last fork recovery interruption")
forked_header_hash, hash_path = fork_state.fork_point_headerhash, fork_state.new_mainchain_hash_path
else:
forked_header_hash, hash_path = self._get_fork_point(block)
fork_state.fork_point_headerhash = forked_header_hash
fork_state.new_mainchain_hash_path.extend(hash_path)
self._state.put_fork_state(fork_state)
rollback_done = False
if fork_state.old_mainchain_hash_path:
b = self._state.get_block(fork_state.old_mainchain_hash_path[-1])
if b and b.prev_headerhash == fork_state.fork_point_headerhash:
rollback_done = True
if not rollback_done:
logger.info("Rolling back")
old_hash_path = self._rollback(forked_header_hash, fork_state)
else:
old_hash_path = fork_state.old_mainchain_hash_path
if not self.add_chain(hash_path[-1::-1], fork_state):
logger.warning(
"Fork Recovery Failed... Recovering back to old mainchain")
# If above condition is true, then it means, the node failed to add_chain
# Thus old chain state, must be retrieved
self._rollback(forked_header_hash)
self.add_chain(old_hash_path[-1::-1],
fork_state) # Restores the old chain state
return False
logger.info("Fork Recovery Finished")
self.trigger_miner = True
return True
def _add_block(self, block, batch=None, check_stale=True) -> (bool, bool):
self.trigger_miner = False
block_size_limit = self.get_block_size_limit(block)
if block_size_limit and block.size > block_size_limit:
logger.info('Block Size greater than threshold limit %s > %s',
block.size, block_size_limit)
return False, False
return self._try_branch_add_block(block, batch, check_stale)
def add_block(self, block: Block, check_stale=True) -> bool:
with self.lock:
if block.block_number < self.height - config.dev.reorg_limit:
logger.debug('Skipping block #%s as beyond re-org limit',
block.block_number)
return False
if self.get_block_is_duplicate(block):
return False
batch = self._state.batch
block_flag, fork_flag = self._add_block(block,
batch=batch,
check_stale=check_stale)
if block_flag:
if not fork_flag:
self._state.write_batch(batch)
logger.info('Added Block #%s %s', block.block_number,
bin2hstr(block.headerhash))
return True
return False
def _add_block_metadata(self, headerhash, block_timestamp,
parent_headerhash, batch):
block_metadata = self._state.get_block_metadata(headerhash)
if not block_metadata:
block_metadata = BlockMetadata.create()
parent_metadata = self._state.get_block_metadata(parent_headerhash)
parent_block_difficulty = parent_metadata.block_difficulty
parent_cumulative_difficulty = parent_metadata.cumulative_difficulty
block_metadata.update_last_headerhashes(
parent_metadata.last_N_headerhashes, parent_headerhash)
measurement = self._state.get_measurement(block_timestamp,
parent_headerhash,
parent_metadata)
block_difficulty, _ = DifficultyTracker.get(
measurement=measurement, parent_difficulty=parent_block_difficulty)
block_cumulative_difficulty = StringToUInt256(
str(
int(UInt256ToString(block_difficulty)) +
int(UInt256ToString(parent_cumulative_difficulty))))
block_metadata.set_block_difficulty(block_difficulty)
block_metadata.set_cumulative_difficulty(block_cumulative_difficulty)
parent_metadata.add_child_headerhash(headerhash)
self._state.put_block_metadata(parent_headerhash, parent_metadata,
batch)
self._state.put_block_metadata(headerhash, block_metadata, batch)
return block_metadata
def _update_block_number_mapping(self, block, batch):
block_number_mapping = qrl_pb2.BlockNumberMapping(
headerhash=block.headerhash, prev_headerhash=block.prev_headerhash)
self._state.put_block_number_mapping(block.block_number,
block_number_mapping, batch)
def test_create(self):
tp = TransactionPool()
self.assertIsNotNone(tp)
class ChainManager:
def __init__(self, state):
self.state = state
self.tx_pool = TransactionPool(None)
self.last_block = Block.from_json(GenesisBlock().to_json())
self.current_difficulty = StringToUInt256(
str(config.dev.genesis_difficulty))
self.trigger_miner = False
@property
def height(self):
return self.last_block.block_number
def set_broadcast_tx(self, broadcast_tx):
self.tx_pool.set_broadcast_tx(broadcast_tx)
def get_last_block(self) -> Block:
return self.last_block
def get_cumulative_difficulty(self):
last_block_metadata = self.state.get_block_metadata(
self.last_block.headerhash)
return last_block_metadata.cumulative_difficulty
def load(self, genesis_block):
height = self.state.get_mainchain_height()
if height == -1:
self.state.put_block(genesis_block, None)
block_number_mapping = qrl_pb2.BlockNumberMapping(
headerhash=genesis_block.headerhash,
prev_headerhash=genesis_block.prev_headerhash)
self.state.put_block_number_mapping(genesis_block.block_number,
block_number_mapping, None)
parent_difficulty = StringToUInt256(
str(config.dev.genesis_difficulty))
self.current_difficulty, _ = DifficultyTracker.get(
measurement=config.dev.mining_setpoint_blocktime,
parent_difficulty=parent_difficulty)
block_metadata = BlockMetadata.create()
block_metadata.set_orphan(False)
block_metadata.set_block_difficulty(self.current_difficulty)
block_metadata.set_cumulative_difficulty(self.current_difficulty)
self.state.put_block_metadata(genesis_block.headerhash,
block_metadata, None)
addresses_state = dict()
for genesis_balance in GenesisBlock().genesis_balance:
bytes_addr = genesis_balance.address
addresses_state[bytes_addr] = AddressState.get_default(
bytes_addr)
addresses_state[
bytes_addr]._data.balance = genesis_balance.balance
for tx_idx in range(1, len(genesis_block.transactions)):
tx = Transaction.from_pbdata(
genesis_block.transactions[tx_idx])
for addr in tx.addrs_to:
addresses_state[addr] = AddressState.get_default(addr)
coinbase_tx = Transaction.from_pbdata(
genesis_block.transactions[0])
if not isinstance(coinbase_tx, CoinBase):
return False
addresses_state[coinbase_tx.addr_to] = AddressState.get_default(
coinbase_tx.addr_to)
if not coinbase_tx.validate_extended():
return False
coinbase_tx.apply_on_state(addresses_state)
for tx_idx in range(1, len(genesis_block.transactions)):
tx = Transaction.from_pbdata(
genesis_block.transactions[tx_idx])
tx.apply_on_state(addresses_state)
self.state.put_addresses_state(addresses_state)
self.state.update_tx_metadata(genesis_block, None)
self.state.update_mainchain_height(0, None)
else:
self.last_block = self.get_block_by_number(height)
self.current_difficulty = self.state.get_block_metadata(
self.last_block.headerhash).block_difficulty
def validate_mining_nonce(self, block, enable_logging=False):
parent_metadata = self.state.get_block_metadata(block.prev_headerhash)
parent_block = self.state.get_block(block.prev_headerhash)
measurement = self.state.get_measurement(block.timestamp,
block.prev_headerhash,
parent_metadata)
diff, target = DifficultyTracker.get(
measurement=measurement,
parent_difficulty=parent_metadata.block_difficulty)
if enable_logging:
logger.debug('-----------------START--------------------')
logger.debug('Validate #%s', block.block_number)
logger.debug('block.timestamp %s', block.timestamp)
logger.debug('parent_block.timestamp %s', parent_block.timestamp)
logger.debug('parent_block.difficulty %s',
UInt256ToString(parent_metadata.block_difficulty))
logger.debug('diff : %s | target : %s', UInt256ToString(diff),
target)
logger.debug('-------------------END--------------------')
if not self.verify_input_cached(block.mining_blob, target):
if enable_logging:
logger.warning("PoW verification failed")
qn = Qryptonight()
tmp_hash = qn.hash(block.mining_blob)
logger.warning("{}".format(tmp_hash))
logger.debug('%s', block.to_json())
return False
return True
@functools.lru_cache(maxsize=5)
def verify_input_cached(self, mining_blob, target):
return PoWHelper.verifyInput(mining_blob, target)
def validate_block(self, block, address_txn) -> bool:
len_transactions = len(block.transactions)
if len_transactions < 1:
return False
coinbase_tx = Transaction.from_pbdata(block.transactions[0])
if not isinstance(coinbase_tx, CoinBase):
return False
if not coinbase_tx.validate_extended():
return False
if not PoWValidator().validate_mining_nonce(self.state,
block.blockheader):
return False
coinbase_tx.apply_on_state(address_txn)
# TODO: check block reward must be equal to coinbase amount
for tx_idx in range(1, len_transactions):
tx = Transaction.from_pbdata(block.transactions[tx_idx])
if isinstance(tx, CoinBase):
return False
if not tx.validate(
): # TODO: Move this validation, before adding txn to pool
return False
addr_from_pk_state = address_txn[tx.addr_from]
addr_from_pk = Transaction.get_slave(tx)
if addr_from_pk:
addr_from_pk_state = address_txn[addr_from_pk]
if not tx.validate_extended(address_txn[tx.addr_from],
addr_from_pk_state):
return False
expected_nonce = addr_from_pk_state.nonce + 1
if tx.nonce != expected_nonce:
logger.warning('nonce incorrect, invalid tx')
logger.warning('subtype: %s', tx.type)
logger.warning('%s actual: %s expected: %s', tx.addr_from,
tx.nonce, expected_nonce)
return False
if addr_from_pk_state.ots_key_reuse(tx.ots_key):
logger.warning('pubkey reuse detected: invalid tx %s',
tx.txhash)
logger.warning('subtype: %s', tx.type)
return False
tx.apply_on_state(address_txn)
return True
def _pre_check(self, block, ignore_duplicate):
if block.block_number < 1:
return False
if not block.validate():
return False
if (not ignore_duplicate) and self.state.get_block(
block.headerhash): # Duplicate block check
logger.info('Duplicate block %s %s', block.block_number,
bin2hstr(block.headerhash))
return False
return True
def _try_orphan_add_block(self, block, batch):
prev_block_metadata = self.state.get_block_metadata(
block.prev_headerhash)
if prev_block_metadata is None or prev_block_metadata.is_orphan:
self.state.put_block(block, batch)
self.add_block_metadata(block.headerhash, block.timestamp,
block.prev_headerhash, batch)
return True
return False
def _try_branch_add_block(self, block, batch=None) -> bool:
parent_block = self.state.get_block(block.prev_headerhash)
if not block.validate_parent_child_relation(parent_block):
logger.warning('Failed to validate blocks parent child relation')
return False
address_set = self.state.prepare_address_list(
block) # Prepare list for current block
if self.last_block.headerhash == block.prev_headerhash:
address_txn = self.state.get_state_mainchain(address_set)
else:
address_txn, rollback_headerhash, hash_path = self.state.get_state(
block.prev_headerhash, address_set)
if self.validate_block(block, address_txn):
self.state.put_block(block, None)
self.add_block_metadata(block.headerhash, block.timestamp,
block.prev_headerhash, None)
last_block_metadata = self.state.get_block_metadata(
self.last_block.headerhash)
new_block_metadata = self.state.get_block_metadata(
block.headerhash)
last_block_difficulty = int(
UInt256ToString(last_block_metadata.cumulative_difficulty))
new_block_difficulty = int(
UInt256ToString(new_block_metadata.cumulative_difficulty))
if new_block_difficulty > last_block_difficulty:
if self.last_block.headerhash != block.prev_headerhash:
self.rollback(rollback_headerhash, hash_path)
self.state.put_addresses_state(address_txn)
self.last_block = block
self._update_mainchain(block, batch)
self.tx_pool.remove_tx_in_block_from_pool(block)
self.tx_pool.check_stale_txn()
self.state.update_mainchain_height(block.block_number, batch)
self.state.update_tx_metadata(block, batch)
self.trigger_miner = True
return True
return False
def remove_block_from_mainchain(self, block: Block, batch):
addresses_set = self.state.prepare_address_list(block)
addresses_state = self.state.get_state_mainchain(addresses_set)
for tx_idx in range(len(block.transactions) - 1, 0, -1):
tx = Transaction.from_pbdata(block.transactions[tx_idx])
tx.unapply_on_state(addresses_state, self.state)
# TODO: Move txn from block to pool
# self.tx_pool.add_tx_in_block_from_pool(block)
self.state.update_mainchain_height(block.block_number - 1, batch)
self.state.rollback_tx_metadata(block, batch)
self.state.remove_blocknumber_mapping(block.block_number, batch)
def rollback(self, rollback_headerhash, hash_path):
while self.last_block.headerhash != rollback_headerhash:
self.remove_block_from_mainchain(self.last_block, None)
self.last_block = self.state.get_block(
self.last_block.prev_headerhash)
for header_hash in hash_path[-1::-1]:
block = self.state.get_block(header_hash)
address_set = self.state.prepare_address_list(
block) # Prepare list for current block
addresses_state = self.state.get_state_mainchain(address_set)
for tx_idx in range(0, len(block.transactions)):
tx = Transaction.from_pbdata(block.transactions[tx_idx])
tx.apply_on_state(addresses_state)
self.state.put_addresses_state(addresses_state)
self.last_block = block
self._update_mainchain(block, None)
self.tx_pool.remove_tx_in_block_from_pool(block)
self.state.update_mainchain_height(block.block_number, None)
self.state.update_tx_metadata(block, None)
self.trigger_miner = True
def _add_block(self, block, ignore_duplicate=False, batch=None):
self.trigger_miner = False
block_size_limit = self.state.get_block_size_limit(block)
if block_size_limit and block.size > block_size_limit:
logger.info('Block Size greater than threshold limit %s > %s',
block.size, block_size_limit)
return False
if not self._pre_check(block, ignore_duplicate):
logger.debug('Failed pre_check')
return False
if self._try_orphan_add_block(block, batch):
return True
if self._try_branch_add_block(block, batch):
return True
return False
def add_block(self, block: Block) -> bool:
if block.block_number < self.height - config.dev.reorg_limit:
logger.debug('Skipping block #%s as beyond re-org limit',
block.block_number)
return False
batch = self.state.get_batch()
if self._add_block(block, batch=batch):
self.state.write_batch(batch)
self.update_child_metadata(block.headerhash)
return True
return False
def update_child_metadata(self, headerhash):
block_metadata = self.state.get_block_metadata(headerhash)
childs = list(block_metadata.child_headerhashes)
while childs:
child_headerhash = childs.pop(0)
block = self.state.get_block(child_headerhash)
if not block:
continue
if not self._add_block(block, True):
self._prune([block.headerhash], None)
continue
block_metadata = self.state.get_block_metadata(child_headerhash)
childs += block_metadata.child_headerhashes
def _prune(self, childs, batch):
while childs:
child_headerhash = childs.pop(0)
block_metadata = self.state.get_block_metadata(child_headerhash)
childs += block_metadata.child_headerhashes
self.state.delete(bin2hstr(child_headerhash).encode(), batch)
self.state.delete(
b'metadata_' + bin2hstr(child_headerhash).encode(), batch)
def add_block_metadata(self, headerhash, block_timestamp,
parent_headerhash, batch):
block_metadata = self.state.get_block_metadata(headerhash)
if not block_metadata:
block_metadata = BlockMetadata.create()
parent_metadata = self.state.get_block_metadata(parent_headerhash)
block_difficulty = (0, ) * 32 # 32 bytes to represent 256 bit of 0
block_cumulative_difficulty = (
0, ) * 32 # 32 bytes to represent 256 bit of 0
if not parent_metadata:
parent_metadata = BlockMetadata.create()
else:
parent_block = self.state.get_block(parent_headerhash)
if parent_block:
parent_block_difficulty = parent_metadata.block_difficulty
parent_cumulative_difficulty = parent_metadata.cumulative_difficulty
if not parent_metadata.is_orphan:
block_metadata.update_last_headerhashes(
parent_metadata.last_N_headerhashes, parent_headerhash)
measurement = self.state.get_measurement(
block_timestamp, parent_headerhash, parent_metadata)
block_difficulty, _ = DifficultyTracker.get(
measurement=measurement,
parent_difficulty=parent_block_difficulty)
block_cumulative_difficulty = StringToUInt256(
str(
int(UInt256ToString(block_difficulty)) +
int(UInt256ToString(parent_cumulative_difficulty)))
)
block_metadata.set_orphan(parent_metadata.is_orphan)
block_metadata.set_block_difficulty(block_difficulty)
block_metadata.set_cumulative_difficulty(block_cumulative_difficulty)
parent_metadata.add_child_headerhash(headerhash)
self.state.put_block_metadata(parent_headerhash, parent_metadata,
batch)
self.state.put_block_metadata(headerhash, block_metadata, batch)
# Call once to populate the cache
self.state.get_block_datapoint(headerhash)
def _update_mainchain(self, block, batch):
block_number_mapping = None
while block_number_mapping is None or block.headerhash != block_number_mapping.headerhash:
block_number_mapping = qrl_pb2.BlockNumberMapping(
headerhash=block.headerhash,
prev_headerhash=block.prev_headerhash)
self.state.put_block_number_mapping(block.block_number,
block_number_mapping, batch)
block = self.state.get_block(block.prev_headerhash)
block_number_mapping = self.state.get_block_number_mapping(
block.block_number)
def get_block_by_number(self, block_number) -> Optional[Block]:
return self.state.get_block_by_number(block_number)
def get_state(self, headerhash):
return self.state.get_state(headerhash, set())
def get_address(self, address):
return self.state.get_address_state(address)
def get_transaction(self, transaction_hash) -> list:
for tx_set in self.tx_pool.transactions:
tx = tx_set[1].transaction
if tx.txhash == transaction_hash:
return [tx, None]
return self.state.get_tx_metadata(transaction_hash)
def get_headerhashes(self, start_blocknumber):
start_blocknumber = max(0, start_blocknumber)
end_blocknumber = min(self.last_block.block_number,
start_blocknumber + 2 * config.dev.reorg_limit)
total_expected_headerhash = end_blocknumber - start_blocknumber + 1
node_header_hash = qrl_pb2.NodeHeaderHash()
node_header_hash.block_number = start_blocknumber
block = self.state.get_block_by_number(end_blocknumber)
block_headerhash = block.headerhash
node_header_hash.headerhashes.append(block_headerhash)
end_blocknumber -= 1
while end_blocknumber >= start_blocknumber:
block_metadata = self.state.get_block_metadata(block_headerhash)
for headerhash in block_metadata.last_N_headerhashes[-1::-1]:
node_header_hash.headerhashes.append(headerhash)
end_blocknumber -= len(block_metadata.last_N_headerhashes)
if len(block_metadata.last_N_headerhashes) == 0:
break
block_headerhash = block_metadata.last_N_headerhashes[0]
node_header_hash.headerhashes[:] = node_header_hash.headerhashes[
-1::-1]
del node_header_hash.headerhashes[:len(node_header_hash.headerhashes) -
total_expected_headerhash]
return node_header_hash
def add_ephemeral_message(self,
encrypted_ephemeral: EncryptedEphemeralMessage):
self.state.update_ephemeral(encrypted_ephemeral)
def setUp(self):
self.txpool = TransactionPool(None)
class TestTransactionPool(TestCase):
"""
TransactionPool sits between incoming Transactions from the network and Blocks.
First, incoming Transactions are pending Transactions and go into TransactionPool.pending_tx_pool.
The TxnProcessor has to validate them. Once they are validated, the TxnProcessor puts them into
TransactionPool.transaction_pool, where they wait to be put into the next mined Block.
"""
def setUp(self):
self.txpool = TransactionPool(None)
def test_add_tx_to_pool(self):
tx = make_tx()
result = self.txpool.add_tx_to_pool(tx, 1, replacement_getTime())
self.assertTrue(result)
self.assertEqual(len(self.txpool.transactions), 1)
@patch('qrl.core.TransactionPool.TransactionPool.is_full_transaction_pool',
autospec=True)
def test_add_tx_to_pool_while_full(self, m_is_full_func):
m_is_full_func.return_value = True
tx = make_tx()
result = self.txpool.add_tx_to_pool(tx, 1, replacement_getTime())
self.assertFalse(result) # refused to add to the pool
self.assertEqual(len(self.txpool.transactions), 0) # remains untouched
@patch('qrl.core.TransactionPool.config', autospec=True)
def test_is_full_transaction_pool(self, m_config):
m_config.user.transaction_pool_size = 2
result = self.txpool.is_full_transaction_pool()
self.assertFalse(result)
tx1 = make_tx(fee=1)
tx2 = make_tx(fee=2)
self.txpool.add_tx_to_pool(tx1, 1, replacement_getTime())
self.txpool.add_tx_to_pool(tx2, 1, replacement_getTime())
result = self.txpool.is_full_transaction_pool()
self.assertTrue(result)
def test_get_tx_index_from_pool(self):
tx1 = make_tx(txhash=b'red')
tx2 = make_tx(txhash=b'blue')
tx3 = make_tx(txhash=b'qrlpink')
self.txpool.add_tx_to_pool(tx1, 1, replacement_getTime())
self.txpool.add_tx_to_pool(tx2, 1, replacement_getTime())
self.txpool.add_tx_to_pool(tx3, 1, replacement_getTime())
idx = self.txpool.get_tx_index_from_pool(b'qrlpink')
self.assertEqual(idx, 2)
idx = self.txpool.get_tx_index_from_pool(b'red')
self.assertEqual(idx, 0)
idx = self.txpool.get_tx_index_from_pool(b'ultraviolet')
self.assertEqual(idx, -1)
def test_remove_tx_from_pool(self):
tx1 = make_tx(txhash=b'red')
tx2 = make_tx(txhash=b'blue')
tx3 = make_tx(txhash=b'qrlpink')
self.txpool.add_tx_to_pool(tx1, 1, replacement_getTime())
# If we try to remove a tx that wasn't there, the transaction pool should be untouched
self.assertEqual(len(self.txpool.transaction_pool), 1)
self.txpool.remove_tx_from_pool(tx2)
self.assertEqual(len(self.txpool.transaction_pool), 1)
# Now let's remove a tx from the heap. The size should decrease.
self.txpool.add_tx_to_pool(tx2, 1, replacement_getTime())
self.txpool.add_tx_to_pool(tx3, 1, replacement_getTime())
self.assertEqual(len(self.txpool.transaction_pool), 3)
self.txpool.remove_tx_from_pool(tx2)
self.assertEqual(len(self.txpool.transaction_pool), 2)
@patch(
'qrl.core.TransactionPool.TransactionPool.is_full_pending_transaction_pool',
autospec=True)
def test_update_pending_tx_pool(self, m_is_full_pending_transaction_pool):
tx1 = make_tx()
ip = '127.0.0.1'
m_is_full_pending_transaction_pool.return_value = False
# Due to the straightforward way the function is written, no special setup is needed to get the tx to go in.
result = self.txpool.update_pending_tx_pool(tx1, ip)
self.assertTrue(result)
# If we try to re-add the same tx to the pending_tx_pool, though, it should fail.
result = self.txpool.update_pending_tx_pool(tx1, ip)
self.assertFalse(result)
@patch(
'qrl.core.TransactionPool.TransactionPool.is_full_pending_transaction_pool',
autospec=True)
def test_update_pending_tx_pool_tx_already_validated(
self, m_is_full_pending_transaction_pool):
"""
If the tx is already in TransactionPool.transaction_pool, do not add it to pending_tx_pool.
"""
tx1 = make_tx()
ip = '127.0.0.1'
m_is_full_pending_transaction_pool.return_value = False
self.txpool.add_tx_to_pool(tx1, 1, replacement_getTime())
result = self.txpool.update_pending_tx_pool(tx1, ip)
self.assertFalse(result)
@patch(
'qrl.core.TransactionPool.TransactionPool.is_full_pending_transaction_pool',
autospec=True)
def test_update_pending_tx_pool_is_full_already(
self, m_is_full_pending_transaction_pool):
tx1 = make_tx()
ip = '127.0.0.1'
m_is_full_pending_transaction_pool.return_value = True
result = self.txpool.update_pending_tx_pool(tx1, ip)
self.assertFalse(result)
@patch('qrl.core.TransactionPool.logger')
@patch(
'qrl.core.TransactionPool.TransactionPool.is_full_pending_transaction_pool',
autospec=True)
def test_update_pending_tx_pool_rejects_coinbase_txs(
self, m_is_full_pending_transaction_pool, m_logger):
tx1 = CoinBase()
ip = '127.0.0.1'
m_is_full_pending_transaction_pool.return_value = False
result = self.txpool.update_pending_tx_pool(tx1, ip)
self.assertFalse(result)
@patch('qrl.core.TransactionPool.config', autospec=True)
def test_is_full_pending_transaction_pool(self, m_config):
"""
pending_transaction_pool_size is 3, and pending_transaction_pool_reserve is subtracted out of that, so it's 2.
Trying to add in 3 transactions with ignore_reserve=True will fail, but if ignore_reserve=False, it will go in.
However, after that, adding even more transactions will always fail.
"""
m_config.user.pending_transaction_pool_size = 3
m_config.user.pending_transaction_pool_reserve = 1
tx4 = make_tx(txhash=b'red')
tx1 = make_tx(txhash=b'green')
tx3 = make_tx(txhash=b'blue')
tx2 = make_tx(txhash=b'pink')
ip = '127.0.0.1'
self.txpool.update_pending_tx_pool(tx1, ip)
self.txpool.update_pending_tx_pool(tx2, ip)
result = self.txpool.update_pending_tx_pool(tx3,
ip,
ignore_reserve=True)
self.assertFalse(result)
result = self.txpool.update_pending_tx_pool(tx3,
ip,
ignore_reserve=False)
self.assertTrue(result)
result = self.txpool.update_pending_tx_pool(tx4,
ip,
ignore_reserve=True)
self.assertFalse(result)
result = self.txpool.update_pending_tx_pool(tx4,
ip,
ignore_reserve=False)
self.assertFalse(result)
@patch('qrl.core.misc.ntp.getTime', new=replacement_getTime)
def test_get_pending_transaction(self):
"""
Getting a pending transaction also removes it from the TransactionPool.
Because it may return a single None, or two variables, a funny hack is used in TxnProcessor where the return
from this function is stored in one variable then unpacked later if it is not None.
"""
tx1 = make_tx()
ip = '127.0.0.1'
self.txpool.update_pending_tx_pool(tx1, ip)
self.assertEqual(len(self.txpool.pending_tx_pool_hash), 1)
tx_timestamp = self.txpool.get_pending_transaction()
self.assertEqual(tx_timestamp[0], tx1)
self.assertEqual(len(self.txpool.pending_tx_pool_hash), 0)
tx_timestamp = self.txpool.get_pending_transaction()
self.assertIsNone(tx_timestamp)
@patch('qrl.core.TransactionPool.logger')
@patch('qrl.core.txs.Transaction.Transaction.from_pbdata',
return_value=make_tx())
@patch('qrl.core.TransactionPool.TransactionPool.add_tx_to_pool',
return_value=True)
def test_add_tx_from_block_to_pool(self, m_add_tx_to_pool, m_from_pbdata,
m_logger):
m_block = Mock(autospec=Block,
block_number=5,
headerhash=b'test block header')
m_block.transactions = [CoinBase(), make_tx(), make_tx()]
self.txpool.add_tx_from_block_to_pool(m_block, 5)
self.assertEqual(m_add_tx_to_pool.call_count,
2) # 2 because the function ignores the Coinbase tx
# If there is a problem adding to the tx_pool, the logger should be invoked.
m_add_tx_to_pool.return_value = False
self.txpool.add_tx_from_block_to_pool(m_block, 5)
m_logger.warning.assert_called()
@patch('qrl.core.txs.Transaction.Transaction.from_pbdata',
new=replacement_from_pbdata)
def test_remove_tx_in_block_from_pool(self):
m_block = Mock(autospec=Block)
tx1 = make_tx(name='Mock TX 1', ots_key=1, PK=b'pk')
tx2 = make_tx(name='Mock TX 2', ots_key=2, PK=b'pk')
m_block.transactions = [CoinBase(), tx1, tx2]
# To remove the tx from the pool we have to add it first!
self.txpool.add_tx_to_pool(tx1, 5)
self.txpool.add_tx_to_pool(tx2, 5)
self.assertEqual(len(self.txpool.transaction_pool), 2)
self.txpool.remove_tx_in_block_from_pool(m_block)
self.assertEqual(len(self.txpool.transaction_pool), 0)
@patch('qrl.core.TransactionInfo.config', autospec=True)
@patch('qrl.core.TransactionPool.TransactionPool.is_full_transaction_pool',
return_value=False)
def test_check_stale_txn(self, m_is_full_transaction_pool, m_config):
"""
Stale Transactions are Transactions that were supposed to go into block 5, but for some reason didn't make it.
They languish in TransactionPool until check_stale_txn() checks the Pool and updates the tx_info to make them
go into a higher block.
For each stale transaction, P2PFactory.broadcast_tx() will be called.
"""
# Redefine at what point should txs be considered stale
m_config.user.stale_transaction_threshold = 2
bob_xmss = get_bob_xmss(4)
alice_xmss = get_alice_xmss(4)
tx1 = TransferTransaction.create(addrs_to=[bob_xmss.address],
amounts=[1000000],
fee=1,
xmss_pk=alice_xmss.pk)
tx1.sign(alice_xmss)
tx2 = TransferTransaction.create(addrs_to=[bob_xmss.address],
amounts=[10000],
fee=1,
xmss_pk=alice_xmss.pk)
tx2.sign(alice_xmss)
m_broadcast_tx = Mock(
name='Mock Broadcast TX function (in P2PFactory)')
self.txpool.add_tx_to_pool(tx1, 5)
self.txpool.add_tx_to_pool(tx2, 5)
self.txpool.set_broadcast_tx(m_broadcast_tx)
with set_qrl_dir('no_data'):
state = State()
self.txpool.check_stale_txn(state, 8)
self.assertEqual(m_broadcast_tx.call_count, 0)
m = MockFunction()
bob_address_state = AddressState.get_default(bob_xmss.address)
bob_address_state.pbdata.balance = 1000000000000
m.put(bob_xmss.address, bob_address_state)
state.get_address_state = m.get
tx3 = TransferTransaction.create(addrs_to=[alice_xmss.address],
amounts=[10000],
fee=1,
xmss_pk=bob_xmss.pk)
tx3.sign(bob_xmss)
self.txpool.add_tx_to_pool(tx3, 5)
self.txpool.check_stale_txn(state, 8)
self.assertEqual(m_broadcast_tx.call_count, 1)
class TestTransactionPoolRemoveTxInBlockFromPool(TestCase):
"""
Up until 4096 (max_ots_tracking_index), the state of each OTS index USED/UNUSED is stored in a bitfield.
Default height of wallet is 12, so 2^12 = 4096 obviously
Above that however, the network only keeps track of the last used OTS index as a number. So the next
tx.ots_index must be 4096 < ots_index < network_ots_index_counter (AddressState.ots_counter).
Suppose you have a Block with two Transactions from the same public address in it, with ots_index=4098 and 4099.
If TransactionPool has 4097, it should be invalidated because 4098 is already used and we are on an counter
method of keeping track of OTS indexes.
Of course, 4098 and 4099 also have to be deleted.
"""
@patch('qrl.core.misc.ntp.getTime', new=replacement_getTime)
def setUp(self):
self.txpool = TransactionPool(None)
self.tx_3907 = make_tx(name='Mock TX 3907',
txhash=b'h3907',
ots_key=3907)
self.tx_4095 = make_tx(name='Mock TX 4095',
txhash=b'h4095',
ots_key=4095)
self.tx_4096 = make_tx(name='Mock TX 4096',
txhash=b'h4096',
ots_key=4096)
self.tx_4097 = make_tx(name='Mock TX 4097',
txhash=b'h4097',
ots_key=4097)
self.tx_4098 = make_tx(name='Mock TX 4098',
txhash=b'h4098',
ots_key=4098)
self.tx_4099 = make_tx(name='Mock TX 4099',
txhash=b'h4099',
ots_key=4099)
self.tx_4100 = make_tx(name='Mock TX 4100',
txhash=b'h4100',
ots_key=4100)
self.tx_4200 = make_tx(name='Mock TX 4200',
txhash=b'h4200',
ots_key=4200)
# To remove the tx from the pool we have to add it first!
self.txpool.add_tx_to_pool(self.tx_4095, 5)
self.txpool.add_tx_to_pool(self.tx_4096, 5)
self.txpool.add_tx_to_pool(self.tx_4097, 5)
self.txpool.add_tx_to_pool(self.tx_4098, 5)
self.txpool.add_tx_to_pool(self.tx_4099, 5)
self.txpool.add_tx_to_pool(self.tx_4100, 5)
self.txpool.add_tx_to_pool(self.tx_4200, 5)
@patch('qrl.core.TransactionPool.config', autospec=True)
@patch('qrl.core.TransactionPool.TransactionPool.is_full_transaction_pool',
return_value=False)
@patch('qrl.core.txs.Transaction.Transaction.from_pbdata',
new=replacement_from_pbdata)
def test_block_4098_4099(self, m_is_full_transaction_pool, m_config):
"""
TxPool = [4095-4100, 4200]
Block = [4098, 4099]
TxPool Afterwards = [4095, 4100, 4200]
"""
# Ensure that a "large OTS index" is 4096
m_config.dev.max_ots_tracking_index = 4096
m_block = Mock(autospec=Block)
m_block.transactions = [CoinBase(), self.tx_4098, self.tx_4099]
self.txpool.remove_tx_in_block_from_pool(m_block)
txs_in_txpool = [
t[1].transaction for t in self.txpool.transaction_pool
]
self.assertEqual(len(self.txpool.transaction_pool), 3)
self.assertNotIn(self.tx_4097, txs_in_txpool)
self.assertNotIn(self.tx_4098, txs_in_txpool)
self.assertNotIn(self.tx_4099, txs_in_txpool)
self.assertIn(self.tx_4095, txs_in_txpool)
self.assertIn(self.tx_4100, txs_in_txpool)
self.assertIn(self.tx_4200, txs_in_txpool)
@patch('qrl.core.TransactionPool.config', autospec=True)
@patch('qrl.core.TransactionPool.TransactionPool.is_full_transaction_pool',
return_value=False)
@patch('qrl.core.txs.Transaction.Transaction.from_pbdata',
new=replacement_from_pbdata)
def test_txpool_3907_block_4098_4099(self, m_is_full_transaction_pool,
m_config):
"""
TxPool = [3907, 4095-4100, 4200]
Block = [4098, 4099]
TxPool Afterwards = [3907, 4095, 4100, 4200]
"""
# Ensure that a "large OTS index" is 4096
m_config.dev.max_ots_tracking_index = 4096
m_block = Mock(autospec=Block)
m_block.transactions = [CoinBase(), self.tx_4098, self.tx_4099]
self.txpool.add_tx_to_pool(self.tx_3907, 5)
self.txpool.remove_tx_in_block_from_pool(m_block)
txs_in_txpool = [
t[1].transaction for t in self.txpool.transaction_pool
]
# 3907 should also be in the Pool since it is exempt from the counter
self.assertEqual(len(self.txpool.transaction_pool), 4)
self.assertNotIn(self.tx_4097, txs_in_txpool)
self.assertNotIn(self.tx_4098, txs_in_txpool)
self.assertNotIn(self.tx_4099, txs_in_txpool)
self.assertIn(self.tx_3907, txs_in_txpool)
self.assertIn(self.tx_4095, txs_in_txpool)
self.assertIn(self.tx_4100, txs_in_txpool)
self.assertIn(self.tx_4200, txs_in_txpool)
@patch('qrl.core.TransactionPool.config', autospec=True)
@patch('qrl.core.TransactionPool.TransactionPool.is_full_transaction_pool',
return_value=False)
@patch('qrl.core.txs.Transaction.Transaction.from_pbdata',
new=replacement_from_pbdata)
def test_block_4200(self, m_is_full_transaction_pool, m_config):
"""
TxPool = [3907, 4095-4100, 4200]
Block = [4200]
TxPool Afterwards = [3907, 4095]
"""
# Ensure that a "large OTS index" is 4096
m_config.dev.max_ots_tracking_index = 4096
m_block = Mock(autospec=Block)
m_block.transactions = [CoinBase(), self.tx_4200]
self.txpool.add_tx_to_pool(self.tx_3907, 5)
self.txpool.remove_tx_in_block_from_pool(m_block)
txs_in_txpool = [
t[1].transaction for t in self.txpool.transaction_pool
]
self.assertEqual(len(self.txpool.transaction_pool), 2)
self.assertIn(self.tx_3907, txs_in_txpool)
self.assertIn(self.tx_4095, txs_in_txpool)
@patch('qrl.core.TransactionPool.config', autospec=True)
@patch('qrl.core.TransactionPool.TransactionPool.is_full_transaction_pool',
return_value=False)
@patch('qrl.core.txs.Transaction.Transaction.from_pbdata',
new=replacement_from_pbdata)
def test_txpool_4095_4096_4097_otherppl_block_4098_4099(
self, m_is_full_transaction_pool, m_config):
"""
TxPool = [4096-4100, 4200, 4095-4097_otherppl]
Block = [4200]
TxPool Afterwards = [4095, 4095-4097_otherppl]
"""
# Ensure that a "large OTS index" is 4096
m_config.dev.max_ots_tracking_index = 4096
m_block = Mock(autospec=Block)
m_block.transactions = [CoinBase(), self.tx_4200]
tx_other_4095 = make_tx(name='Mock TX 4095',
txhash=b'h4095_other',
ots_key=4095,
PK='otherppl')
tx_other_4096 = make_tx(name='Mock TX 4096',
txhash=b'h4096_other',
ots_key=4096,
PK='otherppl')
tx_other_4097 = make_tx(name='Mock TX 4097',
txhash=b'h4097_other',
ots_key=4097,
PK='otherppl')
self.txpool.add_tx_to_pool(tx_other_4095, 5)
self.txpool.add_tx_to_pool(tx_other_4096, 5)
self.txpool.add_tx_to_pool(tx_other_4097, 5)
self.txpool.remove_tx_in_block_from_pool(m_block)
txs_in_txpool = [
t[1].transaction for t in self.txpool.transaction_pool
]
self.assertEqual(len(self.txpool.transaction_pool), 4)
self.assertIn(self.tx_4095, txs_in_txpool)
self.assertIn(tx_other_4095, txs_in_txpool)
self.assertIn(tx_other_4096, txs_in_txpool)
self.assertIn(tx_other_4097, txs_in_txpool)
@patch('qrl.core.TransactionPool.config', autospec=True)
@patch('qrl.core.TransactionPool.TransactionPool.is_full_transaction_pool',
return_value=False)
@patch('qrl.core.txs.Transaction.Transaction.from_pbdata',
new=replacement_from_pbdata)
def test_block_1000(self, m_is_full_transaction_pool, m_config):
"""
TxPool = [4095-4100, 4200]
Block = [1000]
TxPool Afterwards = [4095-4100, 4200]
"""
# Ensure that a "large OTS index" is 4096
m_config.dev.max_ots_tracking_index = 4096
tx_1000 = make_tx(name='Mock TX 1000', txhash=b'h1000', ots_key=1000)
m_block = Mock(autospec=Block)
m_block.transactions = [CoinBase(), tx_1000]
self.assertEqual(7, len(self.txpool.transaction_pool))
self.txpool.remove_tx_in_block_from_pool(m_block)
txs_in_txpool = [
t[1].transaction for t in self.txpool.transaction_pool
]
self.assertEqual(7, len(txs_in_txpool))
class TestMinerWithRealTransactionPool(TestCase):
def setUp(self):
self.m_mining_qaddress = alice.qaddress
self.m_mining_address = parse_qaddress(self.m_mining_qaddress)
self.alice_address_state = Mock(
autospec=OptimizedAddressState,
name='mock alice OptimizedAddressState')
self.chain_manager = Mock(spec=ChainManager)
self.chain_manager.get_block_size_limit.return_value = 500
self.chain_manager.get_address_state.return_value = self.alice_address_state
self.chain_manager.get_config_by_block_number.return_value = config.dev
self.parent_block = Block()
self.parent_difficulty = StringToUInt256(
'0') # tuple (0,0,0,0,0...) length 32
self.m_pre_block_logic = Mock(spec=POW.pre_block_logic, name='hello')
mining_thread_count = 1
self.miner = Miner(self.chain_manager, self.m_pre_block_logic,
self.m_mining_address, mining_thread_count)
self.txpool = TransactionPool(None)
def replacement_set_affected_address(addresses_set):
return addresses_set.add(alice.address)
self.m_tx_args = {
"addr_from": alice.address,
"addrs_to": [bob.address],
"amounts": [10],
"fee": 1,
"PK": alice.pk,
"master_addr": None,
"size": 150,
"validate_extended.return_value": True,
"set_affected_address": replacement_set_affected_address
}
def mock_new_state_container(self):
addresses_state = {alice.address: self.alice_address_state}
self.chain_manager.new_state_container.return_value = StateContainer(
addresses_state=addresses_state,
tokens=Indexer(b'token', None),
slaves=Indexer(b'slave', None),
lattice_pk=Indexer(b'lattice_pk', None),
multi_sig_spend_txs=dict(),
votes_stats=dict(),
block_number=5,
total_coin_supply=0,
current_dev_config=config.dev,
write_access=False,
my_db=None,
batch=None)
@patch('qrl.core.Miner.Block.create')
def test_create_block_with_one_transaction(self, m_create, m_logger):
self.mock_new_state_container()
m_tx = Mock(autospec=TransferTransaction,
name='mock TransferTransaction')
m_tx.fee = 0
m_tx.configure_mock(**self.m_tx_args)
m_create.return_value = Mock(autospec=Block,
name='mock Block',
size=205)
self.txpool.add_tx_to_pool(m_tx, 1, replacement_getTime())
self.miner.create_block(last_block=self.parent_block,
mining_nonce=0,
tx_pool=self.txpool,
miner_address=self.m_mining_address)
seed_block = self.chain_manager.get_block_by_number(
self.miner._qn.get_seed_height(self.parent_block.block_number + 1))
m_create.assert_called_with(dev_config=config.dev,
block_number=1,
prev_headerhash=b'',
prev_timestamp=0,
transactions=[m_tx],
miner_address=alice.address,
seed_height=seed_block.block_number,
seed_hash=seed_block.headerhash)
@patch('qrl.core.Miner.Block.create')
def test_create_block_does_not_include_invalid_txs_from_txpool(
self, m_create, m_logger):
self.mock_new_state_container()
m_tx = Mock(autospec=TransferTransaction,
name='mock TransferTransaction')
m_tx.validate_all.return_value = False
m_tx.configure_mock(**self.m_tx_args)
m_create.return_value = Mock(autospec=Block,
name='mock Block',
size=205)
self.txpool.add_tx_to_pool(m_tx, 1, replacement_getTime())
self.miner.create_block(last_block=self.parent_block,
mining_nonce=0,
tx_pool=self.txpool,
miner_address=self.m_mining_address)
seed_block = self.chain_manager.get_block_by_number(
self.miner._qn.get_seed_height(self.parent_block.block_number + 1))
m_create.assert_called_with(dev_config=config.dev,
block_number=1,
prev_headerhash=b'',
prev_timestamp=0,
transactions=[],
miner_address=alice.address,
seed_height=seed_block.block_number,
seed_hash=seed_block.headerhash)
class TestMinerWithRealTransactionPool(TestCase):
def setUp(self):
self.m_mining_qaddress = alice.qaddress
self.m_mining_address = parse_qaddress(self.m_mining_qaddress)
self.chain_manager = Mock(spec=ChainManager)
self.chain_manager.get_block_size_limit.return_value = 500
self.chain_manager.get_address_state.return_value = Mock(autospec=AddressState, name='mock alice AddressState')
self.parent_block = Block()
self.parent_difficulty = StringToUInt256('0') # tuple (0,0,0,0,0...) length 32
self.m_pre_block_logic = Mock(spec=POW.pre_block_logic, name='hello')
m_add_unprocessed_txn_fn = create_autospec(P2PFactory.add_unprocessed_txn)
mining_thread_count = 1
self.miner = Miner(self.m_pre_block_logic,
self.m_mining_address,
self.chain_manager,
mining_thread_count,
m_add_unprocessed_txn_fn)
self.txpool = TransactionPool(None)
def replacement_set_affected_address(addresses_set):
return addresses_set.add(alice.address)
self.m_tx_args = {"addr_from": alice.address,
"addrs_to": [bob.address],
"amounts": [10],
"fee": 1,
"PK": alice.pk,
"master_addr": None,
"size": 150,
"validate_extended.return_value": True,
"set_affected_address": replacement_set_affected_address
}
@patch('qrl.core.Miner.Block.create')
def test_create_block_with_one_transaction(self, m_create, m_logger):
m_tx = Mock(autospec=TransferTransaction, name='mock TransferTransaction')
m_tx.configure_mock(**self.m_tx_args)
m_create.return_value = Mock(autospec=Block, name='mock Block', size=205)
self.txpool.add_tx_to_pool(m_tx, 1, replacement_getTime())
self.miner.create_block(last_block=self.parent_block, mining_nonce=0, tx_pool=self.txpool,
miner_address=self.m_mining_address)
m_create.assert_called_with(block_number=1, prev_headerhash=b'', prev_timestamp=0, transactions=[m_tx],
miner_address=alice.address)
@patch('qrl.core.Miner.Block.create')
def test_create_block_does_not_include_invalid_txs_from_txpool(self, m_create, m_logger):
self.m_tx_args["validate_extended.return_value"] = False
m_tx = Mock(autospec=TransferTransaction, name='mock TransferTransaction')
m_tx.configure_mock(**self.m_tx_args)
m_create.return_value = Mock(autospec=Block, name='mock Block', size=205)
self.txpool.add_tx_to_pool(m_tx, 1, replacement_getTime())
self.miner.create_block(last_block=self.parent_block, mining_nonce=0, tx_pool=self.txpool,
miner_address=self.m_mining_address)
m_create.assert_called_with(block_number=1, prev_headerhash=b'', prev_timestamp=0, transactions=[],
miner_address=alice.address)
def create_block(self,
last_block,
mining_nonce,
tx_pool: TransactionPool,
miner_address) -> Optional[Block]:
seed_block = self._chain_manager.get_block_by_number(self._qn.get_seed_height(last_block.block_number + 1))
dev_config = self._chain_manager.get_config_by_block_number(block_number=last_block.block_number + 1)
dummy_block = Block.create(dev_config=dev_config,
block_number=last_block.block_number + 1,
prev_headerhash=last_block.headerhash,
prev_timestamp=last_block.timestamp,
transactions=[],
miner_address=miner_address,
seed_height=seed_block.block_number,
seed_hash=seed_block.headerhash)
dummy_block.set_nonces(dev_config, mining_nonce, 0)
t_pool2 = tx_pool.transactions
block_size = dummy_block.size
block_size_limit = self._chain_manager.get_block_size_limit(last_block, dev_config)
transactions = []
state_container = self._chain_manager.new_state_container(set(),
last_block.block_number,
True,
None)
for tx_set in t_pool2:
tx = tx_set[1].transaction
# Skip Transactions for later, which doesn't fit into block
if block_size + tx.size + dev_config.tx_extra_overhead > block_size_limit:
continue
if not self._chain_manager.update_state_container(tx, state_container):
logger.error("[create_block] Error updating state_container")
return None
if not tx.validate_all(state_container, check_nonce=False):
if not state_container.revert_update():
return None
tx_pool.remove_tx_from_pool(tx)
continue
if not self._chain_manager.apply_txn(tx, state_container):
logger.error("[create_block] Failed to apply txn")
if not state_container.revert_update():
return None
continue
addr_from_pk_state = state_container.addresses_state[tx.addr_from]
addr_from_pk = Transaction.get_slave(tx)
if addr_from_pk:
addr_from_pk_state = state_container.addresses_state[addr_from_pk]
tx._data.nonce = addr_from_pk_state.nonce
block_size += tx.size + dev_config.tx_extra_overhead
transactions.append(tx)
block = Block.create(dev_config=dev_config,
block_number=last_block.block_number + 1,
prev_headerhash=last_block.headerhash,
prev_timestamp=last_block.timestamp,
transactions=transactions,
miner_address=miner_address,
seed_height=seed_block.block_number,
seed_hash=seed_block.headerhash)
return block
class ChainManager:
def __init__(self, state):
self.state = state
self.tx_pool = TransactionPool() # TODO: Move to some pool manager
self.last_block = Block.from_json(GenesisBlock().to_json())
self.current_difficulty = StringToUInt256(str(config.dev.genesis_difficulty))
self._difficulty_tracker = DifficultyTracker()
self.trigger_miner = False
@property
def height(self):
return self.last_block.block_number
def get_last_block(self) -> Block:
return self.last_block
def get_cumulative_difficulty(self):
last_block_metadata = self.state.get_block_metadata(self.last_block.headerhash)
return last_block_metadata.cumulative_difficulty
def load(self, genesis_block):
height = self.state.get_mainchain_height()
if height == -1:
self.state.put_block(genesis_block, None)
block_number_mapping = qrl_pb2.BlockNumberMapping(headerhash=genesis_block.headerhash,
prev_headerhash=genesis_block.prev_headerhash)
self.state.put_block_number_mapping(genesis_block.block_number, block_number_mapping, None)
parent_difficulty = StringToUInt256(str(config.dev.genesis_difficulty))
self.current_difficulty, _ = self._difficulty_tracker.get(
measurement=config.dev.mining_setpoint_blocktime,
parent_difficulty=parent_difficulty)
block_metadata = BlockMetadata.create()
block_metadata.set_orphan(False)
block_metadata.set_block_difficulty(self.current_difficulty)
block_metadata.set_cumulative_difficulty(self.current_difficulty)
self.state.put_block_metadata(genesis_block.headerhash, block_metadata, None)
addresses_state = dict()
for genesis_balance in GenesisBlock().genesis_balance:
bytes_addr = genesis_balance.address.encode()
addresses_state[bytes_addr] = AddressState.get_default(bytes_addr)
addresses_state[bytes_addr]._data.balance = genesis_balance.balance
self.state.state_objects.update_current_state(addresses_state)
self.state.state_objects.push(genesis_block.headerhash)
else:
self.last_block = self.get_block_by_number(height)
self.current_difficulty = self.state.get_block_metadata(self.last_block.headerhash).block_difficulty
def validate_mining_nonce(self, block, enable_logging=False):
parent_metadata = self.state.get_block_metadata(block.prev_headerhash)
parent_block = self.state.get_block(block.prev_headerhash)
input_bytes = StringToUInt256(str(block.mining_nonce))[-4:] + tuple(block.mining_hash)
measurement = self.state.get_measurement(block.timestamp, block.prev_headerhash)
diff, target = self._difficulty_tracker.get(
measurement=measurement,
parent_difficulty=parent_metadata.block_difficulty)
if enable_logging:
logger.debug('-----------------START--------------------')
logger.debug('Validate #%s', block.block_number)
logger.debug('block.timestamp %s', block.timestamp)
logger.debug('parent_block.timestamp %s', parent_block.timestamp)
logger.debug('parent_block.difficulty %s', UInt256ToString(parent_metadata.block_difficulty))
logger.debug('input_bytes %s', UInt256ToString(input_bytes))
logger.debug('diff : %s | target : %s', UInt256ToString(diff), target)
logger.debug('-------------------END--------------------')
if not PoWHelper.verifyInput(input_bytes, target):
if enable_logging:
logger.warning("PoW verification failed")
qn = Qryptonight()
tmp_hash = qn.hash(input_bytes)
logger.warning("{}".format(tmp_hash))
logger.debug('%s', block.to_json())
return False
return True
def validate_block(self, block, address_txn) -> bool:
len_transactions = len(block.transactions)
if len_transactions < 1:
return False
coinbase_tx = Transaction.from_pbdata(block.transactions[0])
coinbase_tx.validate()
if not self.validate_mining_nonce(block):
return False
if coinbase_tx.subtype != qrl_pb2.Transaction.COINBASE:
return False
if not coinbase_tx.validate():
return False
coinbase_tx.apply_on_state(address_txn)
addr_from_pk_state = address_txn[coinbase_tx.txto]
addr_from_pk = Transaction.get_slave(coinbase_tx)
if addr_from_pk:
addr_from_pk_state = address_txn[addr_from_pk]
if not coinbase_tx.validate_extended(address_txn[coinbase_tx.txto],
addr_from_pk_state,
[]):
return False
# TODO: check block reward must be equal to coinbase amount
for tx_idx in range(1, len_transactions):
tx = Transaction.from_pbdata(block.transactions[tx_idx])
if tx.subtype == qrl_pb2.Transaction.COINBASE:
return False
if not tx.validate(): # TODO: Move this validation, before adding txn to pool
return False
addr_from_pk_state = address_txn[tx.txfrom]
addr_from_pk = Transaction.get_slave(tx)
if addr_from_pk:
addr_from_pk_state = address_txn[addr_from_pk]
if not tx.validate_extended(address_txn[tx.txfrom], addr_from_pk_state, []):
return False
expected_nonce = address_txn[tx.txfrom].nonce + 1
if tx.nonce != expected_nonce:
logger.warning('nonce incorrect, invalid tx')
logger.warning('subtype: %s', tx.subtype)
logger.warning('%s actual: %s expected: %s', tx.txfrom, tx.nonce, expected_nonce)
return False
if tx.ots_key_reuse(address_txn[tx.txfrom], tx.ots_key):
logger.warning('pubkey reuse detected: invalid tx %s', tx.txhash)
logger.warning('subtype: %s', tx.subtype)
return False
tx.apply_on_state(address_txn)
return True
def _pre_check(self, block, ignore_duplicate):
if block.block_number < 1:
return False
if not block.validate():
return False
if (not ignore_duplicate) and self.state.get_block(block.headerhash): # Duplicate block check
logger.info('Duplicate block %s %s', block.block_number, bin2hstr(block.headerhash))
return False
return True
def _try_orphan_add_block(self, block, batch):
prev_block_metadata = self.state.get_block_metadata(block.prev_headerhash)
self.trigger_miner = False
if prev_block_metadata is None or prev_block_metadata.is_orphan:
self.state.put_block(block, batch)
self.add_block_metadata(block.headerhash, block.timestamp, block.prev_headerhash, batch)
return True
return False
def _try_branch_add_block(self, block, batch=None) -> bool:
parent_block = self.state.get_block(block.prev_headerhash)
if not block.validate_parent_child_relation(parent_block):
logger.warning('Failed to validate blocks parent child relation')
return False
address_set = self.state.prepare_address_list(block) # Prepare list for current block
if self.last_block.headerhash == block.prev_headerhash:
address_txn = self.state.get_state_mainchain(address_set)
else:
address_txn = self.state.get_state(block.prev_headerhash, address_set)
if self.validate_block(block, address_txn):
self.state.put_block(block, None)
self.add_block_metadata(block.headerhash, block.timestamp, block.prev_headerhash, None)
last_block_metadata = self.state.get_block_metadata(self.last_block.headerhash)
new_block_metadata = self.state.get_block_metadata(block.headerhash)
last_block_difficulty = int(UInt256ToString(last_block_metadata.cumulative_difficulty))
new_block_difficulty = int(UInt256ToString(new_block_metadata.cumulative_difficulty))
self.trigger_miner = False
if new_block_difficulty > last_block_difficulty:
if self.last_block.headerhash != block.prev_headerhash:
self.rollback(block)
return True
self.state.update_mainchain_state(address_txn, block.block_number, block.headerhash)
self.last_block = block
self._update_mainchain(block, batch)
self.tx_pool.remove_tx_in_block_from_pool(block)
self.state.update_mainchain_height(block.block_number, batch)
self.state.update_tx_metadata(block, batch)
self.trigger_miner = True
return True
return False
def rollback(self, block):
hash_path = []
while True:
if self.state.state_objects.contains(block.headerhash):
break
hash_path.append(block.headerhash)
new_block = self.state.get_block(block.prev_headerhash)
if not new_block:
logger.warning('No block found %s', block.prev_headerhash)
break
block = new_block
if block.block_number == 0:
del hash_path[-1] # Skip replaying Genesis Block
break
self.state.state_objects.destroy_current_state(None)
block = self.state.get_block(hash_path[-1])
self.state.state_objects.destroy_fork_states(block.block_number, block.headerhash)
for header_hash in hash_path[-1::-1]:
block = self.state.get_block(header_hash)
address_set = self.state.prepare_address_list(block) # Prepare list for current block
address_txn = self.state.get_state_mainchain(address_set)
self.state.update_mainchain_state(address_txn, block.block_number, block.headerhash)
self.last_block = block
self._update_mainchain(block, None)
self.tx_pool.remove_tx_in_block_from_pool(block)
self.state.update_mainchain_height(block.block_number, None)
self.state.update_tx_metadata(block, None)
self.trigger_miner = True
def _add_block(self, block, ignore_duplicate=False, batch=None):
block_size_limit = self.state.get_block_size_limit(block)
if block_size_limit and block.size > block_size_limit:
logger.info('Block Size greater than threshold limit %s > %s', block.size, block_size_limit)
return False
if not self._pre_check(block, ignore_duplicate):
logger.debug('Failed pre_check')
return False
if self._try_orphan_add_block(block, batch):
return True
if self._try_branch_add_block(block, batch):
return True
return False
def add_block(self, block: Block) -> bool:
if block.block_number < self.height - config.dev.reorg_limit:
logger.debug('Skipping block #%s as beyond re-org limit', block.block_number)
return False
batch = self.state.get_batch()
if self._add_block(block, batch=batch):
self.state.write_batch(batch)
self.update_child_metadata(block.headerhash)
return True
return False
def update_child_metadata(self, headerhash):
block_metadata = self.state.get_block_metadata(headerhash)
childs = list(block_metadata.child_headerhashes)
while childs:
child_headerhash = childs.pop(0)
block = self.state.get_block(child_headerhash)
if not block:
continue
if not self._add_block(block, True):
self._prune([block.headerhash], None)
continue
block_metadata = self.state.get_block_metadata(child_headerhash)
childs += block_metadata.child_headerhashes
def _prune(self, childs, batch):
while childs:
child_headerhash = childs.pop(0)
block_metadata = self.state.get_block_metadata(child_headerhash)
childs += block_metadata.child_headerhashes
self.state.delete(bin2hstr(child_headerhash).encode(), batch)
self.state.delete(b'metadata_' + bin2hstr(child_headerhash).encode(), batch)
def add_block_metadata(self,
headerhash,
block_timestamp,
parent_headerhash,
batch):
parent_metadata = self.state.get_block_metadata(parent_headerhash)
block_difficulty = (0,) * 32 # 32 bytes to represent 256 bit of 0
block_cumulative_difficulty = (0,) * 32 # 32 bytes to represent 256 bit of 0
if not parent_metadata:
parent_metadata = BlockMetadata.create()
else:
parent_block = self.state.get_block(parent_headerhash)
if parent_block:
parent_block_difficulty = parent_metadata.block_difficulty
parent_cumulative_difficulty = parent_metadata.cumulative_difficulty
if not parent_metadata.is_orphan:
measurement = self.state.get_measurement(block_timestamp, parent_headerhash)
block_difficulty, _ = self._difficulty_tracker.get(
measurement=measurement,
parent_difficulty=parent_block_difficulty)
block_cumulative_difficulty = StringToUInt256(str(
int(UInt256ToString(block_difficulty)) +
int(UInt256ToString(parent_cumulative_difficulty))))
block_metadata = self.state.get_block_metadata(headerhash)
if not block_metadata:
block_metadata = BlockMetadata.create()
block_metadata.set_orphan(parent_metadata.is_orphan)
block_metadata.set_block_difficulty(block_difficulty)
block_metadata.set_cumulative_difficulty(block_cumulative_difficulty)
parent_metadata.add_child_headerhash(headerhash)
self.state.put_block_metadata(parent_headerhash, parent_metadata, batch)
self.state.put_block_metadata(headerhash, block_metadata, batch)
def _update_mainchain(self, block, batch):
measurement = self.state.get_measurement(block.timestamp, block.prev_headerhash)
self.current_difficulty, _ = self._difficulty_tracker.get(
measurement=measurement,
parent_difficulty=self.current_difficulty)
block_number_mapping = None
while block_number_mapping is None or block.headerhash != block_number_mapping.headerhash:
block_number_mapping = qrl_pb2.BlockNumberMapping(headerhash=block.headerhash,
prev_headerhash=block.prev_headerhash)
self.state.put_block_number_mapping(block.block_number, block_number_mapping, batch)
block = self.state.get_block(block.prev_headerhash)
block_number_mapping = self.state.get_block_number_mapping(block.block_number)
def get_block_by_number(self, block_number) -> Optional[Block]:
return self.state.get_block_by_number(block_number)
def get_state(self, headerhash):
return self.state.get_state(headerhash, set())
def get_address(self, address):
return self.state.get_address(address)
def get_transaction(self, transaction_hash) -> list:
for tx in self.tx_pool.transaction_pool:
if tx.txhash == transaction_hash:
return [tx, None]
return self.state.get_tx_metadata(transaction_hash)
def get_headerhashes(self, start_blocknumber):
start_blocknumber = max(0, start_blocknumber)
end_blocknumber = min(self.last_block.block_number,
start_blocknumber + config.dev.reorg_limit)
node_header_hash = qrl_pb2.NodeHeaderHash()
node_header_hash.block_number = start_blocknumber
for i in range(start_blocknumber, end_blocknumber + 1):
block = self.state.get_block_by_number(i)
node_header_hash.headerhashes.append(block.headerhash)
return node_header_hash
def add_ephemeral_message(self, encrypted_ephemeral: EncryptedEphemeralMessage):
self.state.update_ephemeral(encrypted_ephemeral)