def __init__(self, chain_manager: ChainManager, sync_state: SyncState,
qrl_node):
self.master_mr = MessageReceipt()
self.pow = None
self.sync_state = sync_state
self._ntp = ntp
self._qrl_node = qrl_node
self._chain_manager = chain_manager
self._chain_manager.set_broadcast_tx(self.broadcast_tx)
self._syncing_enabled = False
self._target_channel = None
self._target_node_header_hash = None
self._last_requested_block_number = None
self._genesis_processed = False
self._peer_connections = []
self._txn_processor_running = False
self.peer_blockheight = dict()
reactor.callLater(config.user.monitor_connections_interval,
self.monitor_connections)
self.p2p_msg_priority = p2p_msg_priority
# Maintains the list of ips in the queue that can be tried to form a new p2p connection
self._peer_q = []
def __init__(self, chain_manager: ChainManager, sync_state: SyncState,
qrl_node):
self.master_mr = MessageReceipt()
self.pow = None
self.sync_state = sync_state
self._ntp = ntp
self._qrl_node = qrl_node
self._chain_manager = chain_manager
self._chain_manager.set_broadcast_tx(self.broadcast_tx)
self._syncing_enabled = False
self._target_peer = None
self._target_node_header_hash = None
self._last_requested_block_idx = None
self._genesis_processed = False
self._peer_connections = []
self._synced_peers_protocol = set()
self._txn_processor_running = False
self.peer_blockheight = dict()
reactor.callLater(config.user.monitor_connections_interval,
self.monitor_connections)
self.p2p_msg_priority = p2p_msg_priority
def __init__(self, buffered_chain: BufferedChain, sync_state: SyncState,
qrl_node: QRLNode):
# FIXME: Constructor signature is not consistent with other factory classes
self.master_mr = MessageReceipt()
self.pos = None
self.ntp = ntp
self.buffered_chain = buffered_chain
self.sync_state = sync_state
self.sync = 0
self.genesis_processed = False # FIXME: Accessed by every p2pprotocol instance
self.peer_connections = [
] # FIXME: Accessed by every p2pprotocol instance
self.synced_peers = set(
) # FIXME: Accessed by every p2pprotocol instance
self.qrl_node = qrl_node
self.txn_processor_running = False
self.bkmr_blocknumber = 0 # Blocknumber for which bkmr is being tracked
self.bkmr_priorityq = queue.PriorityQueue()
# Scheduled and cancel the call, just to initialize with IDelayedCall
self.bkmr_processor = reactor.callLater(1, lambda: None, pos=None)
self.bkmr_processor.cancel()
def test_register(self):
mr = MessageReceipt()
msg_hash = str2bin("asdf")
msg_obj = [1, 2, 3, 4]
msg_type = mr.allowed_types[0]
mr.register(msg_type, msg_hash, msg_obj)
def test_register(self):
mr = MessageReceipt()
msg_hash = str2bin("asdf")
msg_obj = [1, 2, 3, 4]
msg_type = mr.allowed_types[0]
mr.register(msg_type, msg_hash, msg_obj)
def test_contains(self):
mr = MessageReceipt()
msg_hash = str2bin("hash_valid")
msg_obj = [1, 2, 3, 4]
msg_type = mr.allowed_types[0]
mr.register(msg_type, msg_hash, msg_obj)
self.assertTrue(mr.contains(msg_hash, msg_type))
def test_contains(self):
mr = MessageReceipt()
msg_hash = str2bin("hash_valid")
msg_obj = [1, 2, 3, 4]
msg_type = mr.allowed_types[0]
mr.register(msg_type, msg_hash, msg_obj)
self.assertTrue(mr.contains(msg_hash, msg_type))
def __init__(self,
chain_manager: ChainManager,
sync_state: SyncState,
qrl_node):
self.master_mr = MessageReceipt()
self.pow = None
self.sync_state = sync_state
self._ntp = ntp
self._qrl_node = qrl_node
self._chain_manager = chain_manager
self._chain_manager.set_broadcast_tx(self.broadcast_tx)
self._syncing_enabled = False
self._target_peer = None
self._target_node_header_hash = None
self._last_requested_block_idx = None
self._genesis_processed = False
self._peer_connections = []
self._synced_peers_protocol = set()
self._txn_processor_running = False
self.peer_blockheight = dict()
reactor.callLater(config.user.monitor_connections_interval, self.monitor_connections)
self.p2p_msg_priority = {
qrllegacy_pb2.LegacyMessage.VE: 0,
qrllegacy_pb2.LegacyMessage.PL: 0,
qrllegacy_pb2.LegacyMessage.PONG: 0,
######################
qrllegacy_pb2.LegacyMessage.MR: 2,
qrllegacy_pb2.LegacyMessage.SFM: 1,
qrllegacy_pb2.LegacyMessage.BK: 1,
qrllegacy_pb2.LegacyMessage.FB: 1,
qrllegacy_pb2.LegacyMessage.PB: 1,
qrllegacy_pb2.LegacyMessage.BH: 1,
############################
qrllegacy_pb2.LegacyMessage.TX: 1,
qrllegacy_pb2.LegacyMessage.MT: 1,
qrllegacy_pb2.LegacyMessage.TK: 1,
qrllegacy_pb2.LegacyMessage.TT: 1,
qrllegacy_pb2.LegacyMessage.LT: 1,
qrllegacy_pb2.LegacyMessage.SL: 1,
qrllegacy_pb2.LegacyMessage.EPH: 3,
qrllegacy_pb2.LegacyMessage.SYNC: 0,
qrllegacy_pb2.LegacyMessage.CHAINSTATE: 0,
qrllegacy_pb2.LegacyMessage.HEADERHASHES: 1,
qrllegacy_pb2.LegacyMessage.P2P_ACK: 0,
}
def test_register_overflow(self):
mr = MessageReceipt()
msg_obj = [1, 2, 3, 4]
msg_type = mr.allowed_types[0]
config.dev.message_q_size = 4
for i in range(config.dev.message_q_size * 2):
msg_hash = str2bin(str(i))
mr.register(msg_type, msg_hash, msg_obj)
self.assertEqual(len(mr._hash_msg), config.dev.message_q_size)
def test_register_overflow(self):
mr = MessageReceipt()
msg_obj = [1, 2, 3, 4]
msg_type = mr.allowed_types[0]
config.dev.message_q_size = 4
for i in range(config.dev.message_q_size * 2):
msg_hash = str2bin(str(i))
mr.register(msg_type, msg_hash, msg_obj)
self.assertEqual(len(mr._hash_msg), config.dev.message_q_size)
def __init__(self,
chain_manager: ChainManager,
sync_state: SyncState,
qrl_node):
self.master_mr = MessageReceipt()
self.pow = None
self.sync_state = sync_state
self._ntp = ntp
self._qrl_node = qrl_node
self._chain_manager = chain_manager
self._syncing_enabled = False
self._target_peer = None
self._target_node_header_hash = None
self._last_requested_block_idx = None
self._genesis_processed = False
self._peer_connections = []
self._synced_peers_protocol = set()
self._txn_processor_running = False
self.peer_blockheight = dict()
reactor.callLater(180, self.monitor_connections)
self.p2p_msg_priority = {
qrllegacy_pb2.LegacyMessage.VE: 0,
qrllegacy_pb2.LegacyMessage.PL: 0,
qrllegacy_pb2.LegacyMessage.PONG: 0,
######################
qrllegacy_pb2.LegacyMessage.MR: 2,
qrllegacy_pb2.LegacyMessage.SFM: 1,
qrllegacy_pb2.LegacyMessage.BK: 1,
qrllegacy_pb2.LegacyMessage.FB: 1,
qrllegacy_pb2.LegacyMessage.PB: 1,
qrllegacy_pb2.LegacyMessage.BH: 1,
############################
qrllegacy_pb2.LegacyMessage.TX: 1,
qrllegacy_pb2.LegacyMessage.MT: 1,
qrllegacy_pb2.LegacyMessage.TK: 1,
qrllegacy_pb2.LegacyMessage.TT: 1,
qrllegacy_pb2.LegacyMessage.LT: 1,
qrllegacy_pb2.LegacyMessage.SL: 1,
qrllegacy_pb2.LegacyMessage.EPH: 3,
qrllegacy_pb2.LegacyMessage.SYNC: 0,
qrllegacy_pb2.LegacyMessage.CHAINSTATE: 0,
qrllegacy_pb2.LegacyMessage.HEADERHASHES: 1,
qrllegacy_pb2.LegacyMessage.P2P_ACK: 0,
}
def __init__(self, chain, p2pFactory, nodeState, ntp):
self.master_mr = MessageReceipt()
self.nodeState = nodeState
self.ntp = ntp
self.chain = chain
self.r1_time_diff = defaultdict(list)
self.r2_time_diff = defaultdict(list)
self.incoming_blocks = {}
self.last_pos_cycle = 0
self.last_selected_height = 0
self.last_bk_time = 0
self.last_pb_time = 0
self.next_header_hash = None
self.next_block_number = None
self.fmbh_allowed_peers = {}
self.fmbh_blockhash_peers = {}
self.p2pFactory = p2pFactory
def test_add_contains_remove(self):
mr = MessageReceipt()
# FIXME: Hashes being are treated as strings
msg_hash = str2bin("hash_valid")
msg_obj = [1, 2, 3, 4]
msg_type = mr.allowed_types[0]
peer = '127.0.0.1'
mr.register(msg_type, msg_hash, msg_obj)
mr.add_peer(msg_hash, msg_type, peer)
self.assertTrue(mr.contains(msg_hash, msg_type))
self.assertFalse(mr.contains(b'hash_invalid', msg_type))
def test_create(self):
mr = MessageReceipt()
self.assertIsNotNone(mr)
self.assertEqual(mr.allowed_types, [LegacyMessage.TX,
LegacyMessage.LT,
LegacyMessage.EPH,
LegacyMessage.BK,
LegacyMessage.MT,
LegacyMessage.TK,
LegacyMessage.TT,
LegacyMessage.SL,
LegacyMessage.MC,
LegacyMessage.MS,
LegacyMessage.MV])
def test_add_contains_remove(self):
mr = MessageReceipt()
# FIXME: Hashes being are treated as strings
msg_hash = str2bin("hash_valid")
msg_obj = [1, 2, 3, 4]
msg_type = mr.allowed_types[0]
peer = '127.0.0.1'
mr.register(msg_type, msg_hash, msg_obj)
mr.add_peer(msg_hash, msg_type, peer)
self.assertTrue(mr.contains(msg_hash, msg_type))
self.assertFalse(mr.contains(b'hash_invalid', msg_type))
class POS:
def __init__(self, chain, p2pFactory, nodeState, ntp):
self.master_mr = MessageReceipt()
self.nodeState = nodeState
self.ntp = ntp
self.chain = chain
self.r1_time_diff = defaultdict(list)
self.r2_time_diff = defaultdict(list)
self.incoming_blocks = {}
self.last_pos_cycle = 0
self.last_selected_height = 0
self.last_bk_time = 0
self.last_pb_time = 0
self.next_header_hash = None
self.next_block_number = None
self.fmbh_allowed_peers = {}
self.fmbh_blockhash_peers = {}
self.p2pFactory = p2pFactory
def update_node_state(self, state):
self.nodeState.state = state
logger.info('Status changed to %s', self.nodeState.state)
if self.nodeState.state == NState.synced:
self.nodeState.epoch_diff = 0
self.last_pos_cycle = time.time()
self.restart_post_block_logic()
elif self.nodeState.state == NState.unsynced:
self.last_bk_time = time.time()
self.restart_unsynced_logic()
elif self.nodeState.state == NState.forked:
self.stop_post_block_logic()
elif self.nodeState.state == NState.syncing:
self.last_pb_time = time.time()
def stop_monitor_bk(self):
try:
reactor.monitor_bk.cancel()
except Exception: # No need to log this exception
pass
def restart_monitor_bk(self, delay=60):
self.stop_monitor_bk()
reactor.monitor_bk = reactor.callLater(delay, self.monitor_bk)
def monitor_bk(self):
time_diff = time.time() - self.last_pos_cycle
if (self.nodeState.state == NState.synced or self.nodeState.state == NState.unsynced) and 90 < time_diff:
if self.nodeState.state == NState.synced:
self.stop_post_block_logic()
self.reset_everything()
self.update_node_state(NState.unsynced)
self.epoch_diff = -1
elif time.time() - self.last_bk_time > 120:
self.last_pos_cycle = time.time()
logger.info(' POS cycle activated by monitor_bk() ')
self.update_node_state(NState.synced)
if self.nodeState.state == NState.syncing and time.time() - self.last_pb_time > 60:
self.stop_post_block_logic()
self.reset_everything()
self.update_node_state(NState.unsynced)
self.epoch_diff = -1
reactor.monitor_bk = reactor.callLater(60, self.monitor_bk)
def peers_blockheight_headerhash(self):
for peer in self.p2pFactory.peers:
peer.fetch_headerhash_n(self.chain.m_blockheight())
def check_fork_status(self):
current_height = self.chain.m_blockheight()
block_hash_counter = Counter()
for peer in self.p2pFactory.peers:
if current_height in peer.blocknumber_headerhash.keys():
block_hash_counter[peer.blocknumber_headerhash[current_height]] += 1
blockhash = block_hash_counter.most_common(1)
if blockhash:
blockhash = blockhash[0][0]
actual_blockhash = self.chain.m_get_block(current_height).blockheader.headerhash
if actual_blockhash != blockhash:
logger.info('Blockhash didnt matched in peers_blockheight()')
logger.info('Local blockhash - %s', actual_blockhash)
logger.info('Consensus blockhash - %s', blockhash)
fork_recovery(current_height, self.chain, self.randomize_headerhash_fetch)
return True
return
def peers_blockheight(self):
if self.nodeState.state == NState.syncing:
return
if self.check_fork_status():
return
block_height_counter = Counter()
for peer in self.p2pFactory.peers:
block_height_counter[peer.blockheight] += 1
blocknumber = block_height_counter.most_common(1)
if not blocknumber:
return # TODO : Re-Schedule with delay
blocknumber = blocknumber[0][0]
if blocknumber > self.chain.height(): # chain.m_blockheight(): len(chain.m_blockchain)
# pending_blocks['target'] = blocknumber
logger.info('Calling downloader from peers_blockheight due to no POS CYCLE %s', blocknumber)
logger.info('Download block from %s to %s', self.chain.height() + 1, blocknumber)
self.last_pb_time = time.time()
self.update_node_state(NState.syncing)
self.randomize_block_fetch(self.chain.height() + 1)
return
def schedule_peers_blockheight(self, delay=100):
try:
reactor.peers_blockheight.cancel()
except Exception: # No need to log this exception
pass
reactor.peers_blockheight = reactor.callLater(delay, self.peers_blockheight)
try:
reactor.peers_blockheight_headerhash.cancel() # No need to log this exception
except Exception as e:
pass
reactor.peers_blockheight_headerhash = reactor.callLater(70, self.peers_blockheight_headerhash)
# pos functions. an asynchronous loop.
# first block 1 is created with the stake list for epoch 0 decided from circulated st transactions
def pre_pos_1(self, data=None): # triggered after genesis for block 1..
logger.info('pre_pos_1')
# are we a staker in the stake list?
if self.chain.mining_address in self.chain.m_blockchain[0].stake_list:
logger.info('mining address: %s in the genesis.stake_list', self.chain.mining_address)
hashchain(self.chain.my[0][1], epoch=0)
self.chain.hash_chain = self.chain.my[0][1].hc
self.chain.block_chain_buffer.hash_chain[0] = self.chain.my[0][1].hc
logger.info('hashchain terminator: %s', self.chain.my[0][1].hc_terminator)
st = StakeTransaction().create(blocknumber=0,
xmss=self.chain.my[0][1],
hashchain_terminator=self.chain.my[0][1].hc_terminator,
first_hash=self.chain.my[0][1].hc[-1][-2],
balance=self.chain.state.state_balance(self.chain.mining_address))
self.chain.wallet.f_save_winfo()
self.chain.add_tx_to_pool(st)
# send the stake tx to generate hashchain terminators for the staker addresses..
self.p2pFactory.send_st_to_peers(st)
logger.info('await delayed call to build staker list from genesis')
reactor.callLater(5, self.pre_pos_2, st)
return
logger.info('not in stake list..no further pre_pos_x calls')
return
def pre_pos_2(self, data=None):
logger.info('pre_pos_2')
if self.chain.height() >= 1:
return
# assign hash terminators to addresses and generate a temporary stake list ordered by st.hash..
tmp_list = []
for tx in self.chain.transaction_pool:
if tx.subtype == transaction.TX_SUBTYPE_STAKE:
if tx.txfrom in self.chain.m_blockchain[0].stake_list:
tmp_list.append([tx.txfrom, tx.hash, 0, tx.first_hash, genesis_info[tx.txfrom]])
# required as doing chain.stake_list.index(s) which will result into different number on different server
self.chain.block_chain_buffer.epoch_seed = self.chain.state.calc_seed(tmp_list)
self.chain.stake_list = sorted(tmp_list,
key=lambda staker: self.chain.score(stake_address=staker[0],
reveal_one=sha256(str(staker[1])),
balance=staker[4],
seed=self.chain.block_chain_buffer.epoch_seed))
logger.info('genesis stakers ready = %s / %s', len(self.chain.stake_list), config.dev.minimum_required_stakers)
logger.info('node address: %s', self.chain.mining_address)
if len(self.chain.stake_list) < config.dev.minimum_required_stakers: # stake pool still not full..reloop..
self.p2pFactory.send_st_to_peers(data)
logger.info('waiting for stakers.. retry in 5s')
reactor.callID = reactor.callLater(5, self.pre_pos_2, data)
return
if self.chain.mining_address == self.chain.stake_list[0][0]:
logger.info('designated to create block 1: building block..')
# create the genesis block 2 here..
my_hash_chain, _ = self.chain.select_hashchain(self.chain.m_blockchain[-1].blockheader.headerhash,
self.chain.mining_address, self.chain.my[0][1].hc,
blocknumber=1)
b = self.chain.m_create_block(my_hash_chain[-2])
self.pre_block_logic(b)
else:
logger.info('await block creation by stake validator: %s', self.chain.stake_list[0][0])
self.last_bk_time = time.time()
self.restart_unsynced_logic()
return
def process_transactions(self, num):
tmp_num = num
for tx in self.chain.pending_tx_pool:
tmp_num -= 1
tx_peer = tx[1]
tx = tx[0]
if not tx.validate_tx():
logger.info('>>>TX %s failed validate_tx', tx.txhash)
continue
block_chain_buffer = self.chain.block_chain_buffer
tx_state = block_chain_buffer.get_stxn_state(blocknumber=block_chain_buffer.height(),
addr=tx.txfrom)
isValidState = tx.state_validate_tx(
tx_state=tx_state,
transaction_pool=self.chain.transaction_pool
)
if not isValidState:
logger.info('>>>TX %s failed state_validate', tx.txhash)
continue
logger.info('>>>TX - %s from - %s relaying..', tx.txhash, tx_peer.transport.getPeer().host)
self.chain.add_tx_to_pool(tx)
txn_msg = tx_peer.wrap_message('TX', tx.transaction_to_json())
for peer in tx_peer.factory.peer_connections:
if peer != tx_peer:
peer.transport.write(txn_msg)
for i in range(num - tmp_num):
del self.chain.pending_tx_pool[0]
del self.chain.pending_tx_pool_hash[0]
# create new block..
def create_new_block(self, winner, reveals, vote_hashes, last_block_number):
logger.info('create_new_block #%s', (last_block_number + 1))
block_obj = self.chain.create_stake_block(winner, reveals, vote_hashes, last_block_number)
return block_obj
def reset_everything(self, data=None):
logger.info('** resetting loops and emptying chain.stake_reveal_one and chain.expected_winner ')
for r in self.chain.stake_reveal_one:
msg_hash = r[5]
self.master_mr.deregister(msg_hash, 'R1')
del self.chain.stake_reveal_one[:]
return
def filter_reveal_one_two(self, blocknumber=None):
if not blocknumber:
blocknumber = self.chain.m_blockchain[-1].blockheader.blocknumber
self.chain.stake_reveal_one = filter(lambda s: s[2] > blocknumber,
self.chain.stake_reveal_one)
return
# TODO: Incomplete fn, use to select the maximum blockheight by consensus
def select_blockheight_by_consensus(self):
block_height_counter = Counter()
# for identity in self.fmbh_allowed_peers:
# block_height_counter[s[2]] += 1
target_block_height = block_height_counter.most_common(1)
if len(target_block_height) == 0:
return None
last_selected_height = target_block_height[0][0]
return last_selected_height
'''
Unsynced Logic
1. Request for maximum blockheight and passes bock number X
2. Peers response chain height with headerhash and the headerhash of block number X
3. Unsynced node, selects most common chain height, matches the headerhash of block number X
4. If headerhash of block number X doesn't match, change state to Forked
5. If headerhash of block number X matches, perform Downloading of blocks from those selected peers
'''
def restart_unsynced_logic(self, delay=0):
try:
reactor.unsynced_logic.cancel()
except Exception: # No need to log this exception
pass
reactor.unsynced_logic = reactor.callLater(delay, self.unsynced_logic)
def unsynced_logic(self):
if self.nodeState.state == NState.synced:
return
self.fmbh_blockhash_peers = {}
self.fmbh_allowed_peers = {}
for peer in self.p2pFactory.peer_connections:
self.fmbh_allowed_peers[peer.identity] = None
peer.fetch_FMBH()
reactor.unsynced_logic = reactor.callLater(20, self.start_download)
def start_download(self):
# add peers and their identity to requested list
# FMBH
if self.nodeState.state == NState.synced:
return
logger.info('Checking Download..')
'''
global fmbh_blockhash_peers
max_height = None
selected_blockhash = None
for blockheaderhash in fmbh_blockhash_peers:
if fmbh_blockhash_peers[blockheaderhash]['blocknumber']>max_height:
max_height = fmbh_blockhash_peers[blockheaderhash]['blocknumber']
selected_blockhash = blockheaderhash
for peer in fmbh_blockhash_peers[selected_blockhash]['peers']:
f.target_peers = {}
f.target_peers[peer.identity] = peer
if max_height == None or max_height<=chain.height():
chain.state.update(NState.synced)
return
chain.state.update(NState.syncing)
pending_blocks['start_block'] = chain.m_blockchain[-1].blockheader.blocknumber
pending_blocks['target'] = fmbh_blockhash_peers[selected_blockhash]['blocknumber']
pending_blocks['headerhash'] = selected_blockhash
randomize_block_fetch(chain.height() + 1)
'''
tmp_max = -1
max_headerhash = None
for headerhash in self.fmbh_blockhash_peers:
if self.fmbh_blockhash_peers[headerhash]['blocknumber'] > self.chain.height():
if len(self.fmbh_blockhash_peers[headerhash]['peers']) > tmp_max:
tmp_max = len(self.fmbh_blockhash_peers[headerhash]['peers'])
max_headerhash = headerhash
# Adding all peers
# TODO only trusted peer
# for peer in self.p2pFactory.peers:
if not max_headerhash:
logger.info('No peers responded FMBH request')
return
for peer in self.fmbh_blockhash_peers[max_headerhash]['peers']:
self.p2pFactory.target_peers[peer.identity] = peer
self.update_node_state(NState.syncing)
self.randomize_block_fetch(self.chain.height() + 1)
def pre_block_logic(self, block, peer_identity=None):
if len(self.chain.m_blockchain) == 0:
self.chain.m_read_chain()
blocknumber = block.blockheader.blocknumber
chain_buffer_height = self.chain.block_chain_buffer.height()
if blocknumber <= self.chain.height():
return False
if self.nodeState.state == NState.synced:
if self.chain.block_chain_buffer.add_block(block):
self.p2pFactory.send_block_to_peers(block, peer_identity)
else:
if chain_buffer_height + 1 == blocknumber:
if blocknumber > 1 and self.chain.block_chain_buffer.add_block(block):
self.p2pFactory.send_block_to_peers(block, peer_identity)
elif blocknumber == 1 and self.chain.block_chain_buffer.add_block_mainchain(block):
self.p2pFactory.send_block_to_peers(block, peer_identity)
self.update_node_state(NState.synced)
else:
self.chain.block_chain_buffer.add_pending_block(block)
if self.nodeState.state == NState.synced:
if chain_buffer_height + 1 == blocknumber:
self.last_pos_cycle = time.time()
block_timestamp = int(block.blockheader.timestamp)
curr_time = int(self.ntp.getTime())
delay = config.dev.POS_delay_after_block - min(config.dev.POS_delay_after_block,
max(0, curr_time - block_timestamp))
self.restart_post_block_logic(delay)
return True
def stop_post_block_logic(self, delay=0):
try:
reactor.post_block_logic.cancel()
reactor.prepare_winners.cancel()
except Exception: # No need to log this exception
pass
def restart_post_block_logic(self, delay=0):
self.stop_post_block_logic()
reactor.post_block_logic = reactor.callLater(delay,
self.post_block_logic)
def post_block_logic(self):
"""
post block logic we initiate the next POS cycle
send R1, send ST, reset POS flags and remove unnecessary
messages in chain.stake_reveal_one and _two..
:return:
"""
self.filter_reveal_one_two()
our_reveal = None
blocknumber = self.chain.block_chain_buffer.height() + 1
if self.p2pFactory.stake:
tmp_stake_list = [
s[0] for s in self.chain.block_chain_buffer.stake_list_get(blocknumber)
]
if self.chain.mining_address in tmp_stake_list:
our_reveal = self.p2pFactory.send_stake_reveal_one(blocknumber)
self.schedule_prepare_winners(our_reveal, blocknumber - 1, 30)
next_stake_list = self.chain.block_chain_buffer.next_stake_list_get(blocknumber)
next_stake_first_hash = {}
for s in next_stake_list:
next_stake_first_hash[s[0]] = s[3]
epoch = blocknumber // config.dev.blocks_per_epoch
epoch_blocknum = blocknumber - epoch * config.dev.blocks_per_epoch
if epoch_blocknum < config.dev.stake_before_x_blocks and self.chain.mining_address not in next_stake_first_hash:
diff = max(1, (
(config.dev.stake_before_x_blocks - epoch_blocknum + 1) * int(1 - config.dev.st_txn_safety_margin)))
if random.randint(1, diff) == 1:
self.make_st_tx(blocknumber, None)
elif epoch_blocknum >= config.dev.stake_before_x_blocks - 1 and self.chain.mining_address in next_stake_first_hash:
if next_stake_first_hash[self.chain.mining_address] is None:
threshold_blocknum = self.chain.state.get_staker_threshold_blocknum(next_stake_list,
self.chain.mining_address)
max_threshold_blocknum = config.dev.blocks_per_epoch
if threshold_blocknum == config.dev.low_staker_first_hash_block:
max_threshold_blocknum = config.dev.high_staker_first_hash_block
if threshold_blocknum - 1 <= epoch_blocknum < max_threshold_blocknum - 1:
diff = max(1, (
(max_threshold_blocknum - epoch_blocknum + 1) * int(1 - config.dev.st_txn_safety_margin)))
if random.randint(1, diff) == 1:
my = deepcopy(self.chain.my[0][1])
hashchain(my, epoch=epoch + 1)
self.make_st_tx(blocknumber, my.hc[-1][-2])
return
def make_st_tx(self, blocknumber, first_hash):
balance = self.chain.state.state_balance(self.chain.mining_address)
if balance < config.dev.minimum_staking_balance_required:
logger.warning('Staking not allowed due to insufficient balance')
logger.warning('Balance %s', balance)
return
st = StakeTransaction().create(
blocknumber,
self.chain.my[0][1],
first_hash=first_hash,
balance=balance
)
self.p2pFactory.send_st_to_peers(st)
self.chain.wallet.f_save_winfo()
for num in range(len(self.chain.transaction_pool)):
t = self.chain.transaction_pool[num]
if t.subtype == transaction.TX_SUBTYPE_STAKE and st.hash == t.hash:
if st.get_message_hash() == t.get_message_hash():
return
self.chain.remove_tx_from_pool(t)
break
self.chain.add_tx_to_pool(st)
def schedule_prepare_winners(self, our_reveal, last_block_number, delay=0):
try:
reactor.prepare_winners.cancel()
except Exception: # No need to log this Exception
pass
reactor.prepare_winners = reactor.callLater(
delay,
self.prepare_winners,
our_reveal=our_reveal,
last_block_number=last_block_number)
def prepare_winners(self, our_reveal, last_block_number):
if not self.nodeState.state == NState.synced:
return
filtered_reveal_one = []
reveals = []
vote_hashes = []
next_block_num = last_block_number + 1
for s in self.chain.stake_reveal_one:
tmp_strongest_headerhash = self.chain.block_chain_buffer.get_strongest_headerhash(last_block_number)
if s[1] == tmp_strongest_headerhash and s[2] == next_block_num:
filtered_reveal_one.append(s)
reveals.append(s[3])
vote_hashes.append(s[5])
self.restart_post_block_logic(30)
if len(filtered_reveal_one) <= 1:
logger.info('only received one reveal for this block.. blocknum #%s', next_block_num)
return
epoch = next_block_num / config.dev.blocks_per_epoch # +1 = next block
seed = self.chain.block_chain_buffer.get_epoch_seed(next_block_num)
winners = self.chain.select_winners(filtered_reveal_one,
topN=3,
seed=seed)
# reactor.process_blocks = reactor.callLater(30, process_blocks, winners=winners, our_reveal=our_reveal)
if not (self.p2pFactory.stake and our_reveal):
return
if our_reveal in winners:
block = self.create_new_block(our_reveal,
reveals,
vote_hashes,
last_block_number)
self.pre_block_logic(block) # broadcast this block
if self.chain.pending_tx_pool:
if len(self.chain.transaction_pool) < 10:
logger.info('Processing TXNs if any')
self.process_transactions(5)
def randomize_block_fetch(self, blocknumber):
if self.nodeState.state != NState.syncing or blocknumber <= self.chain.height():
return
if len(self.p2pFactory.target_peers.keys()) == 0:
logger.info(' No target peers found.. stopping download')
return
reactor.download_monitor = reactor.callLater(20,
self.randomize_block_fetch, blocknumber)
random_peer = self.p2pFactory.target_peers[random.choice(self.p2pFactory.target_peers.keys())]
random_peer.fetch_block_n(blocknumber)
def randomize_headerhash_fetch(self, block_number):
if self.nodeState.state != NState.forked:
return
if block_number not in fork.pending_blocks or fork.pending_blocks[block_number][1] <= 10: # retry only 11 times
headerhash_monitor = reactor.callLater(15, self.randomize_headerhash_fetch, block_number)
if len(self.p2pFactory.peers) > 0:
try:
if len(self.p2pFactory.fork_target_peers) == 0:
for peer in self.p2pFactory.peers:
self.p2pFactory.fork_target_peers[peer.identity] = peer
if len(self.p2pFactory.fork_target_peers) > 0:
random_peer = self.p2pFactory.fork_target_peers[
random.choice(
self.p2pFactory.fork_target_peers.keys()
)
]
count = 0
if block_number in fork.pending_blocks:
count = fork.pending_blocks[block_number][1] + 1
fork.pending_blocks[block_number] = [
random_peer.identity, count, None, headerhash_monitor
]
random_peer.fetch_headerhash_n(block_number)
except Exception as e:
logger.warning('Exception at randomize_headerhash_fetch %s', e)
else:
logger.info('No peers connected.. Will try again... randomize_headerhash_fetch: %s', block_number)
else:
self.update_node_state(NState.unsynced)
def blockheight_map(self):
"""
blockheight map for connected nodes - when the blockheight seems up to date after a sync or error, we check all connected nodes to ensure all on same chain/height..
note - may not return correctly during a block propagation..
once working alter to identify fork better..
:return:
"""
# i = [block_number, headerhash, self.transport.getPeer().host]
logger.info('blockheight_map:')
logger.info(self.chain.blockheight_map)
# first strip out any laggards..
self.chain.blockheight_map = filter(
lambda q: q[0] >= self.chain.m_blockheight(),
self.chain.blockheight_map
)
result = True
# next identify any node entries which are not exactly correct..
for s in self.chain.blockheight_map:
if s[0] == self.chain.m_blockheight():
if s[1] == self.chain.m_blockchain[-1].blockheader.headerhash:
logger.info(('node: ', s[2], '@', s[0], 'w/:', s[1], 'OK'))
elif s[0] > self.chain.m_blockheight():
logger.info(('warning..', s[2], 'at blockheight', s[0]))
result = False
# wipe it..
del self.chain.blockheight_map[:]
return result
class P2PFactory(ServerFactory):
protocol = P2PProtocol
def __init__(self, chain_manager: ChainManager, sync_state: SyncState,
qrl_node):
self.master_mr = MessageReceipt()
self.pow = None
self.sync_state = sync_state
self._ntp = ntp
self._qrl_node = qrl_node
self._chain_manager = chain_manager
self._syncing_enabled = False
self._target_peer = None
self._target_node_header_hash = None
self._last_requested_block_idx = None
self._genesis_processed = False
self._peer_connections = []
self._synced_peers_protocol = set()
self._txn_processor_running = False
self.peer_blockheight = dict()
reactor.callLater(180, self.monitor_connections)
self.p2p_msg_priority = {
qrllegacy_pb2.LegacyMessage.VE: 0,
qrllegacy_pb2.LegacyMessage.PL: 0,
qrllegacy_pb2.LegacyMessage.PONG: 0,
######################
qrllegacy_pb2.LegacyMessage.MR: 2,
qrllegacy_pb2.LegacyMessage.SFM: 1,
qrllegacy_pb2.LegacyMessage.BK: 1,
qrllegacy_pb2.LegacyMessage.FB: 1,
qrllegacy_pb2.LegacyMessage.PB: 1,
qrllegacy_pb2.LegacyMessage.BH: 1,
############################
qrllegacy_pb2.LegacyMessage.TX: 1,
qrllegacy_pb2.LegacyMessage.MT: 1,
qrllegacy_pb2.LegacyMessage.TK: 1,
qrllegacy_pb2.LegacyMessage.TT: 1,
qrllegacy_pb2.LegacyMessage.LT: 1,
qrllegacy_pb2.LegacyMessage.SL: 1,
qrllegacy_pb2.LegacyMessage.EPH: 3,
qrllegacy_pb2.LegacyMessage.SYNC: 0,
qrllegacy_pb2.LegacyMessage.CHAINSTATE: 0,
qrllegacy_pb2.LegacyMessage.HEADERHASHES: 1,
qrllegacy_pb2.LegacyMessage.P2P_ACK: 0,
}
###################################################
###################################################
###################################################
###################################################
###################################################
###################################################
@property
def has_synced_peers(self):
return len(self._synced_peers_protocol) > 0
def get_random_peer(self):
max_cumulative_difficulty = 0
for connection_id in self.peer_blockheight:
max_cumulative_difficulty = max(
max_cumulative_difficulty,
self.peer_blockheight[connection_id][2])
connection_ids = []
for connection_id in self.peer_blockheight:
if self.peer_blockheight[connection_id][
2] == max_cumulative_difficulty:
connection_ids.append(connection_id)
selected_peer_connections = []
for connection_id in connection_ids:
for peer_conn in self._peer_connections:
if peer_conn.connection_id == connection_id:
selected_peer_connections.append(peer_conn)
if len(selected_peer_connections
) == 0 or max_cumulative_difficulty == 0:
return None
return random.sample(selected_peer_connections, 1)[0]
def update_peer_blockheight(self, connection_id, block_number, headerhash,
cumulative_difficulty):
self.peer_blockheight[connection_id] = [
block_number, headerhash,
int(UInt256ToString(cumulative_difficulty))
]
def request_peer_blockheight(self):
for peer in self._peer_connections:
msg = qrllegacy_pb2.LegacyMessage(
func_name=qrllegacy_pb2.LegacyMessage.BH,
bhData=qrl_pb2.BlockHeightData(block_number=0))
peer.send(msg)
def set_peer_synced(self, conn_protocol, synced: bool):
if synced:
self._synced_peers_protocol.add(conn_protocol)
else:
self._synced_peers_protocol.discard(conn_protocol)
###################################################
###################################################
###################################################
###################################################
###################################################
###################################################
###################################################
@property
def connections(self):
return len(self._peer_connections)
@property
def synced(self):
return self.pow.sync_state.state == ESyncState.synced
@property
def reached_conn_limit(self):
return len(self._peer_connections) >= config.user.max_peers_limit
def get_connected_peer_ips(self):
# FIXME: Convert self._peer_connections to set
return set([peer.peer_ip for peer in self._peer_connections])
###################################################
###################################################
###################################################
###################################################
###################################################
###################################################
@property
def chain_height(self):
return self._chain_manager.height
def get_last_block(self):
return self._chain_manager.get_last_block()
def get_headerhashes(self, start_blocknumber):
return self._chain_manager.get_headerhashes(start_blocknumber)
def get_cumulative_difficulty(self):
return self._chain_manager.get_cumulative_difficulty()
def get_block(self, block_number):
return self._chain_manager.get_block_by_number(block_number)
def block_received(self, source, block: Block):
self.pow.last_pb_time = time.time()
logger.info('>>> Received Block #%d %s', block.block_number,
bin2hstr(block.headerhash))
if source != self._target_peer:
logger.warning('Received block from unexpected peer')
logger.warning('Expected peer: %s',
self._target_peer.connection_id)
logger.warning('Found peer: %s', source.connection_id)
return
if block.block_number != self._last_requested_block_idx:
logger.warning('Did not match %s', self._last_requested_block_idx)
return
target_start_blocknumber = self._target_node_header_hash.block_number
expected_headerhash = self._target_node_header_hash.headerhashes[
block.block_number - target_start_blocknumber]
if block.headerhash != expected_headerhash:
logger.warning('Did not match headerhash')
logger.warning('Expected headerhash %s', expected_headerhash)
logger.warning('Found headerhash %s', block.headerhash)
return
# FIXME: This check should not be necessary
if not self._chain_manager.add_block(block):
logger.warning('Failed to Add Block')
return
try:
reactor.download_monitor.cancel()
except Exception as e:
logger.warning("PB: %s", e)
if self.is_syncing_finished():
return
self._last_requested_block_idx += 1
if self.is_syncing_finished():
return
self.peer_fetch_block()
def ban_peer(self, peer_obj):
self._qrl_node.ban_peer(peer_obj)
def is_syncing(self) -> bool:
return self._syncing_enabled
def is_syncing_finished(self, force_finish=False):
curr_index = self._last_requested_block_idx - self._target_node_header_hash.block_number + 1
if curr_index == len(
self._target_node_header_hash.headerhashes) or force_finish:
self._last_requested_block_idx = None
self._target_node_header_hash = None
self._target_peer = None
self._syncing_enabled = False
return True
return False
def peer_fetch_block(self, retry=0):
node_header_hash = self._target_node_header_hash
curr_index = self._last_requested_block_idx - node_header_hash.block_number
block_headerhash = node_header_hash.headerhashes[curr_index]
block = self._chain_manager.state.get_block(block_headerhash)
if not block:
if retry >= 5:
logger.debug('Retry Limit Hit')
self._qrl_node.ban_peer(self._target_peer)
self.is_syncing_finished(force_finish=True)
return
else:
while block and curr_index + 1 < len(
node_header_hash.headerhashes):
self._last_requested_block_idx += 1
curr_index = self._last_requested_block_idx - node_header_hash.block_number
block_headerhash = node_header_hash.headerhashes[curr_index]
block = self._chain_manager.state.get_block(block_headerhash)
retry = 0
if self.is_syncing_finished():
return
self._target_peer.send_fetch_block(self._last_requested_block_idx)
reactor.download_monitor = reactor.callLater(20, self.peer_fetch_block,
retry + 1)
def compare_and_sync(self, peer, node_header_hash: qrl_pb2.NodeHeaderHash):
if self._syncing_enabled:
logger.info('>> Ignoring compare_and_sync Syncing Enabled')
return
last_block = self._chain_manager.get_last_block()
node_last_block_number = node_header_hash.block_number + len(
node_header_hash.headerhashes) - 1
last_block_number = min(last_block.block_number,
node_last_block_number)
if last_block_number < node_header_hash.block_number:
return
fork_block_number = last_block.block_number + 1
fork_found = False
for i in range(last_block_number, node_header_hash.block_number - 1,
-1):
block = self._chain_manager.get_block_by_number(i)
if block.headerhash == node_header_hash.headerhashes[
i - node_header_hash.block_number]:
break
fork_block_number = i
fork_found = True
if fork_found or (last_block.block_number < node_last_block_number):
self._target_peer = peer
self._target_node_header_hash = node_header_hash
self._last_requested_block_idx = fork_block_number
self._syncing_enabled = True
self.peer_fetch_block()
###################################################
###################################################
###################################################
###################################################
###################################################
###################################################
def request_full_message(self, mr_data: qrllegacy_pb2.MRData):
"""
Request Full Message
This function request for the full message against,
the Message Receipt received.
:return:
"""
# FIXME: Again, breaking encasulation
# FIXME: Huge amount of lookups in dictionaries
msg_hash = mr_data.hash
if msg_hash in self.master_mr._hash_msg:
if msg_hash in self.master_mr.requested_hash:
del self.master_mr.requested_hash[msg_hash]
return
if msg_hash not in self.master_mr.requested_hash:
return
peers_list = self.master_mr.requested_hash[
msg_hash].peers_connection_list
message_request = self.master_mr.requested_hash[msg_hash]
for peer in peers_list:
if peer in message_request.already_requested_peers:
continue
message_request.already_requested_peers.append(peer)
msg = qrllegacy_pb2.LegacyMessage(
func_name=qrllegacy_pb2.LegacyMessage.SFM,
mrData=qrllegacy_pb2.MRData(hash=mr_data.hash,
type=mr_data.type))
peer.send(msg)
call_later_obj = reactor.callLater(
config.dev.message_receipt_timeout, self.request_full_message,
mr_data)
message_request.callLater = call_later_obj
return
# If execution reach to this line, then it means no peer was able to provide
# Full message for this hash thus the hash has to be deleted.
# Moreover, negative points could be added to the peers, for this behavior
if msg_hash in self.master_mr.requested_hash:
del self.master_mr.requested_hash[msg_hash]
##############################################
##############################################
##############################################
##############################################
# NOTE: PoW related.. broadcasting, etc. OBSOLETE
def reset_processor_flag(self, _):
self._txn_processor_running = False
def reset_processor_flag_with_err(self, msg):
logger.error('Exception in txn task')
logger.error('%s', msg)
self._txn_processor_running = False
def add_unprocessed_txn(self, tx, ip) -> bool:
if not self._chain_manager.tx_pool.update_pending_tx_pool(tx, ip):
return False
if not self._txn_processor_running:
txn_processor = TxnProcessor(
state=self._chain_manager.state,
transaction_pool_obj=self._chain_manager.tx_pool,
broadcast_tx=self.broadcast_tx)
task_defer = TxnProcessor.create_cooperate(
txn_processor).whenDone()
task_defer.addCallback(self.reset_processor_flag) \
.addErrback(self.reset_processor_flag_with_err)
self._txn_processor_running = True
return True
def broadcast_tx(self, tx: TransferTransaction):
logger.info('<<<Transmitting TX: %s', tx.txhash)
if isinstance(tx, MessageTransaction):
legacy_type = qrllegacy_pb2.LegacyMessage.MT
elif isinstance(tx, TransferTransaction):
legacy_type = qrllegacy_pb2.LegacyMessage.TX
elif isinstance(tx, TokenTransaction):
legacy_type = qrllegacy_pb2.LegacyMessage.TK
elif isinstance(tx, TransferTokenTransaction):
legacy_type = qrllegacy_pb2.LegacyMessage.TT
elif isinstance(tx, LatticePublicKey):
legacy_type = qrllegacy_pb2.LegacyMessage.LT
elif isinstance(tx, SlaveTransaction):
legacy_type = qrllegacy_pb2.LegacyMessage.SL
else:
raise ValueError('Invalid Transaction Type')
self.register_and_broadcast(legacy_type, tx.get_message_hash(),
tx.pbdata)
def broadcast_ephemeral_message(self, encrypted_ephemeral):
logger.info('<<<Broadcasting Encrypted Ephemeral Message')
self._chain_manager.add_ephemeral_message(encrypted_ephemeral)
self.register_and_broadcast(qrllegacy_pb2.LegacyMessage.EPH,
encrypted_ephemeral.get_message_hash(),
encrypted_ephemeral.pbdata)
def broadcast_tx_relay(self, source_peer, tx):
txn_msg = source_peer._wrap_message('TX', tx.to_json())
for peer in self._peer_connections:
if peer != source_peer:
peer.transport.write(txn_msg)
##############################################
##############################################
##############################################
##############################################
def broadcast_block(self, block: Block):
# logger.info('<<<Transmitting block: ', block.headerhash)
data = qrllegacy_pb2.MRData()
data.stake_selector = block.transactions[0].public_key
data.block_number = block.block_number
data.prev_headerhash = bytes(block.prev_headerhash)
self.register_and_broadcast(qrllegacy_pb2.LegacyMessage.BK,
block.headerhash, block.pbdata, data)
##############################################
##############################################
##############################################
##############################################
def register_and_broadcast(self,
msg_type,
msg_hash: bytes,
pbdata,
data=None):
self.master_mr.register(msg_type, msg_hash, pbdata)
self.broadcast(msg_type, msg_hash, data)
def broadcast(self, msg_type, msg_hash: bytes, mr_data=None):
"""
Broadcast
This function sends the Message Receipt to all connected peers.
:return:
"""
ignore_peers = []
if msg_hash in self.master_mr.requested_hash:
ignore_peers = self.master_mr.requested_hash[
msg_hash].peers_connection_list
if not mr_data:
mr_data = qrllegacy_pb2.MRData()
mr_data.hash = msg_hash
mr_data.type = msg_type
data = qrllegacy_pb2.LegacyMessage(
func_name=qrllegacy_pb2.LegacyMessage.MR, mrData=mr_data)
for peer in self._peer_connections:
if peer not in ignore_peers:
peer.send(data)
def broadcast_get_synced_state(self):
# Request all peers to update their synced status
self._synced_peers_protocol = set()
for peer in self._peer_connections:
peer.send_sync()
###################################################
###################################################
###################################################
###################################################
# Event handlers / Comms related
def start_listening(self):
reactor.listenTCP(9000, self)
# NOTE: No need to refactor, it is obsolete
def clientConnectionLost(self, connector, reason): # noqa
logger.debug('connection lost: %s', reason)
def clientConnectionFailed(self, connector, reason):
logger.debug('connection failed: %s', reason)
def startedConnecting(self, connector):
logger.debug('Started connecting: %s', connector)
def add_connection(self, conn_protocol) -> bool:
# TODO: Most of this can go the peer manager
if self._qrl_node.is_banned(conn_protocol.peer_ip):
conn_protocol.loseConnection()
return False
# FIXME: (For AWS) This could be problematic for other users
# FIXME: identify nodes by an GUID?
if config.dev.public_ip and conn_protocol.peer_ip == config.dev.public_ip:
conn_protocol.loseConnection()
return False
if self.reached_conn_limit:
# FIXME: Should we stop listening to avoid unnecessary load due to many connections?
logger.info('Peer limit hit. Disconnecting client %s',
conn_protocol.peer_ip)
conn_protocol.loseConnection()
return False
peer_list = self._qrl_node.peer_addresses
if conn_protocol.peer_ip == conn_protocol.host_ip:
if conn_protocol.peer_ip in peer_list:
logger.info('Self in peer_list, removing..')
peer_list.remove(conn_protocol.peer_ip)
self._qrl_node.peer_manager.update_peer_addresses(peer_list)
conn_protocol.loseConnection()
return False
self._peer_connections.append(conn_protocol)
if conn_protocol.peer_ip not in peer_list:
logger.debug('Adding to peer_list')
peer_list.add(conn_protocol.peer_ip)
self._qrl_node.peer_manager.update_peer_addresses(peer_list)
logger.debug('>>> new peer connection : %s:%s ', conn_protocol.peer_ip,
str(conn_protocol.peer_port))
return True
def remove_connection(self, conn_protocol):
if conn_protocol in self._peer_connections:
self._peer_connections.remove(conn_protocol)
if conn_protocol.connection_id in self.peer_blockheight:
del self.peer_blockheight[conn_protocol.connection_id]
self._synced_peers_protocol.discard(conn_protocol)
def monitor_connections(self):
reactor.callLater(180, self.monitor_connections)
if len(self._peer_connections) == 0:
logger.warning('No Connected Peer Found')
reactor.callLater(60, self._qrl_node.connect_peers)
return
connected_peers_set = set()
for conn_protocol in self._peer_connections:
connected_peers_set.add(conn_protocol.peer_ip)
for ip in config.user.peer_list:
if ip not in connected_peers_set:
self.connect_peer(ip)
def connect_peer(self, peer_address):
if peer_address not in self.get_connected_peer_ips():
reactor.connectTCP(peer_address, 9000, self)
class P2PFactory(ServerFactory):
protocol = P2PProtocol
def __init__(self, chain_manager: ChainManager, sync_state: SyncState,
qrl_node):
self.master_mr = MessageReceipt()
self.pow = None
self.sync_state = sync_state
self._ntp = ntp
self._qrl_node = qrl_node
self._chain_manager = chain_manager
self._chain_manager.set_broadcast_tx(self.broadcast_tx)
self._syncing_enabled = False
self._target_channel = None
self._target_node_header_hash = None
self._last_requested_block_number = None
self._genesis_processed = False
self._peer_connections = []
self._txn_processor_running = False
self.peer_blockheight = dict()
reactor.callLater(config.user.monitor_connections_interval,
self.monitor_connections)
self.p2p_msg_priority = p2p_msg_priority
# Maintains the list of ips in the queue that can be tried to form a new p2p connection
self._peer_q = []
def add_new_peers_to_peer_q(self, peer_list):
"""
Checks ip must not already be in the _peer_q and
connection has not already been established from that ip and port
before adding the new set of peer into _peer_q
"""
peer_set = set(peer_list)
for peer_conn in self._peer_connections:
ip_port = peer_conn.peer.full_address
if ip_port in peer_set:
peer_set.remove(ip_port)
for ip_port in self._peer_q:
if ip_port in peer_set:
peer_set.remove(ip_port)
self._peer_q.extend(peer_set)
###################################################
###################################################
###################################################
###################################################
###################################################
###################################################
def get_random_peer(self):
# FIXME: Used a named tuple to improve readability?
# FIXME: This probably can go the peerManager
max_cumulative_difficulty = 0
for addr_remote in self.peer_blockheight:
max_cumulative_difficulty = max(
max_cumulative_difficulty,
self.peer_blockheight[addr_remote][2])
best_connection_ids = []
for addr_remote in self.peer_blockheight:
if self.peer_blockheight[addr_remote][
2] == max_cumulative_difficulty:
best_connection_ids.append(addr_remote)
selected_peer_connections = []
for addr_remote in best_connection_ids:
for peer_conn in self._peer_connections:
if peer_conn.peer.full_address == addr_remote:
selected_peer_connections.append(peer_conn)
if len(selected_peer_connections
) == 0 or max_cumulative_difficulty == 0:
return None
return random.sample(selected_peer_connections, 1)[0]
def update_peer_blockheight(self, addr_remote, block_number, headerhash,
cumulative_difficulty):
# FIXME: Use a named tuple to improve readability?
self.peer_blockheight[addr_remote] = [
block_number, headerhash,
int(UInt256ToString(cumulative_difficulty))
]
def request_peer_blockheight(self):
for peer in self._peer_connections:
msg = qrllegacy_pb2.LegacyMessage(
func_name=qrllegacy_pb2.LegacyMessage.BH,
bhData=qrl_pb2.BlockHeightData(block_number=0))
peer.send(msg)
###################################################
###################################################
###################################################
###################################################
###################################################
###################################################
###################################################
@property
def num_connections(self):
return len(self._peer_connections)
@property
def connections(self):
return list(self._peer_connections)
@property
def synced(self):
return self.pow.sync_state.state == ESyncState.synced
@property
def reached_conn_limit(self):
return len(self._peer_connections) >= config.user.max_peers_limit
def get_connected_peer_addrs(self):
return set([peer.peer.full_address for peer in self._peer_connections])
###################################################
###################################################
###################################################
###################################################
###################################################
###################################################
@property
def chain_height(self):
return self._chain_manager.height
def get_last_block(self):
return self._chain_manager.last_block
def get_headerhashes(self, start_blocknumber):
return self._chain_manager.get_headerhashes(start_blocknumber)
def get_cumulative_difficulty(self):
return self._chain_manager.get_cumulative_difficulty()
def get_block_by_number(self, block_number):
return self._chain_manager.get_block_by_number(block_number)
def is_block_present(self, header_hash: bytes) -> bool:
if not self._chain_manager.get_block(header_hash):
if header_hash not in self.pow.future_blocks:
return False
return True
def block_received(self, source, block: Block):
self.pow.last_pb_time = ntp.getTime()
logger.info('>>> Received Block #%d %s', block.block_number,
bin2hstr(block.headerhash))
if source != self._target_channel:
if self._target_channel is None:
logger.warning('Received block and target channel is None')
else:
logger.warning('Received block from unexpected peer')
logger.warning('Expected peer: %s', self._target_channel.peer)
logger.warning('Found peer: %s', source.peer)
return
if block.block_number != self._last_requested_block_number:
logger.warning('Did not match %s',
self._last_requested_block_number)
self._qrl_node.peer_manager.ban_channel(source)
return
target_start_blocknumber = self._target_node_header_hash.block_number
expected_headerhash = self._target_node_header_hash.headerhashes[
block.block_number - target_start_blocknumber]
if block.headerhash != expected_headerhash:
logger.warning('Did not match headerhash')
logger.warning('Expected headerhash %s', expected_headerhash)
logger.warning('Found headerhash %s', block.headerhash)
self._qrl_node.peer_manager.ban_channel(source)
return
if not block.validate(self._chain_manager, self.pow.future_blocks):
logger.warning('Syncing Failed: Block Validation Failed')
self._qrl_node.peer_manager.ban_channel(source)
return
if self._chain_manager.add_block(block, check_stale=False):
if self._chain_manager.last_block.headerhash == block.headerhash:
self.pow.suspend_mining_timestamp = ntp.getTime(
) + config.dev.sync_delay_mining
else:
logger.warning('Failed to Add Block')
self._qrl_node.peer_manager.ban_channel(source)
return
try:
reactor.download_monitor.cancel()
except Exception as e:
logger.warning("PB: %s", e)
if self.is_syncing_finished():
return
self._last_requested_block_number += 1
self.peer_fetch_block()
def is_syncing(self) -> bool:
return self._syncing_enabled
def is_syncing_finished(self, force_finish=False):
curr_index = self._last_requested_block_number - self._target_node_header_hash.block_number + 1
if curr_index == len(
self._target_node_header_hash.headerhashes) or force_finish:
self._last_requested_block_number = None
self._target_node_header_hash = None
self._target_channel = None
self._syncing_enabled = False
return True
return False
def peer_fetch_block(self, retry=0):
node_header_hash = self._target_node_header_hash
curr_index = self._last_requested_block_number - node_header_hash.block_number
block_headerhash = node_header_hash.headerhashes[curr_index]
block = self._chain_manager.get_block(block_headerhash)
if retry >= 1:
logger.debug('Retry Limit Hit')
self._qrl_node.peer_manager.ban_channel(self._target_channel)
self.is_syncing_finished(force_finish=True)
return
while block and curr_index + 1 < len(node_header_hash.headerhashes):
self._last_requested_block_number += 1
curr_index = self._last_requested_block_number - node_header_hash.block_number
block_headerhash = node_header_hash.headerhashes[curr_index]
block = self._chain_manager.get_block(block_headerhash)
if block and self.is_syncing_finished():
return
self._target_channel.send_fetch_block(
self._last_requested_block_number)
reactor.download_monitor = reactor.callLater(100,
self.peer_fetch_block,
retry + 1)
def compare_and_sync(self, source_peer,
node_header_hash: qrl_pb2.NodeHeaderHash):
if self._syncing_enabled:
logger.info('>> Ignoring compare_and_sync Syncing Enabled')
return
last_block = self.get_last_block()
node_last_block_number = node_header_hash.block_number + len(
node_header_hash.headerhashes) - 1
last_block_number = min(last_block.block_number,
node_last_block_number)
if last_block_number < node_header_hash.block_number:
return
fork_block_number = last_block.block_number + 1
fork_found = False
for i in range(last_block_number, node_header_hash.block_number - 1,
-1):
block = self._chain_manager.get_block_by_number(i)
if block:
if block.headerhash == node_header_hash.headerhashes[
i - node_header_hash.block_number]:
break
fork_block_number = i
fork_found = True
if fork_found or (last_block.block_number < node_last_block_number):
self._target_channel = source_peer
self._target_node_header_hash = node_header_hash
self._last_requested_block_number = fork_block_number
self._syncing_enabled = True
self.peer_fetch_block()
###################################################
###################################################
###################################################
###################################################
###################################################
###################################################
def request_full_message(self, mr_data: qrllegacy_pb2.MRData):
"""
Request Full Message
This function request for the full message against,
the Message Receipt received.
:return:
"""
# FIXME: Again, breaking encasulation
# FIXME: Huge amount of lookups in dictionaries
msg_hash = mr_data.hash
if msg_hash in self.master_mr._hash_msg:
if msg_hash in self.master_mr.requested_hash:
del self.master_mr.requested_hash[msg_hash]
return
if msg_hash not in self.master_mr.requested_hash:
return
peers_list = self.master_mr.requested_hash[
msg_hash].peers_connection_list
message_request = self.master_mr.requested_hash[msg_hash]
for peer in peers_list:
if peer in message_request.already_requested_peers:
continue
message_request.already_requested_peers.append(peer)
msg = qrllegacy_pb2.LegacyMessage(
func_name=qrllegacy_pb2.LegacyMessage.SFM,
mrData=qrllegacy_pb2.MRData(hash=mr_data.hash,
type=mr_data.type))
peer.send(msg)
call_later_obj = reactor.callLater(
config.dev.message_receipt_timeout, self.request_full_message,
mr_data)
message_request.callLater = call_later_obj
return
# If execution reach to this line, then it means no peer was able to provide
# Full message for this hash thus the hash has to be deleted.
# Moreover, negative points could be added to the peers, for this behavior
if msg_hash in self.master_mr.requested_hash:
del self.master_mr.requested_hash[msg_hash]
##############################################
##############################################
##############################################
##############################################
def reset_processor_flag(self, _):
self._txn_processor_running = False
def reset_processor_flag_with_err(self, msg):
logger.error('Exception in txn task')
logger.error('%s', msg)
self._txn_processor_running = False
def add_unprocessed_txn(self, tx, ip) -> bool:
if tx.fee < config.user.transaction_minimum_fee:
logger.info("Dropping Txn %s", bin2hstr(tx.txhash))
logger.info("Reason: Fee %s is below threshold fee %s", tx.fee,
config.user.transaction_minimum_fee)
return False
if not self._chain_manager.tx_pool.update_pending_tx_pool(tx, ip):
return False
if not self._txn_processor_running:
txn_processor = TxnProcessor(
chain_manager=self._chain_manager,
transaction_pool_obj=self._chain_manager.tx_pool,
broadcast_tx=self.broadcast_tx)
task_defer = TxnProcessor.create_cooperate(
txn_processor).whenDone()
task_defer.addCallback(self.reset_processor_flag) \
.addErrback(self.reset_processor_flag_with_err)
self._txn_processor_running = True
return True
##############################################
##############################################
##############################################
##############################################
def broadcast_tx(self, tx: TransferTransaction):
logger.info('<<<Transmitting TX: %s', bin2hstr(tx.txhash))
if isinstance(tx, MessageTransaction):
legacy_type = qrllegacy_pb2.LegacyMessage.MT
elif isinstance(tx, TransferTransaction):
legacy_type = qrllegacy_pb2.LegacyMessage.TX
elif isinstance(tx, TokenTransaction):
legacy_type = qrllegacy_pb2.LegacyMessage.TK
elif isinstance(tx, TransferTokenTransaction):
legacy_type = qrllegacy_pb2.LegacyMessage.TT
elif isinstance(tx, SlaveTransaction):
legacy_type = qrllegacy_pb2.LegacyMessage.SL
elif isinstance(tx, LatticeTransaction):
legacy_type = qrllegacy_pb2.LegacyMessage.LT
elif isinstance(tx, MultiSigCreate):
legacy_type = qrllegacy_pb2.LegacyMessage.MC
elif isinstance(tx, MultiSigSpend):
legacy_type = qrllegacy_pb2.LegacyMessage.MS
elif isinstance(tx, MultiSigVote):
legacy_type = qrllegacy_pb2.LegacyMessage.MV
else:
raise ValueError('Invalid Transaction Type')
self.register_and_broadcast(legacy_type, tx.get_message_hash(),
tx.pbdata)
def broadcast_block(self, block: Block):
# logger.info('<<<Transmitting block: ', block.headerhash)
data = qrllegacy_pb2.MRData()
data.stake_selector = block.transactions[0].public_key
data.block_number = block.block_number
data.prev_headerhash = bytes(block.prev_headerhash)
self.register_and_broadcast(qrllegacy_pb2.LegacyMessage.BK,
block.headerhash, block.pbdata, data)
def register_and_broadcast(self,
msg_type,
msg_hash: bytes,
pbdata,
data=None):
self.master_mr.register(msg_type, msg_hash, pbdata)
self.broadcast(msg_type, msg_hash, data)
def broadcast(self, msg_type, msg_hash: bytes, mr_data=None):
"""
Broadcast
This function sends the Message Receipt to all connected peers.
:return:
"""
ignore_peers = []
if msg_hash in self.master_mr.requested_hash:
ignore_peers = self.master_mr.requested_hash[
msg_hash].peers_connection_list
if not mr_data:
mr_data = qrllegacy_pb2.MRData()
mr_data.hash = msg_hash
mr_data.type = msg_type
data = qrllegacy_pb2.LegacyMessage(
func_name=qrllegacy_pb2.LegacyMessage.MR, mrData=mr_data)
for peer in self._peer_connections:
if peer not in ignore_peers:
peer.send(data)
def broadcast_get_synced_state(self):
# Request all peers to update their synced status
for peer in self._peer_connections:
peer.send_sync()
###################################################
###################################################
###################################################
###################################################
# Event handlers / Comms related
def start_listening(self):
reactor.listenTCP(config.user.p2p_local_port, self)
def clientConnectionLost(self, connector, reason): # noqa
logger.debug('connection lost: %s', reason)
def clientConnectionFailed(self, connector, reason):
logger.debug('connection failed: %s', reason)
def startedConnecting(self, connector):
logger.debug('Started connecting: %s', connector)
def add_connection(self, conn_protocol) -> bool:
# TODO: Most of this can go peer manager
if self._qrl_node.peer_manager.is_banned(conn_protocol.peer):
return False
redundancy_count = 0
for conn in self._peer_connections:
if conn.peer.ip == conn_protocol.peer.ip:
redundancy_count += 1
if config.user.max_redundant_connections >= 0:
if redundancy_count >= config.user.max_redundant_connections:
logger.info('Redundant Limit. Disconnecting client %s',
conn_protocol.peer)
return False
if self.reached_conn_limit:
# FIXME: Should we stop listening to avoid unnecessary load due to many connections?
logger.info('Peer limit hit. Disconnecting client %s',
conn_protocol.peer)
return False
# Remove your own ip address from the connection
if conn_protocol.peer.ip == conn_protocol.host.ip and conn_protocol.peer.port == config.user.p2p_public_port:
peer_list = [
p for p in self._qrl_node.peer_manager.known_peer_addresses
if p != conn_protocol.peer.full_address
]
self._qrl_node.peer_manager.extend_known_peers(peer_list)
return False
self._peer_connections.append(conn_protocol)
logger.debug('>>> new connection: %s ', conn_protocol.peer)
return True
def remove_connection(self, conn_protocol):
if conn_protocol in self._peer_connections:
self._peer_connections.remove(conn_protocol)
if conn_protocol.peer.full_address in self.peer_blockheight:
del self.peer_blockheight[conn_protocol.peer.full_address]
def monitor_connections(self):
reactor.callLater(config.user.monitor_connections_interval,
self.monitor_connections)
if len(self._peer_connections) == 0:
logger.warning('No Connected Peer Found')
known_peers = self._qrl_node.peer_manager.load_known_peers()
self._peer_q.extend(known_peers)
connected_peers_set = set()
for conn_protocol in self._peer_connections:
connected_peers_set.add(conn_protocol.peer.full_address)
for peer_item in config.user.peer_list:
peer_metadata = IPMetadata.from_full_address(peer_item)
if peer_metadata.full_address in self._peer_q:
self._peer_q.remove(peer_metadata.full_address)
if peer_metadata.full_address not in connected_peers_set:
self.connect_peer([peer_metadata.full_address])
if len(self._peer_connections) >= config.user.max_peers_limit:
return
if len(self._peer_q) == 0:
return
peer_address_list = []
max_length = min(10, config.user.max_peers_limit)
while len(self._peer_q) > 0 and len(peer_address_list) != max_length:
peer_address_list.append(self._peer_q.pop(0))
self.connect_peer(peer_address_list)
def connect_peer(self, full_address_list):
for full_address in full_address_list:
try:
addr = IPMetadata.from_full_address(full_address)
connected_peers = self.get_connected_peer_addrs()
should_connect = addr.full_address not in connected_peers
if should_connect:
reactor.connectTCP(addr.ip, addr.port, self)
except Exception as e:
logger.warning("Could not connect to %s - %s", full_address,
str(e))
class P2PFactory(ServerFactory):
protocol = P2PProtocol
def __init__(self, buffered_chain: BufferedChain, sync_state: SyncState,
qrl_node: QRLNode):
# FIXME: Constructor signature is not consistent with other factory classes
self.master_mr = MessageReceipt()
self.pos = None
self.ntp = ntp
self.buffered_chain = buffered_chain
self.sync_state = sync_state
self.sync = 0
self.genesis_processed = False # FIXME: Accessed by every p2pprotocol instance
self.peer_connections = [
] # FIXME: Accessed by every p2pprotocol instance
self.synced_peers = set(
) # FIXME: Accessed by every p2pprotocol instance
self.qrl_node = qrl_node
self.txn_processor_running = False
self.bkmr_blocknumber = 0 # Blocknumber for which bkmr is being tracked
self.bkmr_priorityq = queue.PriorityQueue()
# Scheduled and cancel the call, just to initialize with IDelayedCall
self.bkmr_processor = reactor.callLater(1, lambda: None, pos=None)
self.bkmr_processor.cancel()
@property
def connections(self):
return len(self.peer_connections)
##############################################
##############################################
##############################################
##############################################
def RFM(self, data):
"""
Request Full Message
This function request for the full message against,
the Message Receipt received.
:return:
"""
# FIXME: Again, breaking encasulation
# FIXME: Huge amount of lookups in dictionaries
msg_hash = data.hash
if msg_hash in self.master_mr.hash_msg:
if msg_hash in self.master_mr.requested_hash:
del self.master_mr.requested_hash[msg_hash]
return
if msg_hash not in self.master_mr.requested_hash:
return
peers_list = self.master_mr.requested_hash[
msg_hash].peers_connection_list
message_request = self.master_mr.requested_hash[msg_hash]
for peer in peers_list:
if peer in message_request.already_requested_peers:
continue
message_request.already_requested_peers.append(peer)
peer.transport.write(peer.wrap_message('SFM', MessageToJson(data)))
call_later_obj = reactor.callLater(
config.dev.message_receipt_timeout, self.RFM, data)
message_request.callLater = call_later_obj
return
# If execution reach to this line, then it means no peer was able to provide
# Full message for this hash thus the hash has to be deleted.
# Moreover, negative points could be added to the peers, for this behavior
if msg_hash in self.master_mr.requested_hash:
del self.master_mr.requested_hash[msg_hash]
def select_best_bkmr(self):
# FIXME: This seems to be a much higher level behavior
blocknumber = self.bkmr_blocknumber
try:
dscore, dhash = self.bkmr_priorityq.get_nowait()
if blocknumber <= self.buffered_chain.height:
oldscore = self.buffered_chain.get_block_score(blocknumber)
if dscore > oldscore:
del self.bkmr_priorityq
self.bkmr_priorityq = queue.PriorityQueue()
return
data = qrl_pb2.MR()
data.hash = dhash
data.type = 'BK'
self.RFM(data)
self.bkmr_processor = reactor.callLater(5, self.select_best_bkmr)
except queue.Empty:
return
except Exception as e:
logger.error('select_best_bkmr Unexpected Exception')
logger.error('%s', e)
##############################################
# NOTE: PoS related.. broadcasting, etc. OBSOLETE
def broadcast_st(self, st: StakeTransaction):
logger.info('<<<Transmitting ST: %s', st.activation_blocknumber)
self.register_and_broadcast('ST', st.get_message_hash(), st.to_json())
def broadcast_vote(self, vote: Vote):
logger.info('<<<Transmitting Vote Txn: %s', vote.blocknumber)
self.register_and_broadcast('VT', vote.get_message_hash(),
vote.to_json())
def broadcast_destake(self, destake_txn: DestakeTransaction):
logger.info('<<<Transmitting Destake Txn: %s', destake_txn.txfrom)
self.register_and_broadcast('DST', destake_txn.get_message_hash(),
destake_txn.to_json())
def broadcast_block(self, block: Block):
# logger.info('<<<Transmitting block: ', block.headerhash)
data = qrl_pb2.MR()
data.stake_selector = block.transactions[0].addr_from
data.block_number = block.block_number
data.prev_headerhash = bytes(block.prev_headerhash)
if block.block_number > 1:
data.reveal_hash = block.reveal_hash
self.register_and_broadcast('BK', block.headerhash, block.to_json(),
data)
def broadcast_tx(self, tx, subtype='TX'):
logger.info('<<<Transmitting TX: %s', tx.txhash)
self.register_and_broadcast(subtype, tx.get_message_hash(),
tx.to_json())
def broadcast_lt(self, lattice_public_key_txn):
logger.info('<<<Transmitting LATTICE txn: %s',
lattice_public_key_txn.txhash)
self.buffered_chain.add_lattice_public_key(lattice_public_key_txn)
self.register_and_broadcast('LT',
lattice_public_key_txn.get_message_hash(),
lattice_public_key_txn.to_json())
def register_and_broadcast(self,
msg_type,
msg_hash: bytes,
msg_json: str,
data=None):
self.master_mr.register(msg_type, msg_hash, msg_json)
self.broadcast(msg_hash, msg_type, data)
def broadcast(self,
msg_hash: bytes,
msg_type,
data=None): # Move to factory
"""
Broadcast
This function sends the Message Receipt to all connected peers.
:return:
"""
ignore_peers = []
if msg_hash in self.master_mr.requested_hash:
ignore_peers = self.master_mr.requested_hash[
msg_hash].peers_connection_list
if not data:
data = qrl_pb2.MR()
data.hash = msg_hash
data.type = msg_type
for peer in self.peer_connections:
if peer not in ignore_peers:
peer.transport.write(
self.protocol.wrap_message('MR', MessageToJson(data)))
def broadcast_get_synced_state(self):
# Request all peers to update their synced status
self.synced_peers = set()
for peer in self.peer_connections:
peer.transport.write(peer.wrap_message('SYNC'))
###################################################
###################################################
###################################################
# NOTE: tx processor related. Obsolete stage 2?
def reset_processor_flag(self, _):
self.txn_processor_running = False
def reset_processor_flag_with_err(self, msg):
logger.error('Exception in txn task')
logger.error('%s', msg)
self.txn_processor_running = False
###################################################
###################################################
###################################################
###################################################
# Event handlers
# NOTE: No need to refactor, it is obsolete
# noinspection PyMethodMayBeStatic
def clientConnectionLost(self, connector, reason):
logger.debug('connection lost: %s', reason)
# TODO: Reconnect has been disabled
# connector.connect()
# noinspection PyMethodMayBeStatic
def clientConnectionFailed(self, connector, reason):
logger.debug('connection failed: %s', reason)
# noinspection PyMethodMayBeStatic
def startedConnecting(self, connector):
logger.debug('Started connecting: %s', connector)
def connect_peers(self):
"""
Will connect to all known peers. This is typically the entry point
It does result in:
- connectionMade in each protocol (session)
- :py:meth:startedConnecting
- :py:meth:clientConnectionFailed
- :py:meth:clientConnectionLost
:return:
:rtype: None
"""
logger.info('<<<Reconnecting to peer list: %s',
self.qrl_node._peer_addresses)
for peer_address in self.qrl_node._peer_addresses:
# FIXME: Refactor search
found = False
for peer_conn in self.peer_connections:
if peer_address == peer_conn.transport.getPeer().host:
found = True
break
if found:
continue
reactor.connectTCP(peer_address, 9000, self)
class P2PFactory(ServerFactory):
protocol = P2PProtocol
def __init__(self,
chain_manager: ChainManager,
sync_state: SyncState,
qrl_node):
self.master_mr = MessageReceipt()
self.pow = None
self.sync_state = sync_state
self._ntp = ntp
self._qrl_node = qrl_node
self._chain_manager = chain_manager
self._syncing_enabled = False
self._target_peer = None
self._target_node_header_hash = None
self._last_requested_block_idx = None
self._genesis_processed = False
self._peer_connections = []
self._synced_peers_protocol = set()
self._txn_processor_running = False
self.peer_blockheight = dict()
reactor.callLater(180, self.monitor_connections)
self.p2p_msg_priority = {
qrllegacy_pb2.LegacyMessage.VE: 0,
qrllegacy_pb2.LegacyMessage.PL: 0,
qrllegacy_pb2.LegacyMessage.PONG: 0,
######################
qrllegacy_pb2.LegacyMessage.MR: 2,
qrllegacy_pb2.LegacyMessage.SFM: 1,
qrllegacy_pb2.LegacyMessage.BK: 1,
qrllegacy_pb2.LegacyMessage.FB: 1,
qrllegacy_pb2.LegacyMessage.PB: 1,
qrllegacy_pb2.LegacyMessage.BH: 1,
############################
qrllegacy_pb2.LegacyMessage.TX: 1,
qrllegacy_pb2.LegacyMessage.MT: 1,
qrllegacy_pb2.LegacyMessage.TK: 1,
qrllegacy_pb2.LegacyMessage.TT: 1,
qrllegacy_pb2.LegacyMessage.LT: 1,
qrllegacy_pb2.LegacyMessage.SL: 1,
qrllegacy_pb2.LegacyMessage.EPH: 3,
qrllegacy_pb2.LegacyMessage.SYNC: 0,
qrllegacy_pb2.LegacyMessage.CHAINSTATE: 0,
qrllegacy_pb2.LegacyMessage.HEADERHASHES: 1,
qrllegacy_pb2.LegacyMessage.P2P_ACK: 0,
}
###################################################
###################################################
###################################################
###################################################
###################################################
###################################################
@property
def has_synced_peers(self):
return len(self._synced_peers_protocol) > 0
def get_random_peer(self):
max_cumulative_difficulty = 0
for connection_id in self.peer_blockheight:
max_cumulative_difficulty = max(max_cumulative_difficulty, self.peer_blockheight[connection_id][2])
connection_ids = []
for connection_id in self.peer_blockheight:
if self.peer_blockheight[connection_id][2] == max_cumulative_difficulty:
connection_ids.append(connection_id)
selected_peer_connections = []
for connection_id in connection_ids:
for peer_conn in self._peer_connections:
if peer_conn.connection_id == connection_id:
selected_peer_connections.append(peer_conn)
if len(selected_peer_connections) == 0 or max_cumulative_difficulty == 0:
return None
return random.sample(selected_peer_connections, 1)[0]
def update_peer_blockheight(self, connection_id, block_number, headerhash, cumulative_difficulty):
self.peer_blockheight[connection_id] = [block_number, headerhash, int(UInt256ToString(cumulative_difficulty))]
def request_peer_blockheight(self):
for peer in self._peer_connections:
msg = qrllegacy_pb2.LegacyMessage(func_name=qrllegacy_pb2.LegacyMessage.BH,
bhData=qrl_pb2.BlockHeightData(block_number=0))
peer.send(msg)
def set_peer_synced(self, conn_protocol, synced: bool):
if synced:
self._synced_peers_protocol.add(conn_protocol)
else:
self._synced_peers_protocol.discard(conn_protocol)
###################################################
###################################################
###################################################
###################################################
###################################################
###################################################
###################################################
@property
def connections(self):
return len(self._peer_connections)
@property
def synced(self):
return self.pow.sync_state.state == ESyncState.synced
@property
def reached_conn_limit(self):
return len(self._peer_connections) >= config.user.max_peers_limit
def get_connected_peer_ips(self):
# FIXME: Convert self._peer_connections to set
return set([peer.peer_ip for peer in self._peer_connections])
###################################################
###################################################
###################################################
###################################################
###################################################
###################################################
@property
def chain_height(self):
return self._chain_manager.height
def get_last_block(self):
return self._chain_manager.get_last_block()
def get_headerhashes(self, start_blocknumber):
return self._chain_manager.get_headerhashes(start_blocknumber)
def get_cumulative_difficulty(self):
return self._chain_manager.get_cumulative_difficulty()
def get_block(self, block_number):
return self._chain_manager.get_block_by_number(block_number)
def block_received(self, source, block: Block):
self.pow.last_pb_time = time.time()
logger.info('>>> Received Block #%d %s', block.block_number, bin2hstr(block.headerhash))
if source != self._target_peer:
logger.warning('Received block from unexpected peer')
logger.warning('Expected peer: %s', self._target_peer.connection_id)
logger.warning('Found peer: %s', source.connection_id)
return
if block.block_number != self._last_requested_block_idx:
logger.warning('Did not match %s', self._last_requested_block_idx)
return
target_start_blocknumber = self._target_node_header_hash.block_number
expected_headerhash = self._target_node_header_hash.headerhashes[block.block_number - target_start_blocknumber]
if block.headerhash != expected_headerhash:
logger.warning('Did not match headerhash')
logger.warning('Expected headerhash %s', expected_headerhash)
logger.warning('Found headerhash %s', block.headerhash)
return
# FIXME: This check should not be necessary
if not self._chain_manager.add_block(block):
logger.warning('Failed to Add Block')
return
try:
reactor.download_monitor.cancel()
except Exception as e:
logger.warning("PB: %s", e)
if self.is_syncing_finished():
return
self._last_requested_block_idx += 1
if self.is_syncing_finished():
return
self.peer_fetch_block()
def is_syncing(self) -> bool:
return self._syncing_enabled
def is_syncing_finished(self, force_finish=False):
if self._last_requested_block_idx == len(self._target_node_header_hash.headerhashes) or force_finish:
self._last_requested_block_idx = None
self._target_node_header_hash = None
self._target_peer = None
self._syncing_enabled = False
return True
return False
def peer_fetch_block(self, retry=0):
node_header_hash = self._target_node_header_hash
curr_index = self._last_requested_block_idx - node_header_hash.block_number
block_headerhash = node_header_hash.headerhashes[curr_index]
block = self._chain_manager.state.get_block(block_headerhash)
if not block:
if retry >= 5:
logger.debug('Retry Limit Hit')
self._qrl_node.ban_peer(self._target_peer)
self.is_syncing_finished(force_finish=True)
return
else:
while block and curr_index < len(node_header_hash.headerhashes):
block_headerhash = node_header_hash.headerhashes[curr_index]
block = self._chain_manager.state.get_block(block_headerhash)
self._last_requested_block_idx += 1
curr_index = self._last_requested_block_idx - node_header_hash.block_number
retry = 0
if self.is_syncing_finished():
return
self._target_peer.send_fetch_block(self._last_requested_block_idx)
reactor.download_monitor = reactor.callLater(20, self.peer_fetch_block, retry+1)
def compare_and_sync(self, peer, node_header_hash: qrl_pb2.NodeHeaderHash):
if self._syncing_enabled:
logger.info('>> Ignoring compare_and_sync Syncing Enabled')
return
last_block = self._chain_manager.get_last_block()
node_last_block_number = node_header_hash.block_number + len(node_header_hash.headerhashes) - 1
last_block_number = min(last_block.block_number, node_last_block_number)
if last_block_number < node_header_hash.block_number:
return
fork_block_number = last_block.block_number + 1
fork_found = False
for i in range(last_block_number, node_header_hash.block_number - 1, -1):
block = self._chain_manager.get_block_by_number(i)
if block.headerhash == node_header_hash.headerhashes[i-node_header_hash.block_number]:
break
fork_block_number = i
fork_found = True
if fork_found or (last_block.block_number < node_last_block_number):
self._target_peer = peer
self._target_node_header_hash = node_header_hash
self._last_requested_block_idx = fork_block_number
self._syncing_enabled = True
self.peer_fetch_block()
###################################################
###################################################
###################################################
###################################################
###################################################
###################################################
def request_full_message(self, mr_data: qrllegacy_pb2.MRData):
"""
Request Full Message
This function request for the full message against,
the Message Receipt received.
:return:
"""
# FIXME: Again, breaking encasulation
# FIXME: Huge amount of lookups in dictionaries
msg_hash = mr_data.hash
if msg_hash in self.master_mr._hash_msg:
if msg_hash in self.master_mr.requested_hash:
del self.master_mr.requested_hash[msg_hash]
return
if msg_hash not in self.master_mr.requested_hash:
return
peers_list = self.master_mr.requested_hash[msg_hash].peers_connection_list
message_request = self.master_mr.requested_hash[msg_hash]
for peer in peers_list:
if peer in message_request.already_requested_peers:
continue
message_request.already_requested_peers.append(peer)
msg = qrllegacy_pb2.LegacyMessage(func_name=qrllegacy_pb2.LegacyMessage.SFM,
mrData=qrllegacy_pb2.MRData(hash=mr_data.hash, type=mr_data.type))
peer.send(msg)
call_later_obj = reactor.callLater(config.dev.message_receipt_timeout,
self.request_full_message,
mr_data)
message_request.callLater = call_later_obj
return
# If execution reach to this line, then it means no peer was able to provide
# Full message for this hash thus the hash has to be deleted.
# Moreover, negative points could be added to the peers, for this behavior
if msg_hash in self.master_mr.requested_hash:
del self.master_mr.requested_hash[msg_hash]
##############################################
##############################################
##############################################
##############################################
# NOTE: PoW related.. broadcasting, etc. OBSOLETE
def reset_processor_flag(self, _):
self._txn_processor_running = False
def reset_processor_flag_with_err(self, msg):
logger.error('Exception in txn task')
logger.error('%s', msg)
self._txn_processor_running = False
def add_unprocessed_txn(self, tx, ip):
self._chain_manager.tx_pool.update_pending_tx_pool(tx, ip)
if not self._txn_processor_running:
txn_processor = TxnProcessor(state=self._chain_manager.state,
transaction_pool_obj=self._chain_manager.tx_pool,
broadcast_tx=self.broadcast_tx)
task_defer = TxnProcessor.create_cooperate(txn_processor).whenDone()
task_defer.addCallback(self.reset_processor_flag) \
.addErrback(self.reset_processor_flag_with_err)
self._txn_processor_running = True
def broadcast_tx(self, tx: TransferTransaction):
logger.info('<<<Transmitting TX: %s', tx.txhash)
if tx.subtype == qrl_pb2.Transaction.MESSAGE:
legacy_type = qrllegacy_pb2.LegacyMessage.MT
elif tx.subtype == qrl_pb2.Transaction.TRANSFER:
legacy_type = qrllegacy_pb2.LegacyMessage.TX
elif tx.subtype == qrl_pb2.Transaction.TOKEN:
legacy_type = qrllegacy_pb2.LegacyMessage.TK
elif tx.subtype == qrl_pb2.Transaction.TRANSFERTOKEN:
legacy_type = qrllegacy_pb2.LegacyMessage.TT
elif tx.subtype == qrl_pb2.Transaction.LATTICE:
legacy_type = qrllegacy_pb2.LegacyMessage.LT
elif tx.subtype == qrl_pb2.Transaction.SLAVE:
legacy_type = qrllegacy_pb2.LegacyMessage.SL
else:
raise ValueError('Invalid Transaction Type')
self.register_and_broadcast(legacy_type, tx.get_message_hash(), tx.pbdata)
def broadcast_ephemeral_message(self, encrypted_ephemeral):
logger.info('<<<Broadcasting Encrypted Ephemeral Message')
self._chain_manager.add_ephemeral_message(encrypted_ephemeral)
self.register_and_broadcast('EPH',
encrypted_ephemeral.get_message_hash(),
encrypted_ephemeral.to_json())
def broadcast_tx_relay(self, source_peer, tx):
txn_msg = source_peer._wrap_message('TX', tx.to_json())
for peer in self._peer_connections:
if peer != source_peer:
peer.transport.write(txn_msg)
##############################################
##############################################
##############################################
##############################################
def broadcast_block(self, block: Block):
# logger.info('<<<Transmitting block: ', block.headerhash)
data = qrllegacy_pb2.MRData()
data.stake_selector = block.transactions[0].public_key
data.block_number = block.block_number
data.prev_headerhash = bytes(block.prev_headerhash)
self.register_and_broadcast(qrllegacy_pb2.LegacyMessage.BK, block.headerhash, block.pbdata, data)
##############################################
##############################################
##############################################
##############################################
def register_and_broadcast(self, msg_type, msg_hash: bytes, pbdata, data=None):
self.master_mr.register(msg_type, msg_hash, pbdata)
self.broadcast(msg_type, msg_hash, data)
def broadcast(self, msg_type, msg_hash: bytes, mr_data=None):
"""
Broadcast
This function sends the Message Receipt to all connected peers.
:return:
"""
ignore_peers = []
if msg_hash in self.master_mr.requested_hash:
ignore_peers = self.master_mr.requested_hash[msg_hash].peers_connection_list
if not mr_data:
mr_data = qrllegacy_pb2.MRData()
mr_data.hash = msg_hash
mr_data.type = msg_type
data = qrllegacy_pb2.LegacyMessage(func_name=qrllegacy_pb2.LegacyMessage.MR,
mrData=mr_data)
for peer in self._peer_connections:
if peer not in ignore_peers:
peer.send(data)
def broadcast_get_synced_state(self):
# Request all peers to update their synced status
self._synced_peers_protocol = set()
for peer in self._peer_connections:
peer.send_sync()
###################################################
###################################################
###################################################
###################################################
# Event handlers / Comms related
def start_listening(self):
reactor.listenTCP(9000, self)
# NOTE: No need to refactor, it is obsolete
def clientConnectionLost(self, connector, reason): # noqa
logger.debug('connection lost: %s', reason)
def clientConnectionFailed(self, connector, reason):
logger.debug('connection failed: %s', reason)
def startedConnecting(self, connector):
logger.debug('Started connecting: %s', connector)
def add_connection(self, conn_protocol) -> bool:
# TODO: Most of this can go the peer manager
if self._qrl_node.is_banned(conn_protocol.peer_ip):
conn_protocol.loseConnection()
return False
# FIXME: (For AWS) This could be problematic for other users
# FIXME: identify nodes by an GUID?
if config.dev.public_ip and conn_protocol.peer_ip == config.dev.public_ip:
conn_protocol.loseConnection()
return False
if self.reached_conn_limit:
# FIXME: Should we stop listening to avoid unnecessary load due to many connections?
logger.info('Peer limit hit. Disconnecting client %s', conn_protocol.peer_ip)
conn_protocol.loseConnection()
return False
peer_list = self._qrl_node.peer_addresses
if conn_protocol.peer_ip == conn_protocol.host_ip:
if conn_protocol.peer_ip in peer_list:
logger.info('Self in peer_list, removing..')
peer_list.remove(conn_protocol.peer_ip)
self._qrl_node.peer_manager.update_peer_addresses(peer_list)
conn_protocol.loseConnection()
return False
self._peer_connections.append(conn_protocol)
if conn_protocol.peer_ip not in peer_list:
logger.debug('Adding to peer_list')
peer_list.add(conn_protocol.peer_ip)
self._qrl_node.peer_manager.update_peer_addresses(peer_list)
logger.debug('>>> new peer connection : %s:%s ', conn_protocol.peer_ip, str(conn_protocol.peer_port))
return True
def remove_connection(self, conn_protocol):
if conn_protocol in self._peer_connections:
self._peer_connections.remove(conn_protocol)
if conn_protocol.connection_id in self.peer_blockheight:
del self.peer_blockheight[conn_protocol.connection_id]
self._synced_peers_protocol.discard(conn_protocol)
def monitor_connections(self):
reactor.callLater(180, self.monitor_connections)
if len(self._peer_connections) == 0:
logger.warning('No Connected Peer Found')
reactor.callLater(60, self._qrl_node.connect_peers)
return
connected_peers_set = set()
for conn_protocol in self._peer_connections:
connected_peers_set.add(conn_protocol.peer_ip)
for ip in config.user.peer_list:
if ip not in connected_peers_set:
self.connect_peer(ip)
def connect_peer(self, peer_address):
if peer_address not in self.get_connected_peer_ips():
reactor.connectTCP(peer_address, 9000, self)
def test_create(self):
mr = MessageReceipt()
self.assertIsNotNone(mr)
self.assertEqual(mr.allowed_types, ['TX', 'ST', 'BK', 'DT'])
