def start_blockheight_loop(self): self.stop_blockheight_loop() self.CurrentBlockheight = BC.Default().Height self.blockheight_loop = LoopingCall(self.BlockheightCheck, clock=self.reactor) self.blockheight_loop_deferred = self.blockheight_loop.start(240, now=False) self.blockheight_loop_deferred.addErrback(self.OnBlockheightcheckError)
def start_check_bcr_loop(self): logger.debug(f"start_check_bcr_loop") if self.check_bcr_loop and self.check_bcr_loop.running: logger.debug("start_check_bcr_loop: still running -> stopping...") self.stop_check_bcr_loop() self.check_bcr_loop = LoopingCall(self.check_bcr_catchup, clock=self.reactor) self.check_bcr_loop_deferred = self.check_bcr_loop.start(5) self.check_bcr_loop_deferred.addErrback(self.OnCheckBcrError)
def start_peer_check_loop(self): logger.debug(f"start_peer_check_loop") if self.peer_check_loop and self.peer_check_loop.running: logger.debug("start_peer_check_loop: still running -> stopping...") self.stop_peer_check_loop() self.peer_check_loop = LoopingCall(self.PeerCheckLoop, clock=self.reactor) self.peer_check_loop_deferred = self.peer_check_loop.start(10, now=False) self.peer_check_loop_deferred.addErrback(self.OnPeerLoopError)
class NodeLeader: _LEAD = None Peers = [] KNOWN_ADDRS = [] DEAD_ADDRS = [] NodeId = None _MissedBlocks = [] BREQPART = 100 BREQMAX = 10000 KnownHashes = [] MissionsGlobal = [] MemPool = {} RelayCache = {} NodeCount = 0 CurrentBlockheight = 0 ServiceEnabled = False peer_check_loop = None peer_check_loop_deferred = None check_bcr_loop = None check_bcr_loop_deferred = None memcheck_loop = None memcheck_loop_deferred = None blockheight_loop = None blockheight_loop_deferred = None task_handles = {} @staticmethod def Instance(reactor=None): """ Get the local node instance. Args: reactor: (optional) custom reactor to use in NodeLeader. Returns: NodeLeader: instance. """ if NodeLeader._LEAD is None: NodeLeader._LEAD = NodeLeader(reactor) return NodeLeader._LEAD def __init__(self, reactor=None): """ Create an instance. This is the equivalent to C#'s LocalNode.cs """ self.Setup() self.ServiceEnabled = settings.SERVICE_ENABLED self.peer_zero_count = 0 # track the number of times PeerCheckLoop saw a Peer count of zero. Reset e.g. after 3 times self.connection_queue = [] self.reactor = twisted_reactor self.incoming_server_running = False self.forced_disconnect_by_us = 0 self.peers_connecting = 0 # for testability if reactor: self.reactor = reactor def start_peer_check_loop(self): logger.debug(f"start_peer_check_loop") if self.peer_check_loop and self.peer_check_loop.running: logger.debug("start_peer_check_loop: still running -> stopping...") self.stop_peer_check_loop() self.peer_check_loop = LoopingCall(self.PeerCheckLoop, clock=self.reactor) self.peer_check_loop_deferred = self.peer_check_loop.start(10, now=False) self.peer_check_loop_deferred.addErrback(self.OnPeerLoopError) def stop_peer_check_loop(self, cancel=True): logger.debug(f"stop_peer_check_loop, cancel: {cancel}") if self.peer_check_loop and self.peer_check_loop.running: logger.debug(f"stop_peer_check_loop, calling stop()") self.peer_check_loop.stop() if cancel and self.peer_check_loop_deferred: logger.debug(f"stop_peer_check_loop, calling cancel()") self.peer_check_loop_deferred.cancel() def start_check_bcr_loop(self): logger.debug(f"start_check_bcr_loop") if self.check_bcr_loop and self.check_bcr_loop.running: logger.debug("start_check_bcr_loop: still running -> stopping...") self.stop_check_bcr_loop() self.check_bcr_loop = LoopingCall(self.check_bcr_catchup, clock=self.reactor) self.check_bcr_loop_deferred = self.check_bcr_loop.start(5) self.check_bcr_loop_deferred.addErrback(self.OnCheckBcrError) def stop_check_bcr_loop(self, cancel=True): logger.debug(f"stop_check_bcr_loop, cancel: {cancel}") if self.check_bcr_loop and self.check_bcr_loop.running: logger.debug(f"stop_check_bcr_loop, calling stop()") self.check_bcr_loop.stop() if cancel and self.check_bcr_loop_deferred: logger.debug(f"stop_check_bcr_loop, calling cancel()") self.check_bcr_loop_deferred.cancel() def start_memcheck_loop(self): self.stop_memcheck_loop() self.memcheck_loop = LoopingCall(self.MempoolCheck, clock=self.reactor) self.memcheck_loop_deferred = self.memcheck_loop.start(240, now=False) self.memcheck_loop_deferred.addErrback(self.OnMemcheckError) def stop_memcheck_loop(self, cancel=True): if self.memcheck_loop and self.memcheck_loop.running: self.memcheck_loop.stop() if cancel and self.memcheck_loop_deferred: self.memcheck_loop_deferred.cancel() def start_blockheight_loop(self): self.stop_blockheight_loop() self.CurrentBlockheight = BC.Default().Height self.blockheight_loop = LoopingCall(self.BlockheightCheck, clock=self.reactor) self.blockheight_loop_deferred = self.blockheight_loop.start(240, now=False) self.blockheight_loop_deferred.addErrback(self.OnBlockheightcheckError) def stop_blockheight_loop(self, cancel=True): if self.blockheight_loop and self.blockheight_loop.running: self.blockheight_loop.stop() if cancel and self.blockheight_loop_deferred: self.blockheight_loop_deferred.cancel() def Setup(self): """ Initialize the local node. Returns: """ self.Peers = [] # active nodes that we're connected to self.KNOWN_ADDRS = [ ] # node addresses that we've learned about from other nodes self.DEAD_ADDRS = [ ] # addresses that were performing poorly or we could not establish a connection to self.MissionsGlobal = [] self.NodeId = random.randint(1294967200, 4294967200) def Restart(self): self.stop_peer_check_loop() self.stop_check_bcr_loop() self.stop_memcheck_loop() self.stop_blockheight_loop() self.peer_check_loop_deferred = None self.check_bcr_loop_deferred = None self.memcheck_loop_deferred = None self.blockheight_loop_deferred = None self.peers_connecting = 0 if len(self.Peers) == 0: # preserve any addresses we know because the peers in the seedlist might have gone bad and then we can't receive new addresses anymore unique_addresses = list(set(self.KNOWN_ADDRS + self.DEAD_ADDRS)) self.KNOWN_ADDRS = unique_addresses self.DEAD_ADDRS = [] self.peer_zero_count = 0 self.connection_queue = [] self.Start(skip_seeds=True) def throttle_sync(self): for peer in self.Peers: # type: NeoNode peer.stop_block_loop(cancel=False) peer.stop_peerinfo_loop(cancel=False) peer.stop_header_loop(cancel=False) # start a loop to check if we've caught up on our requests if not self.check_bcr_loop: self.start_check_bcr_loop() def check_bcr_catchup(self): """we're exceeding data request speed vs receive + process""" logger.debug( f"Checking if BlockRequests has caught up {len(BC.Default().BlockRequests)}" ) # test, perhaps there's some race condition between slow startup and throttle sync, otherwise blocks will never go down for peer in self.Peers: # type: NeoNode peer.stop_block_loop(cancel=False) peer.stop_peerinfo_loop(cancel=False) peer.stop_header_loop(cancel=False) if len(BC.Default().BlockRequests) > 0: for peer in self.Peers: peer.keep_alive() peer.health_check(HEARTBEAT_BLOCKS) peer_bcr_len = len(peer.myblockrequests) # if a peer has cleared its queue then reset heartbeat status to avoid timing out when resuming from "check_bcr" if there's 1 or more really slow peer(s) if peer_bcr_len == 0: peer.start_outstanding_data_request[HEARTBEAT_BLOCKS] = 0 print(f"{peer.prefix} request count: {peer_bcr_len}") if peer_bcr_len == 1: next_hash = BC.Default().GetHeaderHash( self.CurrentBlockheight + 1) print(f"{peer.prefix} {peer.myblockrequests} {next_hash}") else: # we're done catching up. Stop own loop and restart peers self.stop_check_bcr_loop() self.check_bcr_loop = None logger.debug("BlockRequests have caught up...resuming sync") for peer in self.Peers: peer.ProtocolReady() # this starts all loops again # give a little bit of time between startup of peers time.sleep(2) def _process_connection_queue(self): for addr in self.connection_queue: self.SetupConnection(addr) def Start(self, seed_list: List[str] = None, skip_seeds: bool = False) -> None: """ Start connecting to the seed list. Args: seed_list: a list of host:port strings if not supplied use list from `protocol.xxx.json` skip_seeds: skip connecting to seed list """ if not seed_list: seed_list = settings.SEED_LIST logger.debug("Starting up nodeleader") if not skip_seeds: logger.debug("Attempting to connect to seed list...") for bootstrap in seed_list: if not is_ip_address(bootstrap): host, port = bootstrap.split(':') bootstrap = f"{hostname_to_ip(host)}:{port}" addr = Address(bootstrap) self.KNOWN_ADDRS.append(addr) self.SetupConnection(addr) logger.debug( "Starting up nodeleader: starting peer, mempool, and blockheight check loops" ) # check in on peers every 10 seconds self.start_peer_check_loop() self.start_memcheck_loop() self.start_blockheight_loop() if settings.ACCEPT_INCOMING_PEERS and not self.incoming_server_running: class OneShotFactory(Factory): def __init__(self, leader): self.leader = leader def buildProtocol(self, addr): print(f"building new protocol for addr: {addr}") self.leader.AddKnownAddress( Address(f"{addr.host}:{addr.port}")) p = NeoNode(incoming_client=True) p.factory = self return p def listen_err(err): print(f"Failed start listening server for reason: {err.value}") def listen_ok(value): self.incoming_server_running = True logger.debug( f"Starting up nodeleader: setting up listen server on port: {settings.NODE_PORT}" ) server_endpoint = TCP4ServerEndpoint(self.reactor, settings.NODE_PORT) listenport_deferred = server_endpoint.listen( OneShotFactory(leader=self)) listenport_deferred.addCallback(listen_ok) listenport_deferred.addErrback(listen_err) def setBlockReqSizeAndMax(self, breqpart=100, breqmax=10000): if breqpart > 0 and breqpart <= 500 and breqmax > 0 and breqmax > breqpart: self.BREQPART = breqpart self.BREQMAX = breqmax logger.info( "Set each node to request %s blocks per request with a total of %s in queue" % (self.BREQPART, self.BREQMAX)) return True else: raise ValueError( "invalid values. Please specify a block request part and max size for each node, like 30 and 1000" ) def setBlockReqSizeByName(self, name): if name.lower() == 'slow': self.BREQPART = 15 self.BREQMAX = 5000 elif name.lower() == 'normal': self.BREQPART = 100 self.BREQMAX = 10000 elif name.lower() == 'fast': self.BREQPART = 250 self.BREQMAX = 15000 else: logger.info( "configuration name %s not found. use 'slow', 'normal', or 'fast'" % name) return False logger.info( "Set each node to request %s blocks per request with a total of %s in queue" % (self.BREQPART, self.BREQMAX)) return True def RemoteNodePeerReceived(self, host, port, via_node_addr): addr = Address("%s:%s" % (host, port)) if addr not in self.KNOWN_ADDRS and addr not in self.DEAD_ADDRS: logger.debug( f"Adding new address {addr:>21} to known addresses list, received from {via_node_addr}" ) # we always want to save new addresses in case we lose all active connections before we can request a new list self.KNOWN_ADDRS.append(addr) def SetupConnection(self, addr, endpoint=None): if len(self.Peers ) + self.peers_connecting < settings.CONNECTED_PEER_MAX: try: host, port = addr.split(':') if endpoint: point = endpoint else: point = TCP4ClientEndpoint(self.reactor, host, int(port), timeout=5) self.peers_connecting += 1 d = connectProtocol(point, NeoNode()) # type: Deferred d.addErrback(self.clientConnectionFailed, addr) return d except Exception as e: logger.error(f"Setup connection with with {e}") def Shutdown(self): """Disconnect all connected peers.""" logger.debug("Nodeleader shutting down") self.stop_peer_check_loop() self.peer_check_loop_deferred = None self.stop_check_bcr_loop() self.check_bcr_loop_deferred = None self.stop_memcheck_loop() self.memcheck_loop_deferred = None self.stop_blockheight_loop() self.blockheight_loop_deferred = None for p in self.Peers: p.Disconnect() def AddConnectedPeer(self, peer): """ Add a new connect peer to the known peers list. Args: peer (NeoNode): instance. """ # if present self.RemoveFromQueue(peer.address) self.AddKnownAddress(peer.address) if len(self.Peers) > settings.CONNECTED_PEER_MAX: peer.Disconnect("Max connected peers reached", isDead=False) if peer not in self.Peers: self.Peers.append(peer) else: # either peer is already in the list and it has reconnected before it timed out on our side # or it's trying to connect multiple times # or we hit the max connected peer count self.RemoveKnownAddress(peer.address) peer.Disconnect() def RemoveConnectedPeer(self, peer): """ Remove a connected peer from the known peers list. Args: peer (NeoNode): instance. """ if peer in self.Peers: self.Peers.remove(peer) def RemoveFromQueue(self, addr): """ Remove an address from the connection queue Args: addr: Returns: """ if addr in self.connection_queue: self.connection_queue.remove(addr) def RemoveKnownAddress(self, addr): if addr in self.KNOWN_ADDRS: self.KNOWN_ADDRS.remove(addr) def AddKnownAddress(self, addr): if addr not in self.KNOWN_ADDRS: self.KNOWN_ADDRS.append(addr) def AddDeadAddress(self, addr, reason=None): if addr not in self.DEAD_ADDRS: if reason: logger.debug( f"Adding address {addr:>21} to DEAD_ADDRS list. Reason: {reason}" ) else: logger.debug(f"Adding address {addr:>21} to DEAD_ADDRS list.") self.DEAD_ADDRS.append(addr) # something in the dead_addrs list cannot be in the known_addrs list. Which holds either "tested and good" or "untested" addresses self.RemoveKnownAddress(addr) def PeerCheckLoop(self): logger.debug( f"Peer check loop...checking [A:{len(self.KNOWN_ADDRS)} D:{len(self.DEAD_ADDRS)} C:{len(self.Peers)} M:{settings.CONNECTED_PEER_MAX} " f"Q:{len(self.connection_queue)}]") connected = [] peer_to_remove = [] for peer in self.Peers: if peer.endpoint == "": peer_to_remove.append(peer) else: connected.append(peer.address) for p in peer_to_remove: self.Peers.remove(p) self._ensure_peer_tasks_running(connected) self._check_for_queuing_possibilities(connected) self._process_connection_queue() # keep this last, to ensure we first try queueing. self._monitor_for_zero_connected_peers() def _check_for_queuing_possibilities(self, connected): # we sort addresses such that those that we recently disconnected from are last in the list self.KNOWN_ADDRS.sort(key=lambda address: address.last_connection) to_remove = [] for addr in self.KNOWN_ADDRS: if addr in self.DEAD_ADDRS: logger.debug( f"Address {addr} found in DEAD_ADDRS list...skipping") to_remove.append(addr) continue if addr not in connected and addr not in self.connection_queue and len( self.Peers) + len( self.connection_queue) < settings.CONNECTED_PEER_MAX: self.connection_queue.append(addr) logger.debug( f"Queuing {addr:>21} for new connection [in queue: {len(self.connection_queue)} " f"connected: {len(self.Peers)} maxpeers:{settings.CONNECTED_PEER_MAX}]" ) # we couldn't remove addresses found in the DEAD_ADDR list from ADDRS while looping over it # so we do it now to clean up for addr in to_remove: # TODO: might be able to remove. Check if this scenario is still possible since the refactor try: self.KNOWN_ADDRS.remove(addr) except KeyError: pass def _monitor_for_zero_connected_peers(self): """ Track if we lost connection to all peers. Give some retries threshold to allow peers that are in the process of connecting or in the queue to be connected to run """ if len(self.Peers) == 0 and len(self.connection_queue) == 0: if self.peer_zero_count > 2: logger.debug( "Peer count 0 exceeded max retries threshold, restarting..." ) self.Restart() else: logger.debug( f"Peer count is 0, allow for retries or queued connections to be established {self.peer_zero_count}" ) self.peer_zero_count += 1 def _ensure_peer_tasks_running(self, connected): # double check that the peers that are connected are running their tasks # unless we're data throttling # there has been a case where the connection was established, but ProtocolReady() never called nor disconnected. if not self.check_bcr_loop: for peer in self.Peers: if not peer.has_tasks_running() and peer.handshake_complete: peer.start_all_tasks() def InventoryReceived(self, inventory): """ Process a received inventory. Args: inventory (neo.Network.Inventory): expect a Block type. Returns: bool: True if processed and verified. False otherwise. """ if inventory.Hash.ToBytes() in self._MissedBlocks: self._MissedBlocks.remove(inventory.Hash.ToBytes()) if inventory is MinerTransaction: return False if type(inventory) is Block: if BC.Default() is None: return False if BC.Default().ContainsBlock(inventory.Index): return False if not BC.Default().AddBlock(inventory): return False else: if not inventory.Verify(self.MemPool.values()): return False def RelayDirectly(self, inventory): """ Relay the inventory to the remote client. Args: inventory (neo.Network.Inventory): Returns: bool: True if relayed successfully. False otherwise. """ relayed = False self.RelayCache[inventory.Hash.ToBytes()] = inventory for peer in self.Peers: relayed |= peer.Relay(inventory) if len(self.Peers) == 0: if type(BC.Default()) is TestLevelDBBlockchain: # mock a true result for tests return True logger.info("no connected peers") return relayed def Relay(self, inventory): """ Relay the inventory to the remote client. Args: inventory (neo.Network.Inventory): Returns: bool: True if relayed successfully. False otherwise. """ if type(inventory) is MinerTransaction: return False if inventory.Hash.ToBytes() in self.KnownHashes: return False self.KnownHashes.append(inventory.Hash.ToBytes()) if type(inventory) is Block: pass elif type(inventory) is Transaction or issubclass( type(inventory), Transaction): if not self.AddTransaction(inventory): # if we fail to add the transaction for whatever reason, remove it from the known hashes list or we cannot retry the same transaction again try: self.KnownHashes.remove(inventory.Hash.ToBytes()) except ValueError: # it not found pass return False else: # consensus pass relayed = self.RelayDirectly(inventory) return relayed def GetTransaction(self, hash): if hash in self.MemPool.keys(): return self.MemPool[hash] return None def AddTransaction(self, tx): """ Add a transaction to the memory pool. Args: tx (neo.Core.TX.Transaction): instance. Returns: bool: True if successfully added. False otherwise. """ if BC.Default() is None: return False if tx.Hash.ToBytes() in self.MemPool.keys(): return False if BC.Default().ContainsTransaction(tx.Hash): return False if not tx.Verify(self.MemPool.values()): logger.error("Verifying tx result... failed") return False self.MemPool[tx.Hash.ToBytes()] = tx return True def RemoveTransaction(self, tx): """ Remove a transaction from the memory pool if it is found on the blockchain. Args: tx (neo.Core.TX.Transaction): instance. Returns: bool: True if successfully removed. False otherwise. """ if BC.Default() is None: return False if not BC.Default().ContainsTransaction(tx.Hash): return False if tx.Hash.ToBytes() in self.MemPool: del self.MemPool[tx.Hash.ToBytes()] return True return False def MempoolCheck(self): """ Checks the Mempool and removes any tx found on the Blockchain Implemented to resolve https://github.com/CityOfZion/neo-python/issues/703 """ txs = [] values = self.MemPool.values() for tx in values: txs.append(tx) for tx in txs: res = self.RemoveTransaction(tx) if res: logger.debug( "found tx 0x%s on the blockchain ...removed from mempool" % tx.Hash) def BlockheightCheck(self): """ Checks the current blockheight and finds the peer that prevents advancement """ if self.CurrentBlockheight == BC.Default().Height: if len(self.Peers) > 0: logger.debug("Blockheight is not advancing ...") next_hash = BC.Default().GetHeaderHash( self.CurrentBlockheight + 1) culprit_found = False for peer in self.Peers: if next_hash in peer.myblockrequests: culprit_found = True peer.Disconnect() break # this happens when we're connecting to other nodes that are stuck themselves if not culprit_found: for peer in self.Peers: peer.Disconnect() else: self.CurrentBlockheight = BC.Default().Height def clientConnectionFailed(self, err, address: Address): """ Called when we fail to connect to an endpoint Args: err: Twisted Failure instance address: the address we failed to connect to """ if type(err.value) == error.TimeoutError: logger.debug( f"Failed connecting to {address} connection timed out") elif type(err.value) == error.ConnectError: ce = err.value if len(ce.args) > 0: logger.debug( f"Failed connecting to {address} {ce.args[0].value}") else: logger.debug(f"Failed connecting to {address}") else: logger.debug(f"Failed connecting to {address} {err.value}") self.peers_connecting -= 1 self.RemoveKnownAddress(address) self.RemoveFromQueue(address) # if we failed to connect to new addresses, we should always add them to the DEAD_ADDRS list self.AddDeadAddress(address) # for testing return err.type @staticmethod def Reset(): NodeLeader._LEAD = None NodeLeader.Peers = [] NodeLeader.KNOWN_ADDRS = [] NodeLeader.DEAD_ADDRS = [] NodeLeader.NodeId = None NodeLeader._MissedBlocks = [] NodeLeader.BREQPART = 100 NodeLeader.BREQMAX = 10000 NodeLeader.KnownHashes = [] NodeLeader.MissionsGlobal = [] NodeLeader.MemPool = {} NodeLeader.RelayCache = {} NodeLeader.NodeCount = 0 NodeLeader.CurrentBlockheight = 0 NodeLeader.ServiceEnabled = False NodeLeader.peer_check_loop = None NodeLeader.peer_check_loop_deferred = None NodeLeader.check_bcr_loop = None NodeLeader.check_bcr_loop_deferred = None NodeLeader.memcheck_loop = None NodeLeader.memcheck_loop_deferred = None NodeLeader.blockheight_loop = None NodeLeader.blockheight_loop_deferred = None NodeLeader.task_handles = {} def OnSetupConnectionErr(self, err): if type(err.value) == CancelledError: return logger.debug("On setup connection error! %s" % err) def OnCheckBcrError(self, err): if type(err.value) == CancelledError: return logger.debug("On Check BlockRequest error! %s" % err) def OnPeerLoopError(self, err): if type(err.value) == CancelledError: return logger.debug("Error on Peer check loop %s " % err) def OnMemcheckError(self, err): if type(err.value) == CancelledError: return logger.debug("Error on Memcheck check %s " % err) def OnBlockheightcheckError(self, err): if type(err.value) == CancelledError: return logger.debug("Error on Blockheight check loop %s " % err)
def start_memcheck_loop(self): self.stop_memcheck_loop() self.memcheck_loop = LoopingCall(self.MempoolCheck, clock=self.reactor) self.memcheck_loop_deferred = self.memcheck_loop.start(240, now=False) self.memcheck_loop_deferred.addErrback(self.OnMemcheckError)