def getTxStatus(self, txid: str) -> str: with self.chain_lock: peer, port = self._getPort() message = Message(Actions.TxStatusReq, txid, port) with socket.create_connection(peer(), timeout=25) as s: s.sendall(Utils.encode_socket_data(message)) logger.info(f'[EdgeHand] succeed to send TxStatus to {peer}') msg_len = int(binascii.hexlify(s.recv(4) or b'\x00'), 16) data = b'' while msg_len > 0: tdat = s.recv(1024) data += tdat msg_len -= len(tdat) message = Utils.deserialize(data.decode(), self.gs) if data else None if message: logger.info( f'[EdgeHand] received TxStatus from peer {peer}') return message.data else: logger.info( f'[EdgeHand] recv nothing for TxStatus from peer {peer}' ) return None
def load_from_disk(active_chain: BlockChain, utxo_set: UTXO_Set, CHAIN_PATH=Params.CHAIN_FILE): def _connect_block(block: Block, active_chain: BlockChain, utxo_set: UTXO_Set) -> bool: if block.validate_block(active_chain, utxo_set) is None: return False logger.info( f'connecting the {len(active_chain.chain)+1}th block {block.id} to chain {active_chain.idx}' ) active_chain.chain.append(block) for tx in block.txns: if not tx.is_coinbase: for txin in tx.txins: utxo_set.rm_from_utxo(*txin.to_spend) for i, txout in enumerate(tx.txouts): utxo_set.add_to_utxo(txout, tx, i, tx.is_coinbase, len(active_chain.chain)) return True if not os.path.isfile(CHAIN_PATH): logger.info(f'[persistence] chain storage file does not exist') return else: if len(active_chain.chain) > 1: logger.info( f'[persistence] more than the genesis block exists, load_from_disk stopped and return' ) return try: with open(CHAIN_PATH, "rb") as f: block_len = int(f.read(20)) logger.info( f'[persistence] {block_len} is claimed at the head of chain file' ) msg_len = int(f.read(20)) gs = dict() gs['Block'], gs['Transaction'], gs['TxIn'], gs['TxOut'], gs['OutPoint'] = globals()['Block'], \ globals()['Transaction'], globals()['TxIn'], globals()['TxOut'], globals()['OutPoint'] new_blocks = Utils.deserialize(f.read(msg_len), gs) logger.info( f"[persistence] parsing {len(new_blocks)} blocks from disk") for block in new_blocks[1:]: if not _connect_block(block, active_chain, utxo_set): logger.info( f'[persistence] {active_chain.height+1} block connecting failed, load_from_disk stopped and return' ) return logger.info( f'[persistence] loading {len(new_blocks)} blocks successful') except Exception: logger.exception( f'[persistence] failded in loading from chain storage file') return
def getUTXO4Addr(self, wallet_addr: str = None) -> Iterable[UnspentTxOut]: with self.chain_lock: if wallet_addr is None: wallet_addr = self.wallet.my_address peer, port = self._getPort() message = Message(Actions.UTXO4Addr, wallet_addr, port) with socket.create_connection(peer, timeout=25) as s: s.sendall(Utils.encode_socket_data(message)) logger.info(f'[EdgeHand] succeed to send UTXO4Addr to {peer}') msg_len = int(binascii.hexlify(s.recv(4) or b'\x00'), 16) data = b'' while msg_len > 0: tdat = s.recv(1024) data += tdat msg_len -= len(tdat) message = Utils.deserialize(data.decode(), self.gs) if data else None if message: logger.info( f'[EdgeHand] received UTXO4Addr from peer {peer}') return message.data else: logger.info( f'[EdgeHand] recv nothing for UTXO4Addr from peer {peer}' ) return None
def init_peers(cls, peerfile = Params.PEERS_FILE)->Iterable[NamedTuple]: if not os.path.exists(peerfile): peers: Iterable[Peer] =[] for peerlist in Params.PEERS: peer = Peer(str(peerlist[0]), int(peerlist[1])) if peer== Peer('127.0.0.1', Params.PORT_CURRENT) or \ peer == Peer('localhost', Params.PORT_CURRENT) or \ peer.ip == '0.0.0.0' or \ peer == Peer(Params.PUBLIC_IP, Params.PORT_CURRENT): pass else: peers.append(peer) try: with open(peerfile, "wb") as f: logger.info(f"[p2p] saving {len(peers)} hostnames") f.write(Utils.encode_socket_data(list(peers))) except Exception: logger.exception(f'[p2p] saving peers exception') return [] else: try: with open(peerfile, "rb") as f: msg_len = int(binascii.hexlify(f.read(4) or b'\x00'), 16) gs = dict() gs['Peer'] = globals()['Peer'] peers = Utils.deserialize(f.read(msg_len), gs) peers = list(set(peers)) length=len(peers) idx=0 while idx < length: peer = peers[idx] if peer== Peer('127.0.0.1', Params.PORT_CURRENT) or \ peer == Peer('localhost', Params.PORT_CURRENT) or \ peer.ip == '0.0.0.0' or \ peer == Peer(Params.PUBLIC_IP, Params.PORT_CURRENT): del peers[idx] idx -= 1 length -= 1 idx += 1 logger.info(f"[p2p] loading peers with {len(peers)} hostnames") except Exception: logger.exception(f'[p2p] loading peers exception') peers = [] return peers
def work_to_do(): logger.info( f'begin to work in another new thread of initiative_sync') try: with self.peers_lock: self.peerManager.update() peers = self.peerManager.getPeers() Peer.save_peers(peers, Params.PEERS_FILE) except: pass try: time_now = time.time() with self.chain_lock: for block in self.orphan_blocks: if time_now - block.timestamp > Params.MAXIMUM_ALLOWABLE_HEIGHT_DIFF * Params.TIME_BETWEEN_BLOCKS_IN_SECS_TARGET: self.orphan_blocks.remove(block) except: pass getpeers = self.peerManager.getPeers(2) if len(getpeers) > 0: peer = random.sample(getpeers, 1)[0] try: message = Message(Actions.TopBlockReq, None, Params.PORT_CURRENT) with socket.socket( socket.AF_INET, socket.SOCK_STREAM ) as s: #socket.create_connection(peer(), timeout=25) as s: #logger.info(f'[EdgenceChain] begin to connect to {peer}') s.connect(peer()) #logger.info(f'[EdgenceChain] succeed to create socket connection with {peer}, and begin to send data ...') s.sendall(Utils.encode_socket_data(message)) logger.info( f'[EdgenceChain] succeed to send TopBlockReq to {peer}' ) msg_len = int( binascii.hexlify(s.recv(4) or b'\x00'), 16) data = b'' while msg_len > 0: tdat = s.recv(1024) data += tdat msg_len -= len(tdat) s.close() message = Utils.deserialize(data.decode(), self.gs) if data else None if message: logger.info( f'[EdgenceChain] received top block from peer {peer}' ) message = Message(Actions.BlockRev, message.data, Params.PORT_CURRENT, peer) ret = Utils.send_to_peer(message, Peer( '127.0.0.1', Params.PORT_CURRENT), itself=True) if ret != 0: logger.info( f'cannot send data to itself, and its port is {Params.PORT_CURRENT}' ) else: #logger.info(f'[EdgenceChain] send BlockRev to itself') pass else: logger.info( f'[EdgenceChain] recv nothing from peer {peer}' ) except: self.peerManager.addLog(peer, 1)
def initial_block_download(self): self.ibd_done.clear() peers = self.peerManager.getPeers() if peers: logger.info( f'start initial block download from {len(peers)} peers') peer_sample = random.sample(peers, min(len(peers), 6)) message = Message(Actions.BlocksSyncReq, self.active_chain.chain[-1].id, Params.PORT_CURRENT) for peer in peer_sample: try: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.settimeout(70) s.connect(peer()) s.sendall(Utils.encode_socket_data(message)) logger.info( f'sending BLockSyncReq successfully at {self.active_chain.chain[-1].id} to {peer}' ) msg_len = int(binascii.hexlify(s.recv(4) or b'\x00'), 16) data = b'' while msg_len > 0: tdat = s.recv(1024) data += tdat msg_len -= len(tdat) s.close() message = Utils.deserialize(data.decode(), self.gs) if data else None if message: logger.info( f'[EdgenceChain] received blocks in initial_block_download from peer {peer}' ) message = Message(message.action, message.data, Params.PORT_CURRENT, peer) ret = Utils.send_to_peer(message, Peer('127.0.0.1', Params.PORT_CURRENT), itself=True) if ret != 0: logger.info( f'cannot send data to itself, and its port is {Params.PORT_CURRENT}' ) else: logger.info( f'[EdgenceChain] recv no new blocks when in initial_block_download from peer {peer}, and waiting for finishing' ) time.sleep(Params.TIME_BETWEEN_BLOCKS_IN_SECS_TARGET) self.ibd_done.set() break except Exception as e: #logger.exception(f'Error: {repr(e)}, and remove dead peer {peer}') if peer in peers: try: with self.peers_lock: self.peerManager.addLog(peer, 1) except: pass else: logger.info(f'no peer nodes existed, ibd_done is set') self.ibd_done.set()
def load_from_disk(active_chain: BlockChain, utxo_set: UTXO_Set, CHAIN_PATH=Params.CHAIN_FILE): def _connect_block(block: Block, active_chain: BlockChain, utxo_set: UTXO_Set) -> bool: """ Add block after current chain. Return True if block added successfully, else False. """ if block.validate_block(active_chain, utxo_set) is None: return False logger.info( f"connecting the {len(active_chain.chain)+1}th block {block.id} to chain {active_chain.idx}" ) active_chain.chain.append(block) # Minipulate transactions in this block. # Remove txin from utxo_set, add txout to utxo_set. for tx in block.txns: if not tx.is_coinbase: for txin in tx.txins: utxo_set.rm_from_utxo(*txin.to_spend) for i, txout in enumerate(tx.txouts): utxo_set.add_to_utxo( txout, tx, i, tx.is_coinbase, len(active_chain.chain) ) return True if not os.path.isfile(CHAIN_PATH): logger.info(f"[persistence] chain storage file does not exist") return else: # active_chain.chain is a list initialized with genesis_block # so len(active_chain.chain) should equals to 1 if len(active_chain.chain) > 1: logger.info( f"[persistence] more than the genesis block exists, load_from_disk stopped and return" ) return try: with open(CHAIN_PATH, "rb") as f: block_len = int(f.read(20)) logger.info( f"[persistence] {block_len} is claimed at the head of chain file" ) msg_len = int(f.read(20)) gs = dict() gs["Block"], gs["Transaction"], gs["TxIn"], gs["TxOut"], gs["OutPoint"] = ( globals()["Block"], globals()["Transaction"], globals()["TxIn"], globals()["TxOut"], globals()["OutPoint"], ) new_blocks = Utils.deserialize(f.read(msg_len), gs) logger.info(f"[persistence] parsing {len(new_blocks)} blocks from disk") # Connect block after genesis block. for block in new_blocks[1:]: if not _connect_block(block, active_chain, utxo_set): logger.info( f"[persistence] {active_chain.height+1} block connecting failed, load_from_disk stopped and return" ) return logger.info(f"[persistence] loading {len(new_blocks)} blocks successful") except Exception: logger.exception(f"[persistence] failded in loading from chain storage file") return
def initial_block_download(self): """ Download blocks from peers in peerManager. """ self.ibd_done.clear() peers = self.peerManager.getPeers() if peers: logger.info( f"start initial block download from {len(peers)} peers") # Select max 6 peers. peer_sample = random.sample(peers, min(len(peers), 6)) message = Message( Actions.BlocksSyncReq, self.active_chain.chain[-1].id, Params.PORT_CURRENT, ) for peer in peer_sample: try: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.settimeout(70) s.connect(peer()) s.sendall(Utils.encode_socket_data(message)) logger.info( f"sending BLockSyncReq successfully at {self.active_chain.chain[-1].id} to {peer}" ) msg_len = int(binascii.hexlify(s.recv(4) or b"\x00"), 16) data = b"" while msg_len > 0: tdat = s.recv(1024) data += tdat msg_len -= len(tdat) # Useless below. # s.close() message = (Utils.deserialize(data.decode(), self.gs) if data else None) if message: logger.info( f"[EdgenceChain] received blocks in initial_block_download from peer {peer}" ) message = Message(message.action, message.data, Params.PORT_CURRENT, peer) # TODO: simplify it. # Send received data to its TCP server, TCP handler will process received block. ret = Utils.send_to_peer(message, Peer("127.0.0.1", Params.PORT_CURRENT), itself=True) if ret != 0: logger.info( f"cannot send data to itself, and its port is {Params.PORT_CURRENT}" ) else: logger.info( f"[EdgenceChain] recv no new blocks when in initial_block_download from peer {peer}, and waiting for finishing" ) time.sleep(Params.TIME_BETWEEN_BLOCKS_IN_SECS_TARGET) self.ibd_done.set() break except Exception as e: # logger.exception(f'Error: {repr(e)}, and remove dead peer {peer}') if peer in peers: try: with self.peers_lock: # This peer is not success this time, add log: 1 to its log. self.peerManager.addLog(peer, 1) except: pass else: logger.info(f"no peer nodes existed, ibd_done is set") self.ibd_done.set()
CHAIN_PATH = '../' + Params.CHAIN_FILE if not os.path.isfile(CHAIN_PATH): logger.info(f'[persistence] chain storage file does not exist') else: pass try: with open(CHAIN_PATH, "rb") as f: block_len = int(f.read(20)) logger.info( f'[persistence] {block_len} is claimed at the head of chain file') msg_len = int(f.read(20)) gs = dict() gs['Block'], gs['Transaction'], gs['TxIn'], gs['TxOut'] = globals()['Block'], globals()['Transaction'], \ globals()['TxIn'], globals()['TxOut'] new_blocks = Utils.deserialize(f.read(msg_len), gs) logger.info( f'[persistence] loading {len(new_blocks)} blocks successful') except Exception: logger.exception( f'[persistence] failded in loading from chain storage file') for block in new_blocks: print(block.id) block.merkle_hash block.prev_block_hash block.id def check_blocks(blocks: Iterable[Block]) -> bool:
def handleBlockRev(self, block: Block, peer: Peer): if not isinstance(block, Block): logger.info(f'[p2p] {block} is not a Block') return else: if peer != Peer('127.0.0.1', Params.PORT_CURRENT): logger.info( f"[p2p] received block {block.id} from peer {peer}") with self.chain_lock: chain_idx = TCPHandler.check_block_place(block, self.active_chain, self.utxo_set, self.mempool, \ self.side_branches) ret_outside_lock = False if chain_idx is not None and chain_idx >= 0: ret_outside_lock = TCPHandler.do_connect_block_and_after(block, chain_idx, self.active_chain, self.side_branches, \ self.mempool, self.utxo_set, self.mine_interrupt) if not ret_outside_lock: #logger.info(f'####### out of chain_lock: {chain_use_id} of handleBlockRev') return if not Block.locate_block(block.id, self.active_chain, self.side_branches)[0]: return #logger.info(f'####### out of chain_lock: {chain_use_id} of handleBlockRev') if ret_outside_lock is True: if len(self.active_chain.chain ) % Params.SAVE_PER_SIZE == 0 or len( self.active_chain.chain) <= 5: Persistence.save_to_disk(self.active_chain) if chain_idx is not None and chain_idx >= 0: peers = self.peerManager.getPeers() if len(peers) > 0: for _peer in random.sample(peers, min(len(peers), 5)): if _peer != peer: ret = Utils.send_to_peer( Message(Actions.BlockRev, block, Params.PORT_CURRENT), _peer) if ret == 1: if _peer in peers: try: with self.peers_lock: self.peerManager.block(_peer) except: pass elif ret != 0: with self.peers_lock: self.peerManager.addLog(_peer, 1) else: with self.peers_lock: self.peerManager.addLog(_peer, 0) self.sendPeerExtend() elif chain_idx is None: logger.info( f'[p2p] already seen block {block.id}, and do nothing') elif chain_idx == -1: # case of orphan block message = Message(Actions.TopBlocksSyncReq, 50, Params.PORT_CURRENT) if peer == Peer('127.0.0.1', Params.PORT_CURRENT): getpeers = self.peerManager.getPeers(2) if len(getpeers) > 0: peer = random.sample(getpeers, 1)[0] else: return try: with socket.socket( socket.AF_INET, socket.SOCK_STREAM ) as s: #socket.create_connection(peer(), timeout=25) as s: s.connect(peer()) s.sendall(Utils.encode_socket_data(message)) logger.info( f'[p2p] succeed to send TopBlocksSyncReq to {peer}' ) msg_len = int(binascii.hexlify(s.recv(4) or b'\x00'), 16) data = b'' while msg_len > 0: tdat = s.recv(1024) data += tdat msg_len -= len(tdat) s.close() except ConnectionRefusedError: self.peerManager.block(peer) except: self.peerManager.addLog(peer, 1) else: self.peerManager.addLog(peer, 0) message = Utils.deserialize(data.decode(), self.gs) if data else None if message: blocks = message.data if blocks[0].prev_block_hash: if not Block.locate_block( blocks[0].prev_block_hash, self.active_chain, self.side_branches)[0]: logger.info( f"received sync blocks for the orphan block in handleBlockRev, but the first blocks's pre_block_hash cannot be seen on the chains" ) self.peerManager.block(peer) return else: blocks.pop(0) if not Block.locate_block( blocks[0].prev_block_hash, self.active_chain, self.side_branches)[0]: logger.info( f"received sync blocks for the orphan block in handleBlockRev, but the first blocks's pre_block_hash cannot be seen on the chains" ) self.peerManager.block(peer) return message = Message(message.action, message.data, Params.PORT_CURRENT, peer) ret = Utils.send_to_peer(message, Peer('127.0.0.1', Params.PORT_CURRENT), itself=True) if ret != 0: logger.info( f'[p2p] cannot send data to itself, and its current port is {Params.PORT_CURRENT}' ) else: #logger.info(f'[p2p] send BlocksSyncGet to itself') pass else: logger.info(f'[p2p] recv nothing from peer {peer}')
def handleBlockSyncGet(self, blocks: Iterable[Block], peer: Peer): if peer != Peer('127.0.0.1', Params.PORT_CURRENT): logger.info( f"[p2p] receive {len(blocks)} blocks for BlockSyncGet from {peer}" ) new_blocks = [ block for block in blocks if not Block.locate_block( block.id, self.active_chain, self.side_branches)[0] ] logger.info( f'[p2p] {len(new_blocks)} of {len(blocks)} blocks from {peer} is new' ) #if not new_blocks: # logger.info('[p2p] initial block download complete') # self.ibd_done.set() # return #else: # self.ibd_done.clear() if not TCPHandler.check_blocks_headers(new_blocks): return with self.chain_lock: for block in new_blocks: if Block.locate_block(block.id, self.active_chain, self.side_branches)[0]: new_blocks.pop(0) else: break if not new_blocks: return chain_idx = TCPHandler.check_block_place(new_blocks[0], self.active_chain, self.utxo_set, \ self.mempool, self.side_branches) if chain_idx is None: logger.info( f'received blocks have been seen in BlockSyncGet, do nothing and return' ) return if chain_idx <= -1: logger.info(f'[p2p] orphan or wrong blocks') if peer != Peer('127.0.0.1', Params.PORT_CURRENT): with self.peers_lock: self.peerManager.block(peer) return if chain_idx >= 1: # if is side branches, append the blocks (one block left) to the side branches directly logger.info( f'[p2p] just append {len(new_blocks)-1} blocks to side branch {chain_idx}, leaving one block to ' f'be coped with method do_connect_block_and_after') while len(new_blocks) >= 2: self.side_branches[chain_idx - 1].chain.append( new_blocks.pop(0)) for block in new_blocks: if not TCPHandler.do_connect_block_and_after(block, chain_idx, self.active_chain, self.side_branches, \ self.mempool, self.utxo_set, self.mine_interrupt): return if chain_idx == Params.ACTIVE_CHAIN_IDX: if len(self.active_chain.chain) % Params.SAVE_PER_SIZE == 0 or len( self.active_chain.chain) <= 5: Persistence.save_to_disk(self.active_chain) logger.info( f'[p2p] current chain height {self.active_chain.height}, and continue initial block download ... ' ) if peer not in self.peerManager.getAllPeers(): if peer== Peer('127.0.0.1', Params.PORT_CURRENT) or \ peer == Peer('localhost', Params.PORT_CURRENT) or \ peer.ip == '0.0.0.0' or \ peer == Peer(Params.PUBLIC_IP, Params.PORT_CURRENT): return if Utils.is_peer_valid(peer): with self.peers_lock: self.peerManager.add(peer) #self.peers.append(peer) self.sendPeerExtend() else: self.peerManager.block(peer) if peer == Peer('127.0.0.1', Params.PORT_CURRENT): logger.info(f'peer in handleBlockSyncGet cannot be itself') return top_id_chain_idx = new_blocks[-1].id message = Message(Actions.BlocksSyncReq, top_id_chain_idx, Params.PORT_CURRENT) try: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.connect(peer()) s.sendall(Utils.encode_socket_data(message)) logger.info(f'[p2p] succeed to send BlocksSyncReq to {peer}') msg_len = int(binascii.hexlify(s.recv(4) or b'\x00'), 16) data = b'' while msg_len > 0: tdat = s.recv(1024) data += tdat msg_len -= len(tdat) s.close() except ConnectionRefusedError: self.peerManager.block(peer) except: self.peerManager.addLog(peer, 1) else: self.peerManager.addLog(peer, 0) message = Utils.deserialize(data.decode(), self.gs) if data else None if message: logger.info( f'[p2p] received blocks for sync blocks from peer {peer}') message = Message(message.action, message.data, Params.PORT_CURRENT, peer) ret = Utils.send_to_peer(message, Peer('127.0.0.1', Params.PORT_CURRENT), itself=True) if ret != 0: logger.info( f'[p2p] cannot send data to itself, and its current port is {Params.PORT_CURRENT}' ) else: logger.info(f'[p2p] recv nothing from peer {peer}')
def init_peers(cls, peerfile=Params.PEERS_FILE) -> Iterable[namedtuple]: """ If peers_file doesnot exist, read peers from paramater in params/Params.py and save them to peers_file, else read peer from peerfile. """ peers: Iterable[Peer] = [] if not os.path.exists(peerfile): # peers: Iterable[Peer] = [] # read peers from paramater list for peerlist in Params.PEERS: peer = Peer(str(peerlist[0]), int(peerlist[1])) # check whether peer is localhost with localport for this program instance # if so, ignore it if (peer == Peer("127.0.0.1", Params.PORT_CURRENT) or peer == Peer("localhost", Params.PORT_CURRENT) or peer.ip == "0.0.0.0" or peer == Peer( Params.PUBLIC_IP, Params.PORT_CURRENT)): pass else: peers.append(peer) try: with open(peerfile, "wb") as f: logger.info(f"[p2p] saving {len(peers)} hostnames") f.write(Utils.encode_socket_data(list(peers))) except Exception: logger.exception(f"[p2p] saving peers exception") return [] else: try: with open(peerfile, "rb") as f: msg_len = int(binascii.hexlify(f.read(4) or b"\x00"), 16) gs = dict() gs["Peer"] = globals()["Peer"] # peers_from_file may have dupilicated peer info or invalid peer info # which will be checked later. peers_from_file = Utils.deserialize(f.read(msg_len), gs) peers_from_file = list(set(peers_from_file)) # Clear list for peers to ensure it's empty. peers.clear() # ??? can while condition be changed in a while loop ??? # # length = len(peers_from_file) # idx = 0 # while idx < length: # peer = peers[idx] # if ( # peer == Peer("127.0.0.1", Params.PORT_CURRENT) # or peer == Peer("localhost", Params.PORT_CURRENT) # or peer.ip == "0.0.0.0" # or peer == Peer(Params.PUBLIC_IP, Params.PORT_CURRENT) # ): # del peers[idx] # idx -= 1 # length -= 1 # idx += 1 for peer in peers_from_file: if (peer == Peer("127.0.0.1", Params.PORT_CURRENT) or peer == Peer("localhost", Params.PORT_CURRENT) or peer.ip == "0.0.0.0" or peer == Peer( Params.PUBLIC_IP, Params.PORT_CURRENT)): pass else: peers.append(peer) logger.info( f"[p2p] loading peers with {len(peers)} hostnames") except Exception: logger.exception(f"[p2p] loading peers exception") peers.clear() return peers