def add_peer(self, peer: PeerShardConnection): self.peers[peer.cluster_peer_id] = peer Logger.info( "[{}] connected to peer {}".format( Branch(self.full_shard_id).to_str(), peer.cluster_peer_id ) )
def on_wire_protocol_start(self, proto): Logger.info("NODE{} on_wire_protocol_start proto={}".format( self.config["node_num"], proto)) active_peers = self.get_connected_peers() self.app.network.loop.call_soon_threadsafe( asyncio.ensure_future, self.app.network.refresh_connections(active_peers))
def __init__(self, env, diff_calc=None): self.env = env self.root_config = env.quark_chain_config.ROOT if not diff_calc: cutoff = self.root_config.DIFFICULTY_ADJUSTMENT_CUTOFF_TIME diff_factor = self.root_config.DIFFICULTY_ADJUSTMENT_FACTOR min_diff = self.root_config.GENESIS.DIFFICULTY check(cutoff > 0 and diff_factor > 0 and min_diff > 0) diff_calc = EthDifficultyCalculator(cutoff=cutoff, diff_factor=diff_factor, minimum_diff=min_diff) self.diff_calc = diff_calc self.raw_db = env.db self.db = RootDb( self.raw_db, env.quark_chain_config, count_minor_blocks=env.cluster_config.ENABLE_TRANSACTION_HISTORY, ) persisted_tip = self.db.get_tip_header() if persisted_tip: self.tip = persisted_tip Logger.info("Recovered root state with tip height {}".format( self.tip.height)) else: self.__create_genesis_block() Logger.info("Created genesis root block")
def __recover_from_db(self): """ Recover the best chain from local database. """ Logger.info("Recovering root chain from local database...") if b"tipHash" not in self.db: return None r_hash = self.db.get(b"tipHash") r_block = RootBlock.deserialize(self.db.get(b"rblock_" + r_hash)) if r_block.header.height <= 0: return None # use the parent of the tipHash block as the new tip # since it's guaranteed to have been accepted by all the shards # while shards might not have seen the block of tipHash r_hash = r_block.header.hash_prev_block r_block = RootBlock.deserialize(self.db.get(b"rblock_" + r_hash)) self.tip_header = r_block.header # type: RootBlockHeader while len(self.r_header_pool) < self.max_num_blocks_to_recover: self.r_header_pool[r_hash] = r_block.header for m_header in r_block.minor_block_header_list: mtokens = TokenBalanceMap.deserialize( self.db.get(b"mheader_" + m_header.get_hash())).balance_map self.m_hash_dict[m_header.get_hash()] = mtokens if r_block.header.height <= 0: break r_hash = r_block.header.hash_prev_block r_block = RootBlock.deserialize(self.db.get(b"rblock_" + r_hash))
async def __gen(self, num_tx, x_shard_percent, sample_tx: Transaction): Logger.info( "[{}] start generating {} transactions with {}% cross-shard". format(self.shard_id, num_tx, x_shard_percent)) if num_tx <= 0: return start_time = time.time() tx_list = [] total = 0 sample_evm_tx = sample_tx.code.get_evm_transaction() for account in self.accounts: nonce = self.shard.state.get_transaction_count( account.address.recipient) tx = self.create_transaction(account, nonce, x_shard_percent, sample_evm_tx) if not tx: continue tx_list.append(tx) total += 1 if len(tx_list) >= 600 or total >= num_tx: self.shard.add_tx_list(tx_list) tx_list = [] await asyncio.sleep( random.uniform(8, 12) ) # yield CPU so that other stuff won't be held for too long if total >= num_tx: break end_time = time.time() Logger.info("[{}] generated {} transactions in {:.2f} seconds".format( self.shard_id, total, end_time - start_time)) self.running = False
def close(self): for shard in self.shards.values(): for peer_shard_conn in shard.peers.values(): peer_shard_conn.get_forwarding_connection().close() Logger.info("Lost connection with master") return super().close()
def main(): env = parse_args() slave_server = SlaveServer(env) slave_server.start_and_loop() Logger.info("Slave server is shutdown")
def recover_state(self, r_header, m_header): """ When recovering from local database, we can only guarantee the consistency of the best chain. Forking blocks can be in inconsistent state and thus should be pruned from the database so that they can be retried in the future. """ r_hash = r_header.get_hash() while (len(self.r_header_pool) < self.env.quark_chain_config.ROOT.max_root_blocks_in_memory): block = RootBlock.deserialize(self.db.get(b"rblock_" + r_hash)) self.r_minor_header_pool[ r_hash] = self.__get_last_minor_block_in_root_block(block) self.r_header_pool[r_hash] = block.header if (block.header.height <= self.env.quark_chain_config.get_genesis_root_height( self.branch.get_shard_id())): break r_hash = block.header.hash_prev_block m_hash = m_header.get_hash() shard_config = self.env.quark_chain_config.SHARD_LIST[ self.branch.get_shard_id()] while len( self.m_header_pool) < shard_config.max_minor_blocks_in_memory: block = MinorBlock.deserialize(self.db.get(b"mblock_" + m_hash)) self.m_header_pool[m_hash] = block.header self.m_meta_pool[m_hash] = block.meta if block.header.height <= 0: break m_hash = block.header.hash_prev_minor_block Logger.info("[{}] recovered {} minor blocks and {} root blocks".format( self.branch.get_shard_id(), len(self.m_header_pool), len(self.r_header_pool), ))
def __init__(self, env, diff_calc=None): self.env = env self.root_config = env.quark_chain_config.ROOT if not diff_calc: cutoff = self.root_config.DIFFICULTY_ADJUSTMENT_CUTOFF_TIME diff_factor = self.root_config.DIFFICULTY_ADJUSTMENT_FACTOR min_diff = self.root_config.GENESIS.DIFFICULTY check(cutoff > 0 and diff_factor > 0 and min_diff > 0) diff_calc = EthDifficultyCalculator( cutoff=cutoff, diff_factor=diff_factor, minimum_diff=min_diff ) self.diff_calc = diff_calc self.raw_db = env.db self.db = RootDb( self.raw_db, env.quark_chain_config, count_minor_blocks=env.cluster_config.ENABLE_TRANSACTION_HISTORY, ) # header hash -> [coinbase address] during previous blocks (ascending) self.coinbase_addr_cache = LRUCache(maxsize=128) self.tip = self.db.get_tip_header() # type: RootBlockHeader if self.tip: Logger.info( "Recovered root state with tip height {}".format(self.tip.height) ) else: self.tip = self.__create_genesis_block() Logger.info("Created genesis root block")
async def handle_sync_minor_block_list_request(self, req): """ Raises on error""" async def __download_blocks(block_hash_list): op, resp, rpc_id = await peer_shard_conn.write_rpc_request( CommandOp.GET_MINOR_BLOCK_LIST_REQUEST, GetMinorBlockListRequest(block_hash_list), ) return resp.minor_block_list shard = self.shards.get(req.branch, None) if not shard: return SyncMinorBlockListResponse(error_code=errno.EBADMSG) peer_shard_conn = shard.peers.get(req.cluster_peer_id, None) if not peer_shard_conn: return SyncMinorBlockListResponse(error_code=errno.EBADMSG) BLOCK_BATCH_SIZE = 100 block_hash_list = req.minor_block_hash_list # empty if not block_hash_list: return SyncMinorBlockListResponse(error_code=0) try: while len(block_hash_list) > 0: blocks_to_download = block_hash_list[:BLOCK_BATCH_SIZE] try: block_chain = await asyncio.wait_for( __download_blocks(blocks_to_download), TIMEOUT) except asyncio.TimeoutError as e: Logger.info( "[{}] sync request from master failed due to timeout". format(req.branch.get_full_shard_id())) raise e Logger.info( "[{}] sync request from master, downloaded {} blocks ({} - {})" .format( req.branch.get_full_shard_id(), len(block_chain), block_chain[0].header.height, block_chain[-1].header.height, )) check(len(block_chain) == len(blocks_to_download)) add_block_success = await self.slave_server.add_block_list_for_sync( block_chain) if not add_block_success: raise RuntimeError( "Failed to add minor blocks for syncing root block") block_hash_list = block_hash_list[BLOCK_BATCH_SIZE:] branch = block_chain[0].header.branch shard = self.slave_server.shards.get(branch, None) check(shard is not None) return SyncMinorBlockListResponse( error_code=0, shard_stats=shard.state.get_shard_stats()) except Exception: Logger.error_exception() return SyncMinorBlockListResponse(error_code=1)
async def handle_new_block(self, block): """ This is a fast path for block propagation. The block is broadcasted to peers before being added to local state. 0. if local shard is syncing, doesn't make sense to add, skip 1. if block parent is not in local state/new block pool, discard (TODO: is this necessary?) 2. if already in cache or in local state/new block pool, pass 3. validate: check time, difficulty, POW 4. add it to new minor block broadcast cache 5. broadcast to all peers (minus peer that sent it, optional) 6. add_block() to local state (then remove from cache) also, broadcast tip if tip is updated (so that peers can sync if they missed blocks, or are new) """ if self.synchronizer.running: # TODO optional: queue the block if it came from broadcast to so that once sync is over, # catch up immediately return if block.header.get_hash() in self.state.new_block_pool: return if self.state.db.contain_minor_block_by_hash(block.header.get_hash()): return if not self.state.db.contain_minor_block_by_hash( block.header.hash_prev_minor_block): if block.header.hash_prev_minor_block not in self.state.new_block_pool: return # Doing full POSW check requires prev block has been added to the state, which could # slow down block propagation. # TODO: this is a copy of the code in SyncTask.__validate_block_headers. this it a helper try: header = block.header # Note that PoSW may lower diff, so checks here are necessary but not sufficient # More checks happen during block addition shard_config = self.env.quark_chain_config.shards[ header.branch.get_full_shard_id()] consensus_type = shard_config.CONSENSUS_TYPE diff = header.difficulty if shard_config.POSW_CONFIG.ENABLED: diff //= shard_config.POSW_CONFIG.DIFF_DIVIDER validate_seal(header, consensus_type, adjusted_diff=diff) except Exception as e: Logger.warning( "[{}] got block with bad seal in handle_new_block: {}".format( header.branch.to_str(), str(e))) raise e if block.header.create_time > time_ms() // 1000 + 30: return self.state.new_block_pool[block.header.get_hash()] = block Logger.info("[{}/{}] got new block with height {}".format( block.header.branch.get_chain_id(), block.header.branch.get_shard_id(), block.header.height, )) self.broadcast_new_block(block) await self.add_block(block)
def add_block(self, block, write_db=True, skip_if_too_old=True, adjusted_diff: int = None): """ Add new block. return True if a longest block is added, False otherwise There are a couple of optimizations can be done here: - the root block could only contain minor block header hashes as long as the shards fully validate the headers - the header (or hashes) are un-ordered as long as they contains valid sub-chains from previous root block """ if skip_if_too_old and ( self.tip.height - block.header.height > self.root_config.MAX_STALE_ROOT_BLOCK_HEIGHT_DIFF): Logger.info("[R] drop old block {} << {}".format( block.header.height, self.tip.height)) raise ValueError("block is too old {} << {}".format( block.header.height, self.tip.height)) start_ms = time_ms() block_hash, last_minor_block_header_list = self.validate_block( block, adjusted_diff) if write_db: self.db.put_root_block(block, last_minor_block_header_list) tracking_data_str = block.tracking_data.decode("utf-8") if tracking_data_str != "": tracking_data = json.loads(tracking_data_str) sample = { "time": time_ms() // 1000, "shard": "R", "network": self.env.cluster_config.MONITORING.NETWORK_NAME, "cluster": self.env.cluster_config.MONITORING.CLUSTER_ID, "hash": block.header.get_hash().hex(), "height": block.header.height, "original_cluster": tracking_data["cluster"], "inception": tracking_data["inception"], "creation_latency_ms": tracking_data["creation_ms"], "add_block_latency_ms": time_ms() - start_ms, "mined": tracking_data.get("mined", 0), "propagation_latency_ms": start_ms - tracking_data.get("mined", 0), "num_tx": len(block.minor_block_header_list), } asyncio.ensure_future( self.env.cluster_config.kafka_logger.log_kafka_sample_async( self.env.cluster_config.MONITORING.PROPAGATION_TOPIC, sample)) if self.tip.total_difficulty < block.header.total_difficulty: old_tip = self.tip self.tip = block.header # TODO: Atomicity during shutdown self.db.update_tip_hash(block_hash) self.__rewrite_block_index_to(old_tip, block) return True return False
def close(self): if self.state == ConnectionState.ACTIVE: Logger.info("destroying proxy slave connections for {}".format( self.quark_peer.remote)) self.master_server.destroy_peer_cluster_connections( self.cluster_peer_id) super(Connection, self).close() self.quark_peer.close()
def close(self): for shard in self.shards.values(): for peer_shard_conn in shard.peers.values(): peer_shard_conn.get_forwarding_connection().close() Logger.info("Lost connection with master. Shutting down slave ...") self.slave_server.shutdown() return super().close()
def main(): os.chdir(os.path.dirname(os.path.abspath(__file__))) env = parse_args() slave_server = SlaveServer(env) slave_server.start() slave_server.do_loop() Logger.info("Slave server is shutdown")
def loop(self): while True: gevent.sleep(self.REFRESH_INTERVAL) Logger.info("p2p periodic refresh") active_peers = self.get_connected_peers() self.app.network.loop.call_soon_threadsafe( asyncio.ensure_future, self.app.network.refresh_connections(active_peers), )
def start_server(self): coro = asyncio.start_server(self.new_peer, "0.0.0.0", self.port, loop=self.loop) self.server = self.loop.run_until_complete(coro) Logger.info("Self id {}".format(self.self_id.hex())) Logger.info("Listening on {} for p2p".format( self.server.sockets[0].getsockname()))
def start_mining(self, artificial_tx_config): self.artificial_tx_config = artificial_tx_config self.mining = True for branch, shard in self.shards.items(): Logger.info( "[{}] start mining with target minor block time {} seconds". format(branch.get_shard_id(), artificial_tx_config.target_minor_block_time)) shard.miner.start()
async def __start_server(self): """ Run the server until shutdown is called """ self.server = await asyncio.start_server( self.__handle_new_connection, "0.0.0.0", self.env.slave_config.PORT, loop=self.loop, ) Logger.info("Listening on {} for intra-cluster RPC".format( self.server.sockets[0].getsockname()))
def main(): env, unknown_flags = parse_args() FLAGS(sys.argv[:1] + unknown_flags) if FLAGS["verbosity"].using_default_value: FLAGS.verbosity = 0 # INFO level slave_server = SlaveServer(env) slave_server.start_and_loop() Logger.info("Slave server is shutdown")
async def do_sub_proto_handshake(self) -> None: """ overrides BasePeer.do_sub_proto_handshake() """ self.secure_peer = SecurePeer(self) Logger.info("starting peer hello exchange") start_state = await self.secure_peer.start() if start_state: # returns None if successful raise HandshakeFailure( "hello message exchange failed: {}".format(start_state))
async def handle_new_minor_block_header_list_command(self, _op, cmd, _rpc_id): # TODO: allow multiple headers if needed if len(cmd.minor_block_header_list) != 1: self.close_with_error("minor block header list must have only one header") return for m_header in cmd.minor_block_header_list: Logger.info( "[{}] received new header with height {}".format( m_header.branch.get_shard_id(), m_header.height ) ) if m_header.branch != self.shard_state.branch: self.close_with_error("incorrect branch") return if self.best_root_block_header_observed: # check root header is not decreasing if ( cmd.root_block_header.height < self.best_root_block_header_observed.height ): return self.close_with_error( "best observed root header height is decreasing {} < {}".format( cmd.root_block_header.height, self.best_root_block_header_observed.height, ) ) if ( cmd.root_block_header.height == self.best_root_block_header_observed.height ): if cmd.root_block_header != self.best_root_block_header_observed: return self.close_with_error( "best observed root header changed with same height {}".format( self.best_root_block_header_observed.height ) ) # check minor header is not decreasing if m_header.height < self.best_minor_block_header_observed.height: return self.close_with_error( "best observed minor header is decreasing {} < {}".format( m_header.height, self.best_minor_block_header_observed.height, ) ) self.best_root_block_header_observed = cmd.root_block_header self.best_minor_block_header_observed = m_header # Do not download if the new header is not higher than the current tip if self.shard_state.header_tip.height >= m_header.height: return self.shard.synchronizer.add_task(m_header, self)
async def add_block(self, block): """ Returns true if block is successfully added. False on any error. called by 1. local miner (will not run if syncing) 2. SyncTask """ old_tip = self.state.header_tip try: xshard_list = self.state.add_block(block) except Exception as e: Logger.error_exception() return False # only remove from pool if the block successfully added to state, # this may cache failed blocks but prevents them being broadcasted more than needed # TODO add ttl to blocks in new_block_pool self.state.new_block_pool.pop(block.header.get_hash(), None) # block has been added to local state, broadcast tip so that peers can sync if needed try: if old_tip != self.state.header_tip: self.broadcast_new_tip() except Exception: Logger.warning_every_sec("broadcast tip failure", 1) # block already existed in local shard state # but might not have been propagated to other shards and master # let's make sure all the shards and master got it before return if xshard_list is None: future = self.add_block_futures.get(block.header.get_hash(), None) if future: Logger.info( "[{}] {} is being added ... waiting for it to finish". format(block.header.branch.get_shard_id(), block.header.height)) await future return True self.add_block_futures[ block.header.get_hash()] = self.loop.create_future() # Start mining new one before propagating inside cluster # The propagation should be done by the time the new block is mined self.miner.mine_new_block_async() prev_root_height = self.state.db.get_root_block_by_hash( block.header.hash_prev_root_block).header.height await self.slave.broadcast_xshard_tx_list(block, xshard_list, prev_root_height) await self.slave.send_minor_block_header_to_master( block.header, len(block.tx_list), len(xshard_list), self.state.get_shard_stats(), ) self.add_block_futures[block.header.get_hash()].set_result(None) del self.add_block_futures[block.header.get_hash()] return True
def close_dead_peer(self): assert self.id is not None if self.id in self.network.active_peer_pool: del self.network.active_peer_pool[self.id] if self.cluster_peer_id in self.network.cluster_peer_pool: del self.network.cluster_peer_pool[self.cluster_peer_id] Logger.info("Peer {} ({}:{}) disconnected, remaining {}".format( self.id.hex(), self.ip, self.port, len(self.network.active_peer_pool))) self.master_server.destroy_peer_cluster_connections( self.cluster_peer_id) super().close()
def close(self): if self.state == ConnectionState.ACTIVE: assert self.id is not None if self.id in self.network.active_peer_pool: del self.network.active_peer_pool[self.id] if self.cluster_peer_id in self.network.cluster_peer_pool: del self.network.cluster_peer_pool[self.cluster_peer_id] Logger.info("Peer {} disconnected, remaining {}".format( self.id.hex(), len(self.network.active_peer_pool))) self.master_server.destroy_peer_cluster_connections( self.cluster_peer_id) super().close()
async def run() -> None: await loop.create_datagram_endpoint( lambda: discovery, local_addr=("0.0.0.0", args.listen_port) ) try: await discovery.bootstrap() while True: Logger.info("Routing table size={}".format(len(discovery.routing))) await cancel_token.cancellable_wait(asyncio.sleep(5)) except OperationCancelled: pass finally: await discovery.stop()
def get_connected_peers(self): ps = [p for p in self.app.services.peermanager.peers if p] aps = [p for p in ps if not p.is_stopped] Logger.info("I am {} I have {} peers: {}".format( self.app.config["client_version_string"], len(aps), [ p.remote_client_version if p.remote_client_version != "" else "Not Ready" for p in aps ], )) return [ p.remote_client_version.decode("utf-8") for p in aps if p.remote_client_version != "" ]
def __init__(self, env, diff_calc=None): self.env = env self.diff_calc = (diff_calc if diff_calc else EthDifficultyCalculator( cutoff=45, diff_factor=2048, minimum_diff=1000000)) self.raw_db = env.db self.db = RootDb(self.raw_db, env.quark_chain_config.ROOT.max_root_blocks_in_memory) persisted_tip = self.db.get_tip_header() if persisted_tip: self.tip = persisted_tip Logger.info("Recovered root state with tip height {}".format( self.tip.height)) else: self.__create_genesis_block() Logger.info("Created genesis root block")
async def handle_new_block(self, block): """ 0. if local shard is syncing, doesn't make sense to add, skip 1. if block parent is not in local state/new block pool, discard 2. if already in cache or in local state/new block pool, pass 3. validate: check time, difficulty, POW 4. add it to new minor block broadcast cache 5. broadcast to all peers (minus peer that sent it, optional) 6. add_block() to local state (then remove from cache) also, broadcast tip if tip is updated (so that peers can sync if they missed blocks, or are new) """ if self.synchronizer.running: # TODO optinal: queue the block if it came from broadcast to so that once sync is over, catch up immediately return if block.header.get_hash() in self.state.new_block_pool: return if self.state.db.contain_minor_block_by_hash(block.header.get_hash()): return if not self.state.db.contain_minor_block_by_hash( block.header.hash_prev_minor_block): if block.header.hash_prev_minor_block not in self.state.new_block_pool: return full_shard_id = block.header.branch.get_full_shard_id() consensus_type = self.env.quark_chain_config.shards[ full_shard_id].CONSENSUS_TYPE try: validate_seal(block.header, consensus_type) except Exception as e: Logger.warning("[{}] Got block with bad seal: {}".format( full_shard_id, str(e))) return if block.header.create_time > time_ms() // 1000 + 30: return self.state.new_block_pool[block.header.get_hash()] = block Logger.info("[{}/{}] got new block with height {}".format( block.header.branch.get_chain_id(), block.header.branch.get_shard_id(), block.header.height, )) self.broadcast_new_block(block) await self.add_block(block)
def __recover_from_db(self): """ Recover the best chain from local database. """ Logger.info("Recovering root chain from local database...") if b"tipHash" not in self.db: return None r_hash = self.db.get(b"tipHash") r_block = RootBlock.deserialize(self.db.get(b"rblock_" + r_hash)) if r_block.header.height <= 0: return None # use the parent of the tipHash block as the new tip # since it's guaranteed to have been accepted by all the shards # while shards might not have seen the block of tipHash r_hash = r_block.header.hash_prev_block r_block = RootBlock.deserialize(self.db.get(b"rblock_" + r_hash)) self.tip_header = r_block.header # type: RootBlockHeader