def _test(): import argparse from evm.p2p import ecies from evm.chains.ropsten import RopstenChain, ROPSTEN_GENESIS_HEADER from evm.db.backends.level import LevelDB from evm.db.backends.memory import MemoryDB logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') parser = argparse.ArgumentParser() parser.add_argument('-db', type=str, required=True) parser.add_argument('-root-hash', type=str, required=True, help='Hex encoded root hash') args = parser.parse_args() chaindb = BaseChainDB(MemoryDB()) chaindb.persist_header_to_db(ROPSTEN_GENESIS_HEADER) peer_pool = PeerPool(ETHPeer, chaindb, RopstenChain.network_id, ecies.generate_privkey()) asyncio.ensure_future(peer_pool.run()) state_db = LevelDB(args.db) root_hash = decode_hex(args.root_hash) downloader = StateDownloader(state_db, root_hash, peer_pool) loop = asyncio.get_event_loop() try: loop.run_until_complete(downloader.run()) except KeyboardInterrupt: pass loop.run_until_complete(downloader.stop()) loop.run_until_complete(peer_pool.stop()) loop.close()
def _test(): import argparse import signal from evm.p2p import ecies from evm.chains.ropsten import RopstenChain, ROPSTEN_GENESIS_HEADER from evm.db.backends.level import LevelDB from evm.db.backends.memory import MemoryDB logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') parser = argparse.ArgumentParser() parser.add_argument('-db', type=str, required=True) parser.add_argument('-root-hash', type=str, required=True, help='Hex encoded root hash') args = parser.parse_args() chaindb = ChainDB(MemoryDB()) chaindb.persist_header_to_db(ROPSTEN_GENESIS_HEADER) peer_pool = PeerPool(ETHPeer, chaindb, RopstenChain.network_id, ecies.generate_privkey()) asyncio.ensure_future(peer_pool.run()) state_db = LevelDB(args.db) root_hash = decode_hex(args.root_hash) downloader = StateDownloader(state_db, root_hash, peer_pool) loop = asyncio.get_event_loop() for sig in [signal.SIGINT, signal.SIGTERM]: loop.add_signal_handler(sig, downloader.cancel_token.trigger) async def run(): # downloader.run() will run in a loop until the SIGINT/SIGTERM handler triggers its cancel # token, at which point it returns and we stop the pool and downloader. await downloader.run() await peer_pool.stop() await downloader.stop() loop.run_until_complete(run()) loop.close()
class StateDownloader: logger = logging.getLogger("evm.p2p.state.StateDownloader") _pending_nodes = {} # type: Dict[Any, float] _total_processed_nodes = 0 _report_interval = 10 # Number of seconds between progress reports. # TODO: Experiment with different timeout/max_pending values to find the combination that # yields the best results. # FIXME: Should use the # of peers times MAX_STATE_FETCH here _max_pending = 5 * MAX_STATE_FETCH _reply_timeout = 10 # seconds # For simplicity/readability we use 0 here to force a report on the first iteration of the # loop. _last_report_time = 0 def __init__(self, chaindb, state_db, root_hash, network_id, privkey): self.peer_pool = PeerPool(ETHPeer, chaindb, network_id, privkey, self.msg_handler) self.root_hash = root_hash self.scheduler = StateSync(root_hash, state_db, self.logger) def msg_handler(self, peer: BasePeer, cmd: protocol.Command, msg: protocol._DecodedMsgType) -> None: """The callback passed to BasePeer, called for every incoming message.""" peer = cast(ETHPeer, peer) if isinstance(cmd, eth.NodeData): self.logger.debug("Processing NodeData with %d entries" % len(msg)) for node in msg: self._total_processed_nodes += 1 node_key = keccak(node) try: self.scheduler.process([(node_key, node)]) except SyncRequestAlreadyProcessed: # This means we received a node more than once, which can happen when we retry # after a timeout. pass # A node may be received more than once, so pop() with a default value. self._pending_nodes.pop(node_key, None) # FIXME: Need a better criteria to select peers here. async def get_random_peer(self) -> ETHPeer: while len(self.peer_pool.peers) == 0: self.logger.debug("No connected peers, sleeping a bit") await asyncio.sleep(0.5) peer = random.choice(self.peer_pool.peers) return cast(ETHPeer, peer) async def stop(self): await self.peer_pool.stop() async def request_next_batch(self): requests = self.scheduler.next_batch(MAX_STATE_FETCH) if not len(requests): # Although our run() loop frequently yields control to let our msg handler process # received nodes (scheduling new requests), there may be cases when the pending nodes # take a while to arrive thus causing the scheduler to run out of new requests for a # while. self.logger.debug("Scheduler queue is empty, not requesting any nodes") return self.logger.debug("Requesting %d trie nodes" % len(requests)) await self.request_nodes([request.node_key for request in requests]) async def request_nodes(self, node_keys): peer = await self.get_random_peer() now = time.time() for node_key in node_keys: self._pending_nodes[node_key] = now peer.sub_proto.send_get_node_data(node_keys) async def retry_timedout(self): timed_out = [] now = time.time() for node_key, req_time in list(self._pending_nodes.items()): if now - req_time > self._reply_timeout: timed_out.append(node_key) if len(timed_out) == 0: return self.logger.debug("Re-requesting %d trie nodes" % len(timed_out)) await self.request_nodes(timed_out) async def run(self): asyncio.ensure_future(self.peer_pool.run()) self.logger.info("Starting state sync for root hash %s" % encode_hex(self.root_hash)) while self.scheduler.has_pending_requests: # Request new nodes if we haven't reached the limit of pending nodes. if len(self._pending_nodes) < self._max_pending: await self.request_next_batch() # Retry pending nodes that timed out. if len(self._pending_nodes): await self.retry_timedout() if len(self._pending_nodes) > self._max_pending: # Slow down if we've reached the limit of pending nodes. self.logger.debug("Pending trie nodes limit reached, sleeping a bit") await asyncio.sleep(0.3) else: # Yield control to ensure the Peer's msg_handler callback is called to process any # nodes we may have received already. Otherwise we spin too fast and don't process # received nodes often enough. await asyncio.sleep(0) self._maybe_report_progress() self.logger.info("Finished state sync with root hash %s" % encode_hex(self.root_hash)) def _maybe_report_progress(self): if (time.time() - self._last_report_time) >= self._report_interval: self._last_report_time = time.time() self.logger.info("Nodes processed: %d" % self._total_processed_nodes) self.logger.info( "Nodes requested but not received yet: %d" % len(self._pending_nodes)) self.logger.info( "Nodes scheduled but not requested yet: %d" % len(self.scheduler.requests))