async def _sync(self, peer: ETHPeer) -> None: head = await self.chaindb.coro_get_canonical_head() head_td = await self.chaindb.coro_get_score(head.hash) if peer.head_td <= head_td: self.logger.info( "Head TD (%d) announced by %s not higher than ours (%d), not syncing", peer.head_td, peer, head_td) return self.logger.info("Starting sync with %s", peer) # FIXME: Fetch a batch of headers, in reverse order, starting from our current head, and # find the common ancestor between our chain and the peer's. start_at = max(0, head.block_number - eth.MAX_HEADERS_FETCH) while not self._sync_complete.is_set(): self.logger.info("Fetching chain segment starting at #%d", start_at) peer.sub_proto.send_get_block_headers(start_at, eth.MAX_HEADERS_FETCH, reverse=False) try: headers = await wait_with_token( self._new_headers.get(), peer.wait_until_finished(), token=self.cancel_token, timeout=self._reply_timeout) except TimeoutError: self.logger.warn("Timeout waiting for header batch from %s, aborting sync", peer) await peer.stop() break if peer.is_finished(): self.logger.info("%s disconnected, aborting sync", peer) break self.logger.info("Got headers segment starting at #%d", start_at) # TODO: Process headers for consistency. head_number = await self._process_headers(peer, headers) start_at = head_number + 1
async def _handle_msg(self, peer: ETHPeer, cmd: protocol.Command, msg: protocol._DecodedMsgType) -> None: if isinstance(cmd, eth.BlockHeaders): msg = cast(List[BlockHeader], msg) self.logger.debug("Got BlockHeaders from %d to %d", msg[0].block_number, msg[-1].block_number) self._new_headers.put_nowait(msg) elif isinstance(cmd, eth.BlockBodies): await self._handle_block_bodies(peer, cast(List[eth.BlockBody], msg)) elif isinstance(cmd, eth.Receipts): await self._handle_block_receipts( peer, cast(List[List[eth.Receipt]], msg)) elif isinstance(cmd, eth.NewBlock): msg = cast(Dict[str, Any], msg) header = msg['block'][0] actual_head = header.parent_hash actual_td = msg['total_difficulty'] - header.difficulty if actual_td > peer.head_td: peer.head_hash = actual_head peer.head_td = actual_td self._sync_requests.put_nowait(peer) elif isinstance(cmd, eth.Transactions): # TODO: Figure out what to do with those. pass elif isinstance(cmd, eth.NewBlockHashes): # TODO: Figure out what to do with those. pass else: # TODO: There are other msg types we'll want to handle here, but for now just log them # as a warning so we don't forget about it. self.logger.warn("Got unexpected msg: %s (%s)", cmd, msg)
async def _handle_new_block(self, peer: ETHPeer, msg: Dict[str, Any]) -> None: header = msg['block'][0] actual_head = header.parent_hash actual_td = msg['total_difficulty'] - header.difficulty if actual_td > peer.head_td: peer.head_hash = actual_head peer.head_td = actual_td self._sync_requests.put_nowait(peer)
async def _sync(self, peer: ETHPeer) -> None: head = await self.chaindb.coro_get_canonical_head() head_td = await self.chaindb.coro_get_score(head.hash) if peer.head_td <= head_td: self.logger.debug( "Head TD (%d) announced by %s not higher than ours (%d), not syncing", peer.head_td, peer, head_td) return # FIXME: Fetch a batch of headers, in reverse order, starting from our current head, and # find the common ancestor between our chain and the peer's. start_at = max(0, head.block_number - eth.MAX_HEADERS_FETCH) self.logger.debug("Asking %s for header batch starting at %d", peer, start_at) peer.sub_proto.send_get_block_headers(start_at, eth.MAX_HEADERS_FETCH, reverse=False) max_consecutive_timeouts = 3 consecutive_timeouts = 0 while True: try: headers = await wait_with_token( self._new_headers.get(), peer.wait_until_finished(), token=self.cancel_token, timeout=3) except OperationCancelled: break except TimeoutError: self.logger.debug("Timeout waiting for header batch from %s", peer) consecutive_timeouts += 1 if consecutive_timeouts > max_consecutive_timeouts: self.logger.debug( "Too many consecutive timeouts waiting for header batch, aborting sync " "with %s", peer) break continue if peer.is_finished(): self.logger.debug("%s disconnected, stopping sync", peer) break consecutive_timeouts = 0 if headers[-1].block_number <= start_at: self.logger.debug( "Ignoring headers from %d to %d as they've been processed already", headers[0].block_number, headers[-1].block_number) continue # TODO: Process headers for consistency. for header in headers: await self.chaindb.coro_persist_header(header) start_at = header.block_number self._body_requests.put_nowait(headers) self._receipt_requests.put_nowait(headers) self.logger.debug("Asking %s for header batch starting at %d", peer, start_at) # TODO: Instead of requesting sequential batches from a single peer, request a header # skeleton and make concurrent requests, using as many peers as possible, to fill the # skeleton. peer.sub_proto.send_get_block_headers(start_at, eth.MAX_HEADERS_FETCH, reverse=False)
async def handle_msg(self, peer: ETHPeer, cmd: protocol.Command, msg: protocol._DecodedMsgType) -> None: loop = asyncio.get_event_loop() if isinstance(cmd, eth.BlockHeaders): msg = cast(List[BlockHeader], msg) self.logger.debug( "Got BlockHeaders from %d to %d", msg[0].block_number, msg[-1].block_number) self._new_headers.put_nowait(msg) elif isinstance(cmd, eth.BlockBodies): msg = cast(List[eth.BlockBody], msg) self.logger.debug("Got %d BlockBodies from %s", len(msg), peer) iterator = map(make_trie_root_and_nodes, [body.transactions for body in msg]) transactions_tries = await loop.run_in_executor( self.executor, list, iterator) # type: List[Tuple[bytes, Any]] for i in range(len(msg)): body = msg[i] tx_root, trie_dict_data = transactions_tries[i] await self.chaindb.coro_persist_trie_data_dict(trie_dict_data) # TODO: Add transactions to canonical chain; blocked by # https://github.com/ethereum/py-evm/issues/337 uncles_hash = await self.chaindb.coro_persist_uncles(body.uncles) self._pending_bodies.pop((tx_root, uncles_hash), None) elif isinstance(cmd, eth.Receipts): msg = cast(List[List[eth.Receipt]], msg) self.logger.debug("Got Receipts for %d blocks from %s", len(msg), peer) iterator = map(make_trie_root_and_nodes, msg) receipts_tries = await loop.run_in_executor( self.executor, list, iterator) # type: List[Tuple[bytes, Any]] for receipt_root, trie_dict_data in receipts_tries: if receipt_root not in self._pending_receipts: self.logger.warning( "Got unexpected receipt root: %s", encode_hex(receipt_root), ) continue await self.chaindb.coro_persist_trie_data_dict(trie_dict_data) self._pending_receipts.pop(receipt_root) elif isinstance(cmd, eth.NewBlock): msg = cast(Dict[str, Any], msg) header = msg['block'][0] actual_head = header.parent_hash actual_td = msg['total_difficulty'] - header.difficulty if actual_td > peer.head_td: peer.head_hash = actual_head peer.head_td = actual_td self._sync_requests.put_nowait(peer) elif isinstance(cmd, eth.Transactions): # TODO: Figure out what to do with those. pass elif isinstance(cmd, eth.NewBlockHashes): # TODO: Figure out what to do with those. pass else: # TODO: There are other msg types we'll want to handle here, but for now just log them # as a warning so we don't forget about it. self.logger.warn("Got unexpected msg: %s (%s)", cmd, msg)
async def _sync(self, peer: ETHPeer) -> None: head = await self.chaindb.coro_get_canonical_head() head_td = await self.chaindb.coro_get_score(head.hash) if peer.head_td <= head_td: self.logger.info( "Head TD (%d) announced by %s not higher than ours (%d), not syncing", peer.head_td, peer, head_td) return # FIXME: Fetch a batch of headers, in reverse order, starting from our current head, and # find the common ancestor between our chain and the peer's. start_at = max(0, head.block_number - eth.MAX_HEADERS_FETCH) self.logger.debug("Asking %s for header batch starting at %d", peer, start_at) peer.sub_proto.send_get_block_headers(start_at, eth.MAX_HEADERS_FETCH, reverse=False) while True: # TODO: Consider stalling header fetching if there are more than X blocks/receipts # pending, to avoid timeouts caused by us not being able to process (decode/store) # blocks/receipts fast enough. try: headers = await wait_with_token(self._new_headers.get(), peer.wait_until_finished(), token=self.cancel_token, timeout=self._reply_timeout) except OperationCancelled: break except TimeoutError: self.logger.warn( "Timeout waiting for header batch from %s, aborting sync", peer) await peer.stop() break if peer.is_finished(): self.logger.info("%s disconnected, aborting sync", peer) break # TODO: Process headers for consistency. for header in headers: await self.chaindb.coro_persist_header(header) start_at = header.block_number + 1 self._body_requests.put_nowait(headers) self._receipt_requests.put_nowait(headers) self.logger.debug("Asking %s for header batch starting at %d", peer, start_at) # TODO: Instead of requesting sequential batches from a single peer, request a header # skeleton and make concurrent requests, using as many peers as possible, to fill the # skeleton. peer.sub_proto.send_get_block_headers(start_at, eth.MAX_HEADERS_FETCH, reverse=False)
async def handle_msg(self, peer: ETHPeer, cmd: protocol.Command, msg: protocol._DecodedMsgType) -> None: if isinstance(cmd, eth.BlockHeaders): msg = cast(List[BlockHeader], msg) self.logger.debug("Got BlockHeaders from %d to %d", msg[0].block_number, msg[-1].block_number) self._new_headers.put_nowait(msg) elif isinstance(cmd, eth.BlockBodies): msg = cast(List[eth.BlockBody], msg) self.logger.debug("Got %d BlockBodies", len(msg)) for body in msg: tx_root, trie_dict_data = make_trie_root_and_nodes( body.transactions) await self.chaindb.coro_persist_trie_data_dict(trie_dict_data) # TODO: Add transactions to canonical chain; blocked by # https://github.com/ethereum/py-evm/issues/337 uncles_hash = await self.chaindb.coro_persist_uncles( body.uncles) self._pending_bodies.pop((tx_root, uncles_hash), None) elif isinstance(cmd, eth.Receipts): msg = cast(List[List[eth.Receipt]], msg) self.logger.debug("Got Receipts for %d blocks", len(msg)) for block_receipts in msg: receipt_root, trie_dict_data = make_trie_root_and_nodes( block_receipts) await self.chaindb.coro_persist_trie_data_dict(trie_dict_data) self._pending_receipts.pop(receipt_root, None) elif isinstance(cmd, eth.NewBlock): msg = cast(Dict[str, Any], msg) header = msg['block'][0] actual_head = header.parent_hash actual_td = msg['total_difficulty'] - header.difficulty if actual_td > peer.head_td: peer.head_hash = actual_head peer.head_td = actual_td self._sync_requests.put_nowait(peer) elif isinstance(cmd, eth.Transactions): # TODO: Figure out what to do with those. pass elif isinstance(cmd, eth.NewBlockHashes): # TODO: Figure out what to do with those. pass else: # TODO: There are other msg types we'll want to handle here, but for now just log them # as a warning so we don't forget about it. self.logger.warn("Got unexpected msg: %s (%s)", cmd, msg)
async def receive_handshake(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: # Use reader to read the auth_init msg until EOF msg = await reader.read(ENCRYPTED_AUTH_MSG_LEN) # Use HandshakeResponder.decode_authentication(auth_init_message) on auth init msg try: ephem_pubkey, initiator_nonce, initiator_pubkey = decode_authentication( msg, self.privkey) # Try to decode as EIP8 except DecryptionError: msg_size = big_endian_to_int(msg[:2]) remaining_bytes = msg_size - ENCRYPTED_AUTH_MSG_LEN + 2 msg += await reader.read(remaining_bytes) ephem_pubkey, initiator_nonce, initiator_pubkey = decode_authentication( msg, self.privkey) # Get remote's address: IPv4 or IPv6 ip, port, *_ = writer.get_extra_info("peername") remote_address = Address(ip, port) # Create `HandshakeResponder(remote: kademlia.Node, privkey: datatypes.PrivateKey)` instance initiator_remote = Node(initiator_pubkey, remote_address) responder = HandshakeResponder(initiator_remote, self.privkey) # Call `HandshakeResponder.create_auth_ack_message(nonce: bytes)` to create the reply responder_nonce = secrets.token_bytes(HASH_LEN) auth_ack_msg = responder.create_auth_ack_message(nonce=responder_nonce) auth_ack_ciphertext = responder.encrypt_auth_ack_message(auth_ack_msg) # Use the `writer` to send the reply to the remote writer.write(auth_ack_ciphertext) await writer.drain() # Call `HandshakeResponder.derive_shared_secrets()` and use return values to create `Peer` aes_secret, mac_secret, egress_mac, ingress_mac = responder.derive_secrets( initiator_nonce=initiator_nonce, responder_nonce=responder_nonce, remote_ephemeral_pubkey=ephem_pubkey, auth_init_ciphertext=msg, auth_ack_ciphertext=auth_ack_ciphertext) # Create and register peer in peer_pool eth_peer = ETHPeer(remote=initiator_remote, privkey=self.privkey, reader=reader, writer=writer, aes_secret=aes_secret, mac_secret=mac_secret, egress_mac=egress_mac, ingress_mac=ingress_mac, chaindb=self.chaindb, network_id=self.network_id) self.peer_pool.add_peer(eth_peer)
async def _sync(self, peer: ETHPeer) -> None: head = await self.chaindb.coro_get_canonical_head() head_td = await self.chaindb.coro_get_score(head.hash) if peer.head_td <= head_td: self.logger.info( "Head TD (%d) announced by %s not higher than ours (%d), not syncing", peer.head_td, peer, head_td) return self.logger.info("Starting sync with %s", peer) # FIXME: Fetch a batch of headers, in reverse order, starting from our current head, and # find the common ancestor between our chain and the peer's. start_at = max(0, head.block_number - eth.MAX_HEADERS_FETCH) while True: self.logger.info("Fetching chain segment starting at #%d", start_at) peer.sub_proto.send_get_block_headers(start_at, eth.MAX_HEADERS_FETCH, reverse=False) try: headers = await wait_with_token(self._new_headers.get(), peer.wait_until_finished(), token=self.cancel_token, timeout=self._reply_timeout) except TimeoutError: self.logger.warn( "Timeout waiting for header batch from %s, aborting sync", peer) await peer.stop() break if peer.is_finished(): self.logger.info("%s disconnected, aborting sync", peer) break self.logger.info("Got headers segment starting at #%d", start_at) # TODO: Process headers for consistency. await self._download_block_parts( [header for header in headers if not _is_body_empty(header)], self.request_bodies, self._downloaded_bodies, _body_key, 'body') self.logger.info( "Got block bodies for chain segment starting at #%d", start_at) missing_receipts = [ header for header in headers if not _is_receipts_empty(header) ] # Post-Byzantium blocks may have identical receipt roots (e.g. when they have the same # number of transactions and all succeed/failed: ropsten blocks 2503212 and 2503284), # so we do this to avoid requesting the same receipts multiple times. missing_receipts = list(unique(missing_receipts, key=_receipts_key)) await self._download_block_parts(missing_receipts, self.request_receipts, self._downloaded_receipts, _receipts_key, 'receipt') self.logger.info( "Got block receipts for chain segment starting at #%d", start_at) for header in headers: await self.chaindb.coro_persist_header(header) start_at = header.block_number + 1 self.logger.info("Imported chain segment, new head: #%d", start_at - 1) head = await self.chaindb.coro_get_canonical_head() if head.hash == peer.head_hash: self.logger.info("Chain sync with %s completed", peer) self._sync_complete.set() break