async def test_header_gap_fill_detects_invalid_attempt(caplog, event_loop, event_bus, chaindb_with_gaps, chaindb_1000, chaindb_uncle): client_context = ChainContextFactory(headerdb__db=chaindb_with_gaps.db) server_context = ChainContextFactory(headerdb__db=chaindb_uncle.db) peer_pair = LatestETHPeerPairFactory( alice_peer_context=client_context, bob_peer_context=server_context, event_bus=event_bus, ) async with peer_pair as (client_peer, server_peer): client = SequentialHeaderChainGapSyncer( LatestTestChain(chaindb_with_gaps.db), chaindb_with_gaps, MockPeerPoolWithConnectedPeers([client_peer], event_bus=event_bus) ) server_peer_pool = MockPeerPoolWithConnectedPeers([server_peer], event_bus=event_bus) uncle_chaindb = AsyncChainDB(chaindb_uncle.db) async with run_peer_pool_event_server( event_bus, server_peer_pool, handler_type=ETHPeerPoolEventServer ), background_asyncio_service(ETHRequestServer( event_bus, TO_NETWORKING_BROADCAST_CONFIG, uncle_chaindb, )): server_peer.logger.info("%s is serving 1000 blocks", server_peer) client_peer.logger.info("%s is syncing up 1000", client_peer) # We check for 499 because 500 exists from the very beginning (the checkpoint) expected_block_number = 499 async with background_asyncio_service(client): try: await wait_for_head( chaindb_with_gaps, chaindb_1000.get_canonical_block_header_by_number(expected_block_number), sync_timeout=5, ) except asyncio.TimeoutError: assert "Attempted to fill gap with invalid header" in caplog.text # Monkey patch the uncle chaindb to effectively make the attacker peer # switch to the correct chain. uncle_chaindb.db = chaindb_1000.db await wait_for_head( chaindb_with_gaps, chaindb_1000.get_canonical_block_header_by_number(expected_block_number) ) else: raise AssertionError("Succeeded when it was expected to fail")
def make_eth1_request_server(cls, app_config: Eth1AppConfig, base_db: BaseAtomicDB, event_bus: EndpointAPI) -> Service: server: Service if app_config.database_mode is Eth1DbMode.LIGHT: header_db = AsyncHeaderDB(base_db) server = LightRequestServer( event_bus, TO_NETWORKING_BROADCAST_CONFIG, header_db ) elif app_config.database_mode is Eth1DbMode.FULL: chain_db = AsyncChainDB(base_db) server = ETHRequestServer( event_bus, TO_NETWORKING_BROADCAST_CONFIG, chain_db ) else: raise Exception(f"Unsupported Database Mode: {app_config.database_mode}") return server
async def test_proxy_peer_requests(request, event_bus, other_event_bus, event_loop, chaindb_20, client_and_server): server_event_bus = event_bus client_event_bus = other_event_bus client_peer, server_peer = client_and_server client_peer_pool = MockPeerPoolWithConnectedPeers( [client_peer], event_bus=client_event_bus) server_peer_pool = MockPeerPoolWithConnectedPeers( [server_peer], event_bus=server_event_bus) async with AsyncExitStack() as stack: await stack.enter_async_context( run_peer_pool_event_server(client_event_bus, client_peer_pool, handler_type=ETHPeerPoolEventServer)) await stack.enter_async_context( run_peer_pool_event_server(server_event_bus, server_peer_pool, handler_type=ETHPeerPoolEventServer)) await stack.enter_async_context( background_asyncio_service( ETHRequestServer(server_event_bus, TO_NETWORKING_BROADCAST_CONFIG, AsyncChainDB(chaindb_20.db)))) client_proxy_peer_pool = ETHProxyPeerPool( client_event_bus, TO_NETWORKING_BROADCAST_CONFIG) await stack.enter_async_context( background_asyncio_service(client_proxy_peer_pool)) proxy_peer_pool = ETHProxyPeerPool(server_event_bus, TO_NETWORKING_BROADCAST_CONFIG) await stack.enter_async_context( background_asyncio_service(proxy_peer_pool)) proxy_peer = await client_proxy_peer_pool.ensure_proxy_peer( client_peer.session) headers = await proxy_peer.eth_api.get_block_headers(0, 1, 0, False) assert len(headers) == 1 block_header = headers[0] assert block_header.block_number == 0 receipts = await proxy_peer.eth_api.get_receipts(headers) assert len(receipts) == 1 receipt = receipts[0] assert receipt[1][0] == block_header.receipt_root block_bundles = await proxy_peer.eth_api.get_block_bodies(headers) assert len(block_bundles) == 1 first_bundle = block_bundles[0] assert first_bundle[1][0] == block_header.transaction_root node_data = await proxy_peer.eth_api.get_node_data( (block_header.state_root, )) assert node_data[0][0] == block_header.state_root
async def sync(self, args: Namespace, logger: logging.Logger, chain: AsyncChainAPI, base_db: AtomicDatabaseAPI, peer_pool: BasePeerPool, event_bus: EndpointAPI, metrics_service: MetricsServiceAPI) -> None: # create registry for tracking beam sync pivot metrics if enabled if metrics_service == NOOP_METRICS_SERVICE: sync_metrics_registry = None else: sync_metrics_registry = SyncMetricsRegistry(metrics_service) syncer = BeamSyncService( chain, AsyncChainDB(base_db), base_db, cast(ETHPeerPool, peer_pool), event_bus, args.sync_from_checkpoint, args.force_beam_block_number, not args.disable_backfill, sync_metrics_registry, ) async with background_asyncio_service(syncer) as manager: await manager.wait_finished()
async def test_skeleton_syncer(request, event_loop, event_bus, chaindb_fresh, chaindb_1000): client_context = ChainContextFactory(headerdb__db=chaindb_fresh.db) server_context = ChainContextFactory(headerdb__db=chaindb_1000.db) peer_pair = ETHPeerPairFactory( alice_peer_context=client_context, bob_peer_context=server_context, event_bus=event_bus, ) async with peer_pair as (client_peer, server_peer): client_peer_pool = MockPeerPoolWithConnectedPeers([client_peer], event_bus=event_bus) client = FastChainSyncer(LatestTestChain(chaindb_fresh.db), chaindb_fresh, client_peer_pool) server_peer_pool = MockPeerPoolWithConnectedPeers([server_peer], event_bus=event_bus) async with run_peer_pool_event_server( event_bus, server_peer_pool, handler_type=ETHPeerPoolEventServer), run_request_server( event_bus, AsyncChainDB(chaindb_1000.db)): client_peer.logger.info("%s is serving 1000 blocks", client_peer) server_peer.logger.info("%s is syncing up 1000 blocks", server_peer) await asyncio.wait_for(client.run(), timeout=20) head = chaindb_fresh.get_canonical_head() assert head == chaindb_1000.get_canonical_head()
async def test_header_gapfill_syncer(request, event_loop, event_bus, chaindb_with_gaps, chaindb_1000): client_context = ChainContextFactory(headerdb__db=chaindb_with_gaps.db) server_context = ChainContextFactory(headerdb__db=chaindb_1000.db) peer_pair = LatestETHPeerPairFactory( alice_peer_context=client_context, bob_peer_context=server_context, event_bus=event_bus, ) async with peer_pair as (client_peer, server_peer): client = HeaderChainGapSyncer( LatestTestChain(chaindb_with_gaps.db), chaindb_with_gaps, MockPeerPoolWithConnectedPeers([client_peer], event_bus=event_bus) ) server_peer_pool = MockPeerPoolWithConnectedPeers([server_peer], event_bus=event_bus) async with run_peer_pool_event_server( event_bus, server_peer_pool, handler_type=ETHPeerPoolEventServer ), background_asyncio_service(ETHRequestServer( event_bus, TO_NETWORKING_BROADCAST_CONFIG, AsyncChainDB(chaindb_1000.db), )): server_peer.logger.info("%s is serving 1000 blocks", server_peer) client_peer.logger.info("%s is syncing up 1000", client_peer) async with background_asyncio_service(client): await wait_for_head( # We check for 249 because 250 exists from the very beginning (the checkpoint) chaindb_with_gaps, chaindb_1000.get_canonical_block_header_by_number(249))
async def test_regular_syncer_fallback(request, event_loop, event_bus, chaindb_fresh, chaindb_20): """ Test the scenario where a header comes in that's not in memory (but is in the DB) """ client_context = ChainContextFactory(headerdb__db=chaindb_fresh.db) server_context = ChainContextFactory(headerdb__db=chaindb_20.db) peer_pair = LatestETHPeerPairFactory( alice_peer_context=client_context, bob_peer_context=server_context, event_bus=event_bus, ) async with peer_pair as (client_peer, server_peer): client = FallbackTesting_RegularChainSyncer( ByzantiumTestChain(chaindb_fresh.db), chaindb_fresh, MockPeerPoolWithConnectedPeers([client_peer], event_bus=event_bus) ) server_peer_pool = MockPeerPoolWithConnectedPeers([server_peer], event_bus=event_bus) async with run_peer_pool_event_server( event_bus, server_peer_pool, handler_type=ETHPeerPoolEventServer ), background_asyncio_service(ETHRequestServer( event_bus, TO_NETWORKING_BROADCAST_CONFIG, AsyncChainDB(chaindb_20.db) )): server_peer.logger.info("%s is serving 20 blocks", server_peer) client_peer.logger.info("%s is syncing up 20", client_peer) async with background_asyncio_service(client): await wait_for_head(chaindb_fresh, chaindb_20.get_canonical_head()) head = chaindb_fresh.get_canonical_head() assert head.state_root in chaindb_fresh.db
async def test_fast_syncer(request, event_loop, event_bus, chaindb_fresh, chaindb_1000): client_context = ChainContextFactory(headerdb__db=chaindb_fresh.db) server_context = ChainContextFactory(headerdb__db=chaindb_1000.db) peer_pair = LatestETHPeerPairFactory( alice_peer_context=client_context, bob_peer_context=server_context, event_bus=event_bus, ) async with peer_pair as (client_peer, server_peer): client_peer_pool = MockPeerPoolWithConnectedPeers([client_peer], event_bus=event_bus) client = FastChainSyncer(LatestTestChain(chaindb_fresh.db), chaindb_fresh, client_peer_pool) server_peer_pool = MockPeerPoolWithConnectedPeers([server_peer], event_bus=event_bus) async with run_peer_pool_event_server( event_bus, server_peer_pool, handler_type=ETHPeerPoolEventServer ), background_asyncio_service( ETHRequestServer(event_bus, TO_NETWORKING_BROADCAST_CONFIG, AsyncChainDB(chaindb_1000.db))): client_peer.logger.info("%s is serving 1000 blocks", client_peer) server_peer.logger.info("%s is syncing up 1000 blocks", server_peer) async with background_asyncio_service(client) as manager: await asyncio.wait_for(manager.wait_finished(), timeout=20) head = chaindb_fresh.get_canonical_head() assert head == chaindb_1000.get_canonical_head()
async def test_light_syncer(request, event_loop, event_bus, chaindb_fresh, chaindb_20): client_context = ChainContextFactory(headerdb__db=chaindb_fresh.db) server_context = ChainContextFactory(headerdb__db=chaindb_20.db) peer_pair = LESV2PeerPairFactory( alice_peer_context=client_context, bob_peer_context=server_context, event_bus=event_bus, ) async with peer_pair as (client_peer, server_peer): client = LightChainSyncer( LatestTestChain(chaindb_fresh.db), chaindb_fresh, MockPeerPoolWithConnectedPeers([client_peer], event_bus=event_bus)) server_peer_pool = MockPeerPoolWithConnectedPeers([server_peer], event_bus=event_bus) async with run_peer_pool_event_server( event_bus, server_peer_pool, handler_type=LESPeerPoolEventServer ), background_asyncio_service( LightRequestServer( event_bus, TO_NETWORKING_BROADCAST_CONFIG, AsyncChainDB(chaindb_20.db), )): server_peer.logger.info("%s is serving 20 blocks", server_peer) client_peer.logger.info("%s is syncing up 20", client_peer) async with background_asyncio_service(client): await wait_for_head(chaindb_fresh, chaindb_20.get_canonical_head())
async def test_requests_when_peer_in_client_vanishs(request, event_bus, other_event_bus, event_loop, chaindb_20, client_and_server): server_event_bus = event_bus client_event_bus = other_event_bus client_peer, server_peer = client_and_server client_peer_pool = MockPeerPoolWithConnectedPeers( [client_peer], event_bus=client_event_bus) server_peer_pool = MockPeerPoolWithConnectedPeers( [server_peer], event_bus=server_event_bus) async with contextlib.AsyncExitStack() as stack: await stack.enter_async_context( run_peer_pool_event_server(client_event_bus, client_peer_pool, handler_type=ETHPeerPoolEventServer)) await stack.enter_async_context( run_peer_pool_event_server(server_event_bus, server_peer_pool, handler_type=ETHPeerPoolEventServer)) await stack.enter_async_context( background_asyncio_service( ETHRequestServer(server_event_bus, TO_NETWORKING_BROADCAST_CONFIG, MainnetChain.vm_configuration, AsyncChainDB(chaindb_20.db)))) client_proxy_peer_pool = ETHProxyPeerPool( client_event_bus, TO_NETWORKING_BROADCAST_CONFIG) await stack.enter_async_context( background_asyncio_service(client_proxy_peer_pool)) server_proxy_peer_pool = ETHProxyPeerPool( server_event_bus, TO_NETWORKING_BROADCAST_CONFIG) await stack.enter_async_context( background_asyncio_service(server_proxy_peer_pool)) proxy_peer = await client_proxy_peer_pool.ensure_proxy_peer( client_peer.session) # We remove the peer from the client and assume to see PeerConnectionLost exceptions raised client_peer_pool.connected_nodes.pop(client_peer.session) with pytest.raises(PeerConnectionLost): await proxy_peer.eth_api.get_block_headers(0, 1, 0, False) with pytest.raises(PeerConnectionLost): await proxy_peer.eth_api.get_receipts(()) with pytest.raises(PeerConnectionLost): await proxy_peer.eth_api.get_block_bodies(()) with pytest.raises(PeerConnectionLost): await proxy_peer.eth_api.get_node_data(())
async def test_queening_queue_recovers_from_penalty_with_one_peer( event_bus, chaindb_fresh, chaindb_20, has_parallel_peasant_call): local_context = ChainContextFactory(headerdb__db=chaindb_fresh.db) remote_context = ChainContextFactory(headerdb__db=chaindb_20.db) peer_pair = LatestETHPeerPairFactory( alice_peer_context=local_context, bob_peer_context=remote_context, event_bus=event_bus, ) async with peer_pair as (connection_to_local, connection_to_remote): local_peer_pool = MockPeerPoolWithConnectedPeers( [connection_to_remote], event_bus=event_bus, ) async with run_peer_pool_event_server( event_bus, local_peer_pool, handler_type=ETHPeerPoolEventServer ), background_asyncio_service( ETHRequestServer( event_bus, TO_NETWORKING_BROADCAST_CONFIG, AsyncChainDB(chaindb_20.db), )): queue = QueeningQueue(local_peer_pool) async with background_asyncio_service(queue): queen = await asyncio.wait_for(queue.get_queen_peer(), timeout=0.01) assert queen == connection_to_remote queue.penalize_queen(connection_to_remote, delay=0.1) assert queue.queen is None with pytest.raises(asyncio.TimeoutError): # The queen should be penalized for this entire period, and # there are no alternative peers, so this call should hang: await asyncio.wait_for(queue.get_queen_peer(), timeout=0.05) if has_parallel_peasant_call: waiting_on_peasant = asyncio.ensure_future( queue.pop_fastest_peasant()) # But after waiting long enough, even with just one peer, the blocking # call should return. Whether or not there is also a waiting call looking for # a peasant. final_queen = await asyncio.wait_for(queue.get_queen_peer(), timeout=0.075) assert final_queen == connection_to_remote if has_parallel_peasant_call: waiting_on_peasant.cancel() with pytest.raises(asyncio.CancelledError): await waiting_on_peasant
async def sync(self, args: Namespace, logger: logging.Logger, chain: AsyncChainAPI, base_db: AtomicDatabaseAPI, peer_pool: BasePeerPool, event_bus: EndpointAPI) -> None: syncer = FullChainSyncer( chain, AsyncChainDB(base_db), base_db, cast(ETHPeerPool, peer_pool), ) await syncer.run()
async def sync(self, args: Namespace, logger: logging.Logger, chain: AsyncChainAPI, base_db: AtomicDatabaseAPI, peer_pool: BasePeerPool, event_bus: EndpointAPI) -> None: syncer = BeamSyncService(chain, AsyncChainDB(base_db), base_db, cast(ETHPeerPool, peer_pool), event_bus, args.sync_from_checkpoint, args.force_beam_block_number, not args.disable_backfill) async with background_asyncio_service(syncer) as manager: await manager.wait_finished()
async def sync(self, args: Namespace, logger: logging.Logger, chain: AsyncChainAPI, base_db: AtomicDatabaseAPI, peer_pool: BasePeerPool, event_bus: EndpointAPI) -> None: syncer = HeaderChainSyncer( chain, AsyncChainDB(base_db), cast(ETHPeerPool, peer_pool), args.sync_from_checkpoint, ) async with background_asyncio_service(syncer) as manager: await manager.wait_finished()
async def sync(self, args: Namespace, logger: Logger, chain: ChainAPI, base_db: AtomicDatabaseAPI, peer_pool: BasePeerPool, event_bus: EndpointAPI, cancel_token: CancelToken) -> None: syncer = FastThenFullChainSyncer( chain, AsyncChainDB(base_db), base_db, cast(ETHPeerPool, peer_pool), cancel_token, ) await syncer.run()
def get_p2p_server(self) -> LightServer: if self._p2p_server is None: self._p2p_server = LightServer( privkey=self._nodekey, port=self._port, chain=self.get_full_chain(), chaindb=AsyncChainDB(self._base_db), headerdb=self.headerdb, base_db=self._base_db, network_id=self._network_id, max_peers=self._max_peers, event_bus=self.event_bus, metrics_registry=self.metrics_service.registry, ) return self._p2p_server
def get_p2p_server(self) -> LightServer: if self._p2p_server is None: self._p2p_server = LightServer( privkey=self._nodekey, port=self._port, chain=self.get_full_chain(), chaindb=AsyncChainDB(self._base_db), headerdb=self.headerdb, base_db=self._base_db, network_id=self._network_id, max_peers=self._max_peers, token=self.master_cancel_token, event_bus=self.event_bus, ) return self._p2p_server
async def sync(self, args: Namespace, logger: Logger, chain: AsyncChainAPI, base_db: AtomicDatabaseAPI, peer_pool: BasePeerPool, event_bus: EndpointAPI, cancel_token: CancelToken) -> None: syncer = BeamSyncService( chain, AsyncChainDB(base_db), base_db, cast(ETHPeerPool, peer_pool), event_bus, args.beam_from_checkpoint, args.force_beam_block_number, cancel_token, ) await syncer.run()
async def test_proxy_peer_requests(request, event_bus, other_event_bus, event_loop, chaindb_20, client_and_server): server_event_bus = event_bus client_event_bus = other_event_bus client_peer, server_peer = client_and_server client_peer_pool = MockPeerPoolWithConnectedPeers( [client_peer], event_bus=client_event_bus) server_peer_pool = MockPeerPoolWithConnectedPeers( [server_peer], event_bus=server_event_bus) async with run_peer_pool_event_server( client_event_bus, client_peer_pool, handler_type=ETHPeerPoolEventServer), run_peer_pool_event_server( server_event_bus, server_peer_pool, handler_type=ETHPeerPoolEventServer), run_request_server( server_event_bus, AsyncChainDB(chaindb_20.db)), run_proxy_peer_pool( client_event_bus ) as client_proxy_peer_pool, run_proxy_peer_pool( server_event_bus): proxy_peer = await client_proxy_peer_pool.ensure_proxy_peer( client_peer.session) headers = await proxy_peer.requests.get_block_headers(0, 1, 0, False) assert len(headers) == 1 block_header = headers[0] assert block_header.block_number == 0 receipts = await proxy_peer.requests.get_receipts(headers) assert len(receipts) == 1 receipt = receipts[0] assert receipt[1][0] == block_header.receipt_root block_bundles = await proxy_peer.requests.get_block_bodies(headers) assert len(block_bundles) == 1 first_bundle = block_bundles[0] assert first_bundle[1][0] == block_header.transaction_root node_data = await proxy_peer.requests.get_node_data( (block_header.state_root, )) assert node_data[0][0] == block_header.state_root
def get_p2p_server(self) -> FullServer: if self._p2p_server is None: self._p2p_server = FullServer( privkey=self._node_key, port=self._node_port, chain=self.get_full_chain(), chaindb=AsyncChainDB(self._base_db), headerdb=self.headerdb, base_db=self._base_db, network_id=self._network_id, max_peers=self._max_peers, bootstrap_nodes=self._bootstrap_nodes, preferred_nodes=self._preferred_nodes, token=self.cancel_token, event_bus=self.event_bus, ) return self._p2p_server
async def test_regular_syncer_fallback(request, event_loop, event_bus, chaindb_fresh, chaindb_20): """ Test the scenario where a header comes in that's not in memory (but is in the DB) """ client_context = ChainContextFactory(headerdb__db=chaindb_fresh.db) server_context = ChainContextFactory(headerdb__db=chaindb_20.db) peer_pair = ETHPeerPairFactory( alice_peer_context=client_context, bob_peer_context=server_context, event_bus=event_bus, ) async with peer_pair as (client_peer, server_peer): client = FallbackTesting_RegularChainSyncer( ByzantiumTestChain(chaindb_fresh.db), chaindb_fresh, MockPeerPoolWithConnectedPeers([client_peer], event_bus=event_bus)) server_peer_pool = MockPeerPoolWithConnectedPeers([server_peer], event_bus=event_bus) async with run_peer_pool_event_server( event_bus, server_peer_pool, handler_type=ETHPeerPoolEventServer ), background_asyncio_service( ETHRequestServer(event_bus, TO_NETWORKING_BROADCAST_CONFIG, AsyncChainDB(chaindb_20.db))): server_peer.logger.info("%s is serving 20 blocks", server_peer) client_peer.logger.info("%s is syncing up 20", client_peer) def finalizer(): event_loop.run_until_complete(client.cancel()) # Yield control so that client/server.run() returns, otherwise # asyncio will complain. event_loop.run_until_complete(asyncio.sleep(0.1)) request.addfinalizer(finalizer) asyncio.ensure_future(client.run()) await wait_for_head(chaindb_fresh, chaindb_20.get_canonical_head()) head = chaindb_fresh.get_canonical_head() assert head.state_root in chaindb_fresh.db
async def test_requests_when_peer_in_client_vanishs(request, event_bus, other_event_bus, event_loop, chaindb_20, client_and_server): server_event_bus = event_bus client_event_bus = other_event_bus client_peer, server_peer = client_and_server client_peer_pool = MockPeerPoolWithConnectedPeers( [client_peer], event_bus=client_event_bus) server_peer_pool = MockPeerPoolWithConnectedPeers( [server_peer], event_bus=server_event_bus) async with run_peer_pool_event_server( client_event_bus, client_peer_pool, handler_type=ETHPeerPoolEventServer), run_peer_pool_event_server( server_event_bus, server_peer_pool, handler_type=ETHPeerPoolEventServer), run_request_server( server_event_bus, AsyncChainDB(chaindb_20.db)), run_proxy_peer_pool( client_event_bus ) as client_proxy_peer_pool, run_proxy_peer_pool( server_event_bus): proxy_peer = await client_proxy_peer_pool.ensure_proxy_peer( client_peer.session) # We remove the peer from the client and assume to see PeerConnectionLost exceptions raised client_peer_pool.connected_nodes.pop(client_peer.session) with pytest.raises(PeerConnectionLost): await proxy_peer.requests.get_block_headers(0, 1, 0, False) with pytest.raises(PeerConnectionLost): await proxy_peer.requests.get_receipts(()) with pytest.raises(PeerConnectionLost): await proxy_peer.requests.get_block_bodies(()) with pytest.raises(PeerConnectionLost): await proxy_peer.requests.get_node_data(())
async def test_fast_syncer(request, event_bus, event_loop, chaindb_fresh, chaindb_20): client_context = ChainContextFactory(headerdb__db=chaindb_fresh.db) server_context = ChainContextFactory(headerdb__db=chaindb_20.db) peer_pair = ETHPeerPairFactory( alice_peer_context=client_context, bob_peer_context=server_context, event_bus=event_bus, ) async with peer_pair as (client_peer, server_peer): client_peer_pool = MockPeerPoolWithConnectedPeers([client_peer]) client = FastChainSyncer(LatestTestChain(chaindb_fresh.db), chaindb_fresh, client_peer_pool) server_peer_pool = MockPeerPoolWithConnectedPeers([server_peer], event_bus=event_bus) async with run_peer_pool_event_server( event_bus, server_peer_pool, handler_type=ETHPeerPoolEventServer, ), run_request_server( event_bus, AsyncChainDB(chaindb_20.db), ): server_peer.logger.info("%s is serving 20 blocks", server_peer) client_peer.logger.info("%s is syncing up 20", client_peer) # FastChainSyncer.run() will return as soon as it's caught up with the peer. await asyncio.wait_for(client.run(), timeout=5) head = chaindb_fresh.get_canonical_head() assert head == chaindb_20.get_canonical_head() # Now download the state for the chain's head. state_downloader = StateDownloader(chaindb_fresh, chaindb_fresh.db, head.state_root, client_peer_pool) await asyncio.wait_for(state_downloader.run(), timeout=5) assert head.state_root in chaindb_fresh.db
async def test_sequential_header_gapfill_syncer(request, event_loop, event_bus, chaindb_with_gaps, chaindb_1000): client_context = ChainContextFactory(headerdb__db=chaindb_with_gaps.db) server_context = ChainContextFactory(headerdb__db=chaindb_1000.db) peer_pair = LatestETHPeerPairFactory( alice_peer_context=client_context, bob_peer_context=server_context, event_bus=event_bus, ) async with peer_pair as (client_peer, server_peer): chain_with_gaps = LatestTestChain(chaindb_with_gaps.db) client = SequentialHeaderChainGapSyncer( chain_with_gaps, chaindb_with_gaps, MockPeerPoolWithConnectedPeers([client_peer], event_bus=event_bus) ) server_peer_pool = MockPeerPoolWithConnectedPeers([server_peer], event_bus=event_bus) async with run_peer_pool_event_server( event_bus, server_peer_pool, handler_type=ETHPeerPoolEventServer ), background_asyncio_service(ETHRequestServer( event_bus, TO_NETWORKING_BROADCAST_CONFIG, AsyncChainDB(chaindb_1000.db), )): server_peer.logger.info("%s is serving 1000 blocks", server_peer) client_peer.logger.info("%s is syncing up 1000", client_peer) async with background_asyncio_service(client): await wait_for_head( # We check for 499 because 500 is there from the very beginning (the checkpoint) chaindb_with_gaps, chaindb_1000.get_canonical_block_header_by_number(499) ) # This test is supposed to only fill in headers, so the following should fail. # If this ever succeeds it probably means the fixture was re-created with trivial # blocks and the test will fail and remind us what kind of fixture we want here. with pytest.raises(BlockNotFound): chain_with_gaps.get_canonical_block_by_number(499)
async def test_light_syncer(request, event_loop, event_bus, chaindb_fresh, chaindb_20): client_context = ChainContextFactory(headerdb__db=chaindb_fresh.db) server_context = ChainContextFactory(headerdb__db=chaindb_20.db) peer_pair = LESV2PeerPairFactory( alice_peer_context=client_context, bob_peer_context=server_context, event_bus=event_bus, ) async with peer_pair as (client_peer, server_peer): client = LightChainSyncer( LatestTestChain(chaindb_fresh.db), chaindb_fresh, MockPeerPoolWithConnectedPeers([client_peer], event_bus=event_bus)) server_peer_pool = MockPeerPoolWithConnectedPeers([server_peer], event_bus=event_bus) async with run_peer_pool_event_server( event_bus, server_peer_pool, handler_type=LESPeerPoolEventServer ), background_asyncio_service( LightRequestServer( event_bus, TO_NETWORKING_BROADCAST_CONFIG, AsyncChainDB(chaindb_20.db), )): server_peer.logger.info("%s is serving 20 blocks", server_peer) client_peer.logger.info("%s is syncing up 20", client_peer) def finalizer(): event_loop.run_until_complete(client.cancel()) # Yield control so that client/server.run() returns, otherwise # asyncio will complain. event_loop.run_until_complete(asyncio.sleep(0.1)) request.addfinalizer(finalizer) asyncio.ensure_future(client.run()) await wait_for_head(chaindb_fresh, chaindb_20.get_canonical_head())
async def test_no_duplicate_node_data(request, event_loop, event_bus, chaindb_fresh, chaindb_20): """ Test that when a peer calls GetNodeData to ETHRequestServer, with duplicate node hashes, that ETHRequestServer only responds with unique nodes. Note: A nice extension to the test would be to check that a warning is raised about sending the duplicate hashes in the first place. """ client_context = ChainContextFactory(headerdb__db=chaindb_fresh.db) server_context = ChainContextFactory(headerdb__db=chaindb_20.db) peer_pair = LatestETHPeerPairFactory( alice_peer_context=client_context, bob_peer_context=server_context, event_bus=event_bus, ) async with peer_pair as (client_to_server, server_to_client): server_peer_pool = MockPeerPoolWithConnectedPeers([server_to_client], event_bus=event_bus) async with run_peer_pool_event_server( event_bus, server_peer_pool, handler_type=ETHPeerPoolEventServer ), background_asyncio_service( ETHRequestServer( event_bus, TO_NETWORKING_BROADCAST_CONFIG, MainnetChain.vm_configuration, AsyncChainDB(chaindb_20.db), )): root_hash = chaindb_20.get_canonical_head().state_root state_root = chaindb_20.db[root_hash] returned_nodes = await client_to_server.eth_api.get_node_data( (root_hash, root_hash)) assert returned_nodes == ( # Server must not send back duplicates, just the single root node (root_hash, state_root), )
async def test_regular_syncer(request, event_loop, event_bus, chaindb_fresh, chaindb_20): client_context = ChainContextFactory(headerdb__db=chaindb_fresh.db) server_context = ChainContextFactory(headerdb__db=chaindb_20.db) peer_pair = ETHPeerPairFactory( alice_peer_context=client_context, bob_peer_context=server_context, event_bus=event_bus, ) async with peer_pair as (client_peer, server_peer): client = RegularChainSyncer( ByzantiumTestChain(chaindb_fresh.db), chaindb_fresh, MockPeerPoolWithConnectedPeers([client_peer])) server_peer_pool = MockPeerPoolWithConnectedPeers([server_peer], event_bus=event_bus) async with run_peer_pool_event_server( event_bus, server_peer_pool, handler_type=ETHPeerPoolEventServer), run_request_server( event_bus, AsyncChainDB(chaindb_20.db)): server_peer.logger.info("%s is serving 20 blocks", server_peer) client_peer.logger.info("%s is syncing up 20", client_peer) def finalizer(): event_loop.run_until_complete(client.cancel()) # Yield control so that client/server.run() returns, otherwise # asyncio will complain. event_loop.run_until_complete(asyncio.sleep(0.1)) request.addfinalizer(finalizer) asyncio.ensure_future(client.run()) await wait_for_head(chaindb_fresh, chaindb_20.get_canonical_head()) head = chaindb_fresh.get_canonical_head() assert head.state_root in chaindb_fresh.db
async def test_beam_syncer(request, event_loop, event_bus, chaindb_fresh, chaindb_churner, beam_to_block, checkpoint=None): client_context = ChainContextFactory(headerdb__db=chaindb_fresh.db) server_context = ChainContextFactory(headerdb__db=chaindb_churner.db) peer_pair = ETHPeerPairFactory( alice_peer_context=client_context, bob_peer_context=server_context, event_bus=event_bus, ) async with peer_pair as (client_peer, server_peer): # Need a name that will be unique per xdist-process, otherwise # lahja IPC endpoints in each process will clobber each other unique_process_name = uuid.uuid4() # manually add endpoint for beam vm to make requests pausing_config = ConnectionConfig.from_name( f"PausingEndpoint-{unique_process_name}") # manually add endpoint for trie data gatherer to serve requests gatherer_config = ConnectionConfig.from_name( f"GathererEndpoint-{unique_process_name}") client_peer_pool = MockPeerPoolWithConnectedPeers([client_peer]) server_peer_pool = MockPeerPoolWithConnectedPeers([server_peer], event_bus=event_bus) async with run_peer_pool_event_server( event_bus, server_peer_pool, handler_type=ETHPeerPoolEventServer), run_request_server( event_bus, AsyncChainDB(chaindb_churner.db) ), AsyncioEndpoint.serve( pausing_config) as pausing_endpoint, AsyncioEndpoint.serve( gatherer_config) as gatherer_endpoint: client_chain = make_pausing_beam_chain( ((0, PetersburgVM), ), chain_id=999, db=chaindb_fresh.db, event_bus=pausing_endpoint, loop=event_loop, ) client = BeamSyncer( client_chain, chaindb_fresh.db, AsyncChainDB(chaindb_fresh.db), client_peer_pool, gatherer_endpoint, force_beam_block_number=beam_to_block, checkpoint=checkpoint, ) client_peer.logger.info("%s is serving churner blocks", client_peer) server_peer.logger.info("%s is syncing up churner blocks", server_peer) import_server = BlockImportServer( pausing_endpoint, client_chain, token=client.cancel_token, ) asyncio.ensure_future(import_server.run()) await pausing_endpoint.connect_to_endpoints(gatherer_config) asyncio.ensure_future(client.run()) # We can sync at least 10 blocks in 1s at current speeds, (or reach the current one) # Trying to keep the tests short-ish. A fuller test could always set the target header # to the chaindb_churner canonical head, and increase the timeout significantly target_block_number = min(beam_to_block + 10, 129) target_head = chaindb_churner.get_canonical_block_header_by_number( target_block_number) await wait_for_head(chaindb_fresh, target_head, sync_timeout=10) assert target_head.state_root in chaindb_fresh.db # first stop the import server, so it doesn't hang waiting for state data await import_server.cancel() await client.cancel()
async def test_proxy_peer_requests(request, event_bus, other_event_bus, event_loop, chaindb_20, client_and_server): server_event_bus = event_bus client_event_bus = other_event_bus client_peer, server_peer = client_and_server client_peer_pool = MockPeerPoolWithConnectedPeers( [client_peer], event_bus=client_event_bus) server_peer_pool = MockPeerPoolWithConnectedPeers( [server_peer], event_bus=server_event_bus) async with contextlib.AsyncExitStack() as stack: await stack.enter_async_context( run_peer_pool_event_server(client_event_bus, client_peer_pool, handler_type=ETHPeerPoolEventServer)) await stack.enter_async_context( run_peer_pool_event_server(server_event_bus, server_peer_pool, handler_type=ETHPeerPoolEventServer)) base_db = chaindb_20.db await stack.enter_async_context( background_asyncio_service( ETHRequestServer( server_event_bus, TO_NETWORKING_BROADCAST_CONFIG, AsyncChainDB(base_db), ))) await stack.enter_async_context( background_asyncio_service( WitRequestServer( server_event_bus, TO_NETWORKING_BROADCAST_CONFIG, base_db, ))) client_proxy_peer_pool = ETHProxyPeerPool( client_event_bus, TO_NETWORKING_BROADCAST_CONFIG) await stack.enter_async_context( background_asyncio_service(client_proxy_peer_pool)) proxy_peer_pool = ETHProxyPeerPool(server_event_bus, TO_NETWORKING_BROADCAST_CONFIG) await stack.enter_async_context( background_asyncio_service(proxy_peer_pool)) proxy_peer = await client_proxy_peer_pool.ensure_proxy_peer( client_peer.session) headers = await proxy_peer.eth_api.get_block_headers(0, 1, 0, False) assert len(headers) == 1 block_header = headers[0] assert block_header.block_number == 0 receipts = await proxy_peer.eth_api.get_receipts(headers) assert len(receipts) == 1 receipt = receipts[0] assert receipt[1][0] == block_header.receipt_root block_bundles = await proxy_peer.eth_api.get_block_bodies(headers) assert len(block_bundles) == 1 first_bundle = block_bundles[0] assert first_bundle[1][0] == block_header.transaction_root node_data = await proxy_peer.eth_api.get_node_data( (block_header.state_root, )) assert node_data[0][0] == block_header.state_root block_hash = block_header.hash node_hashes = tuple(Hash32Factory.create_batch(5)) # Populate the server's witness DB so that it can reply to our request. wit_db = AsyncWitnessDB(base_db) wit_db.persist_witness_hashes(block_hash, node_hashes) response = await proxy_peer.wit_api.get_block_witness_hashes(block_hash ) assert set(response) == set(node_hashes)
async def _main() -> None: parser = argparse.ArgumentParser() parser.add_argument('-db', type=str, required=True) parser.add_argument('-light', action="store_true") parser.add_argument('-nodekey', type=str) parser.add_argument('-enode', type=str, required=False, help="The enode we should connect to") parser.add_argument('-debug', action="store_true") args = parser.parse_args() logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s: %(message)s', datefmt='%H:%M:%S') log_level = logging.INFO if args.debug: log_level = logging.DEBUG loop = asyncio.get_event_loop() base_db = LevelDB(args.db) headerdb = AsyncHeaderDB(AtomicDB(base_db)) chaindb = AsyncChainDB(AtomicDB(base_db)) try: genesis = chaindb.get_canonical_block_header_by_number(BlockNumber(0)) except HeaderNotFound: genesis = ROPSTEN_GENESIS_HEADER chaindb.persist_header(genesis) peer_pool_class: Type[Union[ETHPeerPool, LESPeerPool]] = ETHPeerPool if args.light: peer_pool_class = LESPeerPool chain_class: Union[Type[AsyncRopstenChain], Type[AsyncMainnetChain]] if genesis.hash == ROPSTEN_GENESIS_HEADER.hash: chain_id = RopstenChain.chain_id vm_config = ROPSTEN_VM_CONFIGURATION chain_class = AsyncRopstenChain elif genesis.hash == MAINNET_GENESIS_HEADER.hash: chain_id = MainnetChain.chain_id vm_config = MAINNET_VM_CONFIGURATION # type: ignore chain_class = AsyncMainnetChain else: raise RuntimeError("Unknown genesis: %s", genesis) if args.nodekey: privkey = load_nodekey(Path(args.nodekey)) else: privkey = ecies.generate_privkey() context = ChainContext( headerdb=headerdb, network_id=chain_id, vm_configuration=vm_config, client_version_string=construct_trinity_client_identifier(), listen_port=30309, p2p_version=DEVP2P_V5, ) peer_pool = peer_pool_class(privkey=privkey, context=context) if args.enode: nodes = tuple([Node.from_uri(args.enode)]) else: nodes = DEFAULT_PREFERRED_NODES[chain_id] async with background_asyncio_service(peer_pool) as manager: manager.run_task(connect_to_peers_loop(peer_pool, nodes)) # type: ignore chain = chain_class(base_db) syncer: Service = None if args.light: syncer = LightChainSyncer(chain, headerdb, cast(LESPeerPool, peer_pool)) else: syncer = RegularChainSyncer(chain, chaindb, cast(ETHPeerPool, peer_pool)) logging.getLogger().setLevel(log_level) sigint_received = asyncio.Event() for sig in [signal.SIGINT, signal.SIGTERM]: loop.add_signal_handler(sig, sigint_received.set) async def exit_on_sigint() -> None: await sigint_received.wait() syncer.get_manager().cancel() asyncio.ensure_future(exit_on_sigint()) async with background_asyncio_service(syncer) as syncer_manager: await syncer_manager.wait_finished()