async def test_endpoint_run(): endpoint = AsyncioEndpoint("test-run") assert endpoint.is_running is False async with endpoint.run(): assert endpoint.is_running is True assert endpoint.is_running is False
async def test_base_wait_until_connections_changed(): config = ConnectionConfig.from_name(generate_unique_name()) async with AsyncioEndpoint.serve(config): async with AsyncioEndpoint("client").run() as client: asyncio.ensure_future(client.connect_to_endpoints(config)) assert not client.is_connected_to(config.name) await asyncio.wait_for(client.wait_until_connections_change(), timeout=0.1) assert client.is_connected_to(config.name)
async def pair_of_endpoints(event_loop, ipc_base_path): config_1 = ConnectionConfig.from_name(generate_unique_name(), base_path=ipc_base_path) config_2 = ConnectionConfig.from_name(generate_unique_name(), base_path=ipc_base_path) async with AsyncioEndpoint.serve(config_1) as endpoint1: async with AsyncioEndpoint.serve(config_2) as endpoint2: await endpoint1.connect_to_endpoints(config_2) await endpoint2.connect_to_endpoints(config_1) yield endpoint1, endpoint2
async def test_endpoint_run_with_error(): endpoint = AsyncioEndpoint("test-run") assert endpoint.is_running is False with pytest.raises(Exception, match="break out of run"): async with endpoint.run(): assert endpoint.is_running is True raise Exception("break out of run") assert endpoint.is_running is False
async def test_base_wait_until_any_endpoint_subscriptions_changed(): config = ConnectionConfig.from_name(generate_unique_name()) async with AsyncioEndpoint.serve(config) as server: async with AsyncioEndpoint("client").run() as client: await client.connect_to_endpoints(config) assert client.is_connected_to(config.name) server.subscribe(SubscriptionEvent, lambda e: None) assert not client.is_any_endpoint_subscribed_to(SubscriptionEvent) await asyncio.wait_for( client.wait_until_endpoint_subscriptions_change(), timeout=0.1 ) assert client.is_any_endpoint_subscribed_to(SubscriptionEvent)
async def client_with_three_connections(ipc_base_path): config_a = ConnectionConfig.from_name("server-a", base_path=ipc_base_path) config_b = ConnectionConfig.from_name("server-b", base_path=ipc_base_path) config_c = ConnectionConfig.from_name("server-c", base_path=ipc_base_path) async with AsyncioEndpoint.serve(config_a) as server_a: async with AsyncioEndpoint.serve(config_b) as server_b: async with AsyncioEndpoint.serve(config_c) as server_c: async with AsyncioEndpoint("client").run() as client: await client.connect_to_endpoints(config_a) await client.connect_to_endpoints(config_b) await client.connect_to_endpoints(config_c) yield client, server_a, server_b, server_c
async def _run(self) -> None: async with AsyncioEndpoint.serve(self._connection_config) as endpoint: self._endpoint = endpoint # signal that the endpoint is now available self._endpoint_available.set() # run background task that automatically connects to newly announced endpoints self.run_daemon_task(self._auto_connect_new_announced_endpoints()) # connect to the *main* endpoint which communicates information # about other endpoints that come online. main_endpoint_config = ConnectionConfig.from_name( MAIN_EVENTBUS_ENDPOINT, self._trinity_config.ipc_dir, ) await endpoint.connect_to_endpoints(main_endpoint_config) # announce ourself to the event bus await endpoint.wait_until_endpoint_subscribed_to( MAIN_EVENTBUS_ENDPOINT, EventBusConnected, ) await endpoint.broadcast( EventBusConnected(self._connection_config), BroadcastConfig(filter_endpoint=MAIN_EVENTBUS_ENDPOINT)) # run until the endpoint exits await self.cancellation()
async def run_proc1(): config = ConnectionConfig.from_name("e1") async with AsyncioEndpoint.serve(config) as server: async for event in server.stream(GetSomethingRequest, num_events=3): await server.broadcast( DeliverSomethingResponse("Yay"), event.broadcast_config() )
async def make_networking_event_bus(): # Tests run concurrently, therefore we need unique IPC paths ipc_path = Path(f"networking-{uuid.uuid4()}.ipc") networking_connection_config = ConnectionConfig( name=NETWORKING_EVENTBUS_ENDPOINT, path=ipc_path) async with AsyncioEndpoint.serve(networking_connection_config) as endpoint: yield endpoint
async def test_endpoint_serve(ipc_base_path): config = ConnectionConfig.from_name(generate_unique_name(), base_path=ipc_base_path) async with AsyncioEndpoint.serve(config) as endpoint: assert endpoint.is_running is True assert endpoint.is_serving is True assert endpoint.is_running is False assert endpoint.is_serving is False
async def test_rejects_duplicates_when_connecting(): own = ConnectionConfig.from_name(generate_unique_name()) async with AsyncioEndpoint.serve(own) as endpoint: await endpoint.connect_to_endpoint(own) assert endpoint.is_connected_to(own.name) with pytest.raises(ConnectionAttemptRejected): await endpoint.connect_to_endpoint(own)
async def test_wait_until_connected_to(endpoint): config = ConnectionConfig.from_name(generate_unique_name()) async with AsyncioEndpoint.serve(config): asyncio.ensure_future(endpoint.connect_to_endpoints(config)) assert not endpoint.is_connected_to(config.name) await endpoint.wait_until_connected_to(config.name) assert endpoint.is_connected_to(config.name)
async def endpoint(event_loop, ipc_base_path): config = ConnectionConfig.from_name(generate_unique_name(), base_path=ipc_base_path) async with AsyncioEndpoint.serve(config) as endpoint: # We need to connect to our own Endpoint if we care about receiving # the events we broadcast. Many tests use the same Endpoint for # broadcasting and receiving which is a valid use case so we hook it up await endpoint.connect_to_endpoint(config) yield endpoint
async def run_network(self): async with AsyncioEndpoint.serve( ConnectionConfig.from_name("network")) as self.network_bus: self.logger.info("Network started") self.network_bus.subscribe(GenericMessageOutgoingEvent, self.send_to_all_nodes) self.logger.info("Network subscribed to all messages") while not self.kill_switch.value: await asyncio.sleep(0) self.logger.info("Network turned off")
async def proc2_worker(): config = ConnectionConfig.from_name("e1") async with AsyncioEndpoint("e2").run() as client: await client.connect_to_endpoints(config) await client.wait_until_any_endpoint_subscribed_to(GetSomethingRequest) for i in range(3): print("Requesting") result = await client.request(GetSomethingRequest()) print(f"Got answer: {result.payload}")
async def run(self) -> None: connection_config = ConnectionConfig.from_name( MAIN_EVENTBUS_ENDPOINT, self._boot_info.trinity_config.ipc_dir) async with AsyncioEndpoint.serve(connection_config) as endpoint: self._endpoint = endpoint # start the background process that tracks and propagates available # endpoints to the other connected endpoints self.manager.run_daemon_task( self._track_and_propagate_available_endpoints) await endpoint.wait_until_any_endpoint_subscribed_to( EventBusConnected) # signal the endpoint is up and running and available self._endpoint_available.set() # instantiate all of the components all_components = tuple( component_cls(self._boot_info) for component_cls in self._component_types) # filter out any components that should not be enabled. enabled_components = tuple(component for component in all_components if component.is_enabled) # a little bit of extra try/finally structure here to produce good # logging messages about the component lifecycle. from p2p.asyncio_utils import create_task, wait_first try: self.logger.info( "Starting components: %s", '/'.join(component.name for component in enabled_components), ) tasks: List[asyncio.Task[Any]] = [] for component in enabled_components: tasks.append( create_task( component.run_in_process(), f'IsolatedComponent/{component.name}/run_in_process' )) tasks.append( asyncio.create_task(self._trigger_component_exit.wait())) self.logger.info("Components started") try: # The timeout is long as our component tasks can do a lot of stuff during # their cleanup. await wait_first(tasks, max_wait_after_cancellation=10) finally: self.logger.info("Stopping components") finally: self.logger.info("Components stopped.") self.manager.cancel()
async def run(self) -> None: connection_config = ConnectionConfig.from_name( MAIN_EVENTBUS_ENDPOINT, self._boot_info.trinity_config.ipc_dir ) async with AsyncioEndpoint.serve(connection_config) as endpoint: self._endpoint = endpoint # start the background process that tracks and propagates available # endpoints to the other connected endpoints self.manager.run_daemon_task(self._track_and_propagate_available_endpoints) self.manager.run_daemon_task(self._handle_shutdown_request) await endpoint.wait_until_any_endpoint_subscribed_to(ShutdownRequest) await endpoint.wait_until_any_endpoint_subscribed_to(EventBusConnected) # signal the endpoint is up and running and available self._endpoint_available.set() # instantiate all of the components all_components = tuple( component_cls(self._boot_info) for component_cls in self._component_types ) # filter out any components that should not be enabled. enabled_components = tuple( component for component in all_components if component.is_enabled ) # a little bit of extra try/finally structure here to produce good # logging messages about the component lifecycle. try: async with AsyncExitStack() as stack: self.logger.info( "Starting components: %s", '/'.join(component.name for component in enabled_components), ) # Concurrently start the components. await asyncio.gather(*( stack.enter_async_context(run_component(component)) for component in enabled_components )) self.logger.info("Components started") try: await self._trigger_component_exit.wait() finally: self.logger.info("Stopping components") finally: self.logger.info("Components stopped.") self.manager.cancel()
async def run(self) -> None: connection_config = ConnectionConfig.from_name( MAIN_EVENTBUS_ENDPOINT, self._boot_info.trinity_config.ipc_dir ) async with AsyncioEndpoint.serve(connection_config) as endpoint: self._endpoint = endpoint # start the background process that tracks and propagates available # endpoints to the other connected endpoints self.manager.run_daemon_task(self._track_and_propagate_available_endpoints) await endpoint.wait_until_any_endpoint_subscribed_to(EventBusConnected) # signal the endpoint is up and running and available self._endpoint_available.set() # instantiate all of the components all_components = tuple( component_cls(self._boot_info) for component_cls in self._component_types ) # filter out any components that should not be enabled. enabled_components = tuple( component for component in all_components if component.is_enabled ) # a little bit of extra try/finally structure here to produce good # logging messages about the component lifecycle. from p2p.logic import wait_first try: self.logger.info( "Starting components: %s", '/'.join(component.name for component in enabled_components), ) context_managers = [component.run() for component in enabled_components] async with AsyncContextGroup(context_managers) as futs: # AsyncContextGroup() yields a Sequence[Any], so we cast to a list of Futures # here to ensure mypy can come to our aid if we forget the create_task() when # adding new entries to the list of Futures we want to wait for. futures = cast(List["asyncio.Future[None]"], list(futs)) futures.append(asyncio.create_task(self._trigger_component_exit.wait())) self.logger.info("Components started") try: await wait_first(futures) finally: self.logger.info("Stopping components") finally: self.logger.info("Components stopped.") self.manager.cancel()
async def test_endpoint_serve_with_error(ipc_base_path): config = ConnectionConfig.from_name(generate_unique_name(), base_path=ipc_base_path) with pytest.raises(Exception, match="break out of serve"): async with AsyncioEndpoint.serve(config) as endpoint: assert endpoint.is_running is True assert endpoint.is_serving is True raise Exception("break out of serve") assert endpoint.is_running is False assert endpoint.is_serving is False
async def run(self, nodes_history): self.nodes_history = nodes_history self.nodes_history[self.addr] = [] async with AsyncioEndpoint( f"node_{self.addr}").run() as self.network_bus: await self.connect_to_network() self.start_time = time.time() while not self.kill_switch.value: # self.logger.debug(f"[{self}] Kill_switch state: {self.kill_switch.value}") await self.main_message_tick() self.logger.info(f"Node_{self.addr} killed") self.network_bus = None
async def proc1_worker(): async with AsyncioEndpoint.serve(ConnectionConfig.from_name("e1")) as server: server.subscribe( SecondThingHappened, lambda event: logging.info( "Received via SUBSCRIBE API in proc1: %s", event.payload ), ) await server.wait_until_any_endpoint_subscribed_to(FirstThingHappened) while True: logging.info("Hello from proc1") await server.broadcast(FirstThingHappened("Hit from proc1")) await asyncio.sleep(2)
async def _run(self) -> None: self._connection_config = ConnectionConfig.from_name( MAIN_EVENTBUS_ENDPOINT, self._boot_info.trinity_config.ipc_dir) async with AsyncioEndpoint.serve(self._connection_config) as endpoint: self._endpoint = endpoint # start the background process that tracks and propagates available # endpoints to the other connected endpoints self.run_daemon_task( self._track_and_propagate_available_endpoints()) # start the plugin manager plugin_manager = PluginManager(endpoint, self._plugins) plugin_manager.prepare(self._boot_info) await self.cancellation()
async def proc2_worker(): config = ConnectionConfig.from_name("e1") async with AsyncioEndpoint("e2").run() as client: await client.connect_to_endpoints(config) asyncio.ensure_future(display_proc1_events(client)) client.subscribe( FirstThingHappened, lambda event: logging.info( "Received via SUBSCRIBE API in proc2: %s", event.payload ), ) await client.wait_until_any_endpoint_subscribed_to(SecondThingHappened) while True: logging.info("Hello from proc2") await client.broadcast(SecondThingHappened("Hit from proc2 ")) await asyncio.sleep(2)
async def test_beam_syncer(request, event_loop, event_bus, chaindb_fresh, chaindb_churner, beam_to_block, checkpoint=None): client_context = ChainContextFactory(headerdb__db=chaindb_fresh.db) server_context = ChainContextFactory(headerdb__db=chaindb_churner.db) peer_pair = ETHPeerPairFactory( alice_peer_context=client_context, bob_peer_context=server_context, event_bus=event_bus, ) async with peer_pair as (client_peer, server_peer): # Need a name that will be unique per xdist-process, otherwise # lahja IPC endpoints in each process will clobber each other unique_process_name = uuid.uuid4() # manually add endpoint for beam vm to make requests pausing_config = ConnectionConfig.from_name( f"PausingEndpoint-{unique_process_name}") # manually add endpoint for trie data gatherer to serve requests gatherer_config = ConnectionConfig.from_name( f"GathererEndpoint-{unique_process_name}") client_peer_pool = MockPeerPoolWithConnectedPeers([client_peer]) server_peer_pool = MockPeerPoolWithConnectedPeers([server_peer], event_bus=event_bus) async with run_peer_pool_event_server( event_bus, server_peer_pool, handler_type=ETHPeerPoolEventServer), run_request_server( event_bus, AsyncChainDB(chaindb_churner.db) ), AsyncioEndpoint.serve( pausing_config) as pausing_endpoint, AsyncioEndpoint.serve( gatherer_config) as gatherer_endpoint: client_chain = make_pausing_beam_chain( ((0, PetersburgVM), ), chain_id=999, db=chaindb_fresh.db, event_bus=pausing_endpoint, loop=event_loop, ) client = BeamSyncer( client_chain, chaindb_fresh.db, AsyncChainDB(chaindb_fresh.db), client_peer_pool, gatherer_endpoint, force_beam_block_number=beam_to_block, checkpoint=checkpoint, ) client_peer.logger.info("%s is serving churner blocks", client_peer) server_peer.logger.info("%s is syncing up churner blocks", server_peer) import_server = BlockImportServer( pausing_endpoint, client_chain, token=client.cancel_token, ) asyncio.ensure_future(import_server.run()) await pausing_endpoint.connect_to_endpoints(gatherer_config) asyncio.ensure_future(client.run()) # We can sync at least 10 blocks in 1s at current speeds, (or reach the current one) # Trying to keep the tests short-ish. A fuller test could always set the target header # to the chaindb_churner canonical head, and increase the timeout significantly target_block_number = min(beam_to_block + 10, 129) target_head = chaindb_churner.get_canonical_block_header_by_number( target_block_number) await wait_for_head(chaindb_fresh, target_head, sync_timeout=10) assert target_head.state_root in chaindb_fresh.db # first stop the import server, so it doesn't hang waiting for state data await import_server.cancel() await client.cancel()
async def test_connect_to_endpoint(endpoint): config = ConnectionConfig.from_name(generate_unique_name()) async with AsyncioEndpoint.serve(config): await endpoint.connect_to_endpoints(config) assert endpoint.is_connected_to(config.name)
def test_pickle_fails(): alice = AsyncioEndpoint("pickle-test") with pytest.raises(Exception): pickle.dumps(alice, protocol=pickle.HIGHEST_PROTOCOL)
async def test_beam_syncer(request, event_loop, event_bus, chaindb_fresh, chaindb_churner, beam_to_block): client_peer, server_peer = await get_directly_linked_peers( request, event_loop, alice_headerdb=FakeAsyncHeaderDB(chaindb_fresh.db), bob_headerdb=FakeAsyncHeaderDB(chaindb_churner.db)) # manually add endpoint for beam vm to make requests pausing_config = ConnectionConfig.from_name("PausingEndpoint") # manually add endpoint for trie data gatherer to serve requests gatherer_config = ConnectionConfig.from_name("GathererEndpoint") client_peer_pool = MockPeerPoolWithConnectedPeers([client_peer]) server_peer_pool = MockPeerPoolWithConnectedPeers([server_peer], event_bus=event_bus) async with run_peer_pool_event_server( event_bus, server_peer_pool, handler_type=ETHPeerPoolEventServer), run_request_server( event_bus, FakeAsyncChainDB(chaindb_churner.db)), AsyncioEndpoint.serve( pausing_config) as pausing_endpoint, AsyncioEndpoint.serve( gatherer_config) as gatherer_endpoint: BeamPetersburgVM = pausing_vm_decorator(PetersburgVM, pausing_endpoint) class BeamPetersburgTestChain(FakeAsyncChain): vm_configuration = ((0, BeamPetersburgVM), ) network_id = 999 client_chain = BeamPetersburgTestChain(chaindb_fresh.db) client = BeamSyncer( client_chain, chaindb_fresh.db, client_chain.chaindb, client_peer_pool, gatherer_endpoint, beam_to_block, ) client_peer.logger.info("%s is serving churner blocks", client_peer) server_peer.logger.info("%s is syncing up churner blocks", server_peer) import_server = BlockImportServer(pausing_endpoint, client_chain, token=client.cancel_token) asyncio.ensure_future(import_server.run()) await pausing_endpoint.connect_to_endpoints(gatherer_config) asyncio.ensure_future(client.run()) # We can sync at least 10 blocks in 1s at current speeds, (or reach the current one) # Trying to keep the tests short-ish. A fuller test could always set the target header # to the chaindb_churner canonical head, and increase the timeout significantly target_block_number = min(beam_to_block + 10, 129) target_head = chaindb_churner.get_canonical_block_header_by_number( target_block_number) await wait_for_head(chaindb_fresh, target_head, sync_timeout=4) assert target_head.state_root in chaindb_fresh.db # first stop the import server, so it doesn't hang waiting for state data await import_server.cancel() await client.cancel()
def test_pickle_fails(): alice = AsyncioEndpoint("pickle-test") with pytest.raises(Exception): pickle.dumps(alice)
async def _beam_syncing( request, event_loop, event_bus, chaindb_fresh, chaindb_churner, beam_to_block, checkpoint=None, VM_at_0=PetersburgVM, enable_state_backfill=False, ): client_context = ChainContextFactory(headerdb__db=chaindb_fresh.db) server_context = ChainContextFactory(headerdb__db=chaindb_churner.db) peer_pair = LatestETHPeerPairFactory( alice_peer_context=client_context, bob_peer_context=server_context, event_bus=event_bus, ) backfiller = LatestETHPeerPairFactory( alice_peer_context=client_context, bob_peer_context=server_context, event_bus=event_bus, ) async with peer_pair as (client_peer, server_peer), backfiller as (client2_peer, backfill_peer): # Need a name that will be unique per xdist-process, otherwise # lahja IPC endpoints in each process will clobber each other unique_process_name = uuid.uuid4() # manually add endpoint for beam vm to make requests pausing_config = ConnectionConfig.from_name( f"PausingEndpoint-{unique_process_name}") # manually add endpoint for trie data gatherer to serve requests gatherer_config = ConnectionConfig.from_name( f"GathererEndpoint-{unique_process_name}") client_peer_pool = MockPeerPoolWithConnectedPeers( [client_peer, backfill_peer], event_bus=event_bus, ) server_peer_pool = MockPeerPoolWithConnectedPeers([server_peer], event_bus=event_bus) backfill_peer_pool = MockPeerPoolWithConnectedPeers( [client2_peer], event_bus=event_bus) async with run_peer_pool_event_server( event_bus, server_peer_pool, handler_type=ETHPeerPoolEventServer ), run_peer_pool_event_server( event_bus, backfill_peer_pool, handler_type=ETHPeerPoolEventServer ), background_asyncio_service( ETHRequestServer(event_bus, TO_NETWORKING_BROADCAST_CONFIG, AsyncChainDB(chaindb_churner.db)) ), AsyncioEndpoint.serve( pausing_config) as pausing_endpoint, AsyncioEndpoint.serve( gatherer_config) as gatherer_endpoint: client_chain = make_pausing_beam_chain( ((0, VM_at_0), ), chain_id=999, consensus_context_class=ConsensusContext, db=chaindb_fresh.db, event_bus=pausing_endpoint, metrics_registry=NoopMetricsRegistry(), loop=event_loop, ) client = BeamSyncer( client_chain, chaindb_fresh.db, AsyncChainDB(chaindb_fresh.db), client_peer_pool, gatherer_endpoint, NoopMetricsRegistry(), force_beam_block_number=beam_to_block, checkpoint=checkpoint, enable_state_backfill=enable_state_backfill, enable_backfill=False, ) client_peer.logger.info("%s is serving churner blocks", client_peer) backfill_peer.logger.info("%s is serving backfill state", backfill_peer) server_peer.logger.info("%s is syncing up churner blocks", server_peer) import_server = BlockImportServer( pausing_endpoint, client_chain, ) async with background_asyncio_service(import_server): await pausing_endpoint.connect_to_endpoints(gatherer_config) async with background_asyncio_service(client): yield client
async def test_connect_to_endpoint(): config = ConnectionConfig.from_name(generate_unique_name()) async with AsyncioEndpoint.serve(config): async with AsyncioEndpoint("client").run() as client: await client.connect_to_endpoint(config) assert client.is_connected_to(config.name)