def test_request_without_subscriber_throws(runner, ipc_base_path): server_config = ConnectionConfig.from_name("server", base_path=ipc_base_path) server = d.driver(d.serve_endpoint(server_config)) client = d.driver( d.run_endpoint("client"), d.throws(d.request(Request()), NoSubscribers) ) runner(server, client)
async def run_network(self): async with AsyncioEndpoint.serve( ConnectionConfig.from_name("network")) as self.network_bus: self.logger.info("Network started") self.network_bus.subscribe(GenericMessageOutgoingEvent, self.send_to_all_nodes) self.logger.info("Network subscribed to all messages") while not self.kill_switch.value: await asyncio.sleep(0) self.logger.info("Network turned off")
async def worker(backend: BaseBackend, name: str, num_events: int) -> None: config = ConnectionConfig.from_name(name) async with backend.Endpoint.serve(config) as event_bus: await event_bus.connect_to_endpoints( ConnectionConfig.from_name(REPORTER_ENDPOINT) ) await event_bus.wait_until_all_remotes_subscribed_to(TotalRecordedEvent) stats = LocalStatistic() events = event_bus.stream(PerfMeasureEvent, num_events=num_events) async for event in events: stats.add( RawMeasureEntry(sent_at=event.sent_at, received_at=time.time()) ) await event_bus.broadcast( TotalRecordedEvent(stats.crunch(event_bus.name)), BroadcastConfig(filter_endpoint=REPORTER_ENDPOINT), )
def __init__(self, trinity_config: TrinityConfig, endpoint_name: str, cancel_token: CancelToken = None, loop: asyncio.AbstractEventLoop = None) -> None: self._trinity_config = trinity_config self._endpoint_available = asyncio.Event() self._connection_config = ConnectionConfig.from_name( endpoint_name, self._trinity_config.ipc_dir) super().__init__(cancel_token, loop)
async def test_base_wait_until_connections_changed(): config = ConnectionConfig.from_name(generate_unique_name()) async with AsyncioEndpoint.serve(config): async with AsyncioEndpoint("client").run() as client: asyncio.ensure_future(client.connect_to_endpoints(config)) assert not client.is_connected_to(config.name) await asyncio.wait_for(client.wait_until_connections_change(), timeout=0.1) assert client.is_connected_to(config.name)
async def proc2_worker(): config = ConnectionConfig.from_name("e1") async with AsyncioEndpoint("e2").run() as client: await client.connect_to_endpoints(config) await client.wait_until_any_endpoint_subscribed_to(GetSomethingRequest) for i in range(3): print("Requesting") result = await client.request(GetSomethingRequest()) print(f"Got answer: {result.payload}")
async def run(self) -> None: connection_config = ConnectionConfig.from_name( MAIN_EVENTBUS_ENDPOINT, self._boot_info.trinity_config.ipc_dir) async with AsyncioEndpoint.serve(connection_config) as endpoint: self._endpoint = endpoint # start the background process that tracks and propagates available # endpoints to the other connected endpoints self.manager.run_daemon_task( self._track_and_propagate_available_endpoints) await endpoint.wait_until_any_endpoint_subscribed_to( EventBusConnected) # signal the endpoint is up and running and available self._endpoint_available.set() # instantiate all of the components all_components = tuple( component_cls(self._boot_info) for component_cls in self._component_types) # filter out any components that should not be enabled. enabled_components = tuple(component for component in all_components if component.is_enabled) # a little bit of extra try/finally structure here to produce good # logging messages about the component lifecycle. from p2p.asyncio_utils import create_task, wait_first try: self.logger.info( "Starting components: %s", '/'.join(component.name for component in enabled_components), ) tasks: List[asyncio.Task[Any]] = [] for component in enabled_components: tasks.append( create_task( component.run_in_process(), f'IsolatedComponent/{component.name}/run_in_process' )) tasks.append( asyncio.create_task(self._trigger_component_exit.wait())) self.logger.info("Components started") try: # The timeout is long as our component tasks can do a lot of stuff during # their cleanup. await wait_first(tasks, max_wait_after_cancellation=10) finally: self.logger.info("Stopping components") finally: self.logger.info("Components stopped.") self.manager.cancel()
async def run(self) -> None: connection_config = ConnectionConfig.from_name( MAIN_EVENTBUS_ENDPOINT, self._boot_info.trinity_config.ipc_dir ) async with AsyncioEndpoint.serve(connection_config) as endpoint: self._endpoint = endpoint # start the background process that tracks and propagates available # endpoints to the other connected endpoints self.manager.run_daemon_task(self._track_and_propagate_available_endpoints) self.manager.run_daemon_task(self._handle_shutdown_request) await endpoint.wait_until_any_endpoint_subscribed_to(ShutdownRequest) await endpoint.wait_until_any_endpoint_subscribed_to(EventBusConnected) # signal the endpoint is up and running and available self._endpoint_available.set() # instantiate all of the components all_components = tuple( component_cls(self._boot_info) for component_cls in self._component_types ) # filter out any components that should not be enabled. enabled_components = tuple( component for component in all_components if component.is_enabled ) # a little bit of extra try/finally structure here to produce good # logging messages about the component lifecycle. try: async with AsyncExitStack() as stack: self.logger.info( "Starting components: %s", '/'.join(component.name for component in enabled_components), ) # Concurrently start the components. await asyncio.gather(*( stack.enter_async_context(run_component(component)) for component in enabled_components )) self.logger.info("Components started") try: await self._trigger_component_exit.wait() finally: self.logger.info("Stopping components") finally: self.logger.info("Components stopped.") self.manager.cancel()
async def test_endpoint_serve_with_error(ipc_base_path): config = ConnectionConfig.from_name(generate_unique_name(), base_path=ipc_base_path) with pytest.raises(Exception, match="break out of serve"): async with AsyncioEndpoint.serve(config) as endpoint: assert endpoint.is_running is True assert endpoint.is_serving is True raise Exception("break out of serve") assert endpoint.is_running is False assert endpoint.is_serving is False
async def run(self) -> None: connection_config = ConnectionConfig.from_name( MAIN_EVENTBUS_ENDPOINT, self._boot_info.trinity_config.ipc_dir ) async with AsyncioEndpoint.serve(connection_config) as endpoint: self._endpoint = endpoint # start the background process that tracks and propagates available # endpoints to the other connected endpoints self.manager.run_daemon_task(self._track_and_propagate_available_endpoints) await endpoint.wait_until_any_endpoint_subscribed_to(EventBusConnected) # signal the endpoint is up and running and available self._endpoint_available.set() # instantiate all of the components all_components = tuple( component_cls(self._boot_info) for component_cls in self._component_types ) # filter out any components that should not be enabled. enabled_components = tuple( component for component in all_components if component.is_enabled ) # a little bit of extra try/finally structure here to produce good # logging messages about the component lifecycle. from p2p.logic import wait_first try: self.logger.info( "Starting components: %s", '/'.join(component.name for component in enabled_components), ) context_managers = [component.run() for component in enabled_components] async with AsyncContextGroup(context_managers) as futs: # AsyncContextGroup() yields a Sequence[Any], so we cast to a list of Futures # here to ensure mypy can come to our aid if we forget the create_task() when # adding new entries to the list of Futures we want to wait for. futures = cast(List["asyncio.Future[None]"], list(futs)) futures.append(asyncio.create_task(self._trigger_component_exit.wait())) self.logger.info("Components started") try: await wait_first(futures) finally: self.logger.info("Stopping components") finally: self.logger.info("Components stopped.") self.manager.cancel()
async def _prepare_start(self) -> None: connection_config = ConnectionConfig.from_name( self.normalized_name, self.boot_info.trinity_config.ipc_dir, ) await self.event_bus.start_serving(connection_config) await self.event_bus.connect_to_endpoints( ConnectionConfig.from_name(MAIN_EVENTBUS_ENDPOINT, self.boot_info.trinity_config.ipc_dir)) # This makes the `main` process aware of this Endpoint which will then propagate the info # so that every other Endpoint can connect directly to the plugin Endpoint await self.event_bus.announce_endpoint() await self.event_bus.broadcast(PluginStartedEvent(type(self))) # Whenever new EventBus Endpoints come up the `main` process broadcasts this event # and we connect to every Endpoint directly asyncio.ensure_future( self.event_bus.auto_connect_new_announced_endpoints()) self.do_start()
async def test_trio_duplicate_endpoint_connection_is_error(ipc_base_path): config = ConnectionConfig.from_name(generate_unique_name(), base_path=ipc_base_path) async with TrioEndpoint.serve(config): async with TrioEndpoint("client").run() as client: await client.connect_to_endpoints(config) assert client.is_connected_to(config.name) with pytest.raises(ConnectionAttemptRejected): await client.connect_to_endpoints(config)
def _prepare_start(self) -> None: log_queue = self.context.boot_kwargs['log_queue'] level = self.context.boot_kwargs.get('log_level', logging.INFO) setup_queue_logging(log_queue, level) connection_config = ConnectionConfig.from_name( self.normalized_name, self.context.trinity_config.ipc_dir) self.event_bus.start_serving_nowait(connection_config) self.event_bus.connect_to_endpoints_blocking( ConnectionConfig.from_name(MAIN_EVENTBUS_ENDPOINT, self.context.trinity_config.ipc_dir)) # This makes the `main` process aware of this Endpoint which will then propagate the info # so that every other Endpoint can connect directly to the plugin Endpoint self.event_bus.announce_endpoint() self.event_bus.broadcast(PluginStartedEvent(type(self))) # Whenever new EventBus Endpoints come up the `main` process broadcasts this event # and we connect to every Endpoint directly self.event_bus.auto_connect_new_announced_endpoints() with self.context.trinity_config.process_id_file(self.normalized_name): self.do_start()
def test_endpoint_broadcast_from_client_to_server(ipc_base_path, runner): server_config = ConnectionConfig.from_name("server", base_path=ipc_base_path) server = d.driver(d.serve_endpoint(server_config), d.wait_for(Event)) client = d.driver( d.run_endpoint("client"), d.connect_to_endpoints(server_config), d.wait_any_then_broadcast(Event()), ) runner(server, client)
async def worker(config: DriverProcessConfig) -> None: conn_config = ConnectionConfig.from_name(DRIVER_ENDPOINT) async with config.backend.Endpoint.serve(conn_config) as event_bus: await event_bus.connect_to_endpoints(*config.connected_endpoints) await event_bus.wait_until_all_remotes_subscribed_to(PerfMeasureEvent) counter = itertools.count() payload = b"\x00" * config.payload_bytes while True: await config.backend.sleep(config.throttle) await event_bus.broadcast( PerfMeasureEvent(payload, next(counter), time.time()) )
async def test_trio_server_endpoint_establishes_reverse_connection_to_client( ipc_base_path): unique_name = generate_unique_name() config = ConnectionConfig.from_name(f"server-{unique_name}", base_path=ipc_base_path) async with TrioEndpoint.serve(config) as server: async with TrioEndpoint(f"client-{unique_name}").run() as client: await client.connect_to_endpoints(config) assert client.is_connected_to(config.name) with trio.fail_after(2): await server.wait_until_connected_to(client.name)
async def launch_node_coro(args: Namespace, trinity_config: TrinityConfig) -> None: networking_connection_config = ConnectionConfig.from_name( NETWORKING_EVENTBUS_ENDPOINT, trinity_config.ipc_dir) async with TrinityEventBusEndpoint.serve( networking_connection_config) as endpoint: NodeClass = trinity_config.get_app_config(Eth1AppConfig).node_class node = NodeClass(endpoint, trinity_config) asyncio.ensure_future(endpoint.auto_connect_new_announced_endpoints()) await endpoint.connect_to_endpoints( ConnectionConfig.from_name(MAIN_EVENTBUS_ENDPOINT, trinity_config.ipc_dir), ) await endpoint.announce_endpoint() # This is a second PluginManager instance governing plugins in a shared process. plugin_manager = PluginManager(SharedProcessScope(endpoint), get_plugins_for_eth1_client()) plugin_manager.prepare(args, trinity_config) asyncio.ensure_future( handle_networking_exit(node, plugin_manager, endpoint)) await node.run()
async def proc1_worker(): async with AsyncioEndpoint.serve(ConnectionConfig.from_name("e1")) as server: server.subscribe( SecondThingHappened, lambda event: logging.info( "Received via SUBSCRIBE API in proc1: %s", event.payload ), ) await server.wait_until_any_endpoint_subscribed_to(FirstThingHappened) while True: logging.info("Hello from proc1") await server.broadcast(FirstThingHappened("Hit from proc1")) await asyncio.sleep(2)
async def worker(logger: logging.Logger, config: ReportingProcessConfig) -> None: conn_config = ConnectionConfig.from_name(REPORTER_ENDPOINT) async with config.backend.Endpoint.serve(conn_config) as event_bus: await event_bus.connect_to_endpoints( ConnectionConfig.from_name(ROOT_ENDPOINT)) global_statistic = GlobalStatistic() events = event_bus.stream(TotalRecordedEvent, num_events=config.num_processes) async for event in events: global_statistic.add(event.total) print_full_report( logger, config.backend, config.num_processes, config.num_events, global_statistic, ) await event_bus.broadcast( ShutdownEvent(), BroadcastConfig(filter_endpoint=ROOT_ENDPOINT))
async def test_base_wait_until_any_endpoint_subscriptions_changed(): config = ConnectionConfig.from_name(generate_unique_name()) async with AsyncioEndpoint.serve(config) as server: async with AsyncioEndpoint("client").run() as client: await client.connect_to_endpoints(config) assert client.is_connected_to(config.name) server.subscribe(SubscriptionEvent, lambda e: None) assert not client.is_any_endpoint_subscribed_to(SubscriptionEvent) await asyncio.wait_for( client.wait_until_endpoint_subscriptions_change(), timeout=0.1 ) assert client.is_any_endpoint_subscribed_to(SubscriptionEvent)
def launch_node(args: Namespace, trinity_config: TrinityConfig) -> None: with trinity_config.process_id_file('networking'): endpoint = TrinityEventBusEndpoint() NodeClass = trinity_config.get_app_config(Eth1AppConfig).node_class node = NodeClass(endpoint, trinity_config) # The `networking` process creates a process pool executor to offload cpu intensive # tasks. We should revisit that when we move the sync in its own process ensure_global_asyncio_executor() loop = node.get_event_loop() networking_connection_config = ConnectionConfig.from_name( NETWORKING_EVENTBUS_ENDPOINT, trinity_config.ipc_dir) endpoint.start_serving_nowait( networking_connection_config, loop, ) endpoint.auto_connect_new_announced_endpoints() endpoint.connect_to_endpoints_blocking( ConnectionConfig.from_name(MAIN_EVENTBUS_ENDPOINT, trinity_config.ipc_dir), # Plugins that run within the networking process broadcast and receive on the # the same endpoint networking_connection_config, ) endpoint.announce_endpoint() # This is a second PluginManager instance governing plugins in a shared process. plugin_manager = setup_plugins(SharedProcessScope(endpoint), get_all_plugins()) plugin_manager.prepare(args, trinity_config) asyncio.ensure_future(handle_networking_exit(node, plugin_manager, endpoint), loop=loop) asyncio.ensure_future(node.run(), loop=loop) loop.run_forever() loop.close()
def test_broadcast_without_listeners_explicitly_allowed(ipc_base_path, runner): server_config = ConnectionConfig.from_name("server", base_path=ipc_base_path) server_done, client_done = d.checkpoint("done") server = d.driver(d.serve_endpoint(server_config), server_done) client = d.driver( d.run_endpoint("client"), d.connect_to_endpoints(server_config), d.wait_until_connected_to("server"), d.broadcast(Event(), BroadcastConfig(require_subscriber=False)), client_done, ) runner(server, client)
async def _run(self) -> None: self._connection_config = ConnectionConfig.from_name( MAIN_EVENTBUS_ENDPOINT, self._boot_info.trinity_config.ipc_dir) async with AsyncioEndpoint.serve(self._connection_config) as endpoint: self._endpoint = endpoint # start the background process that tracks and propagates available # endpoints to the other connected endpoints self.run_daemon_task( self._track_and_propagate_available_endpoints()) # start the plugin manager plugin_manager = PluginManager(endpoint, self._plugins) plugin_manager.prepare(self._boot_info) await self.cancellation()
def test_server_establishes_reverse_connection(ipc_base_path, runner): server_config = ConnectionConfig.from_name("server", base_path=ipc_base_path) server_done, client_done = d.checkpoint("done") server = d.driver( d.serve_endpoint(server_config), d.wait_until_connected_to("client"), server_done, ) client = d.driver(d.run_endpoint("client"), d.connect_to_endpoints(server_config), client_done) runner(server, client)
def test_broadcast_without_listeners_throws(ipc_base_path, runner): server_config = ConnectionConfig.from_name("server", base_path=ipc_base_path) server_done, client_done = d.checkpoint("done") server = d.driver(d.serve_endpoint(server_config), server_done) client = d.driver( d.run_endpoint("client"), d.connect_to_endpoints(server_config), d.wait_until_connected_to("server"), d.throws(d.broadcast(Event()), NoSubscribers), client_done, ) runner(server, client)
def __init__(self, position: Position, addr=None): """ :param position: :param addr: requested address for the node """ super().__init__() self.start_time = 0 self.network = None self.position = position self.addr = self._get_free_address(addr) self.bus_config = ConnectionConfig.from_name("network") self.kill_switch: Value = Value("i", 0) self.nodes_history = {}
async def proc2_worker(): config = ConnectionConfig.from_name("e1") async with AsyncioEndpoint("e2").run() as client: await client.connect_to_endpoints(config) asyncio.ensure_future(display_proc1_events(client)) client.subscribe( FirstThingHappened, lambda event: logging.info( "Received via SUBSCRIBE API in proc2: %s", event.payload ), ) await client.wait_until_any_endpoint_subscribed_to(SecondThingHappened) while True: logging.info("Hello from proc2") await client.broadcast(SecondThingHappened("Hit from proc2 ")) await asyncio.sleep(2)
def test_duplicate_connection_throws_exception(ipc_base_path, runner): server_config = ConnectionConfig.from_name("server", base_path=ipc_base_path) server_done, client_done = d.checkpoint("done") server = d.driver(d.serve_endpoint(server_config), server_done) client = d.driver( d.run_endpoint("client"), d.connect_to_endpoints(server_config), d.wait_until_connected_to("server"), d.throws(d.connect_to_endpoints(server_config), ConnectionAttemptRejected), client_done, ) runner(server, client)
async def trinity_boot_coro(kill_trinity, main_endpoint, trinity_config, # type: ignore plugin_manager, args, extra_kwargs) -> None: # We postpone EventBus connection until here because we don't want one in cases where # a plugin just redefines the `trinity` command such as `trinity fix-unclean-shutdown` main_connection_config = ConnectionConfig.from_name( MAIN_EVENTBUS_ENDPOINT, trinity_config.ipc_dir ) await main_endpoint.start() await main_endpoint.start_server(main_connection_config.path) main_endpoint.track_and_propagate_available_endpoints() main_endpoint.subscribe( ShutdownRequest, lambda ev: kill_trinity(ev.reason) ) plugin_manager.prepare(args, trinity_config, extra_kwargs)
def test_request_response(runner, ipc_base_path): server_config = ConnectionConfig.from_name("server", base_path=ipc_base_path) received = multiprocessing.Event() server = d.driver( d.serve_endpoint(server_config), d.serve_request(Request, get_response=lambda endpoint, event: Response()), ) client = d.driver( d.run_endpoint("client"), d.connect_to_endpoints(server_config), d.wait_until_any_endpoint_subscribed_to(Request), d.request(Request(), on_response=lambda endpoint, event: received.set()), ) runner(server, client) assert received.is_set()