async def launch_node_coro(args: Namespace, trinity_config: TrinityConfig) -> None: endpoint = TrinityEventBusEndpoint() NodeClass = trinity_config.get_app_config(Eth1AppConfig).node_class node = NodeClass(endpoint, trinity_config) networking_connection_config = ConnectionConfig.from_name( NETWORKING_EVENTBUS_ENDPOINT, trinity_config.ipc_dir) await endpoint.start_serving(networking_connection_config) endpoint.auto_connect_new_announced_endpoints() await endpoint.connect_to_endpoints( ConnectionConfig.from_name(MAIN_EVENTBUS_ENDPOINT, trinity_config.ipc_dir), # Plugins that run within the networking process broadcast and receive on the # the same endpoint networking_connection_config, ) await endpoint.announce_endpoint() # This is a second PluginManager instance governing plugins in a shared process. plugin_manager = PluginManager(SharedProcessScope(endpoint), get_all_plugins()) plugin_manager.prepare(args, trinity_config) asyncio.ensure_future( handle_networking_exit(node, plugin_manager, endpoint)) asyncio.ensure_future(node.run())
async def _run_eventbus_for_component( component: Union['TrioIsolatedComponent', 'AsyncioIsolatedComponent'], connect_to_endpoints: Tuple[str, ...], ) -> AsyncIterator[Union[TrioEndpoint, AsyncioEndpoint]]: from trinity.extensibility.trio import TrioIsolatedComponent from trinity.extensibility.asyncio import AsyncioIsolatedComponent if isinstance(component, TrioIsolatedComponent): endpoint_type: Union[Type[TrioEndpoint], Type[AsyncioEndpoint]] = TrioEndpoint elif isinstance(component, AsyncioIsolatedComponent): endpoint_type = AsyncioEndpoint else: raise ValueError("Unknown component type: %s", type(component)) trinity_config = component._boot_info.trinity_config conn_config = ConnectionConfig.from_name(component.get_endpoint_name(), trinity_config.ipc_dir) async with endpoint_type.serve(conn_config) as event_bus: for endpoint in connect_to_endpoints: path = pathlib.Path(endpoint) if not path.is_socket(): raise ValueError("Invalid IPC path: {path}") connection_config = ConnectionConfig(name=path.stem, path=path) logger.info("Attempting to connect to eventbus endpoint at %s", connection_config) await event_bus.connect_to_endpoints(connection_config) yield event_bus
async def proc2_worker(): endpoint = Endpoint() await endpoint.start_serving(ConnectionConfig.from_name('e2')) await endpoint.connect_to_endpoints(ConnectionConfig.from_name('e1'), ) for i in range(3): print("Requesting") result = await endpoint.request(GetSomethingRequest()) print(f"Got answer: {result.payload}")
async def pair_of_endpoints(event_loop, ipc_base_path): config_1 = ConnectionConfig.from_name(generate_unique_name(), base_path=ipc_base_path) config_2 = ConnectionConfig.from_name(generate_unique_name(), base_path=ipc_base_path) async with AsyncioEndpoint.serve(config_1) as endpoint1: async with AsyncioEndpoint.serve(config_2) as endpoint2: await endpoint1.connect_to_endpoints(config_2) await endpoint2.connect_to_endpoints(config_1) yield endpoint1, endpoint2
async def run_proc1(): endpoint = Endpoint() await endpoint.start_serving(ConnectionConfig.from_name('e1')) await endpoint.connect_to_endpoints(ConnectionConfig.from_name('e2'), ) print("subscribing") # Listen for `GetSomethingRequest`'s endpoint.subscribe( GetSomethingRequest, lambda event: # Send a response back to *only* who made that request endpoint.broadcast_nowait(DeliverSomethingResponse("Yay"), event.broadcast_config()))
async def client_with_three_connections(ipc_base_path): config_a = ConnectionConfig.from_name("server-a", base_path=ipc_base_path) config_b = ConnectionConfig.from_name("server-b", base_path=ipc_base_path) config_c = ConnectionConfig.from_name("server-c", base_path=ipc_base_path) async with AsyncioEndpoint.serve(config_a) as server_a: async with AsyncioEndpoint.serve(config_b) as server_b: async with AsyncioEndpoint.serve(config_c) as server_c: async with AsyncioEndpoint("client").run() as client: await client.connect_to_endpoints(config_a) await client.connect_to_endpoints(config_b) await client.connect_to_endpoints(config_c) yield client, server_a, server_b, server_c
async def proc2_worker(): endpoint = Endpoint() await endpoint.start_serving(ConnectionConfig.from_name('e2')) await endpoint.connect_to_endpoints(ConnectionConfig.from_name('e1')) asyncio.ensure_future(display_proc1_events(endpoint)) endpoint.subscribe( FirstThingHappened, lambda event: print( "Received via SUBSCRIBE API in proc2:", event.payload)) while True: print("Hello from proc2") if is_nth_second(2): await endpoint.broadcast( SecondThingHappened("Hit from proc2 ({})".format(time.time()))) await asyncio.sleep(1)
async def worker(cls, name: str, config: ConsumerConfig) -> None: conn_config = ConnectionConfig.from_name(name) async with config.backend.Endpoint.serve(conn_config) as event_bus: await event_bus.connect_to_endpoints( ConnectionConfig.from_name(REPORTER_ENDPOINT)) await event_bus.wait_until_connected_to(DRIVER_ENDPOINT) stats = await cls.do_consumer(event_bus, config) await event_bus.wait_until_endpoint_subscribed_to( REPORTER_ENDPOINT, TotalRecordedEvent) await event_bus.broadcast( TotalRecordedEvent(stats.crunch(event_bus.name)), BroadcastConfig(filter_endpoint=REPORTER_ENDPOINT), )
async def _run(self) -> None: async with AsyncioEndpoint.serve(self._connection_config) as endpoint: self._endpoint = endpoint # signal that the endpoint is now available self._endpoint_available.set() # run background task that automatically connects to newly announced endpoints self.run_daemon_task(self._auto_connect_new_announced_endpoints()) # connect to the *main* endpoint which communicates information # about other endpoints that come online. main_endpoint_config = ConnectionConfig.from_name( MAIN_EVENTBUS_ENDPOINT, self._trinity_config.ipc_dir, ) await endpoint.connect_to_endpoints(main_endpoint_config) # announce ourself to the event bus await endpoint.wait_until_endpoint_subscribed_to( MAIN_EVENTBUS_ENDPOINT, EventBusConnected, ) await endpoint.broadcast( EventBusConnected(self._connection_config), BroadcastConfig(filter_endpoint=MAIN_EVENTBUS_ENDPOINT)) # run until the endpoint exits await self.cancellation()
def create_consumer_endpoint_configs( num_processes: int ) -> Tuple[ConnectionConfig, ...]: return tuple( ConnectionConfig.from_name(create_consumer_endpoint_name(i)) for i in range(num_processes) )
async def make_networking_event_bus(): # Tests run concurrently, therefore we need unique IPC paths ipc_path = Path(f"networking-{uuid.uuid4()}.ipc") networking_connection_config = ConnectionConfig( name=NETWORKING_EVENTBUS_ENDPOINT, path=ipc_path) async with AsyncioEndpoint.serve(networking_connection_config) as endpoint: yield endpoint
async def run_proc1(): config = ConnectionConfig.from_name("e1") async with AsyncioEndpoint.serve(config) as server: async for event in server.stream(GetSomethingRequest, num_events=3): await server.broadcast( DeliverSomethingResponse("Yay"), event.broadcast_config() )
async def run(args: argparse.Namespace, backend: BaseBackend): consumer_endpoint_configs = create_consumer_endpoint_configs(args.num_processes) ( config.path.unlink() for config in consumer_endpoint_configs + tuple( ConnectionConfig.from_name(name) for name in (ROOT_ENDPOINT, REPORTER_ENDPOINT, DRIVER_ENDPOINT) ) ) root_config = ConnectionConfig.from_name(ROOT_ENDPOINT) async with backend.Endpoint.serve(root_config) as root: # The reporter process is collecting statistical events from all consumer processes # For some reason, doing this work in the main process didn't end so well which is # why it was moved into a dedicated process. Notice that this will slightly skew results # as the reporter process will also receive events which we don't account for reporting_config = ReportingProcessConfig( num_events=args.num_events, num_processes=args.num_processes, throttle=args.throttle, payload_bytes=args.payload_bytes, backend=backend, ) reporter = ReportingProcess(reporting_config) reporter.start() for n in range(args.num_processes): consumer_process = ConsumerProcess( create_consumer_endpoint_name(n), args.num_events, backend=backend ) consumer_process.start() # In this benchmark, this is the only process that is flooding events driver_config = DriverProcessConfig( connected_endpoints=consumer_endpoint_configs, num_events=args.num_events, throttle=args.throttle, payload_bytes=args.payload_bytes, backend=backend, ) driver = DriverProcess(driver_config) driver.start() await root.wait_for(ShutdownEvent) driver.stop()
async def test_wait_until_connected_to(endpoint): config = ConnectionConfig.from_name(generate_unique_name()) async with AsyncioEndpoint.serve(config): asyncio.ensure_future(endpoint.connect_to_endpoints(config)) assert not endpoint.is_connected_to(config.name) await endpoint.wait_until_connected_to(config.name) assert endpoint.is_connected_to(config.name)
async def test_connecting_to_other_trio_endpoint(ipc_base_path): config = ConnectionConfig.from_name(generate_unique_name(), base_path=ipc_base_path) async with TrioEndpoint.serve(config): async with TrioEndpoint("client").run() as client: await client.connect_to_endpoints(config) assert client.is_connected_to(config.name)
async def _prepare_start(self) -> None: connection_config = ConnectionConfig.from_name( self.normalized_name, self.boot_info.trinity_config.ipc_dir) await self.event_bus.start_serving(connection_config) await self.event_bus.connect_to_endpoints( ConnectionConfig.from_name(MAIN_EVENTBUS_ENDPOINT, self.boot_info.trinity_config.ipc_dir)) # This makes the `main` process aware of this Endpoint which will then propagate the info # so that every other Endpoint can connect directly to the plugin Endpoint await self.event_bus.announce_endpoint() await self.event_bus.broadcast(PluginStartedEvent(type(self))) # Whenever new EventBus Endpoints come up the `main` process broadcasts this event # and we connect to every Endpoint directly self.event_bus.auto_connect_new_announced_endpoints() self.do_start()
async def test_endpoint_serve(ipc_base_path): config = ConnectionConfig.from_name(generate_unique_name(), base_path=ipc_base_path) async with AsyncioEndpoint.serve(config) as endpoint: assert endpoint.is_running is True assert endpoint.is_serving is True assert endpoint.is_running is False assert endpoint.is_serving is False
async def test_rejects_duplicates_when_connecting(): own = ConnectionConfig.from_name(generate_unique_name()) async with AsyncioEndpoint.serve(own) as endpoint: await endpoint.connect_to_endpoint(own) assert endpoint.is_connected_to(own.name) with pytest.raises(ConnectionAttemptRejected): await endpoint.connect_to_endpoint(own)
async def monitoring(normalized_name: str, trinity_config: TrinityConfig) -> None: event_bus = TrinityEventBusEndpoint("monitoring_ui") connection_config = ConnectionConfig.from_name( normalized_name, trinity_config.ipc_dir, ) await event_bus.start() await event_bus.start_server(connection_config.path) await event_bus.connect_to_endpoints( ConnectionConfig.from_name(MAIN_EVENTBUS_ENDPOINT, trinity_config.ipc_dir)) await event_bus.announce_endpoint() await event_bus.broadcast(PluginStartedEvent(type(MonitoringPlugin))) asyncio.ensure_future(event_bus.auto_connect_new_announced_endpoints()) event_bus.subscribe(NewBlockHashesEvent, lambda event: logging.info(event.msg))
def __init__( self, trinity_config: TrinityConfig, endpoint_name: str, ) -> None: self._trinity_config = trinity_config self._endpoint_available = self.EventType() self._connection_config = ConnectionConfig.from_name( endpoint_name, self._trinity_config.ipc_dir)
async def endpoint(event_loop, ipc_base_path): config = ConnectionConfig.from_name(generate_unique_name(), base_path=ipc_base_path) async with AsyncioEndpoint.serve(config) as endpoint: # We need to connect to our own Endpoint if we care about receiving # the events we broadcast. Many tests use the same Endpoint for # broadcasting and receiving which is a valid use case so we hook it up await endpoint.connect_to_endpoint(config) yield endpoint
async def announce_endpoint(self) -> None: """ Announce this endpoint to the :class:`~trinity.endpoint.TrinityMainEventBusEndpoint` so that it will be further propagated to all other endpoints, allowing them to connect to us. """ await self.broadcast( EventBusConnected( ConnectionConfig(name=self.name, path=self.ipc_path)), BroadcastConfig(filter_endpoint=MAIN_EVENTBUS_ENDPOINT))
async def main() -> None: parser = argparse.ArgumentParser() parser.add_argument('-bootnode', type=str, help="The enode to use as bootnode") parser.add_argument('-networkid', type=int, choices=[ROPSTEN_NETWORK_ID, MAINNET_NETWORK_ID], default=ROPSTEN_NETWORK_ID, help="1 for mainnet, 3 for testnet") parser.add_argument('-l', type=str, help="Log level", default="info") args = parser.parse_args() logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s: %(message)s', datefmt='%H:%M:%S') if args.l == "debug2": # noqa: E741 log_level = DEBUG2_LEVEL_NUM else: log_level = getattr(logging, args.l.upper()) logging.getLogger('p2p').setLevel(log_level) network_cfg = PRECONFIGURED_NETWORKS[args.networkid] # Listen on a port other than 30303 so that we can test against a local geth instance # running on that port. listen_port = 30304 # Use a hard-coded privkey so that our enode is always the same. privkey = keys.PrivateKey( b'~\x054{4\r\xd64\x0f\x98\x1e\x85;\xcc\x08\x1eQ\x10t\x16\xc0\xb0\x7f)=\xc4\x1b\xb7/\x8b&\x83' ) # noqa: E501 addr = kademlia.Address('127.0.0.1', listen_port, listen_port) if args.bootnode: bootstrap_nodes = tuple([kademlia.Node.from_uri(args.bootnode)]) else: bootstrap_nodes = tuple( kademlia.Node.from_uri(enode) for enode in network_cfg.bootnodes) ipc_path = Path(f"networking-{uuid.uuid4()}.ipc") networking_connection_config = ConnectionConfig( name=NETWORKING_EVENTBUS_ENDPOINT, path=ipc_path) headerdb = TrioHeaderDB(AtomicDB(MemoryDB())) headerdb.persist_header(network_cfg.genesis_header) vm_config = network_cfg.vm_configuration enr_field_providers = (functools.partial(generate_eth_cap_enr_field, vm_config, headerdb), ) socket = trio.socket.socket(family=trio.socket.AF_INET, type=trio.socket.SOCK_DGRAM) await socket.bind(('0.0.0.0', listen_port)) async with TrioEndpoint.serve(networking_connection_config) as endpoint: service = DiscoveryService(privkey, addr, bootstrap_nodes, endpoint, socket, enr_field_providers) service.logger.info("Enode: %s", service.this_node.uri()) async with background_trio_service(service): await service.manager.wait_finished()
async def worker(backend: BaseBackend, name: str, num_events: int) -> None: config = ConnectionConfig.from_name(name) async with backend.Endpoint.serve(config) as event_bus: await event_bus.connect_to_endpoints( ConnectionConfig.from_name(REPORTER_ENDPOINT) ) await event_bus.wait_until_all_remotes_subscribed_to(TotalRecordedEvent) stats = LocalStatistic() events = event_bus.stream(PerfMeasureEvent, num_events=num_events) async for event in events: stats.add( RawMeasureEntry(sent_at=event.sent_at, received_at=time.time()) ) await event_bus.broadcast( TotalRecordedEvent(stats.crunch(event_bus.name)), BroadcastConfig(filter_endpoint=REPORTER_ENDPOINT), )
def test_request_without_subscriber_throws(runner, ipc_base_path): server_config = ConnectionConfig.from_name("server", base_path=ipc_base_path) server = d.driver(d.serve_endpoint(server_config)) client = d.driver( d.run_endpoint("client"), d.throws(d.request(Request()), NoSubscribers) ) runner(server, client)
async def proc2_worker(): config = ConnectionConfig.from_name("e1") async with AsyncioEndpoint("e2").run() as client: await client.connect_to_endpoints(config) await client.wait_until_any_endpoint_subscribed_to(GetSomethingRequest) for i in range(3): print("Requesting") result = await client.request(GetSomethingRequest()) print(f"Got answer: {result.payload}")
async def test_base_wait_until_connections_changed(): config = ConnectionConfig.from_name(generate_unique_name()) async with AsyncioEndpoint.serve(config): async with AsyncioEndpoint("client").run() as client: asyncio.ensure_future(client.connect_to_endpoints(config)) assert not client.is_connected_to(config.name) await asyncio.wait_for(client.wait_until_connections_change(), timeout=0.1) assert client.is_connected_to(config.name)
def __init__(self, trinity_config: TrinityConfig, endpoint_name: str, cancel_token: CancelToken = None, loop: asyncio.AbstractEventLoop = None) -> None: self._trinity_config = trinity_config self._endpoint_available = asyncio.Event() self._connection_config = ConnectionConfig.from_name( endpoint_name, self._trinity_config.ipc_dir) super().__init__(cancel_token, loop)
async def run_network(self): async with AsyncioEndpoint.serve( ConnectionConfig.from_name("network")) as self.network_bus: self.logger.info("Network started") self.network_bus.subscribe(GenericMessageOutgoingEvent, self.send_to_all_nodes) self.logger.info("Network subscribed to all messages") while not self.kill_switch.value: await asyncio.sleep(0) self.logger.info("Network turned off")
async def run(self) -> None: connection_config = ConnectionConfig.from_name( MAIN_EVENTBUS_ENDPOINT, self._boot_info.trinity_config.ipc_dir) async with AsyncioEndpoint.serve(connection_config) as endpoint: self._endpoint = endpoint # start the background process that tracks and propagates available # endpoints to the other connected endpoints self.manager.run_daemon_task( self._track_and_propagate_available_endpoints) await endpoint.wait_until_any_endpoint_subscribed_to( EventBusConnected) # signal the endpoint is up and running and available self._endpoint_available.set() # instantiate all of the components all_components = tuple( component_cls(self._boot_info) for component_cls in self._component_types) # filter out any components that should not be enabled. enabled_components = tuple(component for component in all_components if component.is_enabled) # a little bit of extra try/finally structure here to produce good # logging messages about the component lifecycle. from p2p.asyncio_utils import create_task, wait_first try: self.logger.info( "Starting components: %s", '/'.join(component.name for component in enabled_components), ) tasks: List[asyncio.Task[Any]] = [] for component in enabled_components: tasks.append( create_task( component.run_in_process(), f'IsolatedComponent/{component.name}/run_in_process' )) tasks.append( asyncio.create_task(self._trigger_component_exit.wait())) self.logger.info("Components started") try: # The timeout is long as our component tasks can do a lot of stuff during # their cleanup. await wait_first(tasks, max_wait_after_cancellation=10) finally: self.logger.info("Stopping components") finally: self.logger.info("Components stopped.") self.manager.cancel()