def kill_trinity_gracefully(logger: logging.Logger, processes: Iterable[multiprocessing.Process], plugin_manager: PluginManager, main_endpoint: Endpoint, event_bus: EventBus, reason: str = None) -> None: # When a user hits Ctrl+C in the terminal, the SIGINT is sent to all processes in the # foreground *process group*, so both our networking and database processes will terminate # at the same time and not sequentially as we'd like. That shouldn't be a problem but if # we keep getting unhandled BrokenPipeErrors/ConnectionResetErrors like reported in # https://github.com/ethereum/py-evm/issues/827, we might want to change the networking # process' signal handler to wait until the DB process has terminated before doing its # thing. # Notice that we still need the kill_process_gracefully() calls here, for when the user # simply uses 'kill' to send a signal to the main process, but also because they will # perform a non-gracefull shutdown if the process takes too long to terminate. hint = f"({reason})" if reason else f"" logger.info('Shutting down Trinity %s', hint) plugin_manager.shutdown_blocking() main_endpoint.stop() event_bus.stop() for process in processes: # Our sub-processes will have received a SIGINT already (see comment above), so here we # wait 2s for them to finish cleanly, and if they fail we kill them for real. process.join(2) if process.is_alive(): kill_process_gracefully(process, logger) logger.info('%s process (pid=%d) terminated', process.name, process.pid) ArgumentParser().exit(message=f"Trinity shutdown complete {hint}\n")
async def handle_networking_exit(service: BaseService, plugin_manager: PluginManager, endpoint: Endpoint) -> None: async with exit_signal_with_service(service): await plugin_manager.shutdown() endpoint.stop()
def kill_trinity_gracefully(logger: logging.Logger, database_server_process: Any, networking_process: Any, plugin_manager: PluginManager, main_endpoint: Endpoint, event_bus: EventBus, message: str="Trinity shudown complete\n") -> None: # When a user hits Ctrl+C in the terminal, the SIGINT is sent to all processes in the # foreground *process group*, so both our networking and database processes will terminate # at the same time and not sequentially as we'd like. That shouldn't be a problem but if # we keep getting unhandled BrokenPipeErrors/ConnectionResetErrors like reported in # https://github.com/ethereum/py-evm/issues/827, we might want to change the networking # process' signal handler to wait until the DB process has terminated before doing its # thing. # Notice that we still need the kill_process_gracefully() calls here, for when the user # simply uses 'kill' to send a signal to the main process, but also because they will # perform a non-gracefull shutdown if the process takes too long to terminate. logger.info('Keyboard Interrupt: Stopping') plugin_manager.shutdown_blocking() main_endpoint.stop() event_bus.stop() for name, process in [("DB", database_server_process), ("Networking", networking_process)]: # Our sub-processes will have received a SIGINT already (see comment above), so here we # wait 2s for them to finish cleanly, and if they fail we kill them for real. process.join(2) if process.is_alive(): kill_process_gracefully(process, logger) logger.info('%s process (pid=%d) terminated', name, process.pid) # This is required to be within the `kill_trinity_gracefully` so that # plugins can trigger a shutdown of the trinity process. ArgumentParser().exit(message=message)
def launch_node(args: Namespace, trinity_config: TrinityConfig, endpoint: Endpoint) -> None: with trinity_config.process_id_file('networking'): endpoint.connect() NodeClass = trinity_config.node_class # Temporary hack: We setup a second instance of the PluginManager. # The first instance was only to configure the ArgumentParser whereas # for now, the second instance that lives inside the networking process # performs the bulk of the work. In the future, the PluginManager # should probably live in its own process and manage whether plugins # run in the shared plugin process or spawn their own. plugin_manager = setup_plugins(SharedProcessScope(endpoint)) plugin_manager.prepare(args, trinity_config) plugin_manager.broadcast(TrinityStartupEvent( args, trinity_config )) node = NodeClass(plugin_manager, trinity_config) loop = node.get_event_loop() asyncio.ensure_future(handle_networking_exit(node, plugin_manager, endpoint), loop=loop) asyncio.ensure_future(node.run(), loop=loop) loop.run_forever() loop.close()
async def handle_networking_exit(service: BaseService, plugin_manager: PluginManager, endpoint: Endpoint) -> None: async with exit_signal_with_service(service): await plugin_manager.shutdown() endpoint.stop() # Retrieve and shutdown the global executor that was created at startup ensure_global_asyncio_executor().shutdown(wait=True)
def trinity_boot(args: Namespace, trinity_config: TrinityConfig, extra_kwargs: Dict[str, Any], plugin_manager: PluginManager, listener: logging.handlers.QueueListener, event_bus: EventBus, main_endpoint: Endpoint, logger: logging.Logger) -> None: # start the listener thread to handle logs produced by other processes in # the local logger. listener.start() event_bus.start() # First initialize the database process. database_server_process = ctx.Process( name="DB", target=run_database_process, args=( trinity_config, LevelDB, ), kwargs=extra_kwargs, ) # start the processes database_server_process.start() logger.info("Started DB server process (pid=%d)", database_server_process.pid) # networking process needs the IPC socket file provided by the database process try: wait_for_ipc(trinity_config.database_ipc_path) except TimeoutError as e: logger.error("Timeout waiting for database to start. Exiting...") kill_process_gracefully(database_server_process, logger) ArgumentParser().error(message="Timed out waiting for database start") def kill_trinity_with_reason(reason: str) -> None: kill_trinity_gracefully(logger, (database_server_process, ), plugin_manager, main_endpoint, event_bus, reason=reason) main_endpoint.subscribe(ShutdownRequest, lambda ev: kill_trinity_with_reason(ev.reason)) plugin_manager.prepare(args, trinity_config, extra_kwargs) kill_trinity_with_reason("No beacon support yet. SOON!") try: loop = asyncio.get_event_loop() loop.add_signal_handler(signal.SIGTERM, lambda: kill_trinity_with_reason("SIGTERM")) loop.run_forever() loop.close() except KeyboardInterrupt: kill_trinity_with_reason("CTRL+C / Keyboard Interrupt")
async def run_proc1(): endpoint = Endpoint() await endpoint.start_serving(ConnectionConfig.from_name('e1')) await endpoint.connect_to_endpoints(ConnectionConfig.from_name('e2'), ) print("subscribing") # Listen for `GetSomethingRequest`'s endpoint.subscribe( GetSomethingRequest, lambda event: # Send a response back to *only* who made that request endpoint.broadcast_nowait(DeliverSomethingResponse("Yay"), event.broadcast_config()))
async def proc2_worker(): endpoint = Endpoint() await endpoint.start_serving(ConnectionConfig.from_name('e2')) await endpoint.connect_to_endpoints(ConnectionConfig.from_name('e1')) asyncio.ensure_future(display_proc1_events(endpoint)) endpoint.subscribe( FirstThingHappened, lambda event: print( "Received via SUBSCRIBE API in proc2:", event.payload)) while True: print("Hello from proc2") if is_nth_second(2): await endpoint.broadcast( SecondThingHappened("Hit from proc2 ({})".format(time.time()))) await asyncio.sleep(1)
def launch_node(args: Namespace, trinity_config: TrinityConfig, endpoint: Endpoint) -> None: with trinity_config.process_id_file('networking'): NodeClass = trinity_config.node_class node = NodeClass(endpoint, trinity_config) loop = node.get_event_loop() endpoint.connect_no_wait(loop) # This is a second PluginManager instance governing plugins in a shared process. plugin_manager = setup_plugins(SharedProcessScope(endpoint)) plugin_manager.prepare(args, trinity_config) asyncio.ensure_future(handle_networking_exit(node, plugin_manager, endpoint), loop=loop) asyncio.ensure_future(node.run(), loop=loop) loop.run_forever() loop.close()
async def exit_on_signal(service_to_exit: BaseService, endpoint: Endpoint = None) -> None: loop = service_to_exit.get_event_loop() sigint_received = asyncio.Event() for sig in [signal.SIGINT, signal.SIGTERM]: # TODO also support Windows loop.add_signal_handler(sig, sigint_received.set) await sigint_received.wait() try: await service_to_exit.cancel() if endpoint is not None: endpoint.stop() service_to_exit._executor.shutdown(wait=True) finally: loop.stop()
async def proc2_worker(): endpoint = Endpoint() await endpoint.start_serving(ConnectionConfig.from_name('e2')) await endpoint.connect_to_endpoints(ConnectionConfig.from_name('e1'), ) for i in range(3): print("Requesting") result = await endpoint.request(GetSomethingRequest()) print(f"Got answer: {result.payload}")
def launch_node(args: Namespace, chain_config: ChainConfig, endpoint: Endpoint) -> None: with chain_config.process_id_file('networking'): endpoint.connect() NodeClass = chain_config.node_class # Temporary hack: We setup a second instance of the PluginManager. # The first instance was only to configure the ArgumentParser whereas # for now, the second instance that lives inside the networking process # performs the bulk of the work. In the future, the PluginManager # should probably live in its own process and manage whether plugins # run in the shared plugin process or spawn their own. plugin_manager = setup_plugins(SharedProcessScope(endpoint)) plugin_manager.prepare(args, chain_config) plugin_manager.broadcast(TrinityStartupEvent(args, chain_config)) node = NodeClass(plugin_manager, chain_config) run_service_until_quit(node)
def trinity_boot(args: Namespace, trinity_config: TrinityConfig, extra_kwargs: Dict[str, Any], plugin_manager: PluginManager, listener: logging.handlers.QueueListener, event_bus: EventBus, main_endpoint: Endpoint, logger: logging.Logger) -> None: # start the listener thread to handle logs produced by other processes in # the local logger. listener.start() event_bus.start() def kill_trinity_with_reason(reason: str) -> None: kill_trinity_gracefully( logger, (), plugin_manager, main_endpoint, event_bus, reason=reason ) main_endpoint.subscribe( ShutdownRequest, lambda ev: kill_trinity_with_reason(ev.reason) ) plugin_manager.prepare(args, trinity_config, extra_kwargs) try: loop = asyncio.get_event_loop() loop.add_signal_handler(signal.SIGTERM, lambda: kill_trinity_with_reason("SIGTERM")) loop.run_forever() loop.close() except KeyboardInterrupt: kill_trinity_with_reason("CTRL+C / Keyboard Interrupt")
def launch_node(args: Namespace, trinity_config: TrinityConfig, endpoint: Endpoint) -> None: with trinity_config.process_id_file('networking'): NodeClass = trinity_config.get_app_config(Eth1AppConfig).node_class node = NodeClass(endpoint, trinity_config) # The `networking` process creates a process pool executor to offload cpu intensive # tasks. We should revisit that when we move the sync in its own process ensure_global_asyncio_executor() loop = node.get_event_loop() endpoint.connect_no_wait(loop) # This is a second PluginManager instance governing plugins in a shared process. plugin_manager = setup_plugins(SharedProcessScope(endpoint), get_all_plugins()) plugin_manager.prepare(args, trinity_config) asyncio.ensure_future(handle_networking_exit(node, plugin_manager, endpoint), loop=loop) asyncio.ensure_future(node.run(), loop=loop) loop.run_forever() loop.close()
def trinity_boot(args: Namespace, trinity_config: TrinityConfig, extra_kwargs: Dict[str, Any], plugin_manager: PluginManager, listener: logging.handlers.QueueListener, event_bus: EventBus, main_endpoint: Endpoint, logger: logging.Logger) -> None: # start the listener thread to handle logs produced by other processes in # the local logger. listener.start() networking_endpoint = event_bus.create_endpoint(NETWORKING_EVENTBUS_ENDPOINT) event_bus.start() # First initialize the database process. database_server_process = ctx.Process( target=run_database_process, args=( trinity_config, LevelDB, ), kwargs=extra_kwargs, ) networking_process = ctx.Process( target=launch_node, args=(args, trinity_config, networking_endpoint,), kwargs=extra_kwargs, ) # start the processes database_server_process.start() logger.info("Started DB server process (pid=%d)", database_server_process.pid) # networking process needs the IPC socket file provided by the database process try: wait_for_ipc(trinity_config.database_ipc_path) except TimeoutError as e: logger.error("Timeout waiting for database to start. Exiting...") kill_process_gracefully(database_server_process, logger) ArgumentParser().error(message="Timed out waiting for database start") networking_process.start() logger.info("Started networking process (pid=%d)", networking_process.pid) main_endpoint.subscribe( ShutdownRequest, lambda ev: kill_trinity_gracefully( logger, database_server_process, networking_process, plugin_manager, main_endpoint, event_bus ) ) plugin_manager.prepare(args, trinity_config, extra_kwargs) plugin_manager.broadcast(TrinityStartupEvent( args, trinity_config )) try: loop = asyncio.get_event_loop() loop.run_forever() loop.close() except KeyboardInterrupt: kill_trinity_gracefully( logger, database_server_process, networking_process, plugin_manager, main_endpoint, event_bus )
def request_shutdown(event_bus: Endpoint, reason: str) -> None: event_bus.broadcast( ShutdownRequest(reason), BroadcastConfig(filter_endpoint=MAIN_EVENTBUS_ENDPOINT))
async def exit_with_service_and_endpoint(service_to_exit: BaseService, endpoint: Endpoint) -> None: async with exit_signal_with_service(service_to_exit): endpoint.stop()
def helios_boot(args: Namespace, chain_config: ChainConfig, extra_kwargs: Dict[str, Any], plugin_manager: PluginManager, listener: logging.handlers.QueueListener, event_bus: EventBus, main_endpoint: Endpoint, logger: logging.Logger) -> None: # start the listener thread to handle logs produced by other processes in # the local logger. listener.start() logger.info( "Checking for any already running Helios Protocol processes that need shutting down. Remember that you can only run 1 instance at a time." ) fix_unclean_shutdown(chain_config, logger) with chain_config.process_id_file('main'): networking_endpoint = event_bus.create_endpoint( NETWORKING_EVENTBUS_ENDPOINT) event_bus.start() # First initialize the database process. database_server_process = ctx.Process( target=run_database_process, args=( chain_config, LevelDB, ), kwargs=extra_kwargs, ) chain_processes = [] for i in range(chain_config.num_chain_processes): chain_process = ctx.Process( target=run_chain_process, args=(chain_config, i), kwargs=extra_kwargs, ) chain_processes.append(chain_process) networking_process = ctx.Process( target=launch_node, args=( args, chain_config, networking_endpoint, ), kwargs=extra_kwargs, ) # start the processes database_server_process.start() logger.info("Started DB server process (pid=%d)", database_server_process.pid) # networking process needs the IPC socket file provided by the database process try: wait_for_ipc(chain_config.database_ipc_path) except TimeoutError as e: logger.error("Timeout waiting for database to start. Exiting...") kill_process_gracefully(database_server_process, logger) ArgumentParser().error( message="Timed out waiting for database start") for i in range(chain_config.num_chain_processes): chain_process = chain_processes[i] chain_process.start() logger.info("Started chain instance {} process (pid={})".format( i, chain_process.pid)) try: wait_for_ipc(chain_config.get_chain_ipc_path(i)) except TimeoutError as e: logger.error( "Timeout waiting for chain instance {} to start. Exiting..." .format(i)) kill_process_gracefully(chain_process, logger) for j in range(i + 1): kill_process_gracefully(chain_processes[j], logger) ArgumentParser().error( message="Timed out waiting for chain instance {} start". format(i)) networking_process.start() logger.info("Started networking process (pid=%d)", networking_process.pid) main_endpoint.subscribe( ShutdownRequest, lambda ev: kill_helios_gracefully( logger, database_server_process, chain_processes, networking_process, plugin_manager, main_endpoint, event_bus)) plugin_manager.prepare(args, chain_config, extra_kwargs) plugin_manager.broadcast(HeliosStartupEvent(args, chain_config)) try: loop = asyncio.get_event_loop() loop.run_forever() loop.close() except KeyboardInterrupt: kill_helios_gracefully(logger, database_server_process, chain_processes, networking_process, plugin_manager, main_endpoint, event_bus)