async def test_asyncio_isolated_component(boot_info, log_listener): # Test the lifecycle management for isolated process components to be sure # they start and stop as expected component_manager = ComponentManager(boot_info, (AsyncioComponentForTest, ), lambda reason: None) async with background_asyncio_service(component_manager): event_bus = await component_manager.get_event_bus() got_started = asyncio.Future() event_bus.subscribe(IsStarted, lambda ev: got_started.set_result(ev.path)) touch_path = await asyncio.wait_for(got_started, timeout=10) assert not touch_path.exists() component_manager.shutdown('exiting component manager') for _ in range(10000): if not touch_path.exists(): await asyncio.sleep(0.001) else: break else: assert touch_path.exists()
async def test_isolated_component(boot_info, log_listener, component, request, monkeypatch): # On overloaded CI machines it can sometimes take a while for a component's process to start, # so we need a high timeout here. component_timeout = 10 monkeypatch.setenv('ASYNCIO_RUN_IN_PROCESS_STARTUP_TIMEOUT', str(component_timeout)) # Test the lifecycle management for isolated process components to be sure # they start and stop as expected component_manager = ComponentManager(boot_info, (component, )) async with background_asyncio_service(component_manager) as cm_manager: event_bus = await component_manager.get_event_bus() got_started = asyncio.Future() event_bus.subscribe(IsStarted, lambda ev: got_started.set_result(ev.path)) touch_path = await asyncio.wait_for(got_started, timeout=component_timeout) def delete_touch_path(): if touch_path.exists(): touch_path.unlink() request.addfinalizer(delete_touch_path) assert not touch_path.exists() component_manager.shutdown('exiting component manager') await cm_manager.wait_finished() assert touch_path.exists()
async def test_isolated_component(boot_info, log_listener, component, request): # Test the lifecycle management for isolated process components to be sure # they start and stop as expected component_manager = ComponentManager(boot_info, (component, )) async with background_asyncio_service(component_manager) as cm_manager: event_bus = await component_manager.get_event_bus() got_started = asyncio.Future() event_bus.subscribe(IsStarted, lambda ev: got_started.set_result(ev.path)) touch_path = await asyncio.wait_for(got_started, timeout=10) def delete_touch_path(): if touch_path.exists(): touch_path.unlink() request.addfinalizer(delete_touch_path) assert not touch_path.exists() component_manager.shutdown('exiting component manager') await cm_manager.wait_finished() assert touch_path.exists()
class ComponentManagerService(BaseService): _endpoint: EndpointAPI def __init__(self, trinity_boot_info: TrinityBootInfo, components: Sequence[Type[BaseComponent]], kill_trinity_fn: Callable[[str], Any], cancel_token: CancelToken = None, loop: asyncio.AbstractEventLoop = None) -> None: self._boot_info = trinity_boot_info self._components = components self._kill_trinity_fn = kill_trinity_fn super().__init__(cancel_token, loop) async def _run(self) -> None: self._connection_config = ConnectionConfig.from_name( MAIN_EVENTBUS_ENDPOINT, self._boot_info.trinity_config.ipc_dir) async with AsyncioEndpoint.serve(self._connection_config) as endpoint: self._endpoint = endpoint # start the background process that tracks and propagates available # endpoints to the other connected endpoints self.run_daemon_task( self._track_and_propagate_available_endpoints()) self.run_daemon_task(self._handle_shutdown_request()) # start the component manager self.component_manager = ComponentManager(endpoint, self._components) self.component_manager.prepare(self._boot_info) await self.cancellation() async def _handle_shutdown_request(self) -> None: req = await self.wait(self._endpoint.wait_for(ShutdownRequest)) self._kill_trinity_fn(req.reason) self.cancel_nowait() async def _cleanup(self) -> None: self.component_manager.shutdown_blocking() _available_endpoints: Tuple[ConnectionConfig, ...] = () async def _track_and_propagate_available_endpoints(self) -> None: """ Track new announced endpoints and propagate them across all other existing endpoints. """ async for ev in self.wait_iter( self._endpoint.stream(EventBusConnected)): self._available_endpoints = self._available_endpoints + ( ev.connection_config, ) self.logger.debug("New EventBus Endpoint connected %s", ev.connection_config.name) # Broadcast available endpoints to all connected endpoints, giving them # a chance to cross connect await self._endpoint.broadcast( AvailableEndpointsUpdated(self._available_endpoints)) self.logger.debug("Connected EventBus Endpoints %s", self._available_endpoints)
async def _run(self) -> None: self._connection_config = ConnectionConfig.from_name( MAIN_EVENTBUS_ENDPOINT, self._boot_info.trinity_config.ipc_dir) async with AsyncioEndpoint.serve(self._connection_config) as endpoint: self._endpoint = endpoint # start the background process that tracks and propagates available # endpoints to the other connected endpoints self.run_daemon_task( self._track_and_propagate_available_endpoints()) self.run_daemon_task(self._handle_shutdown_request()) # start the component manager self.component_manager = ComponentManager(endpoint, self._components) self.component_manager.prepare(self._boot_info) await self.cancellation()
def run(component_types: Tuple[Type[BaseComponentAPI], ...], boot_info: BootInfo, get_base_db_fn: Callable[[BootInfo], LevelDB]) -> None: runtime_component_types = tuple( cast(Type[BaseIsolatedComponent], component_cls) for component_cls in component_types if issubclass(component_cls, ComponentAPI)) trinity_config = boot_info.trinity_config component_manager_service = ComponentManager( boot_info, runtime_component_types, ) component_manager_manager = AsyncioManager(component_manager_service) loop = asyncio.get_event_loop() loop.add_signal_handler( signal.SIGTERM, component_manager_manager.cancel, 'SIGTERM', ) loop.add_signal_handler( signal.SIGINT, component_manager_service.shutdown, 'CTRL+C', ) logger = logging.getLogger() try: loop.run_until_complete( _run(boot_info, get_base_db_fn, component_manager_manager)) except BaseException: logger.exception("Error during trinity run") raise finally: reason = component_manager_service.reason hint = f" ({reason})" if reason else f"" logger.info('Shutting down Trinity%s', hint) remove_dangling_ipc_files(logger, trinity_config.ipc_dir) argparse.ArgumentParser().exit( message=f"Trinity shutdown complete{hint}\n") if trinity_config.trinity_tmp_root_dir: shutil.rmtree(trinity_config.trinity_root_dir)
async def test_isolated_component_crash(boot_info, log_listener, component): component_manager = ComponentManager(boot_info, (component, )) async with background_asyncio_service(component_manager): event_bus = await component_manager.get_event_bus() component_started = asyncio.Event() event_bus.subscribe(IsStarted, lambda ev: component_started.set()) await asyncio.wait_for(component_started.wait(), timeout=10) try: await asyncio.wait_for( component_manager._trigger_component_exit.wait(), timeout=1) except asyncio.TimeoutError: # XXX: For some reason, when this test fails this AssertionError gets somewhat # obfuscated in the RemoteTraceback raised by asyncio-run-in-process, but the # traceback itself point to this line as the cause of the failure. raise AssertionError( "ComponentManager did not get ShutdownRequest") # Sleep a bit to give the component a chance to terminate, otherwise when shutting down # the ComponentManager we end up double-killing it, which leads to asyncio warnings. await asyncio.sleep(0.5)
def _run_asyncio_components(component_types: Tuple[Type[BaseComponentAPI], ...], boot_info: BootInfo, processes: Tuple[multiprocessing.Process, ...]) -> None: runtime_component_types = tuple( component_cls for component_cls in component_types if issubclass(component_cls, ComponentAPI) ) trinity_config = boot_info.trinity_config component_manager_service = ComponentManager( boot_info, runtime_component_types, ) manager = AsyncioManager(component_manager_service) loop = asyncio.get_event_loop() loop.add_signal_handler( signal.SIGTERM, manager.cancel, 'SIGTERM', ) loop.add_signal_handler( signal.SIGINT, component_manager_service.shutdown, 'CTRL+C', ) try: loop.run_until_complete(manager.run()) except BaseException as err: logger = logging.getLogger() logger.error("Error during trinity run: %r", err) raise finally: kill_trinity_with_reason(trinity_config, processes, component_manager_service.reason) if trinity_config.trinity_tmp_root_dir: shutil.rmtree(trinity_config.trinity_root_dir)
async def test_isolated_component_crash(boot_info, log_listener, component, monkeypatch): # On overloaded CI machines it can sometimes take a while for a component's process to start, # so we need a high timeout here. component_timeout = 10 monkeypatch.setenv('ASYNCIO_RUN_IN_PROCESS_STARTUP_TIMEOUT', str(component_timeout)) component_manager = ComponentManager(boot_info, (component, )) with pytest.raises(ComponentException): async with background_asyncio_service(component_manager): event_bus = await component_manager.get_event_bus() component_started = asyncio.Event() event_bus.subscribe(IsStarted, lambda ev: component_started.set()) await asyncio.wait_for(component_started.wait(), timeout=component_timeout) try: await asyncio.wait_for( component_manager.manager.wait_finished(), timeout=1) except asyncio.TimeoutError: # XXX: For some reason, when this test fails this AssertionError gets somewhat # obfuscated in the RemoteTraceback raised by asyncio-run-in-process, but the # traceback itself point to this line as the cause of the failure. raise AssertionError("ComponentManager did not stop")
def main_entry(trinity_boot: BootFn, app_identifier: str, component_types: Tuple[Type[BaseComponentAPI], ...], sub_configs: Sequence[Type[BaseAppConfig]]) -> None: if is_prerelease(): # this modifies the asyncio logger, but will be overridden by any custom settings below enable_warnings_by_default() for component_cls in component_types: component_cls.configure_parser(parser, subparser) argcomplete.autocomplete(parser) args = parser.parse_args() if not args.genesis and args.network_id not in PRECONFIGURED_NETWORKS: parser.error( f"Unsupported network id: {args.network_id}. To use a network besides " "mainnet or ropsten, you must supply a genesis file with a flag, like " "`--genesis path/to/genesis.json`, also you must specify a data " "directory with `--data-dir path/to/data/directory`" ) # The `common_log_level` is derived from `--log-level <Level>` / `-l <Level>` without # specifying any module. If present, it is used for both `stderr` and `file` logging. common_log_level = args.log_levels and args.log_levels.get(None) has_ambigous_logging_config = (( common_log_level is not None and args.stderr_log_level is not None ) or ( common_log_level is not None and args.file_log_level is not None )) if has_ambigous_logging_config: parser.error( f"""\n Ambiguous logging configuration: The `--log-level (-l)` flag sets the log level for both file and stderr logging. To configure different log level for file and stderr logging, remove the `--log-level` flag and use `--stderr-log-level` and/or `--file-log-level` separately. Alternatively, remove the `--stderr-log-level` and/or `--file-log-level` flags to share one single log level across both handlers. """ ) try: trinity_config = TrinityConfig.from_parser_args(args, app_identifier, sub_configs) except AmbigiousFileSystem: parser.error(TRINITY_AMBIGIOUS_FILESYSTEM_INFO) if not is_data_dir_initialized(trinity_config): # TODO: this will only work as is for chains with known genesis # parameters. Need to flesh out how genesis parameters for custom # chains are defined and passed around. try: initialize_data_dir(trinity_config) except AmbigiousFileSystem: parser.error(TRINITY_AMBIGIOUS_FILESYSTEM_INFO) except MissingPath as e: parser.error( "\n" f"It appears that {e.path} does not exist. " "Trinity does not attempt to create directories outside of its root path. " "Either manually create the path or ensure you are using a data directory " "inside the XDG_TRINITY_ROOT path" ) # +---------------+ # | LOGGING SETUP | # +---------------+ # Setup logging to stderr stderr_logger_level = ( args.stderr_log_level if args.stderr_log_level is not None else (common_log_level if common_log_level is not None else logging.INFO) ) handler_stderr = setup_stderr_logging(stderr_logger_level) # Setup file based logging file_logger_level = ( args.file_log_level if args.file_log_level is not None else (common_log_level if common_log_level is not None else logging.DEBUG) ) handler_file = setup_file_logging(trinity_config.logfile_path, file_logger_level) # Set the individual logger levels that have been specified. logger_levels = {} if args.log_levels is None else args.log_levels set_logger_levels(logger_levels) # get the root logger and set it to the level of the stderr logger. logger = logging.getLogger() logger.setLevel(stderr_logger_level) # This prints out the ASCII "trinity" header in the terminal display_launch_logs(trinity_config) # Setup the log listener which child processes relay their logs through log_listener = IPCListener(handler_stderr, handler_file) # Determine what logging level child processes should use. child_process_log_level = min( stderr_logger_level, file_logger_level, *logger_levels.values(), ) boot_info = BootInfo( args=args, trinity_config=trinity_config, child_process_log_level=child_process_log_level, logger_levels=logger_levels, profile=bool(args.profile), ) # Let the components do runtime validation for component_cls in component_types: component_cls.validate_cli(boot_info) # Components can provide a subcommand with a `func` which does then control # the entire process from here. if hasattr(args, 'func'): args.func(args, trinity_config) return if hasattr(args, 'munge_func'): args.munge_func(args, trinity_config) runtime_component_types = tuple( component_cls for component_cls in component_types if issubclass(component_cls, ComponentAPI) ) with log_listener.run(trinity_config.logging_ipc_path): processes = trinity_boot(boot_info) loop = asyncio.get_event_loop() def kill_trinity_with_reason(reason: str) -> None: kill_trinity_gracefully( trinity_config, logger, processes, reason=reason ) component_manager_service = ComponentManager( boot_info, runtime_component_types, kill_trinity_with_reason, ) manager = AsyncioManager(component_manager_service) loop.add_signal_handler( signal.SIGTERM, manager.cancel, 'SIGTERM', ) loop.add_signal_handler( signal.SIGINT, component_manager_service.shutdown, 'CTRL+C', ) try: loop.run_until_complete(manager.run()) except BaseException as err: logger.error("Error during trinity run: %r", err) raise finally: kill_trinity_with_reason(component_manager_service.reason) if trinity_config.trinity_tmp_root_dir: shutil.rmtree(trinity_config.trinity_root_dir)