def kill_trinity_gracefully(logger: logging.Logger,
database_server_process: Any,
networking_process: Any,
plugin_manager: PluginManager,
main_endpoint: Endpoint,
event_bus: EventBus,
message: str="Trinity shudown complete\n") -> None:
# When a user hits Ctrl+C in the terminal, the SIGINT is sent to all processes in the
# foreground *process group*, so both our networking and database processes will terminate
# at the same time and not sequentially as we'd like. That shouldn't be a problem but if
# we keep getting unhandled BrokenPipeErrors/ConnectionResetErrors like reported in
# https://github.com/ethereum/py-evm/issues/827, we might want to change the networking
# process' signal handler to wait until the DB process has terminated before doing its
# thing.
# Notice that we still need the kill_process_gracefully() calls here, for when the user
# simply uses 'kill' to send a signal to the main process, but also because they will
# perform a non-gracefull shutdown if the process takes too long to terminate.
logger.info('Keyboard Interrupt: Stopping')
plugin_manager.shutdown_blocking()
main_endpoint.stop()
event_bus.stop()
for name, process in [("DB", database_server_process), ("Networking", networking_process)]:
# Our sub-processes will have received a SIGINT already (see comment above), so here we
# wait 2s for them to finish cleanly, and if they fail we kill them for real.
process.join(2)
if process.is_alive():
kill_process_gracefully(process, logger)
logger.info('%s process (pid=%d) terminated', name, process.pid)
# This is required to be within the `kill_trinity_gracefully` so that
# plugins can trigger a shutdown of the trinity process.
ArgumentParser().exit(message=message)
def kill_trinity_gracefully(
logger: logging.Logger,
database_server_process: Any,
networking_process: Any,
plugin_manager: PluginManager,
event_bus: EventBus,
message: str = "Trinity shudown complete\n") -> None:
# When a user hits Ctrl+C in the terminal, the SIGINT is sent to all processes in the
# foreground *process group*, so both our networking and database processes will terminate
# at the same time and not sequentially as we'd like. That shouldn't be a problem but if
# we keep getting unhandled BrokenPipeErrors/ConnectionResetErrors like reported in
# https://github.com/ethereum/py-evm/issues/827, we might want to change the networking
# process' signal handler to wait until the DB process has terminated before doing its
# thing.
# Notice that we still need the kill_process_gracefully() calls here, for when the user
# simply uses 'kill' to send a signal to the main process, but also because they will
# perform a non-gracefull shutdown if the process takes too long to terminate.
logger.info('Keyboard Interrupt: Stopping')
plugin_manager.shutdown()
event_bus.shutdown()
kill_process_gracefully(database_server_process, logger)
logger.info('DB server process (pid=%d) terminated',
database_server_process.pid)
# XXX: This short sleep here seems to avoid us hitting a deadlock when attempting to
# join() the networking subprocess: https://github.com/ethereum/py-evm/issues/940
time.sleep(0.2)
kill_process_gracefully(networking_process, logger)
logger.info('Networking process (pid=%d) terminated',
networking_process.pid)
# This is required to be within the `kill_trinity_gracefully` so that
# plugins can trigger a shutdown of the trinity process.
ArgumentParser().exit(message=message)
def kill_trinity_gracefully(logger: logging.Logger,
processes: Iterable[multiprocessing.Process],
plugin_manager: PluginManager,
main_endpoint: Endpoint,
event_bus: EventBus,
reason: str = None) -> None:
# When a user hits Ctrl+C in the terminal, the SIGINT is sent to all processes in the
# foreground *process group*, so both our networking and database processes will terminate
# at the same time and not sequentially as we'd like. That shouldn't be a problem but if
# we keep getting unhandled BrokenPipeErrors/ConnectionResetErrors like reported in
# https://github.com/ethereum/py-evm/issues/827, we might want to change the networking
# process' signal handler to wait until the DB process has terminated before doing its
# thing.
# Notice that we still need the kill_process_gracefully() calls here, for when the user
# simply uses 'kill' to send a signal to the main process, but also because they will
# perform a non-gracefull shutdown if the process takes too long to terminate.
hint = f"({reason})" if reason else f""
logger.info('Shutting down Trinity %s', hint)
plugin_manager.shutdown_blocking()
main_endpoint.stop()
event_bus.stop()
for process in processes:
# Our sub-processes will have received a SIGINT already (see comment above), so here we
# wait 2s for them to finish cleanly, and if they fail we kill them for real.
process.join(2)
if process.is_alive():
kill_process_gracefully(process, logger)
logger.info('%s process (pid=%d) terminated', process.name,
process.pid)
ArgumentParser().exit(message=f"Trinity shutdown complete {hint}\n")
async def launch_node_coro(args: Namespace,
trinity_config: TrinityConfig) -> None:
endpoint = TrinityEventBusEndpoint()
NodeClass = trinity_config.get_app_config(Eth1AppConfig).node_class
node = NodeClass(endpoint, trinity_config)
networking_connection_config = ConnectionConfig.from_name(
NETWORKING_EVENTBUS_ENDPOINT, trinity_config.ipc_dir)
await endpoint.start_serving(networking_connection_config)
endpoint.auto_connect_new_announced_endpoints()
await endpoint.connect_to_endpoints(
ConnectionConfig.from_name(MAIN_EVENTBUS_ENDPOINT,
trinity_config.ipc_dir),
# Plugins that run within the networking process broadcast and receive on the
# the same endpoint
networking_connection_config,
)
await endpoint.announce_endpoint()
# This is a second PluginManager instance governing plugins in a shared process.
plugin_manager = PluginManager(SharedProcessScope(endpoint),
get_all_plugins())
plugin_manager.prepare(args, trinity_config)
asyncio.ensure_future(
handle_networking_exit(node, plugin_manager, endpoint))
asyncio.ensure_future(node.run())
class PluginManagerService(BaseService):
_endpoint: EndpointAPI
def __init__(self,
trinity_boot_info: TrinityBootInfo,
plugins: Sequence[Type[BasePlugin]],
kill_trinity_fn: Callable[[str], Any],
cancel_token: CancelToken = None,
loop: asyncio.AbstractEventLoop = None) -> None:
self._boot_info = trinity_boot_info
self._plugins = plugins
self._kill_trinity_fn = kill_trinity_fn
super().__init__(cancel_token, loop)
async def _run(self) -> None:
self._connection_config = ConnectionConfig.from_name(
MAIN_EVENTBUS_ENDPOINT, self._boot_info.trinity_config.ipc_dir)
async with AsyncioEndpoint.serve(self._connection_config) as endpoint:
self._endpoint = endpoint
# start the background process that tracks and propagates available
# endpoints to the other connected endpoints
self.run_daemon_task(
self._track_and_propagate_available_endpoints())
self.run_daemon_task(self._handle_shutdown_request())
# start the plugin manager
self.plugin_manager = PluginManager(endpoint, self._plugins)
self.plugin_manager.prepare(self._boot_info)
await self.cancellation()
async def _handle_shutdown_request(self) -> None:
req = await self.wait(self._endpoint.wait_for(ShutdownRequest))
self._kill_trinity_fn(req.reason)
self.cancel_nowait()
async def _cleanup(self) -> None:
self.plugin_manager.shutdown_blocking()
_available_endpoints: Tuple[ConnectionConfig, ...] = ()
async def _track_and_propagate_available_endpoints(self) -> None:
"""
Track new announced endpoints and propagate them across all other existing endpoints.
"""
async for ev in self.wait_iter(
self._endpoint.stream(EventBusConnected)):
self._available_endpoints = self._available_endpoints + (
ev.connection_config, )
self.logger.debug("New EventBus Endpoint connected %s",
ev.connection_config.name)
# Broadcast available endpoints to all connected endpoints, giving them
# a chance to cross connect
await self._endpoint.broadcast(
AvailableEndpointsUpdated(self._available_endpoints))
self.logger.debug("Connected EventBus Endpoints %s",
self._available_endpoints)
def trinity_boot(args: Namespace, trinity_config: TrinityConfig,
extra_kwargs: Dict[str, Any], plugin_manager: PluginManager,
listener: logging.handlers.QueueListener, event_bus: EventBus,
main_endpoint: Endpoint, logger: logging.Logger) -> None:
# start the listener thread to handle logs produced by other processes in
# the local logger.
listener.start()
event_bus.start()
# First initialize the database process.
database_server_process = ctx.Process(
name="DB",
target=run_database_process,
args=(
trinity_config,
LevelDB,
),
kwargs=extra_kwargs,
)
# start the processes
database_server_process.start()
logger.info("Started DB server process (pid=%d)",
database_server_process.pid)
# networking process needs the IPC socket file provided by the database process
try:
wait_for_ipc(trinity_config.database_ipc_path)
except TimeoutError as e:
logger.error("Timeout waiting for database to start. Exiting...")
kill_process_gracefully(database_server_process, logger)
ArgumentParser().error(message="Timed out waiting for database start")
def kill_trinity_with_reason(reason: str) -> None:
kill_trinity_gracefully(logger, (database_server_process, ),
plugin_manager,
main_endpoint,
event_bus,
reason=reason)
main_endpoint.subscribe(ShutdownRequest,
lambda ev: kill_trinity_with_reason(ev.reason))
plugin_manager.prepare(args, trinity_config, extra_kwargs)
kill_trinity_with_reason("No beacon support yet. SOON!")
try:
loop = asyncio.get_event_loop()
loop.add_signal_handler(signal.SIGTERM,
lambda: kill_trinity_with_reason("SIGTERM"))
loop.run_forever()
loop.close()
except KeyboardInterrupt:
kill_trinity_with_reason("CTRL+C / Keyboard Interrupt")
async def _run(self) -> None:
self._connection_config = ConnectionConfig.from_name(
MAIN_EVENTBUS_ENDPOINT, self._boot_info.trinity_config.ipc_dir)
async with AsyncioEndpoint.serve(self._connection_config) as endpoint:
self._endpoint = endpoint
# start the background process that tracks and propagates available
# endpoints to the other connected endpoints
self.run_daemon_task(
self._track_and_propagate_available_endpoints())
# start the plugin manager
plugin_manager = PluginManager(endpoint, self._plugins)
plugin_manager.prepare(self._boot_info)
await self.cancellation()
def trinity_boot(args: Namespace,
trinity_config: TrinityConfig,
extra_kwargs: Dict[str, Any],
plugin_manager: PluginManager,
listener: logging.handlers.QueueListener,
event_bus: EventBus,
main_endpoint: Endpoint,
logger: logging.Logger) -> None:
# start the listener thread to handle logs produced by other processes in
# the local logger.
listener.start()
event_bus.start()
def kill_trinity_with_reason(reason: str) -> None:
kill_trinity_gracefully(
logger,
(),
plugin_manager,
main_endpoint,
event_bus,
reason=reason
)
main_endpoint.subscribe(
ShutdownRequest,
lambda ev: kill_trinity_with_reason(ev.reason)
)
plugin_manager.prepare(args, trinity_config, extra_kwargs)
try:
loop = asyncio.get_event_loop()
loop.add_signal_handler(signal.SIGTERM, lambda: kill_trinity_with_reason("SIGTERM"))
loop.run_forever()
loop.close()
except KeyboardInterrupt:
kill_trinity_with_reason("CTRL+C / Keyboard Interrupt")
async def launch_node_coro(args: Namespace,
trinity_config: TrinityConfig) -> None:
networking_connection_config = ConnectionConfig.from_name(
NETWORKING_EVENTBUS_ENDPOINT, trinity_config.ipc_dir)
async with TrinityEventBusEndpoint.serve(
networking_connection_config) as endpoint:
NodeClass = trinity_config.get_app_config(Eth1AppConfig).node_class
node = NodeClass(endpoint, trinity_config)
asyncio.ensure_future(endpoint.auto_connect_new_announced_endpoints())
await endpoint.connect_to_endpoints(
ConnectionConfig.from_name(MAIN_EVENTBUS_ENDPOINT,
trinity_config.ipc_dir), )
await endpoint.announce_endpoint()
# This is a second PluginManager instance governing plugins in a shared process.
plugin_manager = PluginManager(SharedProcessScope(endpoint),
get_plugins_for_eth1_client())
plugin_manager.prepare(args, trinity_config)
asyncio.ensure_future(
handle_networking_exit(node, plugin_manager, endpoint))
await node.run()
def setup_plugins(scope: BaseManagerProcessScope) -> PluginManager:
plugin_manager = PluginManager(scope)
# TODO: Implement auto-discovery of plugins based on some convention/configuration scheme
plugin_manager.register(ENABLED_PLUGINS)
return plugin_manager
def trinity_boot(args: Namespace,
trinity_config: TrinityConfig,
extra_kwargs: Dict[str, Any],
plugin_manager: PluginManager,
listener: logging.handlers.QueueListener,
event_bus: EventBus,
main_endpoint: Endpoint,
logger: logging.Logger) -> None:
# start the listener thread to handle logs produced by other processes in
# the local logger.
listener.start()
networking_endpoint = event_bus.create_endpoint(NETWORKING_EVENTBUS_ENDPOINT)
event_bus.start()
# First initialize the database process.
database_server_process = ctx.Process(
target=run_database_process,
args=(
trinity_config,
LevelDB,
),
kwargs=extra_kwargs,
)
networking_process = ctx.Process(
target=launch_node,
args=(args, trinity_config, networking_endpoint,),
kwargs=extra_kwargs,
)
# start the processes
database_server_process.start()
logger.info("Started DB server process (pid=%d)", database_server_process.pid)
# networking process needs the IPC socket file provided by the database process
try:
wait_for_ipc(trinity_config.database_ipc_path)
except TimeoutError as e:
logger.error("Timeout waiting for database to start. Exiting...")
kill_process_gracefully(database_server_process, logger)
ArgumentParser().error(message="Timed out waiting for database start")
networking_process.start()
logger.info("Started networking process (pid=%d)", networking_process.pid)
main_endpoint.subscribe(
ShutdownRequest,
lambda ev: kill_trinity_gracefully(
logger,
database_server_process,
networking_process,
plugin_manager,
main_endpoint,
event_bus
)
)
plugin_manager.prepare(args, trinity_config, extra_kwargs)
plugin_manager.broadcast(TrinityStartupEvent(
args,
trinity_config
))
try:
loop = asyncio.get_event_loop()
loop.run_forever()
loop.close()
except KeyboardInterrupt:
kill_trinity_gracefully(
logger,
database_server_process,
networking_process,
plugin_manager,
main_endpoint,
event_bus
)
def setup_plugins(scope: BaseManagerProcessScope,
plugins: Iterable[BasePlugin]) -> PluginManager:
plugin_manager = PluginManager(scope)
plugin_manager.register(plugins)
return plugin_manager
def main_entry(trinity_boot: BootFn,
app_identifier: str,
plugins: Iterable[Type[BasePlugin]],
sub_configs: Iterable[Type[BaseAppConfig]]) -> None:
main_endpoint = TrinityMainEventBusEndpoint()
plugin_manager = PluginManager(
MainAndIsolatedProcessScope(main_endpoint),
plugins
)
plugin_manager.amend_argparser_config(parser, subparser)
args = parser.parse_args()
if not args.genesis and args.network_id not in PRECONFIGURED_NETWORKS:
raise NotImplementedError(
f"Unsupported network id: {args.network_id}. To use a network besides "
"mainnet or ropsten, you must supply a genesis file with a flag, like "
"`--genesis path/to/genesis.json`, also you must specify a data "
"directory with `--data-dir path/to/data/directory`"
)
# The `common_log_level` is derived from `--log-level <Level>` / `-l <Level>` without
# specifying any module. If present, it is used for both `stderr` and `file` logging.
common_log_level = args.log_levels and args.log_levels.get(None)
has_ambigous_logging_config = ((
common_log_level is not None and
args.stderr_log_level is not None
) or (
common_log_level is not None and
args.file_log_level is not None
))
if has_ambigous_logging_config:
parser.error(
f"""\n
Ambiguous logging configuration: The `--log-level (-l)` flag sets the
log level for both file and stderr logging.
To configure different log level for file and stderr logging,
remove the `--log-level` flag and use `--stderr-log-level` and/or
`--file-log-level` separately.
Alternatively, remove the `--stderr-log-level` and/or `--file-log-level`
flags to share one single log level across both handlers.
"""
)
if is_prerelease():
# this modifies the asyncio logger, but will be overridden by any custom settings below
enable_warnings_by_default()
stderr_logger, handler_stream = setup_trinity_stderr_logging(
args.stderr_log_level or common_log_level
)
if args.log_levels:
setup_log_levels(args.log_levels)
try:
trinity_config = TrinityConfig.from_parser_args(args, app_identifier, sub_configs)
except AmbigiousFileSystem:
parser.error(TRINITY_AMBIGIOUS_FILESYSTEM_INFO)
if not is_data_dir_initialized(trinity_config):
# TODO: this will only work as is for chains with known genesis
# parameters. Need to flesh out how genesis parameters for custom
# chains are defined and passed around.
try:
initialize_data_dir(trinity_config)
except AmbigiousFileSystem:
parser.error(TRINITY_AMBIGIOUS_FILESYSTEM_INFO)
except MissingPath as e:
parser.error(
"\n"
f"It appears that {e.path} does not exist. "
"Trinity does not attempt to create directories outside of its root path. "
"Either manually create the path or ensure you are using a data directory "
"inside the XDG_TRINITY_ROOT path"
)
file_logger, log_queue, listener = setup_trinity_file_and_queue_logging(
stderr_logger,
handler_stream,
trinity_config.logfile_path,
args.file_log_level or common_log_level,
)
display_launch_logs(trinity_config)
# compute the minimum configured log level across all configured loggers.
min_configured_log_level = min(
stderr_logger.level,
file_logger.level,
*(args.log_levels or {}).values()
)
extra_kwargs = {
'log_queue': log_queue,
'log_level': min_configured_log_level,
'log_levels': args.log_levels if args.log_levels else {},
'profile': args.profile,
}
# Plugins can provide a subcommand with a `func` which does then control
# the entire process from here.
if hasattr(args, 'func'):
args.func(args, trinity_config)
return
processes = trinity_boot(
args,
trinity_config,
extra_kwargs,
plugin_manager,
listener,
main_endpoint,
stderr_logger,
)
def kill_trinity_with_reason(reason: str) -> None:
kill_trinity_gracefully(
trinity_config,
stderr_logger,
processes,
plugin_manager,
main_endpoint,
reason=reason
)
try:
loop = asyncio.get_event_loop()
asyncio.ensure_future(trinity_boot_coro(
kill_trinity_with_reason,
main_endpoint,
trinity_config,
plugin_manager,
args, extra_kwargs,
))
loop.add_signal_handler(signal.SIGTERM, lambda: kill_trinity_with_reason("SIGTERM"))
loop.run_forever()
loop.close()
except KeyboardInterrupt:
kill_trinity_with_reason("CTRL+C / Keyboard Interrupt")