def kill_trinity_gracefully(logger: logging.Logger,
processes: Iterable[multiprocessing.Process],
plugin_manager: PluginManager,
main_endpoint: Endpoint,
event_bus: EventBus,
reason: str = None) -> None:
# When a user hits Ctrl+C in the terminal, the SIGINT is sent to all processes in the
# foreground *process group*, so both our networking and database processes will terminate
# at the same time and not sequentially as we'd like. That shouldn't be a problem but if
# we keep getting unhandled BrokenPipeErrors/ConnectionResetErrors like reported in
# https://github.com/ethereum/py-evm/issues/827, we might want to change the networking
# process' signal handler to wait until the DB process has terminated before doing its
# thing.
# Notice that we still need the kill_process_gracefully() calls here, for when the user
# simply uses 'kill' to send a signal to the main process, but also because they will
# perform a non-gracefull shutdown if the process takes too long to terminate.
hint = f"({reason})" if reason else f""
logger.info('Shutting down Trinity %s', hint)
plugin_manager.shutdown_blocking()
main_endpoint.stop()
event_bus.stop()
for process in processes:
# Our sub-processes will have received a SIGINT already (see comment above), so here we
# wait 2s for them to finish cleanly, and if they fail we kill them for real.
process.join(2)
if process.is_alive():
kill_process_gracefully(process, logger)
logger.info('%s process (pid=%d) terminated', process.name,
process.pid)
ArgumentParser().exit(message=f"Trinity shutdown complete {hint}\n")
async def handle_networking_exit(service: BaseService,
plugin_manager: PluginManager,
endpoint: Endpoint) -> None:
async with exit_signal_with_service(service):
await plugin_manager.shutdown()
endpoint.stop()
def kill_trinity_gracefully(logger: logging.Logger,
database_server_process: Any,
networking_process: Any,
plugin_manager: PluginManager,
main_endpoint: Endpoint,
event_bus: EventBus,
message: str="Trinity shudown complete\n") -> None:
# When a user hits Ctrl+C in the terminal, the SIGINT is sent to all processes in the
# foreground *process group*, so both our networking and database processes will terminate
# at the same time and not sequentially as we'd like. That shouldn't be a problem but if
# we keep getting unhandled BrokenPipeErrors/ConnectionResetErrors like reported in
# https://github.com/ethereum/py-evm/issues/827, we might want to change the networking
# process' signal handler to wait until the DB process has terminated before doing its
# thing.
# Notice that we still need the kill_process_gracefully() calls here, for when the user
# simply uses 'kill' to send a signal to the main process, but also because they will
# perform a non-gracefull shutdown if the process takes too long to terminate.
logger.info('Keyboard Interrupt: Stopping')
plugin_manager.shutdown_blocking()
main_endpoint.stop()
event_bus.stop()
for name, process in [("DB", database_server_process), ("Networking", networking_process)]:
# Our sub-processes will have received a SIGINT already (see comment above), so here we
# wait 2s for them to finish cleanly, and if they fail we kill them for real.
process.join(2)
if process.is_alive():
kill_process_gracefully(process, logger)
logger.info('%s process (pid=%d) terminated', name, process.pid)
# This is required to be within the `kill_trinity_gracefully` so that
# plugins can trigger a shutdown of the trinity process.
ArgumentParser().exit(message=message)
def launch_node(args: Namespace, trinity_config: TrinityConfig, endpoint: Endpoint) -> None:
with trinity_config.process_id_file('networking'):
endpoint.connect()
NodeClass = trinity_config.node_class
# Temporary hack: We setup a second instance of the PluginManager.
# The first instance was only to configure the ArgumentParser whereas
# for now, the second instance that lives inside the networking process
# performs the bulk of the work. In the future, the PluginManager
# should probably live in its own process and manage whether plugins
# run in the shared plugin process or spawn their own.
plugin_manager = setup_plugins(SharedProcessScope(endpoint))
plugin_manager.prepare(args, trinity_config)
plugin_manager.broadcast(TrinityStartupEvent(
args,
trinity_config
))
node = NodeClass(plugin_manager, trinity_config)
loop = node.get_event_loop()
asyncio.ensure_future(handle_networking_exit(node, plugin_manager, endpoint), loop=loop)
asyncio.ensure_future(node.run(), loop=loop)
loop.run_forever()
loop.close()
async def handle_networking_exit(service: BaseService,
plugin_manager: PluginManager,
endpoint: Endpoint) -> None:
async with exit_signal_with_service(service):
await plugin_manager.shutdown()
endpoint.stop()
# Retrieve and shutdown the global executor that was created at startup
ensure_global_asyncio_executor().shutdown(wait=True)
def trinity_boot(args: Namespace, trinity_config: TrinityConfig,
extra_kwargs: Dict[str, Any], plugin_manager: PluginManager,
listener: logging.handlers.QueueListener, event_bus: EventBus,
main_endpoint: Endpoint, logger: logging.Logger) -> None:
# start the listener thread to handle logs produced by other processes in
# the local logger.
listener.start()
event_bus.start()
# First initialize the database process.
database_server_process = ctx.Process(
name="DB",
target=run_database_process,
args=(
trinity_config,
LevelDB,
),
kwargs=extra_kwargs,
)
# start the processes
database_server_process.start()
logger.info("Started DB server process (pid=%d)",
database_server_process.pid)
# networking process needs the IPC socket file provided by the database process
try:
wait_for_ipc(trinity_config.database_ipc_path)
except TimeoutError as e:
logger.error("Timeout waiting for database to start. Exiting...")
kill_process_gracefully(database_server_process, logger)
ArgumentParser().error(message="Timed out waiting for database start")
def kill_trinity_with_reason(reason: str) -> None:
kill_trinity_gracefully(logger, (database_server_process, ),
plugin_manager,
main_endpoint,
event_bus,
reason=reason)
main_endpoint.subscribe(ShutdownRequest,
lambda ev: kill_trinity_with_reason(ev.reason))
plugin_manager.prepare(args, trinity_config, extra_kwargs)
kill_trinity_with_reason("No beacon support yet. SOON!")
try:
loop = asyncio.get_event_loop()
loop.add_signal_handler(signal.SIGTERM,
lambda: kill_trinity_with_reason("SIGTERM"))
loop.run_forever()
loop.close()
except KeyboardInterrupt:
kill_trinity_with_reason("CTRL+C / Keyboard Interrupt")
async def run_proc1():
endpoint = Endpoint()
await endpoint.start_serving(ConnectionConfig.from_name('e1'))
await endpoint.connect_to_endpoints(ConnectionConfig.from_name('e2'), )
print("subscribing")
# Listen for `GetSomethingRequest`'s
endpoint.subscribe(
GetSomethingRequest,
lambda event:
# Send a response back to *only* who made that request
endpoint.broadcast_nowait(DeliverSomethingResponse("Yay"),
event.broadcast_config()))
async def proc2_worker():
endpoint = Endpoint()
await endpoint.start_serving(ConnectionConfig.from_name('e2'))
await endpoint.connect_to_endpoints(ConnectionConfig.from_name('e1'))
asyncio.ensure_future(display_proc1_events(endpoint))
endpoint.subscribe(
FirstThingHappened, lambda event: print(
"Received via SUBSCRIBE API in proc2:", event.payload))
while True:
print("Hello from proc2")
if is_nth_second(2):
await endpoint.broadcast(
SecondThingHappened("Hit from proc2 ({})".format(time.time())))
await asyncio.sleep(1)
def launch_node(args: Namespace, trinity_config: TrinityConfig, endpoint: Endpoint) -> None:
with trinity_config.process_id_file('networking'):
NodeClass = trinity_config.node_class
node = NodeClass(endpoint, trinity_config)
loop = node.get_event_loop()
endpoint.connect_no_wait(loop)
# This is a second PluginManager instance governing plugins in a shared process.
plugin_manager = setup_plugins(SharedProcessScope(endpoint))
plugin_manager.prepare(args, trinity_config)
asyncio.ensure_future(handle_networking_exit(node, plugin_manager, endpoint), loop=loop)
asyncio.ensure_future(node.run(), loop=loop)
loop.run_forever()
loop.close()
async def exit_on_signal(service_to_exit: BaseService,
endpoint: Endpoint = None) -> None:
loop = service_to_exit.get_event_loop()
sigint_received = asyncio.Event()
for sig in [signal.SIGINT, signal.SIGTERM]:
# TODO also support Windows
loop.add_signal_handler(sig, sigint_received.set)
await sigint_received.wait()
try:
await service_to_exit.cancel()
if endpoint is not None:
endpoint.stop()
service_to_exit._executor.shutdown(wait=True)
finally:
loop.stop()
async def proc2_worker():
endpoint = Endpoint()
await endpoint.start_serving(ConnectionConfig.from_name('e2'))
await endpoint.connect_to_endpoints(ConnectionConfig.from_name('e1'), )
for i in range(3):
print("Requesting")
result = await endpoint.request(GetSomethingRequest())
print(f"Got answer: {result.payload}")
def launch_node(args: Namespace, chain_config: ChainConfig,
endpoint: Endpoint) -> None:
with chain_config.process_id_file('networking'):
endpoint.connect()
NodeClass = chain_config.node_class
# Temporary hack: We setup a second instance of the PluginManager.
# The first instance was only to configure the ArgumentParser whereas
# for now, the second instance that lives inside the networking process
# performs the bulk of the work. In the future, the PluginManager
# should probably live in its own process and manage whether plugins
# run in the shared plugin process or spawn their own.
plugin_manager = setup_plugins(SharedProcessScope(endpoint))
plugin_manager.prepare(args, chain_config)
plugin_manager.broadcast(TrinityStartupEvent(args, chain_config))
node = NodeClass(plugin_manager, chain_config)
run_service_until_quit(node)
def trinity_boot(args: Namespace,
trinity_config: TrinityConfig,
extra_kwargs: Dict[str, Any],
plugin_manager: PluginManager,
listener: logging.handlers.QueueListener,
event_bus: EventBus,
main_endpoint: Endpoint,
logger: logging.Logger) -> None:
# start the listener thread to handle logs produced by other processes in
# the local logger.
listener.start()
event_bus.start()
def kill_trinity_with_reason(reason: str) -> None:
kill_trinity_gracefully(
logger,
(),
plugin_manager,
main_endpoint,
event_bus,
reason=reason
)
main_endpoint.subscribe(
ShutdownRequest,
lambda ev: kill_trinity_with_reason(ev.reason)
)
plugin_manager.prepare(args, trinity_config, extra_kwargs)
try:
loop = asyncio.get_event_loop()
loop.add_signal_handler(signal.SIGTERM, lambda: kill_trinity_with_reason("SIGTERM"))
loop.run_forever()
loop.close()
except KeyboardInterrupt:
kill_trinity_with_reason("CTRL+C / Keyboard Interrupt")
def launch_node(args: Namespace, trinity_config: TrinityConfig,
endpoint: Endpoint) -> None:
with trinity_config.process_id_file('networking'):
NodeClass = trinity_config.get_app_config(Eth1AppConfig).node_class
node = NodeClass(endpoint, trinity_config)
# The `networking` process creates a process pool executor to offload cpu intensive
# tasks. We should revisit that when we move the sync in its own process
ensure_global_asyncio_executor()
loop = node.get_event_loop()
endpoint.connect_no_wait(loop)
# This is a second PluginManager instance governing plugins in a shared process.
plugin_manager = setup_plugins(SharedProcessScope(endpoint),
get_all_plugins())
plugin_manager.prepare(args, trinity_config)
asyncio.ensure_future(handle_networking_exit(node, plugin_manager,
endpoint),
loop=loop)
asyncio.ensure_future(node.run(), loop=loop)
loop.run_forever()
loop.close()
def trinity_boot(args: Namespace,
trinity_config: TrinityConfig,
extra_kwargs: Dict[str, Any],
plugin_manager: PluginManager,
listener: logging.handlers.QueueListener,
event_bus: EventBus,
main_endpoint: Endpoint,
logger: logging.Logger) -> None:
# start the listener thread to handle logs produced by other processes in
# the local logger.
listener.start()
networking_endpoint = event_bus.create_endpoint(NETWORKING_EVENTBUS_ENDPOINT)
event_bus.start()
# First initialize the database process.
database_server_process = ctx.Process(
target=run_database_process,
args=(
trinity_config,
LevelDB,
),
kwargs=extra_kwargs,
)
networking_process = ctx.Process(
target=launch_node,
args=(args, trinity_config, networking_endpoint,),
kwargs=extra_kwargs,
)
# start the processes
database_server_process.start()
logger.info("Started DB server process (pid=%d)", database_server_process.pid)
# networking process needs the IPC socket file provided by the database process
try:
wait_for_ipc(trinity_config.database_ipc_path)
except TimeoutError as e:
logger.error("Timeout waiting for database to start. Exiting...")
kill_process_gracefully(database_server_process, logger)
ArgumentParser().error(message="Timed out waiting for database start")
networking_process.start()
logger.info("Started networking process (pid=%d)", networking_process.pid)
main_endpoint.subscribe(
ShutdownRequest,
lambda ev: kill_trinity_gracefully(
logger,
database_server_process,
networking_process,
plugin_manager,
main_endpoint,
event_bus
)
)
plugin_manager.prepare(args, trinity_config, extra_kwargs)
plugin_manager.broadcast(TrinityStartupEvent(
args,
trinity_config
))
try:
loop = asyncio.get_event_loop()
loop.run_forever()
loop.close()
except KeyboardInterrupt:
kill_trinity_gracefully(
logger,
database_server_process,
networking_process,
plugin_manager,
main_endpoint,
event_bus
)
def request_shutdown(event_bus: Endpoint, reason: str) -> None:
event_bus.broadcast(
ShutdownRequest(reason),
BroadcastConfig(filter_endpoint=MAIN_EVENTBUS_ENDPOINT))
async def exit_with_service_and_endpoint(service_to_exit: BaseService,
endpoint: Endpoint) -> None:
async with exit_signal_with_service(service_to_exit):
endpoint.stop()
def helios_boot(args: Namespace, chain_config: ChainConfig,
extra_kwargs: Dict[str, Any], plugin_manager: PluginManager,
listener: logging.handlers.QueueListener, event_bus: EventBus,
main_endpoint: Endpoint, logger: logging.Logger) -> None:
# start the listener thread to handle logs produced by other processes in
# the local logger.
listener.start()
logger.info(
"Checking for any already running Helios Protocol processes that need shutting down. Remember that you can only run 1 instance at a time."
)
fix_unclean_shutdown(chain_config, logger)
with chain_config.process_id_file('main'):
networking_endpoint = event_bus.create_endpoint(
NETWORKING_EVENTBUS_ENDPOINT)
event_bus.start()
# First initialize the database process.
database_server_process = ctx.Process(
target=run_database_process,
args=(
chain_config,
LevelDB,
),
kwargs=extra_kwargs,
)
chain_processes = []
for i in range(chain_config.num_chain_processes):
chain_process = ctx.Process(
target=run_chain_process,
args=(chain_config, i),
kwargs=extra_kwargs,
)
chain_processes.append(chain_process)
networking_process = ctx.Process(
target=launch_node,
args=(
args,
chain_config,
networking_endpoint,
),
kwargs=extra_kwargs,
)
# start the processes
database_server_process.start()
logger.info("Started DB server process (pid=%d)",
database_server_process.pid)
# networking process needs the IPC socket file provided by the database process
try:
wait_for_ipc(chain_config.database_ipc_path)
except TimeoutError as e:
logger.error("Timeout waiting for database to start. Exiting...")
kill_process_gracefully(database_server_process, logger)
ArgumentParser().error(
message="Timed out waiting for database start")
for i in range(chain_config.num_chain_processes):
chain_process = chain_processes[i]
chain_process.start()
logger.info("Started chain instance {} process (pid={})".format(
i, chain_process.pid))
try:
wait_for_ipc(chain_config.get_chain_ipc_path(i))
except TimeoutError as e:
logger.error(
"Timeout waiting for chain instance {} to start. Exiting..."
.format(i))
kill_process_gracefully(chain_process, logger)
for j in range(i + 1):
kill_process_gracefully(chain_processes[j], logger)
ArgumentParser().error(
message="Timed out waiting for chain instance {} start".
format(i))
networking_process.start()
logger.info("Started networking process (pid=%d)",
networking_process.pid)
main_endpoint.subscribe(
ShutdownRequest, lambda ev: kill_helios_gracefully(
logger, database_server_process, chain_processes,
networking_process, plugin_manager, main_endpoint, event_bus))
plugin_manager.prepare(args, chain_config, extra_kwargs)
plugin_manager.broadcast(HeliosStartupEvent(args, chain_config))
try:
loop = asyncio.get_event_loop()
loop.run_forever()
loop.close()
except KeyboardInterrupt:
kill_helios_gracefully(logger, database_server_process,
chain_processes, networking_process,
plugin_manager, main_endpoint, event_bus)
