def post_fork_child(self):
"""Post-fork child process callback executed via ProcessManager.daemonize()."""
# Set the Exiter exception hook post-fork so as not to affect the pantsd processes exception
# hook with socket-specific behavior.
self._exiter.set_except_hook()
# Set context in the process title.
set_process_title('pantsd-runner [{}]'.format(' '.join(self._args)))
# Broadcast our pid to the remote client so they can send us signals (i.e. SIGINT).
NailgunProtocol.write_chunk(self._socket, ChunkType.PID,
bytes(os.getpid()))
# Setup a SIGINT signal handler.
self._setup_sigint_handler()
# Invoke a Pants run with stdio redirected.
with self._nailgunned_stdio(self._socket):
try:
# Clean global state.
clean_global_runtime_state(reset_subsystem=True)
# Re-raise any deferred exceptions, if present.
self._raise_deferred_exc()
# Otherwise, conduct a normal run.
LocalPantsRunner(self._exiter, self._args, self._env,
self._graph_helper).run()
except KeyboardInterrupt:
self._exiter.exit(1, msg='Interrupted by user.\n')
except Exception:
self._exiter.handle_unhandled_exception(add_newline=True)
else:
self._exiter.exit(0)
def run_sync(self):
"""Synchronously run pantsd."""
os.environ.pop("PYTHONPATH")
global_bootstrap_options = self._bootstrap_options.for_global_scope()
# Set the process name in ps output to 'pantsd' vs './pants compile src/etc:: -ldebug'.
set_process_title(f"pantsd [{self._build_root}]")
# Switch log output to the daemon's log stream, and empty `env` and `argv` to encourage all
# further usage of those variables to happen via engine APIs and options.
self._close_stdio()
with initialize_stdio(global_bootstrap_options), argv_as(
tuple()), hermetic_environment_as():
# Install signal and panic handling.
ExceptionSink.install(
log_location=init_workdir(global_bootstrap_options),
pantsd_instance=True)
native_engine.maybe_set_panic_handler()
self._initialize_metadata()
# Check periodically whether the core is valid, and exit if it is not.
while self._core.is_valid():
time.sleep(self.JOIN_TIMEOUT_SECONDS)
# We're exiting: join the server to avoid interrupting ongoing runs.
self._logger.info(
"Waiting for ongoing runs to complete before exiting...")
native_engine.nailgun_server_await_shutdown(self._server)
self._logger.info("Exiting pantsd")
def post_fork_child(self):
"""Post-fork child process callback executed via ProcessManager.daemonize()."""
# Set the Exiter exception hook post-fork so as not to affect the pantsd processes exception
# hook with socket-specific behavior.
self._exiter.set_except_hook()
# Set context in the process title.
set_process_title('pantsd-runner [{}]'.format(' '.join(self._args)))
# Broadcast our pid to the remote client so they can send us signals (i.e. SIGINT).
NailgunProtocol.write_chunk(self._socket, ChunkType.PID, bytes(os.getpid()))
# Setup a SIGINT signal handler.
self._setup_sigint_handler()
# Invoke a Pants run with stdio redirected.
with self._nailgunned_stdio(self._socket):
try:
# Clean global state.
clean_global_runtime_state(reset_subsystem=True)
# Re-raise any deferred exceptions, if present.
self._raise_deferred_exc()
# Otherwise, conduct a normal run.
LocalPantsRunner(self._exiter, self._args, self._env, self._graph_helper).run()
except KeyboardInterrupt:
self._exiter.exit(1, msg='Interrupted by user.\n')
except Exception:
self._exiter.handle_unhandled_exception(add_newline=True)
else:
self._exiter.exit(0)
def run_sync(self):
"""Synchronously run pantsd."""
# Switch log output to the daemon's log stream from here forward.
self._close_stdio()
with self._pantsd_logging() as log_stream:
# Register an exiter using os._exit to ensure we only close stdio streams once.
ExceptionSink.reset_exiter(Exiter(exiter=os._exit))
# We don't have any stdio streams to log to anymore, but we can get tracebacks of the pantsd
# process by tailing the pantsd log and sending it SIGUSR2.
ExceptionSink.reset_interactive_output_stream(log_stream)
# Reset the log location and the backtrace preference from the global bootstrap options.
global_bootstrap_options = self._bootstrap_options.for_global_scope()
ExceptionSink.reset_should_print_backtrace_to_terminal(
global_bootstrap_options.print_exception_stacktrace)
ExceptionSink.reset_log_location(global_bootstrap_options.pants_workdir)
self._logger.info('pantsd starting, log level is {}'.format(self._log_level))
self._native.set_panic_handler()
# Set the process name in ps output to 'pantsd' vs './pants compile src/etc:: -ldebug'.
set_process_title('pantsd [{}]'.format(self._build_root))
# Write service socket information to .pids.
self._write_named_sockets(self._services.port_map)
# Enter the main service runner loop.
self._setup_services(self._services)
self._run_services(self._services)
def run_sync(self):
"""Synchronously run pantsd."""
os.environ.pop("PYTHONPATH")
# Switch log output to the daemon's log stream from here forward.
self._close_stdio()
with self._pantsd_logging():
# Install signal handling based on global bootstrap options.
global_bootstrap_options = self._bootstrap_options.for_global_scope()
ExceptionSink.install(
log_location=init_workdir(global_bootstrap_options), pantsd_instance=True
)
self._native.set_panic_handler()
# Set the process name in ps output to 'pantsd' vs './pants compile src/etc:: -ldebug'.
set_process_title(f"pantsd [{self._build_root}]")
self._initialize_metadata()
# Check periodically whether the core is valid, and exit if it is not.
while self._core.is_valid():
time.sleep(self.JOIN_TIMEOUT_SECONDS)
# We're exiting: join the server to avoid interrupting ongoing runs.
self._logger.info("Waiting for ongoing runs to complete before exiting...")
self._native.nailgun_server_await_shutdown(self._server)
self._logger.info("Exiting pantsd")
def post_fork_child(self):
"""Post-fork child process callback executed via ProcessManager.daemonize()."""
# Set the Exiter exception hook post-fork so as not to affect the pantsd processes exception
# hook with socket-specific behavior. Note that this intentionally points the faulthandler
# trace stream to sys.stderr, which at this point is still a _LoggerStream object writing to
# the `pantsd.log`. This ensures that in the event of e.g. a hung but detached pantsd-runner
# process that the stacktrace output lands deterministically in a known place vs to a stray
# terminal window.
# TODO: test the above!
ExceptionSink.reset_exiter(self._exiter)
ExceptionSink.reset_interactive_output_stream(
sys.stderr.buffer if PY3 else sys.stderr)
# Ensure anything referencing sys.argv inherits the Pailgun'd args.
sys.argv = self._args
# Set context in the process title.
set_process_title('pantsd-runner [{}]'.format(' '.join(self._args)))
# Broadcast our process group ID (in PID form - i.e. negated) to the remote client so
# they can send signals (e.g. SIGINT) to all processes in the runners process group.
NailgunProtocol.send_pid(self._socket, os.getpid())
NailgunProtocol.send_pgrp(self._socket, os.getpgrp() * -1)
# Stop the services that were paused pre-fork.
for service in self._services.services:
service.terminate()
# Invoke a Pants run with stdio redirected and a proxied environment.
with self.nailgunned_stdio(self._socket, self._env) as finalizer,\
hermetic_environment_as(**self._env):
try:
# Setup the Exiter's finalizer.
self._exiter.set_finalizer(finalizer)
# Clean global state.
clean_global_runtime_state(reset_subsystem=True)
# Re-raise any deferred exceptions, if present.
self._raise_deferred_exc()
# Otherwise, conduct a normal run.
runner = LocalPantsRunner.create(self._exiter, self._args,
self._env, self._target_roots,
self._graph_helper,
self._options_bootstrapper)
runner.set_start_time(
self._maybe_get_client_start_time_from_env(self._env))
runner.run()
except KeyboardInterrupt:
self._exiter.exit_and_fail('Interrupted by user.\n')
except Exception:
ExceptionSink._log_unhandled_exception_and_exit()
else:
self._exiter.exit(0)
def post_fork_child(self):
"""Post-fork child process callback executed via ProcessManager.daemonize()."""
# Set the Exiter exception hook post-fork so as not to affect the pantsd processes exception
# hook with socket-specific behavior. Note that this intentionally points the faulthandler
# trace stream to sys.stderr, which at this point is still a _LoggerStream object writing to
# the `pantsd.log`. This ensures that in the event of e.g. a hung but detached pantsd-runner
# process that the stacktrace output lands deterministically in a known place vs to a stray
# terminal window.
self._exiter.set_except_hook(sys.stderr)
# Ensure anything referencing sys.argv inherits the Pailgun'd args.
sys.argv = self._args
# Set context in the process title.
set_process_title('pantsd-runner [{}]'.format(' '.join(self._args)))
# Setup a SIGINT signal handler.
self._setup_sigint_handler()
# Broadcast our process group ID (in PID form - i.e. negated) to the remote client so
# they can send signals (e.g. SIGINT) to all processes in the runners process group.
pid = str(os.getpgrp() * -1).encode('ascii')
NailgunProtocol.send_pid(self._socket, pid)
# Invoke a Pants run with stdio redirected and a proxied environment.
with self.nailgunned_stdio(self._socket, self._env) as finalizer,\
hermetic_environment_as(**self._env):
try:
# Setup the Exiter's finalizer.
self._exiter.set_finalizer(finalizer)
# Clean global state.
clean_global_runtime_state(reset_subsystem=True)
# Re-raise any deferred exceptions, if present.
self._raise_deferred_exc()
# Otherwise, conduct a normal run.
runner = LocalPantsRunner.create(
self._exiter,
self._args,
self._env,
self._target_roots,
self._graph_helper,
self._options_bootstrapper
)
runner.set_start_time(self._maybe_get_client_start_time_from_env(self._env))
runner.run()
except KeyboardInterrupt:
self._exiter.exit(1, msg='Interrupted by user.\n')
except Exception:
self._exiter.handle_unhandled_exception(add_newline=True)
else:
self._exiter.exit(0)
def run_sync(self):
"""Synchronously run pantsd."""
os.environ.pop("PYTHONPATH")
# Switch log output to the daemon's log stream from here forward.
self._close_stdio()
with self._pantsd_logging() as log_stream:
# We don't have any stdio streams to log to anymore, so we log to a file.
# We don't override the faulthandler destination because the stream we get will proxy things
# via the rust logging code, and faulthandler needs to be writing directly to a real file
# descriptor. When pantsd logging was originally initialised, we already set up faulthandler
# to log to the correct file descriptor, so don't override it.
#
# We can get tracebacks of the pantsd process by tailing the pantsd log and sending it
# SIGUSR2.
ExceptionSink.reset_interactive_output_stream(
log_stream,
override_faulthandler_destination=False,
)
# Reset the log location and the backtrace preference from the global bootstrap options.
global_bootstrap_options = self._bootstrap_options.for_global_scope(
)
ExceptionSink.reset_should_print_backtrace_to_terminal(
global_bootstrap_options.print_exception_stacktrace)
ExceptionSink.reset_log_location(
global_bootstrap_options.pants_workdir)
self._native.set_panic_handler()
# Set the process name in ps output to 'pantsd' vs './pants compile src/etc:: -ldebug'.
set_process_title(f"pantsd [{self._build_root}]")
# Write our pid and the server's port to .pids. Order matters a bit here, because
# technically all that is necessary to connect is the port, and Services are lazily
# initialized by the core when a connection is established. Our pid needs to be on
# disk before that happens.
self._initialize_pid()
self._write_nailgun_port()
# Check periodically whether the core is valid, and exit if it is not.
while self._core.is_valid():
time.sleep(self.JOIN_TIMEOUT_SECONDS)
# We're exiting: join the server to avoid interrupting ongoing runs.
self._logger.info(
"waiting for ongoing runs to complete before exiting...")
self._native.nailgun_server_await_shutdown(self._server)
self._logger.info("exiting.")
def _run(self):
"""Synchronously run pantsd."""
# Switch log output to the daemon's log stream from here forward.
self._setup_logging(self._log_level)
self._logger.info('pantsd starting, log level is {}'.format(self._log_level))
# Purge as much state as possible from the pants run that launched us.
self._clean_runtime_state()
# Set the process name in ps output to 'pantsd' vs './pants compile src/etc:: -ldebug'.
set_process_title('pantsd [{}]'.format(self._build_root))
# Write service socket information to .pids.
self._write_named_sockets(self._socket_map)
# Enter the main service runner loop.
self._run_services(self._services)
def post_fork_child(self):
"""Post-fork child process callback executed via ProcessManager.daemonize()."""
# Set the Exiter exception hook post-fork so as not to affect the pantsd processes exception
# hook with socket-specific behavior. Note that this intentionally points the faulthandler
# trace stream to sys.stderr, which at this point is still a _LoggerStream object writing to
# the `pantsd.log`. This ensures that in the event of e.g. a hung but detached pantsd-runner
# process that the stacktrace output lands deterministically in a known place vs to a stray
# terminal window.
self._exiter.set_except_hook(sys.stderr)
# Ensure anything referencing sys.argv inherits the Pailgun'd args.
sys.argv = self._args
# Set context in the process title.
set_process_title('pantsd-runner [{}]'.format(' '.join(self._args)))
# Broadcast our process group ID (in PID form - i.e. negated) to the remote client so
# they can send signals (e.g. SIGINT) to all processes in the runners process group.
NailgunProtocol.send_pid(self._socket, bytes(os.getpgrp() * -1))
# Setup a SIGINT signal handler.
self._setup_sigint_handler()
# Invoke a Pants run with stdio redirected.
with self._nailgunned_stdio(self._socket) as finalizer:
try:
# Setup the Exiter's finalizer.
self._exiter.set_finalizer(finalizer)
# Clean global state.
clean_global_runtime_state(reset_subsystem=True)
# Re-raise any deferred exceptions, if present.
self._raise_deferred_exc()
# Otherwise, conduct a normal run.
runner = LocalPantsRunner(self._exiter, self._args, self._env,
self._graph_helper)
runner.set_preceding_graph_size(self._preceding_graph_size)
runner.run()
except KeyboardInterrupt:
self._exiter.exit(1, msg='Interrupted by user.\n')
except Exception:
self._exiter.handle_unhandled_exception(add_newline=True)
else:
self._exiter.exit(0)
def _run(self):
"""Synchronously run pantsd."""
# Switch log output to the daemon's log stream from here forward.
self._setup_logging(self._log_level)
self._logger.info('pantsd starting, log level is {}'.format(self._log_level))
# Purge as much state as possible from the pants run that launched us.
clean_global_runtime_state()
# Set the process name in ps output to 'pantsd' vs './pants compile src/etc:: -ldebug'.
set_process_title('pantsd [{}]'.format(self._build_root))
# Write service socket information to .pids.
self._write_named_sockets(self._socket_map)
# Enter the main service runner loop.
self._setup_services(self._services)
self._run_services(self._services)
def run_sync(self):
"""Synchronously run pantsd."""
os.environ.pop("PYTHONPATH")
# Switch log output to the daemon's log stream from here forward.
# Also, register an exiter using os._exit to ensure we only close stdio streams once.
self._close_stdio()
with self._pantsd_logging() as (log_stream,
log_filename), ExceptionSink.exiter_as(
lambda _: Exiter(exiter=os._exit)):
# We don't have any stdio streams to log to anymore, so we log to a file.
# We don't override the faulthandler destination because the stream we get will proxy things
# via the rust logging code, and faulthandler needs to be writing directly to a real file
# descriptor. When pantsd logging was originally initialised, we already set up faulthandler
# to log to the correct file descriptor, so don't override it.
#
# We can get tracebacks of the pantsd process by tailing the pantsd log and sending it
# SIGUSR2.
ExceptionSink.reset_interactive_output_stream(
log_stream,
override_faulthandler_destination=False,
)
# Reset the log location and the backtrace preference from the global bootstrap options.
global_bootstrap_options = self._bootstrap_options.for_global_scope(
)
ExceptionSink.reset_should_print_backtrace_to_terminal(
global_bootstrap_options.print_exception_stacktrace)
ExceptionSink.reset_log_location(
global_bootstrap_options.pants_workdir)
self._native.set_panic_handler()
# Set the process name in ps output to 'pantsd' vs './pants compile src/etc:: -ldebug'.
set_process_title(f"pantsd [{self._build_root}]")
# Write service socket information to .pids.
self._write_named_sockets(self._services.port_map)
# Enter the main service runner loop.
self._setup_services(self._services)
self._run_services(self._services)
def run_sync(self):
"""Synchronously run pantsd."""
# Switch log output to the daemon's log stream from here forward.
self._close_stdio()
with self._pantsd_logging() as log_stream:
self._exiter.set_except_hook(log_stream)
self._logger.info('pantsd starting, log level is {}'.format(self._log_level))
self._native.set_panic_handler()
# Set the process name in ps output to 'pantsd' vs './pants compile src/etc:: -ldebug'.
set_process_title('pantsd [{}]'.format(self._build_root))
# Write service socket information to .pids.
self._write_named_sockets(self._socket_map)
# Enter the main service runner loop.
self._setup_services(self._services)
self._run_services(self._services)
def run_sync(self):
"""Synchronously run pantsd."""
# Switch log output to the daemon's log stream from here forward.
self._close_stdio()
with self._pantsd_logging() as (log_stream, log_filename):
# Register an exiter using os._exit to ensure we only close stdio streams once.
ExceptionSink.reset_exiter(Exiter(exiter=os._exit))
# We don't have any stdio streams to log to anymore, so we log to a file.
# We don't override the faulthandler destination because the stream we get will proxy things
# via the rust logging code, and faulthandler needs to be writing directly to a real file
# descriptor. When pantsd logging was originally initialised, we already set up faulthandler
# to log to the correct file descriptor, so don't override it.
#
# We can get tracebacks of the pantsd process by tailing the pantsd log and sending it
# SIGUSR2.
ExceptionSink.reset_interactive_output_stream(
log_stream,
override_faulthandler_destination=False,
)
# Reset the log location and the backtrace preference from the global bootstrap options.
global_bootstrap_options = self._bootstrap_options.for_global_scope()
ExceptionSink.reset_should_print_backtrace_to_terminal(
global_bootstrap_options.print_exception_stacktrace)
ExceptionSink.reset_log_location(global_bootstrap_options.pants_workdir)
self._native.set_panic_handler()
# Set the process name in ps output to 'pantsd' vs './pants compile src/etc:: -ldebug'.
set_process_title('pantsd [{}]'.format(self._build_root))
# Write service socket information to .pids.
self._write_named_sockets(self._services.port_map)
# Enter the main service runner loop.
self._setup_services(self._services)
self._run_services(self._services)
def run_sync(self):
"""Synchronously run pantsd."""
os.environ.pop("PYTHONPATH")
# Switch log output to the daemon's log stream from here forward.
self._close_stdio()
with self._pantsd_logging():
ExceptionSink.reset_signal_handler(
SignalHandler(pantsd_instance=True))
# Reset the log location and the backtrace preference from the global bootstrap options.
global_bootstrap_options = self._bootstrap_options.for_global_scope(
)
ExceptionSink.reset_log_location(
global_bootstrap_options.pants_workdir)
self._native.set_panic_handler()
# Set the process name in ps output to 'pantsd' vs './pants compile src/etc:: -ldebug'.
set_process_title(f"pantsd [{self._build_root}]")
# Write our pid and the server's port to .pids. Order matters a bit here, because
# technically all that is necessary to connect is the port, and Services are lazily
# initialized by the core when a connection is established. Our pid needs to be on
# disk before that happens.
self._initialize_pid()
self._write_nailgun_port()
# Check periodically whether the core is valid, and exit if it is not.
while self._core.is_valid():
time.sleep(self.JOIN_TIMEOUT_SECONDS)
# We're exiting: join the server to avoid interrupting ongoing runs.
self._logger.info(
"Waiting for ongoing runs to complete before exiting...")
self._native.nailgun_server_await_shutdown(self._server)
self._logger.info("Exiting pantsd")
def post_fork_child(self):
"""Post-fork child process callback executed via ProcessManager.daemonize()."""
# Set the Exiter exception hook post-fork so as not to affect the pantsd processes exception
# hook with socket-specific behavior. Note that this intentionally points the faulthandler
# trace stream to sys.stderr, which at this point is still a _LoggerStream object writing to
# the `pantsd.log`. This ensures that in the event of e.g. a hung but detached pantsd-runner
# process that the stacktrace output lands deterministically in a known place vs to a stray
# terminal window.
# TODO: test the above!
ExceptionSink.reset_exiter(self._exiter)
ExceptionSink.reset_interactive_output_stream(sys.stderr.buffer if PY3 else sys.stderr)
ExceptionSink.reset_signal_handler(DaemonSignalHandler())
# Ensure anything referencing sys.argv inherits the Pailgun'd args.
sys.argv = self._args
# Set context in the process title.
set_process_title('pantsd-runner [{}]'.format(' '.join(self._args)))
# Broadcast our process group ID (in PID form - i.e. negated) to the remote client so
# they can send signals (e.g. SIGINT) to all processes in the runners process group.
NailgunProtocol.send_pid(self._socket, os.getpid())
NailgunProtocol.send_pgrp(self._socket, os.getpgrp() * -1)
# Stop the services that were paused pre-fork.
for service in self._services.services:
service.terminate()
# Invoke a Pants run with stdio redirected and a proxied environment.
with self.nailgunned_stdio(self._socket, self._env) as finalizer,\
hermetic_environment_as(**self._env):
try:
# Setup the Exiter's finalizer.
self._exiter.set_finalizer(finalizer)
# Clean global state.
clean_global_runtime_state(reset_subsystem=True)
# Otherwise, conduct a normal run.
runner = LocalPantsRunner.create(
self._exiter,
self._args,
self._env,
self._target_roots,
self._graph_helper,
self._options_bootstrapper
)
runner.set_start_time(self._maybe_get_client_start_time_from_env(self._env))
# Re-raise any deferred exceptions, if present.
self._raise_deferred_exc()
runner.run()
except KeyboardInterrupt:
self._exiter.exit_and_fail('Interrupted by user.\n')
except _GracefulTerminationException as e:
ExceptionSink.log_exception(
'Encountered graceful termination exception {}; exiting'.format(e))
self._exiter.exit(e.exit_code)
except Exception:
# TODO: We override sys.excepthook above when we call ExceptionSink.set_exiter(). That
# excepthook catches `SignalHandledNonLocalExit`s from signal handlers, which isn't
# happening here, so something is probably overriding the excepthook. By catching Exception
# and calling this method, we emulate the normal, expected sys.excepthook override.
ExceptionSink._log_unhandled_exception_and_exit()
else:
self._exiter.exit(PANTS_SUCCEEDED_EXIT_CODE)
