def _get_log_file_paths(self, workdir, pants_run):
pid_specific_log_file = ExceptionSink.exceptions_log_path(for_pid=pants_run.pid, in_dir=workdir)
self.assertTrue(os.path.isfile(pid_specific_log_file))
shared_log_file = ExceptionSink.exceptions_log_path(in_dir=workdir)
self.assertTrue(os.path.isfile(shared_log_file))
self.assertNotEqual(pid_specific_log_file, shared_log_file)
return (pid_specific_log_file, shared_log_file)
def main():
start_time = time.time()
exiter = Exiter()
ExceptionSink.reset_exiter(exiter)
with maybe_profiled(os.environ.get('PANTSC_PROFILE')):
try:
PantsRunner(exiter, start_time=start_time).run()
except KeyboardInterrupt as e:
exiter.exit_and_fail('Interrupted by user:\n{}'.format(e))
def execute(self):
exit_msg = self._lifecycle_stubs.add_exiter_message
if exit_msg:
new_exiter = MessagingExiter(exit_msg)
ExceptionSink.reset_exiter(new_exiter)
output_file = self._lifecycle_stubs.new_interactive_stream_output_file
if output_file:
file_stream = open(output_file, 'wb')
ExceptionSink.reset_interactive_output_stream(file_stream, output_file)
raise Exception('erroneous!')
def set_start_time(self, start_time):
# Launch RunTracker as early as possible (before .run() is called).
self._run_tracker = RunTracker.global_instance()
self._reporting = Reporting.global_instance()
self._run_start_time = start_time
self._reporting.initialize(self._run_tracker, self._options, start_time=self._run_start_time)
# Capture a repro of the 'before' state for this build, if needed.
self._repro = Reproducer.global_instance().create_repro()
if self._repro:
self._repro.capture(self._run_tracker.run_info.get_as_dict())
# The __call__ method of the Exiter allows for the prototype pattern.
self._exiter = LocalExiter(self._run_tracker, self._repro, exiter=self._exiter)
ExceptionSink.reset_exiter(self._exiter)
def exit(self, result=PANTS_SUCCEEDED_EXIT_CODE, msg=None, *args, **kwargs):
# These strings are prepended to the existing exit message when calling the superclass .exit().
additional_messages = []
try:
if not self._run_tracker.has_ended():
if result == PANTS_SUCCEEDED_EXIT_CODE:
outcome = WorkUnit.SUCCESS
elif result == PANTS_FAILED_EXIT_CODE:
outcome = WorkUnit.FAILURE
else:
run_tracker_msg = ("unrecognized exit code {} provided to {}.exit() -- "
"interpreting as a failure in the run tracker"
.format(result, type(self).__name__))
# Log the unrecognized exit code to the fatal exception log.
ExceptionSink.log_exception(run_tracker_msg)
# Ensure the unrecognized exit code message is also logged to the terminal.
additional_messages.push(run_tracker_msg)
outcome = WorkUnit.FAILURE
self._run_tracker.set_root_outcome(outcome)
run_tracker_result = self._run_tracker.end()
assert result == run_tracker_result, "pants exit code not correctly recorded by run tracker"
except ValueError as e:
# If we have been interrupted by a signal, calling .end() sometimes writes to a closed file,
# so we just log that fact here and keep going.
exception_string = str(e)
ExceptionSink.log_exception(exception_string)
additional_messages.push(exception_string)
finally:
if self._repro:
# TODO: Have Repro capture the 'after' state (as a diff) as well? (in reference to the below
# 'before' state comment)
# NB: this writes to the logger, which is expected to still be alive if we are exiting from
# a signal.
self._repro.log_location_of_repro_file()
if additional_messages:
msg = '{}\n\n{}'.format('\n'.join(additional_messages),
msg or '')
super(LocalExiter, self).exit(result=result, msg=msg, *args, **kwargs)
def execute(self, execution_request):
"""Invoke the engine for the given ExecutionRequest, returning Return and Throw states.
:return: A tuple of (root, Return) tuples and (root, Throw) tuples.
"""
start_time = time.time()
roots = list(zip(execution_request.roots,
self._scheduler._run_and_return_roots(self._session, execution_request.native)))
ExceptionSink.toggle_ignoring_sigint_v2_engine(False)
self._maybe_visualize()
logger.debug(
'computed %s nodes in %f seconds. there are %s total nodes.',
len(roots),
time.time() - start_time,
self._scheduler.graph_len()
)
returns = tuple((root, state) for root, state in roots if type(state) is Return)
throws = tuple((root, state) for root, state in roots if type(state) is Throw)
return returns, throws
def run(self, start_time: float) -> ExitCode:
self._set_start_time(start_time)
with maybe_profiled(self.profile_path):
global_options = self.options.for_global_scope()
streaming_handlers = global_options.streaming_workunits_handlers
report_interval = global_options.streaming_workunits_report_interval
callbacks = Subsystem.get_streaming_workunit_callbacks(
streaming_handlers)
streaming_reporter = StreamingWorkunitHandler(
self.graph_session.scheduler_session,
callbacks=callbacks,
report_interval_seconds=report_interval,
)
if self.options.help_request:
all_help_info = HelpInfoExtracter.get_all_help_info(
self.options,
self.union_membership,
self.graph_session.goal_consumed_subsystem_scopes,
)
help_printer = HelpPrinter(
bin_name=global_options.pants_bin_name,
help_request=self.options.help_request,
all_help_info=all_help_info,
use_color=global_options.colors,
)
return help_printer.print_help()
with streaming_reporter.session():
engine_result = PANTS_FAILED_EXIT_CODE
try:
engine_result = self._run_v2()
except Exception as e:
ExceptionSink.log_exception(e)
run_tracker_result = self._finish_run(engine_result)
return self._merge_exit_codes(engine_result, run_tracker_result)
def run_sync(self):
"""Synchronously run pantsd."""
os.environ.pop("PYTHONPATH")
global_bootstrap_options = self._bootstrap_options.for_global_scope()
# Set the process name in ps output to 'pantsd' vs './pants compile src/etc:: -ldebug'.
set_process_title(f"pantsd [{self._build_root}]")
# Switch log output to the daemon's log stream, and empty `env` and `argv` to encourage all
# further usage of those variables to happen via engine APIs and options.
self._close_stdio(
pants_log_path(PurePath(global_bootstrap_options.pants_workdir)))
with initialize_stdio(global_bootstrap_options), argv_as(
tuple()), hermetic_environment_as():
# Install signal and panic handling.
ExceptionSink.install(
log_location=init_workdir(global_bootstrap_options),
pantsd_instance=True)
native_engine.maybe_set_panic_handler()
self._initialize_metadata()
# Check periodically whether the core is valid, and exit if it is not.
while self._core.is_valid():
time.sleep(self.JOIN_TIMEOUT_SECONDS)
# We're exiting: purge our metadata to prevent new connections, then join the server
# to avoid interrupting ongoing runs.
self.purge_metadata(force=True)
self._logger.info(
"Waiting for ongoing runs to complete before exiting...")
native_engine.nailgun_server_await_shutdown(self._server)
# Then shutdown the PantsDaemonCore, which will shut down any live Scheduler.
self._logger.info(
"Waiting for Sessions to complete before exiting...")
self._core.shutdown()
self._logger.info("Exiting pantsd")
def run(self, start_time: float) -> None:
self.scrub_pythonpath()
# TODO could options-bootstrapper be parsed in the runners?
options_bootstrapper = OptionsBootstrapper.create(env=self.env,
args=self.args)
bootstrap_options = options_bootstrapper.bootstrap_options
global_bootstrap_options = bootstrap_options.for_global_scope()
# Initialize the workdir early enough to ensure that logging has a destination.
workdir_src = init_workdir(global_bootstrap_options)
ExceptionSink.reset_log_location(workdir_src)
# We enable Rust logging here,
# and everything before it will be routed through regular Python logging.
self._enable_rust_logging(global_bootstrap_options)
ExceptionSink.reset_should_print_backtrace_to_terminal(
global_bootstrap_options.print_exception_stacktrace)
if self._should_run_with_pantsd(global_bootstrap_options):
try:
RemotePantsRunner(self._exiter, self.args, self.env,
options_bootstrapper).run()
return
except RemotePantsRunner.Fallback as e:
logger.warning(
"Client exception: {!r}, falling back to non-daemon mode".
format(e))
# N.B. Inlining this import speeds up the python thin client run by about 100ms.
from pants.bin.local_pants_runner import LocalPantsRunner
runner = LocalPantsRunner.create(
env=self.env, options_bootstrapper=options_bootstrapper)
runner.set_start_time(start_time)
runner.run()
def post_fork_child(self):
"""Post-fork child process callback executed via ProcessManager.daemonize()."""
# Set the Exiter exception hook post-fork so as not to affect the pantsd processes exception
# hook with socket-specific behavior. Note that this intentionally points the faulthandler
# trace stream to sys.stderr, which at this point is still a _LoggerStream object writing to
# the `pantsd.log`. This ensures that in the event of e.g. a hung but detached pantsd-runner
# process that the stacktrace output lands deterministically in a known place vs to a stray
# terminal window.
# TODO: test the above!
ExceptionSink.reset_exiter(self._exiter)
ExceptionSink.reset_interactive_output_stream(
sys.stderr.buffer if PY3 else sys.stderr)
# Ensure anything referencing sys.argv inherits the Pailgun'd args.
sys.argv = self._args
# Set context in the process title.
set_process_title('pantsd-runner [{}]'.format(' '.join(self._args)))
# Broadcast our process group ID (in PID form - i.e. negated) to the remote client so
# they can send signals (e.g. SIGINT) to all processes in the runners process group.
NailgunProtocol.send_pid(self._socket, os.getpid())
NailgunProtocol.send_pgrp(self._socket, os.getpgrp() * -1)
# Stop the services that were paused pre-fork.
for service in self._services.services:
service.terminate()
# Invoke a Pants run with stdio redirected and a proxied environment.
with self.nailgunned_stdio(self._socket, self._env) as finalizer,\
hermetic_environment_as(**self._env):
try:
# Setup the Exiter's finalizer.
self._exiter.set_finalizer(finalizer)
# Clean global state.
clean_global_runtime_state(reset_subsystem=True)
# Re-raise any deferred exceptions, if present.
self._raise_deferred_exc()
# Otherwise, conduct a normal run.
runner = LocalPantsRunner.create(self._exiter, self._args,
self._env, self._target_roots,
self._graph_helper,
self._options_bootstrapper)
runner.set_start_time(
self._maybe_get_client_start_time_from_env(self._env))
runner.run()
except KeyboardInterrupt:
self._exiter.exit_and_fail('Interrupted by user.\n')
except Exception:
ExceptionSink._log_unhandled_exception_and_exit()
else:
self._exiter.exit(0)
def run(self):
# Ensure anything referencing sys.argv inherits the Pailgun'd args.
sys.argv = self._args
# Broadcast our process group ID (in PID form - i.e. negated) to the remote client so
# they can send signals (e.g. SIGINT) to all processes in the runners process group.
with self._maybe_shutdown_socket.lock:
NailgunProtocol.send_pid(self._maybe_shutdown_socket.socket, os.getpid())
NailgunProtocol.send_pgrp(self._maybe_shutdown_socket.socket, os.getpgrp() * -1)
# Invoke a Pants run with stdio redirected and a proxied environment.
with self.nailgunned_stdio(self._maybe_shutdown_socket, self._env) as finalizer, \
hermetic_environment_as(**self._env), \
encapsulated_global_logger():
try:
# Raise any exceptions we may have found when precomputing products.
# NB: We raise it here as opposed to earlier because we have setup logging and stdio.
if self._exception is not None:
raise self._exception
# Clean global state.
clean_global_runtime_state(reset_subsystem=True)
# Setup the Exiter's finalizer.
self._exiter.set_finalizer(finalizer)
# Otherwise, conduct a normal run.
runner = LocalPantsRunner.create(
PantsRunFailCheckerExiter(),
self._args,
self._env,
self._target_roots,
self._graph_helper,
self._options_bootstrapper,
)
runner.set_start_time(self._maybe_get_client_start_time_from_env(self._env))
runner.run()
except KeyboardInterrupt:
self._exiter.exit_and_fail('Interrupted by user.\n')
except _PantsRunFinishedWithFailureException as e:
ExceptionSink.log_exception(
'Pants run failed with exception: {}; exiting'.format(e))
self._exiter.exit(e.exit_code)
except _PantsProductPrecomputeFailed as e:
ExceptionSink.log_exception(repr(e))
self._exiter.exit(e.exit_code)
except Exception as e:
# TODO: We override sys.excepthook above when we call ExceptionSink.set_exiter(). That
# excepthook catches `SignalHandledNonLocalExit`s from signal handlers, which isn't
# happening here, so something is probably overriding the excepthook. By catching Exception
# and calling this method, we emulate the normal, expected sys.excepthook override.
ExceptionSink._log_unhandled_exception_and_exit(exc=e)
else:
self._exiter.exit(PANTS_SUCCEEDED_EXIT_CODE)
def launch_repl(self, pex, **pex_run_kwargs):
running_under_pantsd = self.context.options.for_global_scope().enable_pantsd
if not running_under_pantsd:
# While the repl subprocess is synchronously spawned, we rely on process group
# signalling for a SIGINT to reach the repl subprocess directly - and want to
# do nothing in response on the parent side.
with ExceptionSink.trapped_signals(PythonReplSignalHandler()):
self._run_repl(pex, **pex_run_kwargs)
else:
# In pantsd, this task will be running in a non-main thread,
# so we can't override signal handling here.
# That said, this means that under pantsd,
# Ctrl-C will simply crash the repl (and the daemon).
# TODO(#7623) Potential more robust (but more invasive) fix.
self._run_repl(pex, **pex_run_kwargs)
def launch_repl(self, pex, **pex_run_kwargs):
running_under_pantsd = self.context.options.for_global_scope(
).enable_pantsd
if not running_under_pantsd:
# While the repl subprocess is synchronously spawned, we rely on process group
# signalling for a SIGINT to reach the repl subprocess directly - and want to
# do nothing in response on the parent side.
with ExceptionSink.trapped_signals(PythonReplSignalHandler()):
self._run_repl(pex, **pex_run_kwargs)
else:
# In pantsd, this task will be running in a non-main thread,
# so we can't override signal handling here.
# That said, this means that under pantsd,
# Ctrl-C will simply crash the repl (and the daemon).
# TODO(#7623) Potential more robust (but more invasive) fix.
self._run_repl(pex, **pex_run_kwargs)
def run_sync(self):
"""Synchronously run pantsd."""
# Switch log output to the daemon's log stream from here forward.
self._close_stdio()
with self._pantsd_logging() as (log_stream, log_filename):
# Register an exiter using os._exit to ensure we only close stdio streams once.
ExceptionSink.reset_exiter(Exiter(exiter=os._exit))
# We don't have any stdio streams to log to anymore, so we log to a file.
# We don't override the faulthandler destination because the stream we get will proxy things
# via the rust logging code, and faulthandler needs to be writing directly to a real file
# descriptor. When pantsd logging was originally initialised, we already set up faulthandler
# to log to the correct file descriptor, so don't override it.
#
# We can get tracebacks of the pantsd process by tailing the pantsd log and sending it
# SIGUSR2.
ExceptionSink.reset_interactive_output_stream(
log_stream,
override_faulthandler_destination=False,
)
# Reset the log location and the backtrace preference from the global bootstrap options.
global_bootstrap_options = self._bootstrap_options.for_global_scope(
)
ExceptionSink.reset_should_print_backtrace_to_terminal(
global_bootstrap_options.print_exception_stacktrace)
ExceptionSink.reset_log_location(
global_bootstrap_options.pants_workdir)
self._native.set_panic_handler()
# Set the process name in ps output to 'pantsd' vs './pants compile src/etc:: -ldebug'.
set_process_title('pantsd [{}]'.format(self._build_root))
# Write service socket information to .pids.
self._write_named_sockets(self._services.port_map)
# Enter the main service runner loop.
self._setup_services(self._services)
self._run_services(self._services)
def run_sync(self):
"""Synchronously run pantsd."""
os.environ.pop("PYTHONPATH")
# Switch log output to the daemon's log stream from here forward.
self._close_stdio()
with self._pantsd_logging() as log_stream:
# We don't have any stdio streams to log to anymore, so we log to a file.
# We don't override the faulthandler destination because the stream we get will proxy things
# via the rust logging code, and faulthandler needs to be writing directly to a real file
# descriptor. When pantsd logging was originally initialised, we already set up faulthandler
# to log to the correct file descriptor, so don't override it.
#
# We can get tracebacks of the pantsd process by tailing the pantsd log and sending it
# SIGUSR2.
ExceptionSink.reset_interactive_output_stream(
log_stream,
override_faulthandler_destination=False,
)
# Reset the log location and the backtrace preference from the global bootstrap options.
global_bootstrap_options = self._bootstrap_options.for_global_scope(
)
ExceptionSink.reset_should_print_backtrace_to_terminal(
global_bootstrap_options.print_exception_stacktrace)
ExceptionSink.reset_log_location(
global_bootstrap_options.pants_workdir)
self._native.set_panic_handler()
# Set the process name in ps output to 'pantsd' vs './pants compile src/etc:: -ldebug'.
set_process_title(f"pantsd [{self._build_root}]")
# Write our pid and the server's port to .pids. Order matters a bit here, because
# technically all that is necessary to connect is the port, and Services are lazily
# initialized by the core when a connection is established. Our pid needs to be on
# disk before that happens.
self._initialize_pid()
self._write_nailgun_port()
# Check periodically whether the core is valid, and exit if it is not.
while self._core.is_valid():
time.sleep(self.JOIN_TIMEOUT_SECONDS)
# We're exiting: join the server to avoid interrupting ongoing runs.
self._logger.info(
"waiting for ongoing runs to complete before exiting...")
self._native.nailgun_server_await_shutdown(self._server)
self._logger.info("exiting.")
def run_sync(self):
"""Synchronously run pantsd."""
# Switch log output to the daemon's log stream from here forward.
self._close_stdio()
with self._pantsd_logging() as (log_stream, log_filename):
# Register an exiter using os._exit to ensure we only close stdio streams once.
ExceptionSink.reset_exiter(Exiter(exiter=os._exit))
# We don't have any stdio streams to log to anymore, so we log to a file.
# We don't override the faulthandler destination because the stream we get will proxy things
# via the rust logging code, and faulthandler needs to be writing directly to a real file
# descriptor. When pantsd logging was originally initialised, we already set up faulthandler
# to log to the correct file descriptor, so don't override it.
#
# We can get tracebacks of the pantsd process by tailing the pantsd log and sending it
# SIGUSR2.
ExceptionSink.reset_interactive_output_stream(
log_stream,
override_faulthandler_destination=False,
)
# Reset the log location and the backtrace preference from the global bootstrap options.
global_bootstrap_options = self._bootstrap_options.for_global_scope()
ExceptionSink.reset_should_print_backtrace_to_terminal(
global_bootstrap_options.print_exception_stacktrace)
ExceptionSink.reset_log_location(global_bootstrap_options.pants_workdir)
self._native.set_panic_handler()
# Set the process name in ps output to 'pantsd' vs './pants compile src/etc:: -ldebug'.
set_process_title('pantsd [{}]'.format(self._build_root))
# Write service socket information to .pids.
self._write_named_sockets(self._services.port_map)
# Enter the main service runner loop.
self._setup_services(self._services)
self._run_services(self._services)
def _extract_remote_exception(self, pantsd_pid, nailgun_error):
"""Given a NailgunError, returns a Terminated exception with additional info (where possible).
This method will include the entire exception log for either the `pid` in the NailgunError, or
failing that, the `pid` of the pantsd instance.
"""
sources = [pantsd_pid]
if nailgun_error.pid is not None:
sources = [abs(nailgun_error.pid)] + sources
exception_text = None
for source in sources:
log_path = ExceptionSink.exceptions_log_path(for_pid=source)
exception_text = maybe_read_file(log_path)
if exception_text:
break
exception_suffix = '\nRemote exception:\n{}'.format(exception_text) if exception_text else ''
return self.Terminated('abruptly lost active connection to pantsd runner: {!r}{}'.format(
nailgun_error, exception_suffix))
def test_backup_logging_on_fatal_error(self):
sink = self._gen_sink_subclass()
with self.captured_logging(level=logging.ERROR) as captured:
with temporary_dir() as tmpdir:
sink.reset_log_location(tmpdir)
with unittest.mock.patch.object(sink, '_try_write_with_flush', autospec=sink) as mock_write:
mock_write.side_effect = ExceptionSink.ExceptionSinkError('fake write failure')
sink.log_exception('XXX')
errors = list(captured.errors())
self.assertEqual(2, len(errors))
def format_log_rx(log_file_type):
return '.*'.join(re.escape(s) for s in [
"pants.base.exception_sink: Error logging the message 'XXX' to the {log_file_type} file "
"handle for".format(log_file_type=log_file_type),
"at pid {pid}".format(pid=os.getpid()),
"\nfake write failure",
])
self.assertRegex(str(errors[0]), format_log_rx('pid-specific'))
self.assertRegex(str(errors[1]), format_log_rx('shared'))
def run(self):
# Register our exiter at the beginning of the run() method so that any code in this process from
# this point onwards will use that exiter in the case of a fatal error.
ExceptionSink.reset_exiter(self._exiter)
options_bootstrapper = OptionsBootstrapper.create(env=self._env,
args=self._args)
bootstrap_options = options_bootstrapper.bootstrap_options
global_bootstrap_options = bootstrap_options.for_global_scope()
# We enable Rust logging here,
# and everything before it will be routed through regular Python logging.
self._enable_rust_logging(global_bootstrap_options)
ExceptionSink.reset_should_print_backtrace_to_terminal(
global_bootstrap_options.print_exception_stacktrace)
ExceptionSink.reset_log_location(
global_bootstrap_options.pants_workdir)
for message_regexp in global_bootstrap_options.ignore_pants_warnings:
warnings.filterwarnings(action='ignore', message=message_regexp)
# TODO https://github.com/pantsbuild/pants/issues/7205
if self._should_run_with_pantsd(global_bootstrap_options):
try:
return RemotePantsRunner(self._exiter, self._args, self._env,
options_bootstrapper).run()
except RemotePantsRunner.Fallback as e:
logger.warn(
'caught client exception: {!r}, falling back to non-daemon mode'
.format(e))
# N.B. Inlining this import speeds up the python thin client run by about 100ms.
from pants.bin.local_pants_runner import LocalPantsRunner
if self.will_terminate_pantsd():
logger.debug("Pantsd terminating goal detected: {}".format(
self._args))
runner = LocalPantsRunner.create(
self._exiter,
self._args,
self._env,
options_bootstrapper=options_bootstrapper)
runner.set_start_time(self._start_time)
return runner.run()
def _connect_and_execute(
self, pantsd_handle: PantsDaemonClient.Handle) -> ExitCode:
port = pantsd_handle.port
pid = pantsd_handle.pid
global_options = self._bootstrap_options.for_global_scope()
# Merge the nailgun TTY capability environment variables with the passed environment dict.
ng_env = NailgunProtocol.ttynames_to_env(sys.stdin, sys.stdout.buffer,
sys.stderr.buffer)
modified_env = {
**self._env,
**ng_env,
"PANTSD_RUNTRACKER_CLIENT_START_TIME":
str(self._start_time),
"PANTSD_REQUEST_TIMEOUT_LIMIT":
str(global_options.pantsd_timeout_when_multiple_invocations),
}
# Instantiate a NailgunClient.
client = NailgunClient(
port=port,
remote_pid=pid,
ins=sys.stdin,
out=sys.stdout.buffer,
err=sys.stderr.buffer,
exit_on_broken_pipe=True,
metadata_base_dir=pantsd_handle.metadata_base_dir,
)
timeout = global_options.pantsd_pailgun_quit_timeout
pantsd_signal_handler = PailgunClientSignalHandler(client,
pid=pid,
timeout=timeout)
with ExceptionSink.trapped_signals(
pantsd_signal_handler), STTYSettings.preserved():
# Execute the command on the pailgun.
return client.execute(self._args[0], self._args[1:], modified_env)
def test_backup_logging_on_fatal_error(caplog):
sink = _gen_sink_subclass()
with temporary_dir() as tmpdir:
sink.reset_log_location(tmpdir)
with unittest.mock.patch.object(sink, "_try_write_with_flush", autospec=sink) as mock_write:
mock_write.side_effect = ExceptionSink.ExceptionSinkError("fake write failure")
sink._log_exception("XXX")
errors = [record for record in caplog.records if record.levelname == "ERROR"]
assert len(errors) == 2
def assert_log(log_file_type: str, log):
assert bool(
re.search(
fr"Error logging the message 'XXX' to the {log_file_type} file handle for .* at "
fr"pid {os.getpid()}",
log.msg,
)
)
assert log.filename == "exception_sink.py"
assert_log("pid-specific", errors[0])
assert_log("shared", errors[1])
def run(self):
self.scrub_pythonpath()
options_bootstrapper = OptionsBootstrapper.create(env=self._env,
args=self._args)
bootstrap_options = options_bootstrapper.bootstrap_options
global_bootstrap_options = bootstrap_options.for_global_scope()
# Initialize the workdir early enough to ensure that logging has a destination.
workdir_src = init_workdir(global_bootstrap_options)
ExceptionSink.reset_log_location(workdir_src)
# We enable Rust logging here,
# and everything before it will be routed through regular Python logging.
self._enable_rust_logging(global_bootstrap_options)
ExceptionSink.reset_should_print_backtrace_to_terminal(
global_bootstrap_options.print_exception_stacktrace)
ExceptionSink.reset_log_location(
global_bootstrap_options.pants_workdir)
# TODO https://github.com/pantsbuild/pants/issues/7205
if self._should_run_with_pantsd(global_bootstrap_options):
try:
return RemotePantsRunner(self._exiter, self._args, self._env,
options_bootstrapper).run()
except RemotePantsRunner.Fallback as e:
logger.warning(
"caught client exception: {!r}, falling back to non-daemon mode"
.format(e))
# N.B. Inlining this import speeds up the python thin client run by about 100ms.
from pants.bin.local_pants_runner import LocalPantsRunner
if self.will_terminate_pantsd():
logger.debug("Pantsd terminating goal detected: {}".format(
self._args))
runner = LocalPantsRunner.create(
self._args, self._env, options_bootstrapper=options_bootstrapper)
runner.set_start_time(self._start_time)
return runner.run()
def run(self):
# Register our exiter at the beginning of the run() method so that any code in this process from
# this point onwards will use that exiter in the case of a fatal error.
ExceptionSink.reset_exiter(self._exiter)
options_bootstrapper = OptionsBootstrapper.create(env=self._env, args=self._args)
bootstrap_options = options_bootstrapper.bootstrap_options
global_bootstrap_options = bootstrap_options.for_global_scope()
# We enable Rust logging here,
# and everything before it will be routed through regular Python logging.
self._enable_rust_logging(global_bootstrap_options)
ExceptionSink.reset_should_print_backtrace_to_terminal(global_bootstrap_options.print_exception_stacktrace)
ExceptionSink.reset_log_location(global_bootstrap_options.pants_workdir)
for message_regexp in global_bootstrap_options.ignore_pants_warnings:
warnings.filterwarnings(action='ignore', message=message_regexp)
if global_bootstrap_options.enable_pantsd:
try:
return RemotePantsRunner(self._exiter, self._args, self._env, options_bootstrapper).run()
except RemotePantsRunner.Fallback as e:
logger.warn('caught client exception: {!r}, falling back to non-daemon mode'.format(e))
# N.B. Inlining this import speeds up the python thin client run by about 100ms.
from pants.bin.local_pants_runner import LocalPantsRunner
runner = LocalPantsRunner.create(
self._exiter,
self._args,
self._env,
options_bootstrapper=options_bootstrapper
)
runner.set_start_time(self._start_time)
return runner.run()
def create_native_pantsd_file_log_handler(level, native, native_filename):
fd = native.setup_pantsd_logger(native_filename, get_numeric_level(level))
ExceptionSink.reset_interactive_output_stream(os.fdopen(os.dup(fd), "a"))
return NativeHandler(level, native, native_filename=native_filename)
def post_fork_child(self):
"""Post-fork child process callback executed via ProcessManager.daemonize()."""
# Set the Exiter exception hook post-fork so as not to affect the pantsd processes exception
# hook with socket-specific behavior. Note that this intentionally points the faulthandler
# trace stream to sys.stderr, which at this point is still a _LoggerStream object writing to
# the `pantsd.log`. This ensures that in the event of e.g. a hung but detached pantsd-runner
# process that the stacktrace output lands deterministically in a known place vs to a stray
# terminal window.
# TODO: test the above!
ExceptionSink.reset_exiter(self._exiter)
ExceptionSink.reset_interactive_output_stream(sys.stderr.buffer if PY3 else sys.stderr)
ExceptionSink.reset_signal_handler(DaemonSignalHandler())
# Ensure anything referencing sys.argv inherits the Pailgun'd args.
sys.argv = self._args
# Set context in the process title.
set_process_title('pantsd-runner [{}]'.format(' '.join(self._args)))
# Broadcast our process group ID (in PID form - i.e. negated) to the remote client so
# they can send signals (e.g. SIGINT) to all processes in the runners process group.
NailgunProtocol.send_pid(self._socket, os.getpid())
NailgunProtocol.send_pgrp(self._socket, os.getpgrp() * -1)
# Stop the services that were paused pre-fork.
for service in self._services.services:
service.terminate()
# Invoke a Pants run with stdio redirected and a proxied environment.
with self.nailgunned_stdio(self._socket, self._env) as finalizer,\
hermetic_environment_as(**self._env):
try:
# Setup the Exiter's finalizer.
self._exiter.set_finalizer(finalizer)
# Clean global state.
clean_global_runtime_state(reset_subsystem=True)
# Otherwise, conduct a normal run.
runner = LocalPantsRunner.create(
self._exiter,
self._args,
self._env,
self._target_roots,
self._graph_helper,
self._options_bootstrapper
)
runner.set_start_time(self._maybe_get_client_start_time_from_env(self._env))
# Re-raise any deferred exceptions, if present.
self._raise_deferred_exc()
runner.run()
except KeyboardInterrupt:
self._exiter.exit_and_fail('Interrupted by user.\n')
except _GracefulTerminationException as e:
ExceptionSink.log_exception(
'Encountered graceful termination exception {}; exiting'.format(e))
self._exiter.exit(e.exit_code)
except Exception:
# TODO: We override sys.excepthook above when we call ExceptionSink.set_exiter(). That
# excepthook catches `SignalHandledNonLocalExit`s from signal handlers, which isn't
# happening here, so something is probably overriding the excepthook. By catching Exception
# and calling this method, we emulate the normal, expected sys.excepthook override.
ExceptionSink._log_unhandled_exception_and_exit()
else:
self._exiter.exit(PANTS_SUCCEEDED_EXIT_CODE)
def _log_exception(self, msg): ExceptionSink.log_exception(msg)
def create_native_pantsd_file_log_handler(level, native, native_filename): fd = native.setup_pantsd_logger(native_filename, get_numeric_level(level)) ExceptionSink.reset_interactive_output_stream(os.fdopen(os.dup(fd), 'a')) return NativeHandler(level, native, native_filename=native_filename)
def wrap_global_exiter(cls, run_tracker, repro):
with ExceptionSink.exiter_as(lambda previous_exiter: cls(run_tracker, repro, previous_exiter)):
yield
def run(self, request: InteractiveProcess) -> InteractiveProcessResult:
ExceptionSink.toggle_ignoring_sigint_v2_engine(True)
return self._scheduler.run_local_interactive_process(request)
def override_global_exiter(cls, maybe_shutdown_socket: MaybeShutdownSocket,
finalizer: Callable[[], None]) -> None:
with ExceptionSink.exiter_as(lambda previous_exiter: cls(
maybe_shutdown_socket, finalizer, previous_exiter)):
yield
async def run_lifecycle_stubs(opts: LifecycleStubsSubsystem) -> LifecycleStubsGoal:
output_file = opts.options.new_interactive_stream_output_file
if output_file:
file_stream = open(output_file, "wb")
ExceptionSink.reset_interactive_output_stream(file_stream, output_file)
raise Exception("erroneous!")
def run(self, request: InteractiveProcess) -> InteractiveProcessResult:
if request.forward_signals_to_process:
with ExceptionSink.ignoring_sigint():
return self._scheduler.run_local_interactive_process(request)
return self._scheduler.run_local_interactive_process(request)
def create_native_pantsd_file_log_handler(
log_level: LogLevel, native: Native,
native_filename: str) -> NativeHandler:
fd = native.setup_pantsd_logger(native_filename, log_level.level)
ExceptionSink.reset_interactive_output_stream(os.fdopen(os.dup(fd), "a"))
return NativeHandler(log_level, native, native_filename=native_filename)
def run(self):
# Ensure anything referencing sys.argv inherits the Pailgun'd args.
sys.argv = self.args
# Invoke a Pants run with stdio redirected and a proxied environment.
with self.nailgunned_stdio(
self.maybe_shutdown_socket,
self.env) as finalizer, DaemonExiter.override_global_exiter(
self.maybe_shutdown_socket,
finalizer), hermetic_environment_as(**self.env):
exit_code = PANTS_SUCCEEDED_EXIT_CODE
try:
# Clean global state.
clean_global_runtime_state(reset_subsystem=True)
options_bootstrapper = OptionsBootstrapper.create(
args=self.args, env=self.env)
options, build_config = LocalPantsRunner.parse_options(
options_bootstrapper)
global_options = options.for_global_scope()
session = self.scheduler_service.prepare_graph(options)
specs = SpecsCalculator.create(
options=options,
session=session.scheduler_session,
exclude_patterns=tuple(
global_options.exclude_target_regexp),
tags=tuple(global_options.tag) if global_options.tag else
(),
)
if options.help_request:
help_printer = HelpPrinter(
options=options,
union_membership=UnionMembership(
build_config.union_rules()),
)
exit_code = help_printer.print_help()
else:
exit_code = self.scheduler_service.graph_run_v2(
session, specs, options, options_bootstrapper)
# self.scheduler_service.graph_run_v2 will already run v2 or ambiguous goals. We should
# only enter this code path if v1 is set.
if global_options.v1:
with ExceptionSink.exiter_as_until_exception(
lambda _: PantsRunFailCheckerExiter()):
runner = LocalPantsRunner.create(
self.env, options_bootstrapper, specs, session)
env_start_time = self.env.pop(
"PANTSD_RUNTRACKER_CLIENT_START_TIME", None)
start_time = float(
env_start_time) if env_start_time else None
runner.set_start_time(start_time)
runner.run()
except KeyboardInterrupt:
self._exiter.exit_and_fail("Interrupted by user.\n")
except _PantsRunFinishedWithFailureException as e:
ExceptionSink.log_exception(
"Pants run failed with exception: {}; exiting".format(e))
self._exiter.exit(e.exit_code)
except Exception as e:
# TODO: We override sys.excepthook above when we call ExceptionSink.set_exiter(). That
# excepthook catches `SignalHandledNonLocalExit`s from signal handlers, which isn't
# happening here, so something is probably overriding the excepthook. By catching Exception
# and calling this method, we emulate the normal, expected sys.excepthook override.
ExceptionSink._log_unhandled_exception_and_exit(exc=e)
else:
self._exiter.exit(exit_code)
