def run(self):
# Ensure anything referencing sys.argv inherits the Pailgun'd args.
sys.argv = self._args
# Broadcast our process group ID (in PID form - i.e. negated) to the remote client so
# they can send signals (e.g. SIGINT) to all processes in the runners process group.
with self._maybe_shutdown_socket.lock:
NailgunProtocol.send_pid(self._maybe_shutdown_socket.socket,
os.getpid())
NailgunProtocol.send_pgrp(self._maybe_shutdown_socket.socket,
os.getpgrp() * -1)
# Invoke a Pants run with stdio redirected and a proxied environment.
with self.nailgunned_stdio(
self._maybe_shutdown_socket,
self._env) as finalizer, DaemonExiter.override_global_exiter(
self._maybe_shutdown_socket,
finalizer), hermetic_environment_as(
**self._env), encapsulated_global_logger():
try:
# Clean global state.
clean_global_runtime_state(reset_subsystem=True)
options, _, options_bootstrapper = LocalPantsRunner.parse_options(
self._args, self._env)
graph_helper, specs, exit_code = self._scheduler_service.prepare_v1_graph_run_v2(
options,
options_bootstrapper,
)
self.exit_code = exit_code
# Otherwise, conduct a normal run.
with ExceptionSink.exiter_as_until_exception(
lambda _: PantsRunFailCheckerExiter()):
runner = LocalPantsRunner.create(
self._args,
self._env,
specs,
graph_helper,
options_bootstrapper,
)
runner.set_start_time(
self._maybe_get_client_start_time_from_env(self._env))
runner.run()
except KeyboardInterrupt:
self._exiter.exit_and_fail("Interrupted by user.\n")
except _PantsRunFinishedWithFailureException as e:
ExceptionSink.log_exception(
"Pants run failed with exception: {}; exiting".format(e))
self._exiter.exit(e.exit_code)
except Exception as e:
# TODO: We override sys.excepthook above when we call ExceptionSink.set_exiter(). That
# excepthook catches `SignalHandledNonLocalExit`s from signal handlers, which isn't
# happening here, so something is probably overriding the excepthook. By catching Exception
# and calling this method, we emulate the normal, expected sys.excepthook override.
ExceptionSink._log_unhandled_exception_and_exit(exc=e)
else:
self._exiter.exit(self.exit_code if self.
exit_code else PANTS_SUCCEEDED_EXIT_CODE)
def create(cls, sock, args, env, services, scheduler_service):
try:
# N.B. This will temporarily redirect stdio in the daemon's pre-fork context
# to the nailgun session. We'll later do this a second time post-fork, because
# threads.
with cls.nailgunned_stdio(sock, env, handle_stdin=False):
options, _, options_bootstrapper = LocalPantsRunner.parse_options(args, env)
subprocess_dir = options.for_global_scope().pants_subprocessdir
graph_helper, target_roots, exit_code = scheduler_service.prefork(options, options_bootstrapper)
deferred_exc = None if exit_code == PANTS_SUCCEEDED_EXIT_CODE else _GracefulTerminationException(exit_code)
except Exception:
deferred_exc = sys.exc_info()
graph_helper = None
target_roots = None
options_bootstrapper = None
# N.B. This will be overridden with the correct value if options
# parsing is successful - otherwise it permits us to run just far
# enough to raise the deferred exception.
subprocess_dir = os.path.join(get_buildroot(), '.pids')
return cls(
sock,
args,
env,
graph_helper,
target_roots,
services,
subprocess_dir,
options_bootstrapper,
deferred_exc
)
def create(cls, sock, args, env, services, scheduler_service):
maybe_shutdown_socket = MaybeShutdownSocket(sock)
try:
# N.B. This will redirect stdio in the daemon's context to the nailgun session.
with cls.nailgunned_stdio(maybe_shutdown_socket,
env,
handle_stdin=False) as finalizer:
options, _, options_bootstrapper = LocalPantsRunner.parse_options(
args, env)
subprocess_dir = options.for_global_scope().pants_subprocessdir
graph_helper, target_roots, exit_code = scheduler_service.prefork(
options, options_bootstrapper)
finalizer()
except Exception:
graph_helper = None
target_roots = None
options_bootstrapper = None
# TODO: this should no longer be necessary, remove the creation of subprocess_dir
subprocess_dir = os.path.join(get_buildroot(), '.pids')
exit_code = 1
# TODO This used to raise the _GracefulTerminationException, and maybe it should again, or notify in some way that the prefork has failed.
return cls(maybe_shutdown_socket, args, env, graph_helper,
target_roots, services, subprocess_dir,
options_bootstrapper, exit_code)
def run(self, start_time: float) -> ExitCode:
self.scrub_pythonpath()
options_bootstrapper = OptionsBootstrapper.create(
env=self.env, args=self.args, allow_pantsrc=True
)
with warnings.catch_warnings(record=True):
bootstrap_options = options_bootstrapper.bootstrap_options
global_bootstrap_options = bootstrap_options.for_global_scope()
# Initialize the workdir early enough to ensure that logging has a destination.
workdir_src = init_workdir(global_bootstrap_options)
ExceptionSink.reset_log_location(workdir_src)
setup_warning_filtering(global_bootstrap_options.ignore_pants_warnings or [])
# We enable logging here, and everything before it will be routed through regular
# Python logging.
setup_logging(global_bootstrap_options, stderr_logging=True)
if self._should_run_with_pantsd(global_bootstrap_options):
try:
remote_runner = RemotePantsRunner(self.args, self.env, options_bootstrapper)
return remote_runner.run()
except RemotePantsRunner.Fallback as e:
logger.warning("Client exception: {!r}, falling back to non-daemon mode".format(e))
# N.B. Inlining this import speeds up the python thin client run by about 100ms.
from pants.bin.local_pants_runner import LocalPantsRunner
runner = LocalPantsRunner.create(env=self.env, options_bootstrapper=options_bootstrapper)
return runner.run(start_time)
def _run(self,
is_remote,
exiter,
args,
env,
process_metadata_dir=None,
options_bootstrapper=None):
if is_remote:
try:
return RemotePantsRunner(exiter, args, env,
process_metadata_dir).run()
except RemotePantsRunner.RECOVERABLE_EXCEPTIONS as e:
# N.B. RemotePantsRunner will raise one of RECOVERABLE_EXCEPTIONS in the event we
# encounter a failure while discovering or initially connecting to the pailgun. In
# this case, we fall back to LocalPantsRunner which seamlessly executes the requested
# run and bootstraps pantsd for use in subsequent runs.
logger.debug(
'caught client exception: {!r}, falling back to LocalPantsRunner'
.format(e))
# N.B. Inlining this import speeds up the python thin client run by about 100ms.
from pants.bin.local_pants_runner import LocalPantsRunner
return LocalPantsRunner(
exiter, args, env,
options_bootstrapper=options_bootstrapper).run()
def post_fork_child(self):
"""Post-fork child process callback executed via ProcessManager.daemonize()."""
# Set the Exiter exception hook post-fork so as not to affect the pantsd processes exception
# hook with socket-specific behavior.
self._exiter.set_except_hook()
# Set context in the process title.
set_process_title('pantsd-runner [{}]'.format(' '.join(self._args)))
# Broadcast our pid to the remote client so they can send us signals (i.e. SIGINT).
NailgunProtocol.write_chunk(self._socket, ChunkType.PID,
bytes(os.getpid()))
# Setup a SIGINT signal handler.
self._setup_sigint_handler()
# Invoke a Pants run with stdio redirected.
with self._nailgunned_stdio(self._socket):
try:
# Clean global state.
clean_global_runtime_state(reset_subsystem=True)
# Re-raise any deferred exceptions, if present.
self._raise_deferred_exc()
# Otherwise, conduct a normal run.
LocalPantsRunner(self._exiter, self._args, self._env,
self._graph_helper).run()
except KeyboardInterrupt:
self._exiter.exit(1, msg='Interrupted by user.\n')
except Exception:
self._exiter.handle_unhandled_exception(add_newline=True)
else:
self._exiter.exit(0)
def run(self):
options_bootstrapper = OptionsBootstrapper(env=self._env,
args=self._args)
bootstrap_options = options_bootstrapper.get_bootstrap_options()
ExceptionSink.set_destination(
bootstrap_options.for_global_scope().pants_workdir)
if bootstrap_options.for_global_scope().enable_pantsd:
try:
return RemotePantsRunner(self._exiter, self._args, self._env,
bootstrap_options).run()
except RemotePantsRunner.Fallback as e:
logger.warn(
'caught client exception: {!r}, falling back to non-daemon mode'
.format(e))
# N.B. Inlining this import speeds up the python thin client run by about 100ms.
from pants.bin.local_pants_runner import LocalPantsRunner
runner = LocalPantsRunner.create(
self._exiter,
self._args,
self._env,
options_bootstrapper=options_bootstrapper)
runner.set_start_time(self._start_time)
return runner.run()
def create(cls, sock, args, env, services, scheduler_service):
try:
# N.B. This will temporarily redirect stdio in the daemon's pre-fork context
# to the nailgun session. We'll later do this a second time post-fork, because
# threads.
with cls.nailgunned_stdio(sock, env, handle_stdin=False):
options, _, options_bootstrapper = LocalPantsRunner.parse_options(
args, env)
subprocess_dir = options.for_global_scope().pants_subprocessdir
graph_helper, target_roots, exit_code = scheduler_service.prefork(
options, options_bootstrapper)
deferred_exc = None if exit_code == PANTS_SUCCEEDED_EXIT_CODE else _GracefulTerminationException(
exit_code)
except Exception:
deferred_exc = sys.exc_info()
graph_helper = None
target_roots = None
options_bootstrapper = None
# N.B. This will be overridden with the correct value if options
# parsing is successful - otherwise it permits us to run just far
# enough to raise the deferred exception.
subprocess_dir = os.path.join(get_buildroot(), '.pids')
return cls(sock, args, env, graph_helper, target_roots, services,
subprocess_dir, options_bootstrapper, deferred_exc)
def run(self):
# Register our exiter at the beginning of the run() method so that any code in this process from
# this point onwards will use that exiter in the case of a fatal error.
ExceptionSink.reset_exiter(self._exiter)
options_bootstrapper = OptionsBootstrapper(env=self._env,
args=self._args)
bootstrap_options = options_bootstrapper.get_bootstrap_options()
global_bootstrap_options = bootstrap_options.for_global_scope()
ExceptionSink.reset_should_print_backtrace_to_terminal(
global_bootstrap_options.print_exception_stacktrace)
ExceptionSink.reset_log_location(
global_bootstrap_options.pants_workdir)
if global_bootstrap_options.enable_pantsd:
try:
return RemotePantsRunner(self._exiter, self._args, self._env,
bootstrap_options).run()
except RemotePantsRunner.Fallback as e:
logger.warn(
'caught client exception: {!r}, falling back to non-daemon mode'
.format(e))
# N.B. Inlining this import speeds up the python thin client run by about 100ms.
from pants.bin.local_pants_runner import LocalPantsRunner
runner = LocalPantsRunner.create(
self._exiter,
self._args,
self._env,
options_bootstrapper=options_bootstrapper)
runner.set_start_time(self._start_time)
return runner.run()
def run(self, start_time: float) -> ExitCode:
self.scrub_pythonpath()
options_bootstrapper = OptionsBootstrapper.create(env=self.env,
args=self.args,
allow_pantsrc=True)
with warnings.catch_warnings(record=True):
bootstrap_options = options_bootstrapper.bootstrap_options
global_bootstrap_options = bootstrap_options.for_global_scope()
# We enable logging here, and everything before it will be routed through regular
# Python logging.
setup_logging(global_bootstrap_options, stderr_logging=True)
if self._should_run_with_pantsd(global_bootstrap_options):
try:
remote_runner = RemotePantsRunner(self.args, self.env,
options_bootstrapper)
return remote_runner.run()
except RemotePantsRunner.Fallback as e:
logger.warning(
"Client exception: {!r}, falling back to non-daemon mode".
format(e))
# N.B. Inlining this import speeds up the python thin client run by about 100ms.
from pants.bin.local_pants_runner import LocalPantsRunner
# We only install signal handling via ExceptionSink if the run will execute in this process.
ExceptionSink.install(
log_location=init_workdir(global_bootstrap_options),
pantsd_instance=False)
runner = LocalPantsRunner.create(
env=self.env, options_bootstrapper=options_bootstrapper)
return runner.run(start_time)
def post_fork_child(self):
"""Post-fork child process callback executed via ProcessManager.daemonize()."""
# Set the Exiter exception hook post-fork so as not to affect the pantsd processes exception
# hook with socket-specific behavior. Note that this intentionally points the faulthandler
# trace stream to sys.stderr, which at this point is still a _LoggerStream object writing to
# the `pantsd.log`. This ensures that in the event of e.g. a hung but detached pantsd-runner
# process that the stacktrace output lands deterministically in a known place vs to a stray
# terminal window.
self._exiter.set_except_hook(sys.stderr)
# Ensure anything referencing sys.argv inherits the Pailgun'd args.
sys.argv = self._args
# Set context in the process title.
set_process_title('pantsd-runner [{}]'.format(' '.join(self._args)))
# Broadcast our process group ID (in PID form - i.e. negated) to the remote client so
# they can send signals (e.g. SIGINT) to all processes in the runners process group.
NailgunProtocol.send_pid(self._socket, bytes(os.getpgrp() * -1))
# Setup a SIGINT signal handler.
self._setup_sigint_handler()
# Invoke a Pants run with stdio redirected and a proxied environment.
with self._nailgunned_stdio(self._socket) as finalizer, hermetic_environment_as(**self._env):
try:
# Setup the Exiter's finalizer.
self._exiter.set_finalizer(finalizer)
# Clean global state.
clean_global_runtime_state(reset_subsystem=True)
# Re-raise any deferred exceptions, if present.
self._raise_deferred_exc()
# Otherwise, conduct a normal run.
runner = LocalPantsRunner(self._exiter, self._args, self._env, self._graph_helper)
runner.set_preceding_graph_size(self._preceding_graph_size)
runner.run()
except KeyboardInterrupt:
self._exiter.exit(1, msg='Interrupted by user.\n')
except Exception:
self._exiter.handle_unhandled_exception(add_newline=True)
else:
self._exiter.exit(0)
def post_fork_child(self):
"""Post-fork child process callback executed via ProcessManager.daemonize()."""
# Set the Exiter exception hook post-fork so as not to affect the pantsd processes exception
# hook with socket-specific behavior. Note that this intentionally points the faulthandler
# trace stream to sys.stderr, which at this point is still a _LoggerStream object writing to
# the `pantsd.log`. This ensures that in the event of e.g. a hung but detached pantsd-runner
# process that the stacktrace output lands deterministically in a known place vs to a stray
# terminal window.
self._exiter.set_except_hook(sys.stderr)
# Ensure anything referencing sys.argv inherits the Pailgun'd args.
sys.argv = self._args
# Set context in the process title.
set_process_title('pantsd-runner [{}]'.format(' '.join(self._args)))
# Broadcast our process group ID (in PID form - i.e. negated) to the remote client so
# they can send signals (e.g. SIGINT) to all processes in the runners process group.
NailgunProtocol.send_pid(self._socket, bytes(os.getpgrp() * -1))
# Setup a SIGINT signal handler.
self._setup_sigint_handler()
# Invoke a Pants run with stdio redirected and a proxied environment.
with self._nailgunned_stdio(
self._socket) as finalizer, hermetic_environment_as(
**self._env):
try:
# Setup the Exiter's finalizer.
self._exiter.set_finalizer(finalizer)
# Clean global state.
clean_global_runtime_state(reset_subsystem=True)
# Re-raise any deferred exceptions, if present.
self._raise_deferred_exc()
# Otherwise, conduct a normal run.
runner = LocalPantsRunner(
self._exiter,
self._args,
self._env,
target_roots=self._target_roots,
daemon_build_graph=self._graph_helper)
runner.set_start_time(
self._maybe_get_client_start_time_from_env(self._env))
runner.set_preceding_graph_size(self._preceding_graph_size)
runner.run()
except KeyboardInterrupt:
self._exiter.exit(1, msg='Interrupted by user.\n')
except Exception:
self._exiter.handle_unhandled_exception(add_newline=True)
else:
self._exiter.exit(0)
def run(self):
# Ensure anything referencing sys.argv inherits the Pailgun'd args.
sys.argv = self._args
# Broadcast our process group ID (in PID form - i.e. negated) to the remote client so
# they can send signals (e.g. SIGINT) to all processes in the runners process group.
with self._maybe_shutdown_socket.lock:
NailgunProtocol.send_pid(self._maybe_shutdown_socket.socket, os.getpid())
NailgunProtocol.send_pgrp(self._maybe_shutdown_socket.socket, os.getpgrp() * -1)
# Invoke a Pants run with stdio redirected and a proxied environment.
with self.nailgunned_stdio(self._maybe_shutdown_socket, self._env) as finalizer, \
hermetic_environment_as(**self._env), \
encapsulated_global_logger():
try:
# Raise any exceptions we may have found when precomputing products.
# NB: We raise it here as opposed to earlier because we have setup logging and stdio.
if self._exception is not None:
raise self._exception
# Clean global state.
clean_global_runtime_state(reset_subsystem=True)
# Setup the Exiter's finalizer.
self._exiter.set_finalizer(finalizer)
# Otherwise, conduct a normal run.
runner = LocalPantsRunner.create(
PantsRunFailCheckerExiter(),
self._args,
self._env,
self._target_roots,
self._graph_helper,
self._options_bootstrapper,
)
runner.set_start_time(self._maybe_get_client_start_time_from_env(self._env))
runner.run()
except KeyboardInterrupt:
self._exiter.exit_and_fail('Interrupted by user.\n')
except _PantsRunFinishedWithFailureException as e:
ExceptionSink.log_exception(
'Pants run failed with exception: {}; exiting'.format(e))
self._exiter.exit(e.exit_code)
except _PantsProductPrecomputeFailed as e:
ExceptionSink.log_exception(repr(e))
self._exiter.exit(e.exit_code)
except Exception as e:
# TODO: We override sys.excepthook above when we call ExceptionSink.set_exiter(). That
# excepthook catches `SignalHandledNonLocalExit`s from signal handlers, which isn't
# happening here, so something is probably overriding the excepthook. By catching Exception
# and calling this method, we emulate the normal, expected sys.excepthook override.
ExceptionSink._log_unhandled_exception_and_exit(exc=e)
else:
self._exiter.exit(PANTS_SUCCEEDED_EXIT_CODE)
def run(self):
options_bootstrapper = OptionsBootstrapper(env=self._env, args=self._args)
bootstrap_options = options_bootstrapper.get_bootstrap_options()
if bootstrap_options.for_global_scope().enable_pantsd:
try:
return RemotePantsRunner(self._exiter, self._args, self._env, bootstrap_options).run()
except RemotePantsRunner.Fallback as e:
logger.debug('caught client exception: {!r}, falling back to non-daemon mode'.format(e))
# N.B. Inlining this import speeds up the python thin client run by about 100ms.
from pants.bin.local_pants_runner import LocalPantsRunner
runner = LocalPantsRunner(
self._exiter,
self._args,
self._env,
options_bootstrapper=options_bootstrapper
)
runner.set_start_time(self._start_time)
return runner.run()
def _run(self, is_remote, exiter, args, env, options_bootstrapper=None):
if is_remote:
try:
return RemotePantsRunner(exiter, args, env).run()
except RemotePantsRunner.RECOVERABLE_EXCEPTIONS as e:
# N.B. RemotePantsRunner will raise one of RECOVERABLE_EXCEPTIONS in the event we
# encounter a failure while discovering or initially connecting to the pailgun. In
# this case, we fall back to LocalPantsRunner which seamlessly executes the requested
# run and bootstraps pantsd for use in subsequent runs.
logger.debug(
'caught client exception: {!r}, falling back to LocalPantsRunner'
.format(e))
return LocalPantsRunner(
exiter, args, env,
options_bootstrapper=options_bootstrapper).run()
def single_daemonized_run(
self,
args: Tuple[str, ...],
env: Dict[str, str],
cancellation_latch: PySessionCancellationLatch,
) -> ExitCode:
"""Run a single daemonized run of Pants.
All aspects of the `sys` global should already have been replaced in `__call__`, so this
method should not need any special handling for the fact that it's running in a proxied
environment.
"""
try:
logger.debug("Connected to pantsd")
# Capture the client's start time, which we propagate here in order to get an accurate
# view of total time.
env_start_time = env.get("PANTSD_RUNTRACKER_CLIENT_START_TIME", None)
if not env_start_time:
# NB: We warn rather than erroring here because it eases use of non-Pants nailgun
# clients for testing.
logger.warning(
"No start time was reported by the client! Metrics may be inaccurate."
)
start_time = float(env_start_time) if env_start_time else time.time()
options_bootstrapper = OptionsBootstrapper.create(
env=env, args=args, allow_pantsrc=True
)
# Run using the pre-warmed Session.
complete_env = CompleteEnvironment(env)
scheduler, options_initializer = self._core.prepare(options_bootstrapper, complete_env)
runner = LocalPantsRunner.create(
complete_env,
options_bootstrapper,
scheduler=scheduler,
options_initializer=options_initializer,
cancellation_latch=cancellation_latch,
)
return runner.run(start_time)
except Exception as e:
logger.exception(e)
return PANTS_FAILED_EXIT_CODE
except KeyboardInterrupt:
print("Interrupted by user.\n", file=sys.stderr)
return PANTS_FAILED_EXIT_CODE
def run(self):
# Register our exiter at the beginning of the run() method so that any code in this process from
# this point onwards will use that exiter in the case of a fatal error.
ExceptionSink.reset_exiter(self._exiter)
options_bootstrapper = OptionsBootstrapper.create(env=self._env,
args=self._args)
bootstrap_options = options_bootstrapper.bootstrap_options
global_bootstrap_options = bootstrap_options.for_global_scope()
# We enable Rust logging here,
# and everything before it will be routed through regular Python logging.
self._enable_rust_logging(global_bootstrap_options)
ExceptionSink.reset_should_print_backtrace_to_terminal(
global_bootstrap_options.print_exception_stacktrace)
ExceptionSink.reset_log_location(
global_bootstrap_options.pants_workdir)
for message_regexp in global_bootstrap_options.ignore_pants_warnings:
warnings.filterwarnings(action='ignore', message=message_regexp)
# TODO https://github.com/pantsbuild/pants/issues/7205
if self._should_run_with_pantsd(global_bootstrap_options):
try:
return RemotePantsRunner(self._exiter, self._args, self._env,
options_bootstrapper).run()
except RemotePantsRunner.Fallback as e:
logger.warn(
'caught client exception: {!r}, falling back to non-daemon mode'
.format(e))
# N.B. Inlining this import speeds up the python thin client run by about 100ms.
from pants.bin.local_pants_runner import LocalPantsRunner
if self.will_terminate_pantsd():
logger.debug("Pantsd terminating goal detected: {}".format(
self._args))
runner = LocalPantsRunner.create(
self._exiter,
self._args,
self._env,
options_bootstrapper=options_bootstrapper)
runner.set_start_time(self._start_time)
return runner.run()
def single_daemonized_run(
self, working_dir: str,
cancellation_latch: PySessionCancellationLatch) -> ExitCode:
"""Run a single daemonized run of Pants.
All aspects of the `sys` global should already have been replaced in `__call__`, so this
method should not need any special handling for the fact that it's running in a proxied
environment.
"""
# Capture the client's start time, which we propagate here in order to get an accurate
# view of total time.
env_start_time = os.environ.get("PANTSD_RUNTRACKER_CLIENT_START_TIME",
None)
start_time = float(env_start_time) if env_start_time else time.time()
# Clear global mutable state before entering `LocalPantsRunner`. Note that we use
# `sys.argv` and `os.environ`, since they have been mutated to maintain the illusion
# of a local run: once we allow for concurrent runs, this information should be
# propagated down from the caller.
# see https://github.com/pantsbuild/pants/issues/7654
BuildConfigInitializer.reset()
options_bootstrapper = OptionsBootstrapper.create(env=os.environ,
args=sys.argv,
allow_pantsrc=True)
bootstrap_options = options_bootstrapper.bootstrap_options
global_bootstrap_options = bootstrap_options.for_global_scope()
# Run using the pre-warmed Session.
with self._stderr_logging(global_bootstrap_options):
try:
scheduler = self._core.prepare_scheduler(options_bootstrapper)
runner = LocalPantsRunner.create(
os.environ,
options_bootstrapper,
scheduler=scheduler,
cancellation_latch=cancellation_latch,
)
return runner.run(start_time)
except Exception as e:
logger.exception(e)
return PANTS_FAILED_EXIT_CODE
except KeyboardInterrupt:
print("Interrupted by user.\n", file=sys.stderr)
return PANTS_FAILED_EXIT_CODE
def run(self, start_time: float) -> ExitCode:
self.scrub_pythonpath()
options_bootstrapper = OptionsBootstrapper.create(env=self.env,
args=self.args,
allow_pantsrc=True)
with warnings.catch_warnings(record=True):
bootstrap_options = options_bootstrapper.bootstrap_options
global_bootstrap_options = bootstrap_options.for_global_scope()
# We enable logging here, and everything before it will be routed through regular
# Python logging.
stdin_fileno = sys.stdin.fileno()
stdout_fileno = sys.stdout.fileno()
stderr_fileno = sys.stderr.fileno()
with initialize_stdio(global_bootstrap_options), stdio_destination(
stdin_fileno=stdin_fileno,
stdout_fileno=stdout_fileno,
stderr_fileno=stderr_fileno,
):
# N.B. We inline imports to speed up the python thin client run, and avoids importing
# engine types until after the runner has had a chance to set PANTS_BIN_NAME.
if self._should_run_with_pantsd(global_bootstrap_options):
from pants.bin.remote_pants_runner import RemotePantsRunner
try:
remote_runner = RemotePantsRunner(self.args, self.env,
options_bootstrapper)
return remote_runner.run(start_time)
except RemotePantsRunner.Fallback as e:
logger.warning(
f"Client exception: {e!r}, falling back to non-daemon mode"
)
from pants.bin.local_pants_runner import LocalPantsRunner
# We only install signal handling via ExceptionSink if the run will execute in this process.
ExceptionSink.install(
log_location=init_workdir(global_bootstrap_options),
pantsd_instance=False)
runner = LocalPantsRunner.create(
env=CompleteEnvironment(self.env),
options_bootstrapper=options_bootstrapper)
return runner.run(start_time)
def run(self):
self.scrub_pythonpath()
options_bootstrapper = OptionsBootstrapper.create(env=self._env,
args=self._args)
bootstrap_options = options_bootstrapper.bootstrap_options
global_bootstrap_options = bootstrap_options.for_global_scope()
# Initialize the workdir early enough to ensure that logging has a destination.
workdir_src = init_workdir(global_bootstrap_options)
ExceptionSink.reset_log_location(workdir_src)
# We enable Rust logging here,
# and everything before it will be routed through regular Python logging.
self._enable_rust_logging(global_bootstrap_options)
ExceptionSink.reset_should_print_backtrace_to_terminal(
global_bootstrap_options.print_exception_stacktrace)
ExceptionSink.reset_log_location(
global_bootstrap_options.pants_workdir)
# TODO https://github.com/pantsbuild/pants/issues/7205
if self._should_run_with_pantsd(global_bootstrap_options):
try:
return RemotePantsRunner(self._exiter, self._args, self._env,
options_bootstrapper).run()
except RemotePantsRunner.Fallback as e:
logger.warning(
"caught client exception: {!r}, falling back to non-daemon mode"
.format(e))
# N.B. Inlining this import speeds up the python thin client run by about 100ms.
from pants.bin.local_pants_runner import LocalPantsRunner
if self.will_terminate_pantsd():
logger.debug("Pantsd terminating goal detected: {}".format(
self._args))
runner = LocalPantsRunner.create(
self._args, self._env, options_bootstrapper=options_bootstrapper)
runner.set_start_time(self._start_time)
return runner.run()
def run(self, start_time: float) -> ExitCode:
self.scrub_pythonpath()
options_bootstrapper = OptionsBootstrapper.create(env=self.env, args=self.args)
bootstrap_options = options_bootstrapper.bootstrap_options
global_bootstrap_options = bootstrap_options.for_global_scope()
# Initialize the workdir early enough to ensure that logging has a destination.
workdir_src = init_workdir(global_bootstrap_options)
ExceptionSink.reset_log_location(workdir_src)
# We enable logging here, and everything before it will be routed through regular
# Python logging.
setup_logging(global_bootstrap_options)
ExceptionSink.reset_should_print_backtrace_to_terminal(
global_bootstrap_options.print_exception_stacktrace
)
# TODO: When we remove this deprecation, we'll change the default for the option to true.
deprecated_conditional(
lambda: global_bootstrap_options.is_default("enable_pantsd"),
removal_version="1.30.0.dev0",
entity_description="--enable-pantsd defaulting to False",
hint_message=(
"Pantsd improves runtime performance and will be enabled by default in the 1.30.x "
"stable releases. To prepare for that change, we recommend setting the "
"`[GLOBAL] enable_pantsd` setting to `True` in your pants.toml or pants.ini file."
),
)
if self._should_run_with_pantsd(global_bootstrap_options):
try:
return RemotePantsRunner(self.args, self.env, options_bootstrapper).run(start_time)
except RemotePantsRunner.Fallback as e:
logger.warning("Client exception: {!r}, falling back to non-daemon mode".format(e))
# N.B. Inlining this import speeds up the python thin client run by about 100ms.
from pants.bin.local_pants_runner import LocalPantsRunner
runner = LocalPantsRunner.create(env=self.env, options_bootstrapper=options_bootstrapper)
return runner.run(start_time)
def post_fork_child(self):
"""Post-fork child process callback executed via ProcessManager.daemonize()."""
# Set the Exiter exception hook post-fork so as not to affect the pantsd processes exception
# hook with socket-specific behavior. Note that this intentionally points the faulthandler
# trace stream to sys.stderr, which at this point is still a _LoggerStream object writing to
# the `pantsd.log`. This ensures that in the event of e.g. a hung but detached pantsd-runner
# process that the stacktrace output lands deterministically in a known place vs to a stray
# terminal window.
self._exiter.set_except_hook(sys.stderr)
# Ensure anything referencing sys.argv inherits the Pailgun'd args.
sys.argv = self._args
# Set context in the process title.
set_process_title('pantsd-runner [{}]'.format(' '.join(self._args)))
# Broadcast our pid to the remote client so they can send us signals (i.e. SIGINT).
NailgunProtocol.write_chunk(self._socket, ChunkType.PID,
bytes(os.getpid()))
# Setup a SIGINT signal handler.
self._setup_sigint_handler()
# Invoke a Pants run with stdio redirected.
with self._nailgunned_stdio(self._socket):
try:
# Clean global state.
clean_global_runtime_state(reset_subsystem=True)
# Re-raise any deferred exceptions, if present.
self._raise_deferred_exc()
# Otherwise, conduct a normal run.
LocalPantsRunner(self._exiter, self._args, self._env,
self._graph_helper).run()
except KeyboardInterrupt:
self._exiter.exit(1, msg='Interrupted by user.\n')
except Exception:
self._exiter.handle_unhandled_exception(add_newline=True)
else:
self._exiter.exit(0)
def create(cls, sock, args, env, services, scheduler_service):
maybe_shutdown_socket = MaybeShutdownSocket(sock)
exception = None
exit_code = PANTS_SUCCEEDED_EXIT_CODE
# TODO(#8002) This can probably be moved to the try:except block in DaemonPantsRunner.run() function,
# Making exception handling a lot easier.
try:
# N.B. This will redirect stdio in the daemon's context to the nailgun session.
with cls.nailgunned_stdio(maybe_shutdown_socket, env, handle_stdin=False) as finalizer:
options, _, options_bootstrapper = LocalPantsRunner.parse_options(args, env)
subprocess_dir = options.for_global_scope().pants_subprocessdir
graph_helper, target_roots, exit_code = scheduler_service.prefork(options, options_bootstrapper)
finalizer()
except Exception as e:
graph_helper = None
target_roots = None
options_bootstrapper = None
# TODO: this should no longer be necessary, remove the creation of subprocess_dir
subprocess_dir = os.path.join(get_buildroot(), '.pids')
exception = _PantsProductPrecomputeFailed(e)
# NB: If a scheduler_service.prefork finishes with a non-0 exit code but doesn't raise an exception
# (e.g. ./pants list-and-die-for-testing ...). We still want to know about it.
if exception is None and exit_code != PANTS_SUCCEEDED_EXIT_CODE:
exception = _PantsProductPrecomputeFailed(
_PantsRunFinishedWithFailureException(exit_code=exit_code)
)
return cls(
maybe_shutdown_socket,
args,
env,
graph_helper,
target_roots,
services,
subprocess_dir,
options_bootstrapper,
exception
)
def run(self):
options_bootstrapper = OptionsBootstrapper(env=self._env,
args=self._args)
bootstrap_options = options_bootstrapper.get_bootstrap_options()
if bootstrap_options.for_global_scope().enable_pantsd:
try:
return RemotePantsRunner(self._exiter, self._args, self._env,
bootstrap_options).run()
except RemotePantsRunner.Fallback as e:
logger.debug(
'caught client exception: {!r}, falling back to non-daemon mode'
.format(e))
# N.B. Inlining this import speeds up the python thin client run by about 100ms.
from pants.bin.local_pants_runner import LocalPantsRunner
return LocalPantsRunner(
self._exiter,
self._args,
self._env,
options_bootstrapper=options_bootstrapper).run()
def run(self, start_time: float) -> None:
self.scrub_pythonpath()
# TODO could options-bootstrapper be parsed in the runners?
options_bootstrapper = OptionsBootstrapper.create(env=self.env,
args=self.args)
bootstrap_options = options_bootstrapper.bootstrap_options
global_bootstrap_options = bootstrap_options.for_global_scope()
# Initialize the workdir early enough to ensure that logging has a destination.
workdir_src = init_workdir(global_bootstrap_options)
ExceptionSink.reset_log_location(workdir_src)
# We enable Rust logging here,
# and everything before it will be routed through regular Python logging.
self._enable_rust_logging(global_bootstrap_options)
ExceptionSink.reset_should_print_backtrace_to_terminal(
global_bootstrap_options.print_exception_stacktrace)
if self._should_run_with_pantsd(global_bootstrap_options):
try:
RemotePantsRunner(self._exiter, self.args, self.env,
options_bootstrapper).run()
return
except RemotePantsRunner.Fallback as e:
logger.warning(
"Client exception: {!r}, falling back to non-daemon mode".
format(e))
# N.B. Inlining this import speeds up the python thin client run by about 100ms.
from pants.bin.local_pants_runner import LocalPantsRunner
runner = LocalPantsRunner.create(
env=self.env, options_bootstrapper=options_bootstrapper)
runner.set_start_time(start_time)
runner.run()
def run(self):
# Register our exiter at the beginning of the run() method so that any code in this process from
# this point onwards will use that exiter in the case of a fatal error.
ExceptionSink.reset_exiter(self._exiter)
options_bootstrapper = OptionsBootstrapper.create(env=self._env, args=self._args)
bootstrap_options = options_bootstrapper.bootstrap_options
global_bootstrap_options = bootstrap_options.for_global_scope()
# We enable Rust logging here,
# and everything before it will be routed through regular Python logging.
self._enable_rust_logging(global_bootstrap_options)
ExceptionSink.reset_should_print_backtrace_to_terminal(global_bootstrap_options.print_exception_stacktrace)
ExceptionSink.reset_log_location(global_bootstrap_options.pants_workdir)
for message_regexp in global_bootstrap_options.ignore_pants_warnings:
warnings.filterwarnings(action='ignore', message=message_regexp)
if global_bootstrap_options.enable_pantsd:
try:
return RemotePantsRunner(self._exiter, self._args, self._env, options_bootstrapper).run()
except RemotePantsRunner.Fallback as e:
logger.warn('caught client exception: {!r}, falling back to non-daemon mode'.format(e))
# N.B. Inlining this import speeds up the python thin client run by about 100ms.
from pants.bin.local_pants_runner import LocalPantsRunner
runner = LocalPantsRunner.create(
self._exiter,
self._args,
self._env,
options_bootstrapper=options_bootstrapper
)
runner.set_start_time(self._start_time)
return runner.run()
def post_fork_child(self):
"""Post-fork child process callback executed via ProcessManager.daemonize()."""
# Set the Exiter exception hook post-fork so as not to affect the pantsd processes exception
# hook with socket-specific behavior. Note that this intentionally points the faulthandler
# trace stream to sys.stderr, which at this point is still a _LoggerStream object writing to
# the `pantsd.log`. This ensures that in the event of e.g. a hung but detached pantsd-runner
# process that the stacktrace output lands deterministically in a known place vs to a stray
# terminal window.
# TODO: test the above!
ExceptionSink.reset_exiter(self._exiter)
ExceptionSink.reset_interactive_output_stream(
sys.stderr.buffer if PY3 else sys.stderr)
ExceptionSink.reset_signal_handler(DaemonSignalHandler())
# Ensure anything referencing sys.argv inherits the Pailgun'd args.
sys.argv = self._args
# Set context in the process title.
set_process_title('pantsd-runner [{}]'.format(' '.join(self._args)))
# Broadcast our process group ID (in PID form - i.e. negated) to the remote client so
# they can send signals (e.g. SIGINT) to all processes in the runners process group.
NailgunProtocol.send_pid(self._socket, os.getpid())
NailgunProtocol.send_pgrp(self._socket, os.getpgrp() * -1)
# Stop the services that were paused pre-fork.
for service in self._services.services:
service.terminate()
# Invoke a Pants run with stdio redirected and a proxied environment.
with self.nailgunned_stdio(self._socket, self._env) as finalizer,\
hermetic_environment_as(**self._env):
try:
# Setup the Exiter's finalizer.
self._exiter.set_finalizer(finalizer)
# Clean global state.
clean_global_runtime_state(reset_subsystem=True)
# Otherwise, conduct a normal run.
runner = LocalPantsRunner.create(self._exiter, self._args,
self._env, self._target_roots,
self._graph_helper,
self._options_bootstrapper)
runner.set_start_time(
self._maybe_get_client_start_time_from_env(self._env))
# Re-raise any deferred exceptions, if present.
self._raise_deferred_exc()
runner.run()
except KeyboardInterrupt:
self._exiter.exit_and_fail('Interrupted by user.\n')
except _GracefulTerminationException as e:
ExceptionSink.log_exception(
'Encountered graceful termination exception {}; exiting'.
format(e))
self._exiter.exit(e.exit_code)
except Exception:
# TODO: We override sys.excepthook above when we call ExceptionSink.set_exiter(). That
# excepthook catches `SignalHandledNonLocalExit`s from signal handlers, which isn't
# happening here, so something is probably overriding the excepthook. By catching Exception
# and calling this method, we emulate the normal, expected sys.excepthook override.
ExceptionSink._log_unhandled_exception_and_exit()
else:
self._exiter.exit(PANTS_SUCCEEDED_EXIT_CODE)
def run(self):
# Ensure anything referencing sys.argv inherits the Pailgun'd args.
sys.argv = self.args
# Invoke a Pants run with stdio redirected and a proxied environment.
with self.nailgunned_stdio(
self.maybe_shutdown_socket,
self.env) as finalizer, DaemonExiter.override_global_exiter(
self.maybe_shutdown_socket,
finalizer), hermetic_environment_as(**self.env):
exit_code = PANTS_SUCCEEDED_EXIT_CODE
try:
# Clean global state.
clean_global_runtime_state(reset_subsystem=True)
options_bootstrapper = OptionsBootstrapper.create(
args=self.args, env=self.env)
options, build_config = LocalPantsRunner.parse_options(
options_bootstrapper)
global_options = options.for_global_scope()
session = self.scheduler_service.prepare_graph(options)
specs = SpecsCalculator.create(
options=options,
session=session.scheduler_session,
exclude_patterns=tuple(
global_options.exclude_target_regexp),
tags=tuple(global_options.tag) if global_options.tag else
(),
)
if options.help_request:
help_printer = HelpPrinter(
options=options,
union_membership=UnionMembership(
build_config.union_rules()),
)
exit_code = help_printer.print_help()
else:
exit_code = self.scheduler_service.graph_run_v2(
session, specs, options, options_bootstrapper)
# self.scheduler_service.graph_run_v2 will already run v2 or ambiguous goals. We should
# only enter this code path if v1 is set.
if global_options.v1:
with ExceptionSink.exiter_as_until_exception(
lambda _: PantsRunFailCheckerExiter()):
runner = LocalPantsRunner.create(
self.env, options_bootstrapper, specs, session)
env_start_time = self.env.pop(
"PANTSD_RUNTRACKER_CLIENT_START_TIME", None)
start_time = float(
env_start_time) if env_start_time else None
runner.set_start_time(start_time)
runner.run()
except KeyboardInterrupt:
self._exiter.exit_and_fail("Interrupted by user.\n")
except _PantsRunFinishedWithFailureException as e:
ExceptionSink.log_exception(
"Pants run failed with exception: {}; exiting".format(e))
self._exiter.exit(e.exit_code)
except Exception as e:
# TODO: We override sys.excepthook above when we call ExceptionSink.set_exiter(). That
# excepthook catches `SignalHandledNonLocalExit`s from signal handlers, which isn't
# happening here, so something is probably overriding the excepthook. By catching Exception
# and calling this method, we emulate the normal, expected sys.excepthook override.
ExceptionSink._log_unhandled_exception_and_exit(exc=e)
else:
self._exiter.exit(exit_code)
def post_fork_child(self):
"""Post-fork child process callback executed via ProcessManager.daemonize()."""
# Set the Exiter exception hook post-fork so as not to affect the pantsd processes exception
# hook with socket-specific behavior. Note that this intentionally points the faulthandler
# trace stream to sys.stderr, which at this point is still a _LoggerStream object writing to
# the `pantsd.log`. This ensures that in the event of e.g. a hung but detached pantsd-runner
# process that the stacktrace output lands deterministically in a known place vs to a stray
# terminal window.
# TODO: test the above!
ExceptionSink.reset_exiter(self._exiter)
ExceptionSink.reset_interactive_output_stream(sys.stderr.buffer if PY3 else sys.stderr)
ExceptionSink.reset_signal_handler(DaemonSignalHandler())
# Ensure anything referencing sys.argv inherits the Pailgun'd args.
sys.argv = self._args
# Set context in the process title.
set_process_title('pantsd-runner [{}]'.format(' '.join(self._args)))
# Broadcast our process group ID (in PID form - i.e. negated) to the remote client so
# they can send signals (e.g. SIGINT) to all processes in the runners process group.
NailgunProtocol.send_pid(self._socket, os.getpid())
NailgunProtocol.send_pgrp(self._socket, os.getpgrp() * -1)
# Stop the services that were paused pre-fork.
for service in self._services.services:
service.terminate()
# Invoke a Pants run with stdio redirected and a proxied environment.
with self.nailgunned_stdio(self._socket, self._env) as finalizer,\
hermetic_environment_as(**self._env):
try:
# Setup the Exiter's finalizer.
self._exiter.set_finalizer(finalizer)
# Clean global state.
clean_global_runtime_state(reset_subsystem=True)
# Otherwise, conduct a normal run.
runner = LocalPantsRunner.create(
self._exiter,
self._args,
self._env,
self._target_roots,
self._graph_helper,
self._options_bootstrapper
)
runner.set_start_time(self._maybe_get_client_start_time_from_env(self._env))
# Re-raise any deferred exceptions, if present.
self._raise_deferred_exc()
runner.run()
except KeyboardInterrupt:
self._exiter.exit_and_fail('Interrupted by user.\n')
except _GracefulTerminationException as e:
ExceptionSink.log_exception(
'Encountered graceful termination exception {}; exiting'.format(e))
self._exiter.exit(e.exit_code)
except Exception:
# TODO: We override sys.excepthook above when we call ExceptionSink.set_exiter(). That
# excepthook catches `SignalHandledNonLocalExit`s from signal handlers, which isn't
# happening here, so something is probably overriding the excepthook. By catching Exception
# and calling this method, we emulate the normal, expected sys.excepthook override.
ExceptionSink._log_unhandled_exception_and_exit()
else:
self._exiter.exit(PANTS_SUCCEEDED_EXIT_CODE)