def run(self, start_time: float) -> ExitCode:
run_tracker = RunTracker.global_instance()
self._start_run(run_tracker, start_time)
with maybe_profiled(self.profile_path):
global_options = self.options.for_global_scope()
if self.options.help_request:
return self._print_help(self.options.help_request)
streaming_handlers = global_options.streaming_workunits_handlers
callbacks = Subsystem.get_streaming_workunit_callbacks(
streaming_handlers)
streaming_reporter = StreamingWorkunitHandler(
self.graph_session.scheduler_session,
callbacks=callbacks,
report_interval_seconds=global_options.
streaming_workunits_report_interval,
)
goals = tuple(self.options.goals)
with streaming_reporter.session():
engine_result = PANTS_FAILED_EXIT_CODE
try:
engine_result = self._run_v2(goals)
except Exception as e:
ExceptionSink.log_exception(e)
self._finish_run(run_tracker, engine_result)
return engine_result
def run(self, start_time: float) -> ExitCode:
spec_parser = SpecsParser(get_buildroot())
specs = [str(spec_parser.parse_spec(spec)) for spec in self.options.specs]
self.run_tracker.start(run_start_time=start_time, specs=specs)
with maybe_profiled(self.profile_path):
global_options = self.options.for_global_scope()
if self.options.help_request:
return self._print_help(self.options.help_request)
streaming_reporter = StreamingWorkunitHandler(
self.graph_session.scheduler_session,
run_tracker=self.run_tracker,
callbacks=self._get_workunits_callbacks(),
report_interval_seconds=global_options.streaming_workunits_report_interval,
)
goals = tuple(self.options.goals)
with streaming_reporter.session():
if not goals:
return PANTS_SUCCEEDED_EXIT_CODE
engine_result = PANTS_FAILED_EXIT_CODE
try:
engine_result = self._perform_run(goals)
except Exception as e:
ExceptionSink.log_exception(e)
metrics = self.graph_session.scheduler_session.metrics()
self.run_tracker.set_pantsd_scheduler_metrics(metrics)
self.run_tracker.end_run(engine_result)
return engine_result
def run(self, start_time: float) -> ExitCode:
self._set_start_time(start_time)
with maybe_profiled(self.profile_path):
global_options = self.options.for_global_scope()
streaming_handlers = global_options.streaming_workunits_handlers
report_interval = global_options.streaming_workunits_report_interval
callbacks = Subsystem.get_streaming_workunit_callbacks(
streaming_handlers)
streaming_reporter = StreamingWorkunitHandler(
self.graph_session.scheduler_session,
callbacks=callbacks,
report_interval_seconds=report_interval,
)
if self.options.help_request:
help_printer = HelpPrinter(
options=self.options,
union_membership=self.union_membership)
return help_printer.print_help()
v1 = global_options.v1
v2 = global_options.v2
with streaming_reporter.session():
engine_result, goal_runner_result = PANTS_FAILED_EXIT_CODE, PANTS_FAILED_EXIT_CODE
try:
engine_result = self._maybe_run_v2(v2)
goal_runner_result = self._maybe_run_v1(v1)
except Exception as e:
ExceptionSink.log_exception(e)
run_tracker_result = self._finish_run(
self._merge_exit_codes(engine_result, goal_runner_result))
return self._merge_exit_codes(engine_result, goal_runner_result,
run_tracker_result)
def run(self):
# Ensure anything referencing sys.argv inherits the Pailgun'd args.
sys.argv = self._args
# Broadcast our process group ID (in PID form - i.e. negated) to the remote client so
# they can send signals (e.g. SIGINT) to all processes in the runners process group.
with self._maybe_shutdown_socket.lock:
NailgunProtocol.send_pid(self._maybe_shutdown_socket.socket,
os.getpid())
NailgunProtocol.send_pgrp(self._maybe_shutdown_socket.socket,
os.getpgrp() * -1)
# Invoke a Pants run with stdio redirected and a proxied environment.
with self.nailgunned_stdio(
self._maybe_shutdown_socket,
self._env) as finalizer, DaemonExiter.override_global_exiter(
self._maybe_shutdown_socket,
finalizer), hermetic_environment_as(
**self._env), encapsulated_global_logger():
try:
# Clean global state.
clean_global_runtime_state(reset_subsystem=True)
options, _, options_bootstrapper = LocalPantsRunner.parse_options(
self._args, self._env)
graph_helper, specs, exit_code = self._scheduler_service.prepare_v1_graph_run_v2(
options,
options_bootstrapper,
)
self.exit_code = exit_code
# Otherwise, conduct a normal run.
with ExceptionSink.exiter_as_until_exception(
lambda _: PantsRunFailCheckerExiter()):
runner = LocalPantsRunner.create(
self._args,
self._env,
specs,
graph_helper,
options_bootstrapper,
)
runner.set_start_time(
self._maybe_get_client_start_time_from_env(self._env))
runner.run()
except KeyboardInterrupt:
self._exiter.exit_and_fail("Interrupted by user.\n")
except _PantsRunFinishedWithFailureException as e:
ExceptionSink.log_exception(
"Pants run failed with exception: {}; exiting".format(e))
self._exiter.exit(e.exit_code)
except Exception as e:
# TODO: We override sys.excepthook above when we call ExceptionSink.set_exiter(). That
# excepthook catches `SignalHandledNonLocalExit`s from signal handlers, which isn't
# happening here, so something is probably overriding the excepthook. By catching Exception
# and calling this method, we emulate the normal, expected sys.excepthook override.
ExceptionSink._log_unhandled_exception_and_exit(exc=e)
else:
self._exiter.exit(self.exit_code if self.
exit_code else PANTS_SUCCEEDED_EXIT_CODE)
def run(self):
# Ensure anything referencing sys.argv inherits the Pailgun'd args.
sys.argv = self._args
# Broadcast our process group ID (in PID form - i.e. negated) to the remote client so
# they can send signals (e.g. SIGINT) to all processes in the runners process group.
with self._maybe_shutdown_socket.lock:
NailgunProtocol.send_pid(self._maybe_shutdown_socket.socket, os.getpid())
NailgunProtocol.send_pgrp(self._maybe_shutdown_socket.socket, os.getpgrp() * -1)
# Invoke a Pants run with stdio redirected and a proxied environment.
with self.nailgunned_stdio(self._maybe_shutdown_socket, self._env) as finalizer, \
hermetic_environment_as(**self._env), \
encapsulated_global_logger():
try:
# Raise any exceptions we may have found when precomputing products.
# NB: We raise it here as opposed to earlier because we have setup logging and stdio.
if self._exception is not None:
raise self._exception
# Clean global state.
clean_global_runtime_state(reset_subsystem=True)
# Setup the Exiter's finalizer.
self._exiter.set_finalizer(finalizer)
# Otherwise, conduct a normal run.
runner = LocalPantsRunner.create(
PantsRunFailCheckerExiter(),
self._args,
self._env,
self._target_roots,
self._graph_helper,
self._options_bootstrapper,
)
runner.set_start_time(self._maybe_get_client_start_time_from_env(self._env))
runner.run()
except KeyboardInterrupt:
self._exiter.exit_and_fail('Interrupted by user.\n')
except _PantsRunFinishedWithFailureException as e:
ExceptionSink.log_exception(
'Pants run failed with exception: {}; exiting'.format(e))
self._exiter.exit(e.exit_code)
except _PantsProductPrecomputeFailed as e:
ExceptionSink.log_exception(repr(e))
self._exiter.exit(e.exit_code)
except Exception as e:
# TODO: We override sys.excepthook above when we call ExceptionSink.set_exiter(). That
# excepthook catches `SignalHandledNonLocalExit`s from signal handlers, which isn't
# happening here, so something is probably overriding the excepthook. By catching Exception
# and calling this method, we emulate the normal, expected sys.excepthook override.
ExceptionSink._log_unhandled_exception_and_exit(exc=e)
else:
self._exiter.exit(PANTS_SUCCEEDED_EXIT_CODE)
def _finish_run(self, code: ExitCode) -> ExitCode:
"""Checks that the RunTracker is in good shape to exit, and then returns its exit code.
TODO: The RunTracker's exit code will likely not be relevant in v2: the exit codes of
individual `@goal_rule`s are everything in that case.
"""
run_tracker_result = PANTS_SUCCEEDED_EXIT_CODE
# These strings are prepended to the existing exit message when calling the superclass .exit().
additional_messages = []
try:
self._update_stats()
if code == PANTS_SUCCEEDED_EXIT_CODE:
outcome = WorkUnit.SUCCESS
elif code == PANTS_FAILED_EXIT_CODE:
outcome = WorkUnit.FAILURE
else:
run_tracker_msg = (
"unrecognized exit code {} provided to {}.exit() -- "
"interpreting as a failure in the run tracker".format(
code,
type(self).__name__))
# Log the unrecognized exit code to the fatal exception log.
ExceptionSink.log_exception(exc=Exception(run_tracker_msg))
# Ensure the unrecognized exit code message is also logged to the terminal.
additional_messages.append(run_tracker_msg)
outcome = WorkUnit.FAILURE
self._run_tracker.set_root_outcome(outcome)
run_tracker_result = self._run_tracker.end()
except ValueError as e:
# If we have been interrupted by a signal, calling .end() sometimes writes to a closed file,
# so we just log that fact here and keep going.
ExceptionSink.log_exception(exc=e)
finally:
if self._repro:
# TODO: Have Repro capture the 'after' state (as a diff) as well? (in reference to the below
# 'before' state comment)
# NB: this writes to the logger, which is expected to still be alive if we are exiting from
# a signal.
self._repro.log_location_of_repro_file()
if additional_messages:
# NB: We do not log to the exceptions log in this case, because we expect that these are
# higher level unstructed errors: strutured versions will already have been written at
# various places.
logger.error("\n".join(additional_messages))
return run_tracker_result
def _run_inner(self) -> ExitCode:
goals = tuple(self.options.goals)
if self.options.help_request:
return self._print_help(self.options.help_request)
if not goals:
return PANTS_SUCCEEDED_EXIT_CODE
try:
return self._perform_run(goals)
except Exception as e:
ExceptionSink.log_exception(e)
return PANTS_FAILED_EXIT_CODE
except KeyboardInterrupt:
print("Interrupted by user.\n", file=sys.stderr)
return PANTS_FAILED_EXIT_CODE
def exit(self,
result=PANTS_SUCCEEDED_EXIT_CODE,
msg=None,
*args,
**kwargs):
# These strings are prepended to the existing exit message when calling the superclass .exit().
additional_messages = []
try:
if not self._run_tracker.has_ended():
if result == PANTS_SUCCEEDED_EXIT_CODE:
outcome = WorkUnit.SUCCESS
elif result == PANTS_FAILED_EXIT_CODE:
outcome = WorkUnit.FAILURE
else:
run_tracker_msg = (
"unrecognized exit code {} provided to {}.exit() -- "
"interpreting as a failure in the run tracker".format(
result,
type(self).__name__))
# Log the unrecognized exit code to the fatal exception log.
ExceptionSink.log_exception(run_tracker_msg)
# Ensure the unrecognized exit code message is also logged to the terminal.
additional_messages.append(run_tracker_msg)
outcome = WorkUnit.FAILURE
self._run_tracker.set_root_outcome(outcome)
run_tracker_result = self._run_tracker.end()
assert (
result == run_tracker_result
), "pants exit code not correctly recorded by run tracker"
except ValueError as e:
# If we have been interrupted by a signal, calling .end() sometimes writes to a closed file,
# so we just log that fact here and keep going.
exception_string = str(e)
ExceptionSink.log_exception(exception_string)
additional_messages.append(exception_string)
finally:
if self._repro:
# TODO: Have Repro capture the 'after' state (as a diff) as well? (in reference to the below
# 'before' state comment)
# NB: this writes to the logger, which is expected to still be alive if we are exiting from
# a signal.
self._repro.log_location_of_repro_file()
if additional_messages:
msg = "{}\n\n{}".format("\n".join(additional_messages), msg or "")
super().exit(result=result, msg=msg, *args, **kwargs)
def exit(self, result=PANTS_SUCCEEDED_EXIT_CODE, msg=None, *args, **kwargs):
# These strings are prepended to the existing exit message when calling the superclass .exit().
additional_messages = []
try:
if not self._run_tracker.has_ended():
if result == PANTS_SUCCEEDED_EXIT_CODE:
outcome = WorkUnit.SUCCESS
elif result == PANTS_FAILED_EXIT_CODE:
outcome = WorkUnit.FAILURE
else:
run_tracker_msg = ("unrecognized exit code {} provided to {}.exit() -- "
"interpreting as a failure in the run tracker"
.format(result, type(self).__name__))
# Log the unrecognized exit code to the fatal exception log.
ExceptionSink.log_exception(run_tracker_msg)
# Ensure the unrecognized exit code message is also logged to the terminal.
additional_messages.push(run_tracker_msg)
outcome = WorkUnit.FAILURE
self._run_tracker.set_root_outcome(outcome)
run_tracker_result = self._run_tracker.end()
assert result == run_tracker_result, "pants exit code not correctly recorded by run tracker"
except ValueError as e:
# If we have been interrupted by a signal, calling .end() sometimes writes to a closed file,
# so we just log that fact here and keep going.
exception_string = str(e)
ExceptionSink.log_exception(exception_string)
additional_messages.push(exception_string)
finally:
if self._repro:
# TODO: Have Repro capture the 'after' state (as a diff) as well? (in reference to the below
# 'before' state comment)
# NB: this writes to the logger, which is expected to still be alive if we are exiting from
# a signal.
self._repro.log_location_of_repro_file()
if additional_messages:
msg = '{}\n\n{}'.format('\n'.join(additional_messages),
msg or '')
super(LocalExiter, self).exit(result=result, msg=msg, *args, **kwargs)
def _finish_run(self, code: ExitCode) -> ExitCode:
"""Checks that the RunTracker is in good shape to exit, and then returns its exit code.
TODO: The RunTracker's exit code will likely not be relevant in v2: the exit codes of
individual `@goal_rule`s are everything in that case.
"""
run_tracker_result = PANTS_SUCCEEDED_EXIT_CODE
scheduler_session = self.graph_session.scheduler_session
try:
metrics = scheduler_session.metrics()
self._run_tracker.pantsd_stats.set_scheduler_metrics(metrics)
outcome = WorkUnit.SUCCESS if code == PANTS_SUCCEEDED_EXIT_CODE else WorkUnit.FAILURE
self._run_tracker.set_root_outcome(outcome)
run_tracker_result = self._run_tracker.end()
except ValueError as e:
# If we have been interrupted by a signal, calling .end() sometimes writes to a closed
# file, so we just log that fact here and keep going.
ExceptionSink.log_exception(exc=e)
return run_tracker_result
def run(self, start_time: float) -> ExitCode:
self._set_start_time(start_time)
with maybe_profiled(self.profile_path):
global_options = self.options.for_global_scope()
streaming_handlers = global_options.streaming_workunits_handlers
report_interval = global_options.streaming_workunits_report_interval
callbacks = Subsystem.get_streaming_workunit_callbacks(
streaming_handlers)
streaming_reporter = StreamingWorkunitHandler(
self.graph_session.scheduler_session,
callbacks=callbacks,
report_interval_seconds=report_interval,
)
if self.options.help_request:
all_help_info = HelpInfoExtracter.get_all_help_info(
self.options,
self.union_membership,
self.graph_session.goal_consumed_subsystem_scopes,
)
help_printer = HelpPrinter(
bin_name=global_options.pants_bin_name,
help_request=self.options.help_request,
all_help_info=all_help_info,
use_color=global_options.colors,
)
return help_printer.print_help()
with streaming_reporter.session():
engine_result = PANTS_FAILED_EXIT_CODE
try:
engine_result = self._run_v2()
except Exception as e:
ExceptionSink.log_exception(e)
run_tracker_result = self._finish_run(engine_result)
return self._merge_exit_codes(engine_result, run_tracker_result)
def _log_exception(self, msg): ExceptionSink.log_exception(msg)
def post_fork_child(self):
"""Post-fork child process callback executed via ProcessManager.daemonize()."""
# Set the Exiter exception hook post-fork so as not to affect the pantsd processes exception
# hook with socket-specific behavior. Note that this intentionally points the faulthandler
# trace stream to sys.stderr, which at this point is still a _LoggerStream object writing to
# the `pantsd.log`. This ensures that in the event of e.g. a hung but detached pantsd-runner
# process that the stacktrace output lands deterministically in a known place vs to a stray
# terminal window.
# TODO: test the above!
ExceptionSink.reset_exiter(self._exiter)
ExceptionSink.reset_interactive_output_stream(
sys.stderr.buffer if PY3 else sys.stderr)
ExceptionSink.reset_signal_handler(DaemonSignalHandler())
# Ensure anything referencing sys.argv inherits the Pailgun'd args.
sys.argv = self._args
# Set context in the process title.
set_process_title('pantsd-runner [{}]'.format(' '.join(self._args)))
# Broadcast our process group ID (in PID form - i.e. negated) to the remote client so
# they can send signals (e.g. SIGINT) to all processes in the runners process group.
NailgunProtocol.send_pid(self._socket, os.getpid())
NailgunProtocol.send_pgrp(self._socket, os.getpgrp() * -1)
# Stop the services that were paused pre-fork.
for service in self._services.services:
service.terminate()
# Invoke a Pants run with stdio redirected and a proxied environment.
with self.nailgunned_stdio(self._socket, self._env) as finalizer,\
hermetic_environment_as(**self._env):
try:
# Setup the Exiter's finalizer.
self._exiter.set_finalizer(finalizer)
# Clean global state.
clean_global_runtime_state(reset_subsystem=True)
# Otherwise, conduct a normal run.
runner = LocalPantsRunner.create(self._exiter, self._args,
self._env, self._target_roots,
self._graph_helper,
self._options_bootstrapper)
runner.set_start_time(
self._maybe_get_client_start_time_from_env(self._env))
# Re-raise any deferred exceptions, if present.
self._raise_deferred_exc()
runner.run()
except KeyboardInterrupt:
self._exiter.exit_and_fail('Interrupted by user.\n')
except _GracefulTerminationException as e:
ExceptionSink.log_exception(
'Encountered graceful termination exception {}; exiting'.
format(e))
self._exiter.exit(e.exit_code)
except Exception:
# TODO: We override sys.excepthook above when we call ExceptionSink.set_exiter(). That
# excepthook catches `SignalHandledNonLocalExit`s from signal handlers, which isn't
# happening here, so something is probably overriding the excepthook. By catching Exception
# and calling this method, we emulate the normal, expected sys.excepthook override.
ExceptionSink._log_unhandled_exception_and_exit()
else:
self._exiter.exit(PANTS_SUCCEEDED_EXIT_CODE)
def run(self):
# Ensure anything referencing sys.argv inherits the Pailgun'd args.
sys.argv = self.args
# Invoke a Pants run with stdio redirected and a proxied environment.
with self.nailgunned_stdio(
self.maybe_shutdown_socket,
self.env) as finalizer, DaemonExiter.override_global_exiter(
self.maybe_shutdown_socket,
finalizer), hermetic_environment_as(**self.env):
exit_code = PANTS_SUCCEEDED_EXIT_CODE
try:
# Clean global state.
clean_global_runtime_state(reset_subsystem=True)
options_bootstrapper = OptionsBootstrapper.create(
args=self.args, env=self.env)
options, build_config = LocalPantsRunner.parse_options(
options_bootstrapper)
global_options = options.for_global_scope()
session = self.scheduler_service.prepare_graph(options)
specs = SpecsCalculator.create(
options=options,
session=session.scheduler_session,
exclude_patterns=tuple(
global_options.exclude_target_regexp),
tags=tuple(global_options.tag) if global_options.tag else
(),
)
if options.help_request:
help_printer = HelpPrinter(
options=options,
union_membership=UnionMembership(
build_config.union_rules()),
)
exit_code = help_printer.print_help()
else:
exit_code = self.scheduler_service.graph_run_v2(
session, specs, options, options_bootstrapper)
# self.scheduler_service.graph_run_v2 will already run v2 or ambiguous goals. We should
# only enter this code path if v1 is set.
if global_options.v1:
with ExceptionSink.exiter_as_until_exception(
lambda _: PantsRunFailCheckerExiter()):
runner = LocalPantsRunner.create(
self.env, options_bootstrapper, specs, session)
env_start_time = self.env.pop(
"PANTSD_RUNTRACKER_CLIENT_START_TIME", None)
start_time = float(
env_start_time) if env_start_time else None
runner.set_start_time(start_time)
runner.run()
except KeyboardInterrupt:
self._exiter.exit_and_fail("Interrupted by user.\n")
except _PantsRunFinishedWithFailureException as e:
ExceptionSink.log_exception(
"Pants run failed with exception: {}; exiting".format(e))
self._exiter.exit(e.exit_code)
except Exception as e:
# TODO: We override sys.excepthook above when we call ExceptionSink.set_exiter(). That
# excepthook catches `SignalHandledNonLocalExit`s from signal handlers, which isn't
# happening here, so something is probably overriding the excepthook. By catching Exception
# and calling this method, we emulate the normal, expected sys.excepthook override.
ExceptionSink._log_unhandled_exception_and_exit(exc=e)
else:
self._exiter.exit(exit_code)
def post_fork_child(self):
"""Post-fork child process callback executed via ProcessManager.daemonize()."""
# Set the Exiter exception hook post-fork so as not to affect the pantsd processes exception
# hook with socket-specific behavior. Note that this intentionally points the faulthandler
# trace stream to sys.stderr, which at this point is still a _LoggerStream object writing to
# the `pantsd.log`. This ensures that in the event of e.g. a hung but detached pantsd-runner
# process that the stacktrace output lands deterministically in a known place vs to a stray
# terminal window.
# TODO: test the above!
ExceptionSink.reset_exiter(self._exiter)
ExceptionSink.reset_interactive_output_stream(sys.stderr.buffer if PY3 else sys.stderr)
ExceptionSink.reset_signal_handler(DaemonSignalHandler())
# Ensure anything referencing sys.argv inherits the Pailgun'd args.
sys.argv = self._args
# Set context in the process title.
set_process_title('pantsd-runner [{}]'.format(' '.join(self._args)))
# Broadcast our process group ID (in PID form - i.e. negated) to the remote client so
# they can send signals (e.g. SIGINT) to all processes in the runners process group.
NailgunProtocol.send_pid(self._socket, os.getpid())
NailgunProtocol.send_pgrp(self._socket, os.getpgrp() * -1)
# Stop the services that were paused pre-fork.
for service in self._services.services:
service.terminate()
# Invoke a Pants run with stdio redirected and a proxied environment.
with self.nailgunned_stdio(self._socket, self._env) as finalizer,\
hermetic_environment_as(**self._env):
try:
# Setup the Exiter's finalizer.
self._exiter.set_finalizer(finalizer)
# Clean global state.
clean_global_runtime_state(reset_subsystem=True)
# Otherwise, conduct a normal run.
runner = LocalPantsRunner.create(
self._exiter,
self._args,
self._env,
self._target_roots,
self._graph_helper,
self._options_bootstrapper
)
runner.set_start_time(self._maybe_get_client_start_time_from_env(self._env))
# Re-raise any deferred exceptions, if present.
self._raise_deferred_exc()
runner.run()
except KeyboardInterrupt:
self._exiter.exit_and_fail('Interrupted by user.\n')
except _GracefulTerminationException as e:
ExceptionSink.log_exception(
'Encountered graceful termination exception {}; exiting'.format(e))
self._exiter.exit(e.exit_code)
except Exception:
# TODO: We override sys.excepthook above when we call ExceptionSink.set_exiter(). That
# excepthook catches `SignalHandledNonLocalExit`s from signal handlers, which isn't
# happening here, so something is probably overriding the excepthook. By catching Exception
# and calling this method, we emulate the normal, expected sys.excepthook override.
ExceptionSink._log_unhandled_exception_and_exit()
else:
self._exiter.exit(PANTS_SUCCEEDED_EXIT_CODE)
