Beispiel #1
0
    def run(self):
        # Ensure anything referencing sys.argv inherits the Pailgun'd args.
        sys.argv = self._args

        # Broadcast our process group ID (in PID form - i.e. negated) to the remote client so
        # they can send signals (e.g. SIGINT) to all processes in the runners process group.
        with self._maybe_shutdown_socket.lock:
            NailgunProtocol.send_pid(self._maybe_shutdown_socket.socket,
                                     os.getpid())
            NailgunProtocol.send_pgrp(self._maybe_shutdown_socket.socket,
                                      os.getpgrp() * -1)

        # Invoke a Pants run with stdio redirected and a proxied environment.
        with self.nailgunned_stdio(
                self._maybe_shutdown_socket,
                self._env) as finalizer, DaemonExiter.override_global_exiter(
                    self._maybe_shutdown_socket,
                    finalizer), hermetic_environment_as(
                        **self._env), encapsulated_global_logger():
            try:
                # Clean global state.
                clean_global_runtime_state(reset_subsystem=True)

                options, _, options_bootstrapper = LocalPantsRunner.parse_options(
                    self._args, self._env)
                graph_helper, specs, exit_code = self._scheduler_service.prepare_v1_graph_run_v2(
                    options,
                    options_bootstrapper,
                )
                self.exit_code = exit_code

                # Otherwise, conduct a normal run.
                with ExceptionSink.exiter_as_until_exception(
                        lambda _: PantsRunFailCheckerExiter()):
                    runner = LocalPantsRunner.create(
                        self._args,
                        self._env,
                        specs,
                        graph_helper,
                        options_bootstrapper,
                    )
                    runner.set_start_time(
                        self._maybe_get_client_start_time_from_env(self._env))

                    runner.run()
            except KeyboardInterrupt:
                self._exiter.exit_and_fail("Interrupted by user.\n")
            except _PantsRunFinishedWithFailureException as e:
                ExceptionSink.log_exception(
                    "Pants run failed with exception: {}; exiting".format(e))
                self._exiter.exit(e.exit_code)
            except Exception as e:
                # TODO: We override sys.excepthook above when we call ExceptionSink.set_exiter(). That
                # excepthook catches `SignalHandledNonLocalExit`s from signal handlers, which isn't
                # happening here, so something is probably overriding the excepthook. By catching Exception
                # and calling this method, we emulate the normal, expected sys.excepthook override.
                ExceptionSink._log_unhandled_exception_and_exit(exc=e)
            else:
                self._exiter.exit(self.exit_code if self.
                                  exit_code else PANTS_SUCCEEDED_EXIT_CODE)
Beispiel #2
0
    def run(self):
        options_bootstrapper = OptionsBootstrapper(env=self._env,
                                                   args=self._args)
        bootstrap_options = options_bootstrapper.get_bootstrap_options()
        ExceptionSink.set_destination(
            bootstrap_options.for_global_scope().pants_workdir)

        if bootstrap_options.for_global_scope().enable_pantsd:
            try:
                return RemotePantsRunner(self._exiter, self._args, self._env,
                                         bootstrap_options).run()
            except RemotePantsRunner.Fallback as e:
                logger.warn(
                    'caught client exception: {!r}, falling back to non-daemon mode'
                    .format(e))

        # N.B. Inlining this import speeds up the python thin client run by about 100ms.
        from pants.bin.local_pants_runner import LocalPantsRunner

        runner = LocalPantsRunner.create(
            self._exiter,
            self._args,
            self._env,
            options_bootstrapper=options_bootstrapper)
        runner.set_start_time(self._start_time)
        return runner.run()
Beispiel #3
0
    def run(self, start_time: float) -> ExitCode:
        self.scrub_pythonpath()

        options_bootstrapper = OptionsBootstrapper.create(
            env=self.env, args=self.args, allow_pantsrc=True
        )
        with warnings.catch_warnings(record=True):
            bootstrap_options = options_bootstrapper.bootstrap_options
            global_bootstrap_options = bootstrap_options.for_global_scope()

        # Initialize the workdir early enough to ensure that logging has a destination.
        workdir_src = init_workdir(global_bootstrap_options)
        ExceptionSink.reset_log_location(workdir_src)

        setup_warning_filtering(global_bootstrap_options.ignore_pants_warnings or [])
        # We enable logging here, and everything before it will be routed through regular
        # Python logging.
        setup_logging(global_bootstrap_options, stderr_logging=True)

        if self._should_run_with_pantsd(global_bootstrap_options):
            try:
                remote_runner = RemotePantsRunner(self.args, self.env, options_bootstrapper)
                return remote_runner.run()
            except RemotePantsRunner.Fallback as e:
                logger.warning("Client exception: {!r}, falling back to non-daemon mode".format(e))

        # N.B. Inlining this import speeds up the python thin client run by about 100ms.
        from pants.bin.local_pants_runner import LocalPantsRunner

        runner = LocalPantsRunner.create(env=self.env, options_bootstrapper=options_bootstrapper)
        return runner.run(start_time)
Beispiel #4
0
    def run(self, start_time: float) -> ExitCode:
        self.scrub_pythonpath()

        options_bootstrapper = OptionsBootstrapper.create(env=self.env,
                                                          args=self.args,
                                                          allow_pantsrc=True)
        with warnings.catch_warnings(record=True):
            bootstrap_options = options_bootstrapper.bootstrap_options
            global_bootstrap_options = bootstrap_options.for_global_scope()

        # We enable logging here, and everything before it will be routed through regular
        # Python logging.
        setup_logging(global_bootstrap_options, stderr_logging=True)

        if self._should_run_with_pantsd(global_bootstrap_options):
            try:
                remote_runner = RemotePantsRunner(self.args, self.env,
                                                  options_bootstrapper)
                return remote_runner.run()
            except RemotePantsRunner.Fallback as e:
                logger.warning(
                    "Client exception: {!r}, falling back to non-daemon mode".
                    format(e))

        # N.B. Inlining this import speeds up the python thin client run by about 100ms.
        from pants.bin.local_pants_runner import LocalPantsRunner

        # We only install signal handling via ExceptionSink if the run will execute in this process.
        ExceptionSink.install(
            log_location=init_workdir(global_bootstrap_options),
            pantsd_instance=False)
        runner = LocalPantsRunner.create(
            env=self.env, options_bootstrapper=options_bootstrapper)
        return runner.run(start_time)
Beispiel #5
0
    def run(self):
        # Register our exiter at the beginning of the run() method so that any code in this process from
        # this point onwards will use that exiter in the case of a fatal error.
        ExceptionSink.reset_exiter(self._exiter)

        options_bootstrapper = OptionsBootstrapper(env=self._env,
                                                   args=self._args)
        bootstrap_options = options_bootstrapper.get_bootstrap_options()
        global_bootstrap_options = bootstrap_options.for_global_scope()

        ExceptionSink.reset_should_print_backtrace_to_terminal(
            global_bootstrap_options.print_exception_stacktrace)
        ExceptionSink.reset_log_location(
            global_bootstrap_options.pants_workdir)

        if global_bootstrap_options.enable_pantsd:
            try:
                return RemotePantsRunner(self._exiter, self._args, self._env,
                                         bootstrap_options).run()
            except RemotePantsRunner.Fallback as e:
                logger.warn(
                    'caught client exception: {!r}, falling back to non-daemon mode'
                    .format(e))

        # N.B. Inlining this import speeds up the python thin client run by about 100ms.
        from pants.bin.local_pants_runner import LocalPantsRunner

        runner = LocalPantsRunner.create(
            self._exiter,
            self._args,
            self._env,
            options_bootstrapper=options_bootstrapper)
        runner.set_start_time(self._start_time)
        return runner.run()
Beispiel #6
0
    def post_fork_child(self):
        """Post-fork child process callback executed via ProcessManager.daemonize()."""
        # Set the Exiter exception hook post-fork so as not to affect the pantsd processes exception
        # hook with socket-specific behavior. Note that this intentionally points the faulthandler
        # trace stream to sys.stderr, which at this point is still a _LoggerStream object writing to
        # the `pantsd.log`. This ensures that in the event of e.g. a hung but detached pantsd-runner
        # process that the stacktrace output lands deterministically in a known place vs to a stray
        # terminal window.
        # TODO: test the above!
        ExceptionSink.reset_exiter(self._exiter)

        ExceptionSink.reset_interactive_output_stream(
            sys.stderr.buffer if PY3 else sys.stderr)

        # Ensure anything referencing sys.argv inherits the Pailgun'd args.
        sys.argv = self._args

        # Set context in the process title.
        set_process_title('pantsd-runner [{}]'.format(' '.join(self._args)))

        # Broadcast our process group ID (in PID form - i.e. negated) to the remote client so
        # they can send signals (e.g. SIGINT) to all processes in the runners process group.
        NailgunProtocol.send_pid(self._socket, os.getpid())
        NailgunProtocol.send_pgrp(self._socket, os.getpgrp() * -1)

        # Stop the services that were paused pre-fork.
        for service in self._services.services:
            service.terminate()

        # Invoke a Pants run with stdio redirected and a proxied environment.
        with self.nailgunned_stdio(self._socket, self._env) as finalizer,\
             hermetic_environment_as(**self._env):
            try:
                # Setup the Exiter's finalizer.
                self._exiter.set_finalizer(finalizer)

                # Clean global state.
                clean_global_runtime_state(reset_subsystem=True)

                # Re-raise any deferred exceptions, if present.
                self._raise_deferred_exc()

                # Otherwise, conduct a normal run.
                runner = LocalPantsRunner.create(self._exiter, self._args,
                                                 self._env, self._target_roots,
                                                 self._graph_helper,
                                                 self._options_bootstrapper)
                runner.set_start_time(
                    self._maybe_get_client_start_time_from_env(self._env))
                runner.run()
            except KeyboardInterrupt:
                self._exiter.exit_and_fail('Interrupted by user.\n')
            except Exception:
                ExceptionSink._log_unhandled_exception_and_exit()
            else:
                self._exiter.exit(0)
  def run(self):
    # Ensure anything referencing sys.argv inherits the Pailgun'd args.
    sys.argv = self._args

    # Broadcast our process group ID (in PID form - i.e. negated) to the remote client so
    # they can send signals (e.g. SIGINT) to all processes in the runners process group.
    with self._maybe_shutdown_socket.lock:
      NailgunProtocol.send_pid(self._maybe_shutdown_socket.socket, os.getpid())
      NailgunProtocol.send_pgrp(self._maybe_shutdown_socket.socket, os.getpgrp() * -1)

    # Invoke a Pants run with stdio redirected and a proxied environment.
    with self.nailgunned_stdio(self._maybe_shutdown_socket, self._env) as finalizer, \
      hermetic_environment_as(**self._env), \
      encapsulated_global_logger():
      try:
        # Raise any exceptions we may have found when precomputing products.
        # NB: We raise it here as opposed to earlier because we have setup logging and stdio.
        if self._exception is not None:
          raise self._exception

        # Clean global state.
        clean_global_runtime_state(reset_subsystem=True)

        # Setup the Exiter's finalizer.
        self._exiter.set_finalizer(finalizer)

        # Otherwise, conduct a normal run.
        runner = LocalPantsRunner.create(
          PantsRunFailCheckerExiter(),
          self._args,
          self._env,
          self._target_roots,
          self._graph_helper,
          self._options_bootstrapper,
        )
        runner.set_start_time(self._maybe_get_client_start_time_from_env(self._env))

        runner.run()
      except KeyboardInterrupt:
        self._exiter.exit_and_fail('Interrupted by user.\n')
      except _PantsRunFinishedWithFailureException as e:
        ExceptionSink.log_exception(
          'Pants run failed with exception: {}; exiting'.format(e))
        self._exiter.exit(e.exit_code)
      except _PantsProductPrecomputeFailed as e:
        ExceptionSink.log_exception(repr(e))
        self._exiter.exit(e.exit_code)
      except Exception as e:
        # TODO: We override sys.excepthook above when we call ExceptionSink.set_exiter(). That
        # excepthook catches `SignalHandledNonLocalExit`s from signal handlers, which isn't
        # happening here, so something is probably overriding the excepthook. By catching Exception
        # and calling this method, we emulate the normal, expected sys.excepthook override.
        ExceptionSink._log_unhandled_exception_and_exit(exc=e)
      else:
        self._exiter.exit(PANTS_SUCCEEDED_EXIT_CODE)
  def post_fork_child(self):
    """Post-fork child process callback executed via ProcessManager.daemonize()."""
    # Set the Exiter exception hook post-fork so as not to affect the pantsd processes exception
    # hook with socket-specific behavior. Note that this intentionally points the faulthandler
    # trace stream to sys.stderr, which at this point is still a _LoggerStream object writing to
    # the `pantsd.log`. This ensures that in the event of e.g. a hung but detached pantsd-runner
    # process that the stacktrace output lands deterministically in a known place vs to a stray
    # terminal window.
    self._exiter.set_except_hook(sys.stderr)

    # Ensure anything referencing sys.argv inherits the Pailgun'd args.
    sys.argv = self._args

    # Set context in the process title.
    set_process_title('pantsd-runner [{}]'.format(' '.join(self._args)))

    # Setup a SIGINT signal handler.
    self._setup_sigint_handler()

    # Broadcast our process group ID (in PID form - i.e. negated) to the remote client so
    # they can send signals (e.g. SIGINT) to all processes in the runners process group.
    pid = str(os.getpgrp() * -1).encode('ascii')
    NailgunProtocol.send_pid(self._socket, pid)

    # Invoke a Pants run with stdio redirected and a proxied environment.
    with self.nailgunned_stdio(self._socket, self._env) as finalizer,\
         hermetic_environment_as(**self._env):
      try:
        # Setup the Exiter's finalizer.
        self._exiter.set_finalizer(finalizer)

        # Clean global state.
        clean_global_runtime_state(reset_subsystem=True)

        # Re-raise any deferred exceptions, if present.
        self._raise_deferred_exc()

        # Otherwise, conduct a normal run.
        runner = LocalPantsRunner.create(
          self._exiter,
          self._args,
          self._env,
          self._target_roots,
          self._graph_helper,
          self._options_bootstrapper
        )
        runner.set_start_time(self._maybe_get_client_start_time_from_env(self._env))
        runner.run()
      except KeyboardInterrupt:
        self._exiter.exit(1, msg='Interrupted by user.\n')
      except Exception:
        self._exiter.handle_unhandled_exception(add_newline=True)
      else:
        self._exiter.exit(0)
Beispiel #9
0
    def single_daemonized_run(
        self,
        args: Tuple[str, ...],
        env: Dict[str, str],
        cancellation_latch: PySessionCancellationLatch,
    ) -> ExitCode:
        """Run a single daemonized run of Pants.

        All aspects of the `sys` global should already have been replaced in `__call__`, so this
        method should not need any special handling for the fact that it's running in a proxied
        environment.
        """

        try:
            logger.debug("Connected to pantsd")
            # Capture the client's start time, which we propagate here in order to get an accurate
            # view of total time.
            env_start_time = env.get("PANTSD_RUNTRACKER_CLIENT_START_TIME", None)
            if not env_start_time:
                # NB: We warn rather than erroring here because it eases use of non-Pants nailgun
                # clients for testing.
                logger.warning(
                    "No start time was reported by the client! Metrics may be inaccurate."
                )
            start_time = float(env_start_time) if env_start_time else time.time()

            options_bootstrapper = OptionsBootstrapper.create(
                env=env, args=args, allow_pantsrc=True
            )

            # Run using the pre-warmed Session.
            complete_env = CompleteEnvironment(env)
            scheduler, options_initializer = self._core.prepare(options_bootstrapper, complete_env)
            runner = LocalPantsRunner.create(
                complete_env,
                options_bootstrapper,
                scheduler=scheduler,
                options_initializer=options_initializer,
                cancellation_latch=cancellation_latch,
            )
            return runner.run(start_time)
        except Exception as e:
            logger.exception(e)
            return PANTS_FAILED_EXIT_CODE
        except KeyboardInterrupt:
            print("Interrupted by user.\n", file=sys.stderr)
            return PANTS_FAILED_EXIT_CODE
Beispiel #10
0
    def run(self):
        # Register our exiter at the beginning of the run() method so that any code in this process from
        # this point onwards will use that exiter in the case of a fatal error.
        ExceptionSink.reset_exiter(self._exiter)

        options_bootstrapper = OptionsBootstrapper.create(env=self._env,
                                                          args=self._args)
        bootstrap_options = options_bootstrapper.bootstrap_options
        global_bootstrap_options = bootstrap_options.for_global_scope()

        # We enable Rust logging here,
        # and everything before it will be routed through regular Python logging.
        self._enable_rust_logging(global_bootstrap_options)

        ExceptionSink.reset_should_print_backtrace_to_terminal(
            global_bootstrap_options.print_exception_stacktrace)
        ExceptionSink.reset_log_location(
            global_bootstrap_options.pants_workdir)

        for message_regexp in global_bootstrap_options.ignore_pants_warnings:
            warnings.filterwarnings(action='ignore', message=message_regexp)

        # TODO https://github.com/pantsbuild/pants/issues/7205
        if self._should_run_with_pantsd(global_bootstrap_options):
            try:
                return RemotePantsRunner(self._exiter, self._args, self._env,
                                         options_bootstrapper).run()
            except RemotePantsRunner.Fallback as e:
                logger.warn(
                    'caught client exception: {!r}, falling back to non-daemon mode'
                    .format(e))

        # N.B. Inlining this import speeds up the python thin client run by about 100ms.
        from pants.bin.local_pants_runner import LocalPantsRunner

        if self.will_terminate_pantsd():
            logger.debug("Pantsd terminating goal detected: {}".format(
                self._args))

        runner = LocalPantsRunner.create(
            self._exiter,
            self._args,
            self._env,
            options_bootstrapper=options_bootstrapper)
        runner.set_start_time(self._start_time)
        return runner.run()
Beispiel #11
0
    def single_daemonized_run(
            self, working_dir: str,
            cancellation_latch: PySessionCancellationLatch) -> ExitCode:
        """Run a single daemonized run of Pants.

        All aspects of the `sys` global should already have been replaced in `__call__`, so this
        method should not need any special handling for the fact that it's running in a proxied
        environment.
        """

        # Capture the client's start time, which we propagate here in order to get an accurate
        # view of total time.
        env_start_time = os.environ.get("PANTSD_RUNTRACKER_CLIENT_START_TIME",
                                        None)
        start_time = float(env_start_time) if env_start_time else time.time()

        # Clear global mutable state before entering `LocalPantsRunner`. Note that we use
        # `sys.argv` and `os.environ`, since they have been mutated to maintain the illusion
        # of a local run: once we allow for concurrent runs, this information should be
        # propagated down from the caller.
        #   see https://github.com/pantsbuild/pants/issues/7654
        BuildConfigInitializer.reset()
        options_bootstrapper = OptionsBootstrapper.create(env=os.environ,
                                                          args=sys.argv,
                                                          allow_pantsrc=True)
        bootstrap_options = options_bootstrapper.bootstrap_options
        global_bootstrap_options = bootstrap_options.for_global_scope()

        # Run using the pre-warmed Session.
        with self._stderr_logging(global_bootstrap_options):
            try:
                scheduler = self._core.prepare_scheduler(options_bootstrapper)
                runner = LocalPantsRunner.create(
                    os.environ,
                    options_bootstrapper,
                    scheduler=scheduler,
                    cancellation_latch=cancellation_latch,
                )
                return runner.run(start_time)
            except Exception as e:
                logger.exception(e)
                return PANTS_FAILED_EXIT_CODE
            except KeyboardInterrupt:
                print("Interrupted by user.\n", file=sys.stderr)
                return PANTS_FAILED_EXIT_CODE
Beispiel #12
0
    def run(self, start_time: float) -> ExitCode:
        self.scrub_pythonpath()

        options_bootstrapper = OptionsBootstrapper.create(env=self.env,
                                                          args=self.args,
                                                          allow_pantsrc=True)
        with warnings.catch_warnings(record=True):
            bootstrap_options = options_bootstrapper.bootstrap_options
            global_bootstrap_options = bootstrap_options.for_global_scope()

        # We enable logging here, and everything before it will be routed through regular
        # Python logging.
        stdin_fileno = sys.stdin.fileno()
        stdout_fileno = sys.stdout.fileno()
        stderr_fileno = sys.stderr.fileno()
        with initialize_stdio(global_bootstrap_options), stdio_destination(
                stdin_fileno=stdin_fileno,
                stdout_fileno=stdout_fileno,
                stderr_fileno=stderr_fileno,
        ):
            # N.B. We inline imports to speed up the python thin client run, and avoids importing
            # engine types until after the runner has had a chance to set PANTS_BIN_NAME.

            if self._should_run_with_pantsd(global_bootstrap_options):
                from pants.bin.remote_pants_runner import RemotePantsRunner

                try:
                    remote_runner = RemotePantsRunner(self.args, self.env,
                                                      options_bootstrapper)
                    return remote_runner.run(start_time)
                except RemotePantsRunner.Fallback as e:
                    logger.warning(
                        f"Client exception: {e!r}, falling back to non-daemon mode"
                    )

            from pants.bin.local_pants_runner import LocalPantsRunner

            # We only install signal handling via ExceptionSink if the run will execute in this process.
            ExceptionSink.install(
                log_location=init_workdir(global_bootstrap_options),
                pantsd_instance=False)
            runner = LocalPantsRunner.create(
                env=CompleteEnvironment(self.env),
                options_bootstrapper=options_bootstrapper)
            return runner.run(start_time)
Beispiel #13
0
    def run(self, start_time: float) -> ExitCode:
        self.scrub_pythonpath()

        options_bootstrapper = OptionsBootstrapper.create(env=self.env, args=self.args)
        bootstrap_options = options_bootstrapper.bootstrap_options
        global_bootstrap_options = bootstrap_options.for_global_scope()

        # Initialize the workdir early enough to ensure that logging has a destination.
        workdir_src = init_workdir(global_bootstrap_options)
        ExceptionSink.reset_log_location(workdir_src)

        # We enable logging here, and everything before it will be routed through regular
        # Python logging.
        setup_logging(global_bootstrap_options)

        ExceptionSink.reset_should_print_backtrace_to_terminal(
            global_bootstrap_options.print_exception_stacktrace
        )

        # TODO: When we remove this deprecation, we'll change the default for the option to true.
        deprecated_conditional(
            lambda: global_bootstrap_options.is_default("enable_pantsd"),
            removal_version="1.30.0.dev0",
            entity_description="--enable-pantsd defaulting to False",
            hint_message=(
                "Pantsd improves runtime performance and will be enabled by default in the 1.30.x "
                "stable releases. To prepare for that change, we recommend setting the "
                "`[GLOBAL] enable_pantsd` setting to `True` in your pants.toml or pants.ini file."
            ),
        )

        if self._should_run_with_pantsd(global_bootstrap_options):
            try:
                return RemotePantsRunner(self.args, self.env, options_bootstrapper).run(start_time)
            except RemotePantsRunner.Fallback as e:
                logger.warning("Client exception: {!r}, falling back to non-daemon mode".format(e))

        # N.B. Inlining this import speeds up the python thin client run by about 100ms.
        from pants.bin.local_pants_runner import LocalPantsRunner

        runner = LocalPantsRunner.create(env=self.env, options_bootstrapper=options_bootstrapper)
        return runner.run(start_time)
Beispiel #14
0
    def run(self):
        self.scrub_pythonpath()

        options_bootstrapper = OptionsBootstrapper.create(env=self._env,
                                                          args=self._args)
        bootstrap_options = options_bootstrapper.bootstrap_options
        global_bootstrap_options = bootstrap_options.for_global_scope()

        # Initialize the workdir early enough to ensure that logging has a destination.
        workdir_src = init_workdir(global_bootstrap_options)
        ExceptionSink.reset_log_location(workdir_src)

        # We enable Rust logging here,
        # and everything before it will be routed through regular Python logging.
        self._enable_rust_logging(global_bootstrap_options)

        ExceptionSink.reset_should_print_backtrace_to_terminal(
            global_bootstrap_options.print_exception_stacktrace)
        ExceptionSink.reset_log_location(
            global_bootstrap_options.pants_workdir)

        # TODO https://github.com/pantsbuild/pants/issues/7205
        if self._should_run_with_pantsd(global_bootstrap_options):
            try:
                return RemotePantsRunner(self._exiter, self._args, self._env,
                                         options_bootstrapper).run()
            except RemotePantsRunner.Fallback as e:
                logger.warning(
                    "caught client exception: {!r}, falling back to non-daemon mode"
                    .format(e))

        # N.B. Inlining this import speeds up the python thin client run by about 100ms.
        from pants.bin.local_pants_runner import LocalPantsRunner

        if self.will_terminate_pantsd():
            logger.debug("Pantsd terminating goal detected: {}".format(
                self._args))

        runner = LocalPantsRunner.create(
            self._args, self._env, options_bootstrapper=options_bootstrapper)
        runner.set_start_time(self._start_time)
        return runner.run()
Beispiel #15
0
  def run(self):
    options_bootstrapper = OptionsBootstrapper(env=self._env, args=self._args)
    bootstrap_options = options_bootstrapper.get_bootstrap_options()

    if bootstrap_options.for_global_scope().enable_pantsd:
      try:
        return RemotePantsRunner(self._exiter, self._args, self._env, bootstrap_options).run()
      except RemotePantsRunner.Fallback as e:
        logger.warn('caught client exception: {!r}, falling back to non-daemon mode'.format(e))

    # N.B. Inlining this import speeds up the python thin client run by about 100ms.
    from pants.bin.local_pants_runner import LocalPantsRunner

    runner = LocalPantsRunner.create(
        self._exiter,
        self._args,
        self._env,
        options_bootstrapper=options_bootstrapper
    )
    runner.set_start_time(self._start_time)
    return runner.run()
Beispiel #16
0
    def run(self, start_time: float) -> None:
        self.scrub_pythonpath()

        # TODO could options-bootstrapper be parsed in the runners?
        options_bootstrapper = OptionsBootstrapper.create(env=self.env,
                                                          args=self.args)
        bootstrap_options = options_bootstrapper.bootstrap_options
        global_bootstrap_options = bootstrap_options.for_global_scope()

        # Initialize the workdir early enough to ensure that logging has a destination.
        workdir_src = init_workdir(global_bootstrap_options)
        ExceptionSink.reset_log_location(workdir_src)

        # We enable Rust logging here,
        # and everything before it will be routed through regular Python logging.
        self._enable_rust_logging(global_bootstrap_options)

        ExceptionSink.reset_should_print_backtrace_to_terminal(
            global_bootstrap_options.print_exception_stacktrace)

        if self._should_run_with_pantsd(global_bootstrap_options):
            try:
                RemotePantsRunner(self._exiter, self.args, self.env,
                                  options_bootstrapper).run()
                return
            except RemotePantsRunner.Fallback as e:
                logger.warning(
                    "Client exception: {!r}, falling back to non-daemon mode".
                    format(e))

        # N.B. Inlining this import speeds up the python thin client run by about 100ms.
        from pants.bin.local_pants_runner import LocalPantsRunner

        runner = LocalPantsRunner.create(
            env=self.env, options_bootstrapper=options_bootstrapper)
        runner.set_start_time(start_time)
        runner.run()
Beispiel #17
0
  def run(self):
    # Register our exiter at the beginning of the run() method so that any code in this process from
    # this point onwards will use that exiter in the case of a fatal error.
    ExceptionSink.reset_exiter(self._exiter)

    options_bootstrapper = OptionsBootstrapper.create(env=self._env, args=self._args)
    bootstrap_options = options_bootstrapper.bootstrap_options
    global_bootstrap_options = bootstrap_options.for_global_scope()

    # We enable Rust logging here,
    # and everything before it will be routed through regular Python logging.
    self._enable_rust_logging(global_bootstrap_options)

    ExceptionSink.reset_should_print_backtrace_to_terminal(global_bootstrap_options.print_exception_stacktrace)
    ExceptionSink.reset_log_location(global_bootstrap_options.pants_workdir)

    for message_regexp in global_bootstrap_options.ignore_pants_warnings:
      warnings.filterwarnings(action='ignore', message=message_regexp)

    if global_bootstrap_options.enable_pantsd:
      try:
        return RemotePantsRunner(self._exiter, self._args, self._env, options_bootstrapper).run()
      except RemotePantsRunner.Fallback as e:
        logger.warn('caught client exception: {!r}, falling back to non-daemon mode'.format(e))

    # N.B. Inlining this import speeds up the python thin client run by about 100ms.
    from pants.bin.local_pants_runner import LocalPantsRunner

    runner = LocalPantsRunner.create(
        self._exiter,
        self._args,
        self._env,
        options_bootstrapper=options_bootstrapper
    )
    runner.set_start_time(self._start_time)
    return runner.run()
Beispiel #18
0
    def run(self):
        # Ensure anything referencing sys.argv inherits the Pailgun'd args.
        sys.argv = self.args

        # Invoke a Pants run with stdio redirected and a proxied environment.
        with self.nailgunned_stdio(
                self.maybe_shutdown_socket,
                self.env) as finalizer, DaemonExiter.override_global_exiter(
                    self.maybe_shutdown_socket,
                    finalizer), hermetic_environment_as(**self.env):

            exit_code = PANTS_SUCCEEDED_EXIT_CODE
            try:
                # Clean global state.
                clean_global_runtime_state(reset_subsystem=True)

                options_bootstrapper = OptionsBootstrapper.create(
                    args=self.args, env=self.env)
                options, build_config = LocalPantsRunner.parse_options(
                    options_bootstrapper)

                global_options = options.for_global_scope()
                session = self.scheduler_service.prepare_graph(options)

                specs = SpecsCalculator.create(
                    options=options,
                    session=session.scheduler_session,
                    exclude_patterns=tuple(
                        global_options.exclude_target_regexp),
                    tags=tuple(global_options.tag) if global_options.tag else
                    (),
                )

                if options.help_request:
                    help_printer = HelpPrinter(
                        options=options,
                        union_membership=UnionMembership(
                            build_config.union_rules()),
                    )
                    exit_code = help_printer.print_help()
                else:
                    exit_code = self.scheduler_service.graph_run_v2(
                        session, specs, options, options_bootstrapper)

                # self.scheduler_service.graph_run_v2 will already run v2 or ambiguous goals. We should
                # only enter this code path if v1 is set.
                if global_options.v1:
                    with ExceptionSink.exiter_as_until_exception(
                            lambda _: PantsRunFailCheckerExiter()):
                        runner = LocalPantsRunner.create(
                            self.env, options_bootstrapper, specs, session)

                        env_start_time = self.env.pop(
                            "PANTSD_RUNTRACKER_CLIENT_START_TIME", None)
                        start_time = float(
                            env_start_time) if env_start_time else None
                        runner.set_start_time(start_time)
                        runner.run()

            except KeyboardInterrupt:
                self._exiter.exit_and_fail("Interrupted by user.\n")
            except _PantsRunFinishedWithFailureException as e:
                ExceptionSink.log_exception(
                    "Pants run failed with exception: {}; exiting".format(e))
                self._exiter.exit(e.exit_code)
            except Exception as e:
                # TODO: We override sys.excepthook above when we call ExceptionSink.set_exiter(). That
                # excepthook catches `SignalHandledNonLocalExit`s from signal handlers, which isn't
                # happening here, so something is probably overriding the excepthook. By catching Exception
                # and calling this method, we emulate the normal, expected sys.excepthook override.
                ExceptionSink._log_unhandled_exception_and_exit(exc=e)
            else:
                self._exiter.exit(exit_code)
  def post_fork_child(self):
    """Post-fork child process callback executed via ProcessManager.daemonize()."""
    # Set the Exiter exception hook post-fork so as not to affect the pantsd processes exception
    # hook with socket-specific behavior. Note that this intentionally points the faulthandler
    # trace stream to sys.stderr, which at this point is still a _LoggerStream object writing to
    # the `pantsd.log`. This ensures that in the event of e.g. a hung but detached pantsd-runner
    # process that the stacktrace output lands deterministically in a known place vs to a stray
    # terminal window.
    # TODO: test the above!
    ExceptionSink.reset_exiter(self._exiter)

    ExceptionSink.reset_interactive_output_stream(sys.stderr.buffer if PY3 else sys.stderr)
    ExceptionSink.reset_signal_handler(DaemonSignalHandler())

    # Ensure anything referencing sys.argv inherits the Pailgun'd args.
    sys.argv = self._args

    # Set context in the process title.
    set_process_title('pantsd-runner [{}]'.format(' '.join(self._args)))

    # Broadcast our process group ID (in PID form - i.e. negated) to the remote client so
    # they can send signals (e.g. SIGINT) to all processes in the runners process group.
    NailgunProtocol.send_pid(self._socket, os.getpid())
    NailgunProtocol.send_pgrp(self._socket, os.getpgrp() * -1)

    # Stop the services that were paused pre-fork.
    for service in self._services.services:
      service.terminate()

    # Invoke a Pants run with stdio redirected and a proxied environment.
    with self.nailgunned_stdio(self._socket, self._env) as finalizer,\
         hermetic_environment_as(**self._env):
      try:
        # Setup the Exiter's finalizer.
        self._exiter.set_finalizer(finalizer)

        # Clean global state.
        clean_global_runtime_state(reset_subsystem=True)

        # Otherwise, conduct a normal run.
        runner = LocalPantsRunner.create(
          self._exiter,
          self._args,
          self._env,
          self._target_roots,
          self._graph_helper,
          self._options_bootstrapper
        )
        runner.set_start_time(self._maybe_get_client_start_time_from_env(self._env))

        # Re-raise any deferred exceptions, if present.
        self._raise_deferred_exc()

        runner.run()
      except KeyboardInterrupt:
        self._exiter.exit_and_fail('Interrupted by user.\n')
      except _GracefulTerminationException as e:
        ExceptionSink.log_exception(
          'Encountered graceful termination exception {}; exiting'.format(e))
        self._exiter.exit(e.exit_code)
      except Exception:
        # TODO: We override sys.excepthook above when we call ExceptionSink.set_exiter(). That
        # excepthook catches `SignalHandledNonLocalExit`s from signal handlers, which isn't
        # happening here, so something is probably overriding the excepthook. By catching Exception
        # and calling this method, we emulate the normal, expected sys.excepthook override.
        ExceptionSink._log_unhandled_exception_and_exit()
      else:
        self._exiter.exit(PANTS_SUCCEEDED_EXIT_CODE)