def run(self, start_time: float) -> ExitCode: self.scrub_pythonpath() options_bootstrapper = OptionsBootstrapper.create(env=self.env, args=self.args, allow_pantsrc=True) with warnings.catch_warnings(record=True): bootstrap_options = options_bootstrapper.bootstrap_options global_bootstrap_options = bootstrap_options.for_global_scope() # We enable logging here, and everything before it will be routed through regular # Python logging. setup_logging(global_bootstrap_options, stderr_logging=True) if self._should_run_with_pantsd(global_bootstrap_options): try: remote_runner = RemotePantsRunner(self.args, self.env, options_bootstrapper) return remote_runner.run() except RemotePantsRunner.Fallback as e: logger.warning( "Client exception: {!r}, falling back to non-daemon mode". format(e)) # N.B. Inlining this import speeds up the python thin client run by about 100ms. from pants.bin.local_pants_runner import LocalPantsRunner # We only install signal handling via ExceptionSink if the run will execute in this process. ExceptionSink.install( log_location=init_workdir(global_bootstrap_options), pantsd_instance=False) runner = LocalPantsRunner.create( env=self.env, options_bootstrapper=options_bootstrapper) return runner.run(start_time)
def _pantsd_logging(self) -> Iterator[None]: """A context manager that runs with pantsd logging. Asserts that stdio (represented by file handles 0, 1, 2) is closed to ensure that we can safely reuse those fd numbers. """ # Ensure that stdio is closed so that we can safely reuse those file descriptors. for fd in (0, 1, 2): try: os.fdopen(fd) raise AssertionError( f"pantsd logging cannot initialize while stdio is open: {fd}" ) except OSError: pass # Redirect stdio to /dev/null for the rest of the run to reserve those file descriptors. with stdio_as(stdin_fd=-1, stdout_fd=-1, stderr_fd=-1): # Reinitialize logging for the daemon context. global_options = self._bootstrap_options.for_global_scope() setup_logging(global_options, stderr_logging=False) log_dir = os.path.join(self._work_dir, self.name) setup_logging_to_file(global_options.level, log_dir=log_dir, log_filename=self.LOG_NAME) self._logger.debug("Logging reinitialized in pantsd context") yield
def run(self, start_time: float) -> ExitCode: self.scrub_pythonpath() options_bootstrapper = OptionsBootstrapper.create( env=self.env, args=self.args, allow_pantsrc=True ) with warnings.catch_warnings(record=True): bootstrap_options = options_bootstrapper.bootstrap_options global_bootstrap_options = bootstrap_options.for_global_scope() # Initialize the workdir early enough to ensure that logging has a destination. workdir_src = init_workdir(global_bootstrap_options) ExceptionSink.reset_log_location(workdir_src) setup_warning_filtering(global_bootstrap_options.ignore_pants_warnings or []) # We enable logging here, and everything before it will be routed through regular # Python logging. setup_logging(global_bootstrap_options, stderr_logging=True) if self._should_run_with_pantsd(global_bootstrap_options): try: remote_runner = RemotePantsRunner(self.args, self.env, options_bootstrapper) return remote_runner.run() except RemotePantsRunner.Fallback as e: logger.warning("Client exception: {!r}, falling back to non-daemon mode".format(e)) # N.B. Inlining this import speeds up the python thin client run by about 100ms. from pants.bin.local_pants_runner import LocalPantsRunner runner = LocalPantsRunner.create(env=self.env, options_bootstrapper=options_bootstrapper) return runner.run(start_time)
def _pantsd_logging(self): """A context manager that runs with pantsd logging. Asserts that stdio (represented by file handles 0, 1, 2) is closed to ensure that we can safely reuse those fd numbers. """ # Ensure that stdio is closed so that we can safely reuse those file descriptors. for fd in (0, 1, 2): try: os.fdopen(fd) raise AssertionError( 'pantsd logging cannot initialize while stdio is open: {}'. format(fd)) except OSError: pass # Redirect stdio to /dev/null for the rest of the run, to reserve those file descriptors # for further forks. with stdio_as(stdin_fd=-1, stdout_fd=-1, stderr_fd=-1): # Reinitialize logging for the daemon context. result = setup_logging(self._log_level, log_dir=self._log_dir, log_name=self.LOG_NAME) # Do a python-level redirect of stdout/stderr, which will not disturb `0,1,2`. # TODO: Consider giving these pipes/actual fds, in order to make them "deep" replacements # for `1,2`, and allow them to be used via `stdio_as`. sys.stdout = _LoggerStream(logging.getLogger(), logging.INFO, result.log_handler) sys.stderr = _LoggerStream(logging.getLogger(), logging.WARN, result.log_handler) self._logger.debug('logging initialized') yield result.log_handler.stream
def _pantsd_logging(self): """A context manager that runs with pantsd logging. Asserts that stdio (represented by file handles 0, 1, 2) is closed to ensure that we can safely reuse those fd numbers. """ # Ensure that stdio is closed so that we can safely reuse those file descriptors. for fd in (0, 1, 2): try: os.fdopen(fd) raise AssertionError( 'pantsd logging cannot initialize while stdio is open: {}'.format(fd)) except OSError: pass # Redirect stdio to /dev/null for the rest of the run, to reserve those file descriptors # for further forks. with stdio_as(stdin_fd=-1, stdout_fd=-1, stderr_fd=-1): # Reinitialize logging for the daemon context. init_rust_logger(self._log_level, self._log_show_rust_3rdparty) result = setup_logging(self._log_level, log_dir=self._log_dir, log_name=self.LOG_NAME, native=self._native) self._native.override_thread_logging_destination_to_just_pantsd() # Do a python-level redirect of stdout/stderr, which will not disturb `0,1,2`. # TODO: Consider giving these pipes/actual fds, in order to make them "deep" replacements # for `1,2`, and allow them to be used via `stdio_as`. sys.stdout = _LoggerStream(logging.getLogger(), logging.INFO, result.log_handler) sys.stderr = _LoggerStream(logging.getLogger(), logging.WARN, result.log_handler) self._logger.debug('logging initialized') yield (result.log_handler.stream, result.log_handler.native_filename)
def logger(self, log_level: LogLevel) -> Iterator[Tuple[Logger, FileLoggingSetupResult]]: native = self.scheduler._scheduler._native logger = logging.getLogger("my_file_logger") with temporary_dir() as log_dir: logging_setup_result = setup_logging( log_level, log_dir=log_dir, scope=logger.name, native=native ) yield logger, logging_setup_result
def logger(self, level): native = self.scheduler._scheduler._native logger = logging.getLogger('my_file_logger') with temporary_dir() as log_dir: logging_setup_result = setup_logging(level, log_dir=log_dir, scope=logger.name, native=native) yield logger, logging_setup_result
def logger(self, level, file_logging=False): logger = logging.getLogger(str(uuid.uuid4())) with closing(StringIO()) as stream: with self.log_dir(file_logging) as log_dir: log_file = setup_logging(level, console_stream=stream, log_dir=log_dir, scope=logger.name) yield logger, stream, log_file.log_filename
def _stderr_logging(self, global_bootstrap_options): """Temporarily replaces existing handlers (ie, the pantsd handler) with a stderr handler. In the context of pantsd, there will be an existing handler for the pantsd log, which we temporarily replace. Making them additive would cause per-run logs to go to pantsd, which we don't want. TODO: It would be good to handle logging destinations entirely via the threadlocal state rather than via handler mutations. """ handlers = get_logging_handlers() try: clear_logging_handlers() Native().override_thread_logging_destination_to_just_stderr() setup_logging(global_bootstrap_options, stderr_logging=True) yield finally: Native().override_thread_logging_destination_to_just_pantsd() set_logging_handlers(handlers)
def run(self, start_time: float) -> ExitCode: self.scrub_pythonpath() options_bootstrapper = OptionsBootstrapper.create(env=self.env, args=self.args) bootstrap_options = options_bootstrapper.bootstrap_options global_bootstrap_options = bootstrap_options.for_global_scope() # Initialize the workdir early enough to ensure that logging has a destination. workdir_src = init_workdir(global_bootstrap_options) ExceptionSink.reset_log_location(workdir_src) # We enable logging here, and everything before it will be routed through regular # Python logging. setup_logging(global_bootstrap_options) ExceptionSink.reset_should_print_backtrace_to_terminal( global_bootstrap_options.print_exception_stacktrace ) # TODO: When we remove this deprecation, we'll change the default for the option to true. deprecated_conditional( lambda: global_bootstrap_options.is_default("enable_pantsd"), removal_version="1.30.0.dev0", entity_description="--enable-pantsd defaulting to False", hint_message=( "Pantsd improves runtime performance and will be enabled by default in the 1.30.x " "stable releases. To prepare for that change, we recommend setting the " "`[GLOBAL] enable_pantsd` setting to `True` in your pants.toml or pants.ini file." ), ) if self._should_run_with_pantsd(global_bootstrap_options): try: return RemotePantsRunner(self.args, self.env, options_bootstrapper).run(start_time) except RemotePantsRunner.Fallback as e: logger.warning("Client exception: {!r}, falling back to non-daemon mode".format(e)) # N.B. Inlining this import speeds up the python thin client run by about 100ms. from pants.bin.local_pants_runner import LocalPantsRunner runner = LocalPantsRunner.create(env=self.env, options_bootstrapper=options_bootstrapper) return runner.run(start_time)
def _pantsd_logging(self): """A context manager that runs with pantsd logging. Asserts that stdio (represented by file handles 0, 1, 2) is closed to ensure that we can safely reuse those fd numbers. """ # Ensure that stdio is closed so that we can safely reuse those file descriptors. for fd in (0, 1, 2): try: os.fdopen(fd) raise AssertionError( f"pantsd logging cannot initialize while stdio is open: {fd}" ) except OSError: pass # Redirect stdio to /dev/null for the rest of the run, to reserve those file descriptors # for further forks. with stdio_as(stdin_fd=-1, stdout_fd=-1, stderr_fd=-1): # Reinitialize logging for the daemon context. init_rust_logger(self._log_level, self._log_show_rust_3rdparty) result = setup_logging( self._log_level, native=self._native, log_dir=self._log_dir, log_name=self.LOG_NAME, warnings_filter_regexes=self._bootstrap_options. for_global_scope(), ) self._native.override_thread_logging_destination_to_just_pantsd() # Do a python-level redirect of stdout/stderr, which will not disturb `0,1,2`. # TODO: Consider giving these pipes/actual fds, in order to make them "deep" replacements # for `1,2`, and allow them to be used via `stdio_as`. sys.stdout = _LoggerStream(logging.getLogger(), logging.INFO, result.log_handler) sys.stderr = _LoggerStream(logging.getLogger(), logging.WARN, result.log_handler) self._logger.debug("logging initialized") yield (result.log_handler.stream, result.log_handler.native_filename)
def logger(self, level, file_logging=False): logger = logging.getLogger(str(uuid.uuid4())) with closing(six.StringIO()) as stream: with self.log_dir(file_logging) as log_dir: log_file = setup_logging(level, console_stream=stream, log_dir=log_dir, scope=logger.name) yield logger, stream, log_file.log_filename