def run(self, start_time: float) -> ExitCode:
self.scrub_pythonpath()
options_bootstrapper = OptionsBootstrapper.create(env=self.env,
args=self.args,
allow_pantsrc=True)
with warnings.catch_warnings(record=True):
bootstrap_options = options_bootstrapper.bootstrap_options
global_bootstrap_options = bootstrap_options.for_global_scope()
# We enable logging here, and everything before it will be routed through regular
# Python logging.
setup_logging(global_bootstrap_options, stderr_logging=True)
if self._should_run_with_pantsd(global_bootstrap_options):
try:
remote_runner = RemotePantsRunner(self.args, self.env,
options_bootstrapper)
return remote_runner.run()
except RemotePantsRunner.Fallback as e:
logger.warning(
"Client exception: {!r}, falling back to non-daemon mode".
format(e))
# N.B. Inlining this import speeds up the python thin client run by about 100ms.
from pants.bin.local_pants_runner import LocalPantsRunner
# We only install signal handling via ExceptionSink if the run will execute in this process.
ExceptionSink.install(
log_location=init_workdir(global_bootstrap_options),
pantsd_instance=False)
runner = LocalPantsRunner.create(
env=self.env, options_bootstrapper=options_bootstrapper)
return runner.run(start_time)
def _pantsd_logging(self) -> Iterator[None]:
"""A context manager that runs with pantsd logging.
Asserts that stdio (represented by file handles 0, 1, 2) is closed to ensure that we can
safely reuse those fd numbers.
"""
# Ensure that stdio is closed so that we can safely reuse those file descriptors.
for fd in (0, 1, 2):
try:
os.fdopen(fd)
raise AssertionError(
f"pantsd logging cannot initialize while stdio is open: {fd}"
)
except OSError:
pass
# Redirect stdio to /dev/null for the rest of the run to reserve those file descriptors.
with stdio_as(stdin_fd=-1, stdout_fd=-1, stderr_fd=-1):
# Reinitialize logging for the daemon context.
global_options = self._bootstrap_options.for_global_scope()
setup_logging(global_options, stderr_logging=False)
log_dir = os.path.join(self._work_dir, self.name)
setup_logging_to_file(global_options.level,
log_dir=log_dir,
log_filename=self.LOG_NAME)
self._logger.debug("Logging reinitialized in pantsd context")
yield
def run(self, start_time: float) -> ExitCode:
self.scrub_pythonpath()
options_bootstrapper = OptionsBootstrapper.create(
env=self.env, args=self.args, allow_pantsrc=True
)
with warnings.catch_warnings(record=True):
bootstrap_options = options_bootstrapper.bootstrap_options
global_bootstrap_options = bootstrap_options.for_global_scope()
# Initialize the workdir early enough to ensure that logging has a destination.
workdir_src = init_workdir(global_bootstrap_options)
ExceptionSink.reset_log_location(workdir_src)
setup_warning_filtering(global_bootstrap_options.ignore_pants_warnings or [])
# We enable logging here, and everything before it will be routed through regular
# Python logging.
setup_logging(global_bootstrap_options, stderr_logging=True)
if self._should_run_with_pantsd(global_bootstrap_options):
try:
remote_runner = RemotePantsRunner(self.args, self.env, options_bootstrapper)
return remote_runner.run()
except RemotePantsRunner.Fallback as e:
logger.warning("Client exception: {!r}, falling back to non-daemon mode".format(e))
# N.B. Inlining this import speeds up the python thin client run by about 100ms.
from pants.bin.local_pants_runner import LocalPantsRunner
runner = LocalPantsRunner.create(env=self.env, options_bootstrapper=options_bootstrapper)
return runner.run(start_time)
def _pantsd_logging(self):
"""A context manager that runs with pantsd logging.
Asserts that stdio (represented by file handles 0, 1, 2) is closed to ensure that
we can safely reuse those fd numbers.
"""
# Ensure that stdio is closed so that we can safely reuse those file descriptors.
for fd in (0, 1, 2):
try:
os.fdopen(fd)
raise AssertionError(
'pantsd logging cannot initialize while stdio is open: {}'.
format(fd))
except OSError:
pass
# Redirect stdio to /dev/null for the rest of the run, to reserve those file descriptors
# for further forks.
with stdio_as(stdin_fd=-1, stdout_fd=-1, stderr_fd=-1):
# Reinitialize logging for the daemon context.
result = setup_logging(self._log_level,
log_dir=self._log_dir,
log_name=self.LOG_NAME)
# Do a python-level redirect of stdout/stderr, which will not disturb `0,1,2`.
# TODO: Consider giving these pipes/actual fds, in order to make them "deep" replacements
# for `1,2`, and allow them to be used via `stdio_as`.
sys.stdout = _LoggerStream(logging.getLogger(), logging.INFO,
result.log_handler)
sys.stderr = _LoggerStream(logging.getLogger(), logging.WARN,
result.log_handler)
self._logger.debug('logging initialized')
yield result.log_handler.stream
def _pantsd_logging(self):
"""A context manager that runs with pantsd logging.
Asserts that stdio (represented by file handles 0, 1, 2) is closed to ensure that
we can safely reuse those fd numbers.
"""
# Ensure that stdio is closed so that we can safely reuse those file descriptors.
for fd in (0, 1, 2):
try:
os.fdopen(fd)
raise AssertionError(
'pantsd logging cannot initialize while stdio is open: {}'.format(fd))
except OSError:
pass
# Redirect stdio to /dev/null for the rest of the run, to reserve those file descriptors
# for further forks.
with stdio_as(stdin_fd=-1, stdout_fd=-1, stderr_fd=-1):
# Reinitialize logging for the daemon context.
init_rust_logger(self._log_level, self._log_show_rust_3rdparty)
result = setup_logging(self._log_level, log_dir=self._log_dir, log_name=self.LOG_NAME, native=self._native)
self._native.override_thread_logging_destination_to_just_pantsd()
# Do a python-level redirect of stdout/stderr, which will not disturb `0,1,2`.
# TODO: Consider giving these pipes/actual fds, in order to make them "deep" replacements
# for `1,2`, and allow them to be used via `stdio_as`.
sys.stdout = _LoggerStream(logging.getLogger(), logging.INFO, result.log_handler)
sys.stderr = _LoggerStream(logging.getLogger(), logging.WARN, result.log_handler)
self._logger.debug('logging initialized')
yield (result.log_handler.stream, result.log_handler.native_filename)
def logger(self, log_level: LogLevel) -> Iterator[Tuple[Logger, FileLoggingSetupResult]]:
native = self.scheduler._scheduler._native
logger = logging.getLogger("my_file_logger")
with temporary_dir() as log_dir:
logging_setup_result = setup_logging(
log_level, log_dir=log_dir, scope=logger.name, native=native
)
yield logger, logging_setup_result
def logger(self, level):
native = self.scheduler._scheduler._native
logger = logging.getLogger('my_file_logger')
with temporary_dir() as log_dir:
logging_setup_result = setup_logging(level,
log_dir=log_dir,
scope=logger.name,
native=native)
yield logger, logging_setup_result
def logger(self, level, file_logging=False):
logger = logging.getLogger(str(uuid.uuid4()))
with closing(StringIO()) as stream:
with self.log_dir(file_logging) as log_dir:
log_file = setup_logging(level,
console_stream=stream,
log_dir=log_dir,
scope=logger.name)
yield logger, stream, log_file.log_filename
def _stderr_logging(self, global_bootstrap_options):
"""Temporarily replaces existing handlers (ie, the pantsd handler) with a stderr handler.
In the context of pantsd, there will be an existing handler for the pantsd log, which we
temporarily replace. Making them additive would cause per-run logs to go to pantsd, which
we don't want.
TODO: It would be good to handle logging destinations entirely via the threadlocal state
rather than via handler mutations.
"""
handlers = get_logging_handlers()
try:
clear_logging_handlers()
Native().override_thread_logging_destination_to_just_stderr()
setup_logging(global_bootstrap_options, stderr_logging=True)
yield
finally:
Native().override_thread_logging_destination_to_just_pantsd()
set_logging_handlers(handlers)
def run(self, start_time: float) -> ExitCode:
self.scrub_pythonpath()
options_bootstrapper = OptionsBootstrapper.create(env=self.env, args=self.args)
bootstrap_options = options_bootstrapper.bootstrap_options
global_bootstrap_options = bootstrap_options.for_global_scope()
# Initialize the workdir early enough to ensure that logging has a destination.
workdir_src = init_workdir(global_bootstrap_options)
ExceptionSink.reset_log_location(workdir_src)
# We enable logging here, and everything before it will be routed through regular
# Python logging.
setup_logging(global_bootstrap_options)
ExceptionSink.reset_should_print_backtrace_to_terminal(
global_bootstrap_options.print_exception_stacktrace
)
# TODO: When we remove this deprecation, we'll change the default for the option to true.
deprecated_conditional(
lambda: global_bootstrap_options.is_default("enable_pantsd"),
removal_version="1.30.0.dev0",
entity_description="--enable-pantsd defaulting to False",
hint_message=(
"Pantsd improves runtime performance and will be enabled by default in the 1.30.x "
"stable releases. To prepare for that change, we recommend setting the "
"`[GLOBAL] enable_pantsd` setting to `True` in your pants.toml or pants.ini file."
),
)
if self._should_run_with_pantsd(global_bootstrap_options):
try:
return RemotePantsRunner(self.args, self.env, options_bootstrapper).run(start_time)
except RemotePantsRunner.Fallback as e:
logger.warning("Client exception: {!r}, falling back to non-daemon mode".format(e))
# N.B. Inlining this import speeds up the python thin client run by about 100ms.
from pants.bin.local_pants_runner import LocalPantsRunner
runner = LocalPantsRunner.create(env=self.env, options_bootstrapper=options_bootstrapper)
return runner.run(start_time)
def _pantsd_logging(self):
"""A context manager that runs with pantsd logging.
Asserts that stdio (represented by file handles 0, 1, 2) is closed to ensure that we can
safely reuse those fd numbers.
"""
# Ensure that stdio is closed so that we can safely reuse those file descriptors.
for fd in (0, 1, 2):
try:
os.fdopen(fd)
raise AssertionError(
f"pantsd logging cannot initialize while stdio is open: {fd}"
)
except OSError:
pass
# Redirect stdio to /dev/null for the rest of the run, to reserve those file descriptors
# for further forks.
with stdio_as(stdin_fd=-1, stdout_fd=-1, stderr_fd=-1):
# Reinitialize logging for the daemon context.
init_rust_logger(self._log_level, self._log_show_rust_3rdparty)
result = setup_logging(
self._log_level,
native=self._native,
log_dir=self._log_dir,
log_name=self.LOG_NAME,
warnings_filter_regexes=self._bootstrap_options.
for_global_scope(),
)
self._native.override_thread_logging_destination_to_just_pantsd()
# Do a python-level redirect of stdout/stderr, which will not disturb `0,1,2`.
# TODO: Consider giving these pipes/actual fds, in order to make them "deep" replacements
# for `1,2`, and allow them to be used via `stdio_as`.
sys.stdout = _LoggerStream(logging.getLogger(), logging.INFO,
result.log_handler)
sys.stderr = _LoggerStream(logging.getLogger(), logging.WARN,
result.log_handler)
self._logger.debug("logging initialized")
yield (result.log_handler.stream,
result.log_handler.native_filename)
def logger(self, level):
native = self.scheduler._scheduler._native
logger = logging.getLogger('my_file_logger')
with temporary_dir() as log_dir:
logging_setup_result = setup_logging(level, log_dir=log_dir, scope=logger.name, native=native)
yield logger, logging_setup_result
def logger(self, level, file_logging=False):
logger = logging.getLogger(str(uuid.uuid4()))
with closing(six.StringIO()) as stream:
with self.log_dir(file_logging) as log_dir:
log_file = setup_logging(level, console_stream=stream, log_dir=log_dir, scope=logger.name)
yield logger, stream, log_file.log_filename
