def _init_graph_session(
cls,
options_bootstrapper: OptionsBootstrapper,
build_config: BuildConfiguration,
options: Options,
scheduler: Optional[GraphScheduler] = None,
) -> GraphSession:
native = Native()
native.set_panic_handler()
graph_scheduler_helper = scheduler or EngineInitializer.setup_graph(
options_bootstrapper, build_config)
try:
global_scope = options.for_global_scope()
except UnknownFlagsError as err:
cls._handle_unknown_flags(err, options_bootstrapper)
raise
stream_workunits = len(
options.for_global_scope().streaming_workunits_handlers) != 0
return graph_scheduler_helper.new_session(
RunTracker.global_instance().run_id,
dynamic_ui=global_scope.dynamic_ui,
use_colors=global_scope.get("colors", True),
should_report_workunits=stream_workunits,
session_values=SessionValues({
OptionsBootstrapper:
options_bootstrapper,
PantsEnvironment:
PantsEnvironment(os.environ),
}),
)
def _init_graph_session(
cls,
options_initializer: OptionsInitializer,
options_bootstrapper: OptionsBootstrapper,
build_config: BuildConfiguration,
env: CompleteEnvironment,
run_id: str,
options: Options,
scheduler: Optional[GraphScheduler] = None,
cancellation_latch: Optional[PySessionCancellationLatch] = None,
) -> GraphSession:
native = Native()
native.set_panic_handler()
graph_scheduler_helper = scheduler or EngineInitializer.setup_graph(
options_bootstrapper, build_config, env
)
with options_initializer.handle_unknown_flags(options_bootstrapper, env, raise_=True):
global_options = options.for_global_scope()
return graph_scheduler_helper.new_session(
run_id,
dynamic_ui=global_options.dynamic_ui,
use_colors=global_options.get("colors", True),
session_values=SessionValues(
{
OptionsBootstrapper: options_bootstrapper,
CompleteEnvironment: env,
}
),
cancellation_latch=cancellation_latch,
)
def _init_graph_session(
cls,
options_bootstrapper: OptionsBootstrapper,
build_config: BuildConfiguration,
run_id: str,
options: Options,
scheduler: Optional[GraphScheduler] = None,
cancellation_latch: Optional[PySessionCancellationLatch] = None,
) -> GraphSession:
native = Native()
native.set_panic_handler()
graph_scheduler_helper = scheduler or EngineInitializer.setup_graph(
options_bootstrapper, build_config
)
try:
global_scope = options.for_global_scope()
except UnknownFlagsError as err:
cls._handle_unknown_flags(err, options_bootstrapper)
raise
return graph_scheduler_helper.new_session(
run_id,
dynamic_ui=global_scope.dynamic_ui,
use_colors=global_scope.get("colors", True),
session_values=SessionValues(
{
OptionsBootstrapper: options_bootstrapper,
PantsEnvironment: PantsEnvironment(os.environ),
}
),
cancellation_latch=cancellation_latch,
)
def _init_graph_session(
options_bootstrapper: OptionsBootstrapper,
build_config: BuildConfiguration,
options: Options,
) -> LegacyGraphSession:
native = Native()
native.set_panic_handler()
graph_scheduler_helper = EngineInitializer.setup_legacy_graph(
native, options_bootstrapper, build_config)
v2_ui = options.for_global_scope().get("v2_ui", False)
zipkin_trace_v2 = options.for_scope("reporting").zipkin_trace_v2
# TODO(#8658) This should_report_workunits flag must be set to True for
# StreamingWorkunitHandler to receive WorkUnits. It should eventually
# be merged with the zipkin_trace_v2 flag, since they both involve most
# of the same engine functionality, but for now is separate to avoid
# breaking functionality associated with zipkin tracing while iterating on streaming workunit reporting.
stream_workunits = len(
options.for_global_scope().streaming_workunits_handlers) != 0
return graph_scheduler_helper.new_session(
zipkin_trace_v2,
RunTracker.global_instance().run_id,
v2_ui,
should_report_workunits=stream_workunits,
)
def _connect_and_execute(
self, pantsd_handle: PantsDaemonClient.Handle) -> ExitCode:
native = Native()
global_options = self._bootstrap_options.for_global_scope()
executor = PyExecutor(
*OptionsInitializer.compute_executor_arguments(global_options))
# Merge the nailgun TTY capability environment variables with the passed environment dict.
ng_env = NailgunProtocol.ttynames_to_env(sys.stdin, sys.stdout.buffer,
sys.stderr.buffer)
modified_env = {
**self._env,
**ng_env,
"PANTSD_RUNTRACKER_CLIENT_START_TIME":
str(self._start_time),
"PANTSD_REQUEST_TIMEOUT_LIMIT":
str(global_options.pantsd_timeout_when_multiple_invocations),
}
command = self._args[0]
args = self._args[1:]
retries = 3
attempt = 1
while True:
port = pantsd_handle.port
logger.debug(
f"Connecting to pantsd on port {port} attempt {attempt}/{retries}"
)
# We preserve TTY settings since the server might write directly to the TTY, and we'd like
# to clean up any side effects before exiting.
#
# We ignore keyboard interrupts because the nailgun client will handle them.
with STTYSettings.preserved(), interrupts_ignored():
try:
return native.new_nailgun_client(executor=executor,
port=port).execute(
command, args,
modified_env)
# NailgunConnectionException represents a failure connecting to pantsd, so we retry
# up to the retry limit.
except native.lib.NailgunConnectionException as e:
if attempt > retries:
raise self.Fallback(e)
# Wait one second before retrying
logger.warning(
f"Pantsd was unresponsive on port {port}, retrying.")
time.sleep(1)
# One possible cause of the daemon being non-responsive during an attempt might be if a
# another lifecycle operation is happening concurrently (incl teardown). To account for
# this, we won't begin attempting restarts until at least 1 attempt has passed.
if attempt > 1:
pantsd_handle = self._client.restart()
attempt += 1
def _run_pants(self, sock, arguments, environment):
"""Execute a given run with a pants runner."""
# For the pants run, we want to log to stderr.
# TODO Might be worth to make contextmanagers for this?
Native().override_thread_logging_destination_to_just_stderr()
self.server.runner_factory(sock, arguments, environment).run()
Native().override_thread_logging_destination_to_just_pantsd()
def create(cls, options_bootstrapper) -> "PantsDaemon":
"""
:param OptionsBootstrapper options_bootstrapper: The bootstrap options.
:param bool full_init: Whether or not to fully initialize an engine et al for the purposes
of spawning a new daemon. `full_init=False` is intended primarily
for lightweight lifecycle checks (since there is a ~1s overhead to
initialize the engine). See the impl of `maybe_launch` for an example
of the intended usage.
"""
native = Native()
native.override_thread_logging_destination_to_just_pantsd()
bootstrap_options = options_bootstrapper.bootstrap_options
bootstrap_options_values = bootstrap_options.for_global_scope()
core = PantsDaemonCore(cls._setup_services)
server = native.new_nailgun_server(
bootstrap_options_values.pantsd_pailgun_port,
DaemonPantsRunner(core),
)
return PantsDaemon(
native=native,
work_dir=bootstrap_options_values.pants_workdir,
log_level=bootstrap_options_values.level,
server=server,
core=core,
metadata_base_dir=bootstrap_options_values.pants_subprocessdir,
bootstrap_options=bootstrap_options,
)
def create(cls, options_bootstrapper: OptionsBootstrapper) -> PantsDaemon:
# Any warnings that would be triggered here are re-triggered later per-run of Pants, so we
# silence them.
with warnings.catch_warnings(record=True):
bootstrap_options = options_bootstrapper.bootstrap_options
bootstrap_options_values = bootstrap_options.for_global_scope()
native = Native()
executor = PyExecutor(*GlobalOptions.compute_executor_arguments(bootstrap_options_values))
core = PantsDaemonCore(options_bootstrapper, executor, cls._setup_services)
server = native.new_nailgun_server(
executor,
bootstrap_options_values.pantsd_pailgun_port,
DaemonPantsRunner(core),
)
return PantsDaemon(
native=native,
work_dir=bootstrap_options_values.pants_workdir,
log_level=bootstrap_options_values.level,
server=server,
core=core,
metadata_base_dir=bootstrap_options_values.pants_subprocessdir,
bootstrap_options=bootstrap_options,
)
def test_log_filtering_by_rule() -> None:
native = Native()
native.init_rust_logging(
level=LogLevel.INFO.level,
log_show_rust_3rdparty=False,
use_color=False,
show_target=True,
log_levels_by_target={
"debug_target": LogLevel.DEBUG,
},
)
with temporary_dir() as tmpdir:
setup_logging_to_file(LogLevel.INFO, log_dir=tmpdir)
log_file = Path(tmpdir, "pants.log")
native.write_log(msg="log msg one",
level=LogLevel.INFO.level,
target="some.target")
native.write_log(msg="log msg two",
level=LogLevel.DEBUG.level,
target="some.other.target")
native.write_log(msg="log msg three",
level=LogLevel.DEBUG.level,
target="debug_target")
loglines = log_file.read_text().splitlines()
assert "[INFO] (some.target) log msg one" in loglines[0]
assert "[DEBUG] (debug_target) log msg three" in loglines[1]
assert len(loglines) == 2
def test_file_logging() -> None:
native = Native()
native.init_rust_logging(
level=LogLevel.INFO.level, # Tests assume a log level of INFO
log_show_rust_3rdparty=False,
use_color=False,
show_target=False,
log_levels_by_target={},
)
logger = logging.getLogger("my_file_logger")
with temporary_dir() as tmpdir:
setup_logging_to_file(LogLevel.INFO, log_dir=tmpdir)
log_file = Path(tmpdir, "pants.log")
cat = "🐈"
logger.warning("this is a warning")
logger.info("this is some info")
logger.debug("this is some debug info")
logger.info(f"unicode: {cat}")
loglines = log_file.read_text().splitlines()
print(loglines)
assert len(loglines) == 3
assert "[WARN] this is a warning" in loglines[0]
assert "[INFO] this is some info" in loglines[1]
assert f"[INFO] unicode: {cat}" in loglines[2]
def __init__(self, options: Options):
"""
:API: public
"""
self.native = Native()
self._has_started: bool = False
self._has_ended: bool = False
# Select a globally unique ID for the run, that sorts by time.
run_timestamp = time.time()
run_uuid = uuid.uuid4().hex
str_time = time.strftime("%Y_%m_%d_%H_%M_%S",
time.localtime(run_timestamp))
millis = int((run_timestamp * 1000) % 1000)
self.run_id = f"pants_run_{str_time}_{millis}_{run_uuid}"
self._all_options = options
info_dir = os.path.join(
self._all_options.for_global_scope().pants_workdir, "run-tracker")
self._run_info: Dict[str, Any] = {}
# pantsd stats.
self._pantsd_metrics: Dict[str, int] = dict()
self.run_logs_file = Path(info_dir, self.run_id, "logs")
safe_mkdir_for(str(self.run_logs_file))
self.native.set_per_run_log_path(str(self.run_logs_file))
# Initialized in `start()`.
self._run_start_time: Optional[float] = None
self._run_total_duration: Optional[float] = None
def create(cls,
options_bootstrapper: OptionsBootstrapper) -> "PantsDaemon":
with warnings.catch_warnings(record=True):
bootstrap_options = options_bootstrapper.bootstrap_options
bootstrap_options_values = bootstrap_options.for_global_scope()
setup_warning_filtering(bootstrap_options_values.ignore_pants_warnings
or [])
native = Native()
native.override_thread_logging_destination_to_just_pantsd()
core = PantsDaemonCore(cls._setup_services)
server = native.new_nailgun_server(
bootstrap_options_values.pantsd_pailgun_port,
DaemonPantsRunner(core),
)
return PantsDaemon(
native=native,
work_dir=bootstrap_options_values.pants_workdir,
log_level=bootstrap_options_values.level,
server=server,
core=core,
metadata_base_dir=bootstrap_options_values.pants_subprocessdir,
bootstrap_options=bootstrap_options,
)
def __init__(self, *args, **kwargs):
"""
:API: public
"""
super().__init__(*args, **kwargs)
self._run_timestamp = time.time()
self._cmd_line = " ".join(["pants"] + sys.argv[1:])
self._v2_goal_rule_names: Tuple[str, ...] = tuple()
self.run_uuid = uuid.uuid4().hex
# Select a globally unique ID for the run, that sorts by time.
millis = int((self._run_timestamp * 1000) % 1000)
# run_uuid is used as a part of run_id and also as a trace_id for Zipkin tracing
str_time = time.strftime("%Y_%m_%d_%H_%M_%S",
time.localtime(self._run_timestamp))
self.run_id = f"pants_run_{str_time}_{millis}_{self.run_uuid}"
# Initialized in `initialize()`.
self.run_info_dir = None
self.run_info = None
self.cumulative_timings = None
self.self_timings = None
# Initialized in `start()`.
self.report = None
self._main_root_workunit = None
self._all_options = None
# A lock to ensure that adding to stats at the end of a workunit
# operates thread-safely.
self._stats_lock = threading.Lock()
# Log of success/failure/aborted for each workunit.
self.outcomes = {}
# Number of threads for foreground work.
self._num_foreground_workers = self.options.num_foreground_workers
# Number of threads for background work.
self._num_background_workers = self.options.num_background_workers
# self._threadlocal.current_workunit contains the current workunit for the calling thread.
# Note that multiple threads may share a name (e.g., all the threads in a pool).
self._threadlocal = threading.local()
# For background work. Created lazily if needed.
self._background_root_workunit = None
# Trigger subproc pool init while our memory image is still clean (see SubprocPool docstring).
SubprocPool.set_num_processes(self._num_foreground_workers)
SubprocPool.foreground()
self._aborted = False
self._end_memoized_result: Optional[ExitCode] = None
self.native = Native()
self.run_logs_file: Optional[Path] = None
def __init__(self,
log_level: LogLevel,
native_filename: Optional[str] = None) -> None:
super().__init__(None)
self.native = Native()
self.native_filename = native_filename
self.setLevel(log_level.level)
if not self.native_filename:
self.native.setup_stderr_logger()
def _setup_services(
build_root: str,
bootstrap_options: OptionValueContainer,
legacy_graph_scheduler: LegacyGraphScheduler,
native: Native,
watchman: Watchman,
union_membership: UnionMembership,
):
"""Initialize pantsd services.
:returns: A PantsServices instance.
"""
native.override_thread_logging_destination_to_just_pantsd()
fs_event_service = (FSEventService(
watchman,
scheduler=legacy_graph_scheduler.scheduler,
build_root=build_root)
if bootstrap_options.watchman_enable else None)
invalidation_globs = OptionsInitializer.compute_pantsd_invalidation_globs(
build_root, bootstrap_options)
scheduler_service = SchedulerService(
fs_event_service=fs_event_service,
legacy_graph_scheduler=legacy_graph_scheduler,
build_root=build_root,
invalidation_globs=invalidation_globs,
union_membership=union_membership,
max_memory_usage_in_bytes=bootstrap_options.
pantsd_max_memory_usage,
)
pailgun_service = PailgunService(
bootstrap_options.pantsd_pailgun_port,
DaemonPantsRunner(scheduler_service),
scheduler_service,
)
store_gc_service = StoreGCService(legacy_graph_scheduler.scheduler)
return PantsServices(
services=tuple(service for service in (
fs_event_service,
scheduler_service,
pailgun_service,
store_gc_service,
) if service is not None),
port_map=dict(pailgun=pailgun_service.pailgun_port()),
)
def test_close_stdio(mock_close):
mock_options = unittest.mock.Mock()
mock_options_values = unittest.mock.Mock()
mock_options.for_global_scope.return_value = mock_options_values
mock_options_values.pants_subprocessdir = "non_existent_dir"
mock_server = unittest.mock.Mock()
def create_services(bootstrap_options, legacy_graph_scheduler):
return PantsServices()
pantsd = PantsDaemon(
native=Native(),
work_dir="test_work_dir",
log_level=logging.INFO,
server=mock_server,
core=PantsDaemonCore(create_services),
metadata_base_dir="/tmp/pants_test_metadata_dir",
bootstrap_options=mock_options,
)
with stdio_as(-1, -1, -1):
handles = (sys.stdin, sys.stdout, sys.stderr)
fds = [h.fileno() for h in handles]
pantsd._close_stdio()
mock_close.assert_has_calls(unittest.mock.call(x) for x in fds)
for handle in handles:
assert handle.closed is True
def setup_legacy_graph(
options_bootstrapper: OptionsBootstrapper,
build_configuration: BuildConfiguration,
) -> LegacyGraphScheduler:
"""Construct and return the components necessary for LegacyBuildGraph construction."""
native = Native()
build_root = get_buildroot()
bootstrap_options = options_bootstrapper.bootstrap_options.for_global_scope(
)
use_gitignore = bootstrap_options.pants_ignore_use_gitignore
return EngineInitializer.setup_legacy_graph_extended(
OptionsInitializer.compute_pants_ignore(build_root,
bootstrap_options),
use_gitignore,
bootstrap_options.local_store_dir,
bootstrap_options.local_execution_root_dir,
bootstrap_options.named_caches_dir,
bootstrap_options.build_file_prelude_globs,
options_bootstrapper,
build_configuration,
build_root=build_root,
native=native,
glob_match_error_behavior=(
bootstrap_options.files_not_found_behavior.
to_glob_match_error_behavior()),
build_ignore_patterns=bootstrap_options.build_ignore,
exclude_target_regexps=bootstrap_options.exclude_target_regexp,
subproject_roots=bootstrap_options.subproject_roots,
include_trace_on_error=bootstrap_options.
print_exception_stacktrace,
execution_options=ExecutionOptions.from_bootstrap_options(
bootstrap_options),
)
def setup_graph(
options: Options,
build_configuration: BuildConfiguration,
executor: Optional[PyExecutor] = None,
) -> GraphScheduler:
native = Native()
build_root = get_buildroot()
bootstrap_options = options.bootstrap_option_values()
assert bootstrap_options is not None
executor = executor or PyExecutor(
*OptionsInitializer.compute_executor_arguments(bootstrap_options))
return EngineInitializer.setup_graph_extended(
build_configuration,
ExecutionOptions.from_options(options),
native=native,
executor=executor,
pants_ignore_patterns=OptionsInitializer.compute_pants_ignore(
build_root, bootstrap_options),
use_gitignore=bootstrap_options.pants_ignore_use_gitignore,
local_store_dir=bootstrap_options.local_store_dir,
local_execution_root_dir=bootstrap_options.
local_execution_root_dir,
named_caches_dir=bootstrap_options.named_caches_dir,
ca_certs_path=bootstrap_options.ca_certs_path,
build_root=build_root,
include_trace_on_error=bootstrap_options.print_stacktrace,
native_engine_visualize_to=bootstrap_options.
native_engine_visualize_to,
)
def setup_graph(
options_bootstrapper: OptionsBootstrapper,
build_configuration: BuildConfiguration,
) -> GraphScheduler:
native = Native()
build_root = get_buildroot()
bootstrap_options = options_bootstrapper.bootstrap_options.for_global_scope(
)
return EngineInitializer.setup_graph_extended(
options_bootstrapper,
build_configuration,
ExecutionOptions.from_bootstrap_options(bootstrap_options),
pants_ignore_patterns=OptionsInitializer.compute_pants_ignore(
build_root, bootstrap_options),
use_gitignore=bootstrap_options.pants_ignore_use_gitignore,
local_store_dir=bootstrap_options.local_store_dir,
local_execution_root_dir=bootstrap_options.
local_execution_root_dir,
named_caches_dir=bootstrap_options.named_caches_dir,
ca_certs_path=bootstrap_options.ca_certs_path,
build_root=build_root,
native=native,
include_trace_on_error=bootstrap_options.
print_exception_stacktrace,
)
def _init_engine(self, local_store_dir: Optional[str] = None) -> None:
if self._scheduler is not None:
return
options_bootstrapper = OptionsBootstrapper.create(
env={},
args=["--pants-config-files=[]", *self.additional_options],
allow_pantsrc=False)
global_options = options_bootstrapper.bootstrap_options.for_global_scope(
)
local_store_dir = local_store_dir or global_options.local_store_dir
local_execution_root_dir = global_options.local_execution_root_dir
named_caches_dir = global_options.named_caches_dir
graph_session = EngineInitializer.setup_graph_extended(
pants_ignore_patterns=[],
use_gitignore=False,
local_store_dir=local_store_dir,
local_execution_root_dir=local_execution_root_dir,
named_caches_dir=named_caches_dir,
native=Native(),
options_bootstrapper=options_bootstrapper,
build_root=self.build_root,
build_configuration=self.build_config(),
execution_options=ExecutionOptions.from_bootstrap_options(
global_options),
).new_session(build_id="buildid_for_test",
should_report_workunits=True)
self._scheduler = graph_session.scheduler_session
def process_request_thread(self, request, client_address):
"""Override of ThreadingMixIn.process_request_thread() that delegates to the request
handler."""
# Instantiate the request handler.
Native().override_thread_logging_destination_to_just_pantsd()
handler = self.RequestHandlerClass(request, client_address, self)
_, _, _, environment = handler.parsed_request()
try:
with self.ensure_request_is_exclusive(environment, request):
# Attempt to handle a request with the handler.
handler.handle_request()
self.request_complete_callback()
except BrokenPipeError as e:
# The client has closed the connection, most likely from a SIGINT
self.logger.error(
f"Request {request} abruptly closed with {type(e)}, probably because the client crashed or was sent a SIGINT."
)
except Exception as e:
# If that fails, (synchronously) handle the error with the error handler sans-fork.
try:
self.logger.error(f"Request {request} errored with {type(e)}({e!r})")
handler.handle_error(e)
finally:
# Shutdown the socket since we don't expect a fork() in the exception context.
self.shutdown_request(request)
else:
# At this point, we expect a fork() has taken place - the parent side will return, and so we
# close the request here from the parent without explicitly shutting down the socket. The
# child half of this will perform an os._exit() before it gets to this point and is also
# responsible for shutdown and closing of the socket when its execution is complete.
self.close_request(request)
def setup_graph(
options_bootstrapper: OptionsBootstrapper,
build_configuration: BuildConfiguration,
env: CompleteEnvironment,
executor: Optional[PyExecutor] = None,
local_only: bool = False,
) -> GraphScheduler:
native = Native()
build_root = get_buildroot()
bootstrap_options = options_bootstrapper.bootstrap_options.for_global_scope(
)
options = options_bootstrapper.full_options(build_configuration)
assert bootstrap_options is not None
executor = executor or PyExecutor(
*GlobalOptions.compute_executor_arguments(bootstrap_options))
execution_options = ExecutionOptions.from_options(
options, env, local_only=local_only)
return EngineInitializer.setup_graph_extended(
build_configuration,
execution_options,
native=native,
executor=executor,
pants_ignore_patterns=GlobalOptions.compute_pants_ignore(
build_root, bootstrap_options),
use_gitignore=bootstrap_options.pants_ignore_use_gitignore,
local_store_dir=bootstrap_options.local_store_dir,
local_execution_root_dir=bootstrap_options.
local_execution_root_dir,
named_caches_dir=bootstrap_options.named_caches_dir,
ca_certs_path=bootstrap_options.ca_certs_path,
build_root=build_root,
include_trace_on_error=bootstrap_options.print_stacktrace,
native_engine_visualize_to=bootstrap_options.
native_engine_visualize_to,
)
def init_rust_logger(
log_level: LogLevel,
log_show_rust_3rdparty: bool,
use_color: bool,
show_target: bool,
log_levels_by_target: Dict[str, LogLevel] = {},
) -> None:
Native().init_rust_logging(log_level.level, log_show_rust_3rdparty,
use_color, show_target, log_levels_by_target)
def __init__(
self,
*,
rules: Optional[Iterable] = None,
target_types: Optional[Iterable[Type[Target]]] = None,
objects: Optional[Dict[str, Any]] = None,
context_aware_object_factories: Optional[Dict[str, Any]] = None,
) -> None:
self.build_root = os.path.realpath(mkdtemp(suffix="_BUILD_ROOT"))
safe_mkdir(self.build_root, clean=True)
safe_mkdir(self.pants_workdir)
BuildRoot().path = self.build_root
# TODO: Redesign rule registration for tests to be more ergonomic and to make this less
# special-cased.
all_rules = (
*(rules or ()),
*source_root.rules(),
*pants_environment.rules(),
QueryRule(WrappedTarget, (Address,)),
)
build_config_builder = BuildConfiguration.Builder()
build_config_builder.register_aliases(
BuildFileAliases(
objects=objects, context_aware_object_factories=context_aware_object_factories
)
)
build_config_builder.register_rules(all_rules)
build_config_builder.register_target_types(target_types or ())
self.build_config = build_config_builder.create()
options_bootstrapper = create_options_bootstrapper()
global_options = options_bootstrapper.bootstrap_options.for_global_scope()
local_store_dir = global_options.local_store_dir
local_execution_root_dir = global_options.local_execution_root_dir
named_caches_dir = global_options.named_caches_dir
graph_session = EngineInitializer.setup_graph_extended(
pants_ignore_patterns=[],
use_gitignore=False,
local_store_dir=local_store_dir,
local_execution_root_dir=local_execution_root_dir,
named_caches_dir=named_caches_dir,
native=Native(),
options_bootstrapper=options_bootstrapper,
build_root=self.build_root,
build_configuration=self.build_config,
execution_options=ExecutionOptions.from_bootstrap_options(global_options),
).new_session(
build_id="buildid_for_test",
session_values=SessionValues(
{OptionsBootstrapper: options_bootstrapper, PantsEnvironment: PantsEnvironment()}
),
should_report_workunits=True,
)
self.scheduler = graph_session.scheduler_session
def _stderr_logging(self, global_bootstrap_options):
"""Temporarily replaces existing handlers (ie, the pantsd handler) with a stderr handler.
In the context of pantsd, there will be an existing handler for the pantsd log, which we
temporarily replace. Making them additive would cause per-run logs to go to pantsd, which
we don't want.
TODO: It would be good to handle logging destinations entirely via the threadlocal state
rather than via handler mutations.
"""
handlers = get_logging_handlers()
try:
clear_logging_handlers()
Native().override_thread_logging_destination_to_just_stderr()
setup_logging(global_bootstrap_options, stderr_logging=True)
yield
finally:
Native().override_thread_logging_destination_to_just_pantsd()
set_logging_handlers(handlers)
def setUpClass(cls) -> None:
super().setUpClass()
# NB: We must set this up at the class level, rather than per-test level, because
# `init_rust_logging` must never be called more than once. The Rust logger is global and static,
# and initializing it twice in the same test class results in a SIGABRT.
Native().init_rust_logging(
level=LogLevel.INFO.level, # Tests assume a log level of INFO
log_show_rust_3rdparty=False,
use_color=False,
)
class NativeWriter:
scheduler_session: SchedulerSession
native: Native = Native()
def write(self, payload: str) -> None:
raise NotImplementedError
def flush(self):
"""flush() doesn't need to do anything for NativeWriter."""
pass
def setup_logging_to_file(
level: LogLevel,
*,
log_dir: str,
log_filename: str = "pants.log",
) -> NativeHandler:
native = Native()
logger = logging.getLogger(None)
_common_logging_setup(level)
safe_mkdir(log_dir)
log_path = os.path.join(log_dir, log_filename)
native.setup_pantsd_logger(log_path)
handler = NativeHandler(level, native_filename=log_path)
logger.addHandler(handler)
return handler
def _init_graph_session(
options_bootstrapper: OptionsBootstrapper,
build_config: BuildConfiguration,
options: Options,
scheduler: Optional[LegacyGraphScheduler] = None,
) -> LegacyGraphSession:
native = Native()
native.set_panic_handler()
graph_scheduler_helper = scheduler or EngineInitializer.setup_legacy_graph(
options_bootstrapper, build_config)
global_scope = options.for_global_scope()
if global_scope.v2:
dynamic_ui = resolve_conflicting_options(
old_option="v2_ui",
new_option="dynamic_ui",
old_scope=GLOBAL_SCOPE,
new_scope=GLOBAL_SCOPE,
old_container=global_scope,
new_container=global_scope,
)
else:
dynamic_ui = False
use_colors = global_scope.get("colors", True)
zipkin_trace_v2 = options.for_scope("reporting").zipkin_trace_v2
# TODO(#8658) This should_report_workunits flag must be set to True for
# StreamingWorkunitHandler to receive WorkUnits. It should eventually
# be merged with the zipkin_trace_v2 flag, since they both involve most
# of the same engine functionality, but for now is separate to avoid
# breaking functionality associated with zipkin tracing while iterating on streaming workunit reporting.
stream_workunits = len(
options.for_global_scope().streaming_workunits_handlers) != 0
return graph_scheduler_helper.new_session(
zipkin_trace_v2,
RunTracker.global_instance().run_id,
dynamic_ui=dynamic_ui,
use_colors=use_colors,
should_report_workunits=stream_workunits,
)
def __init__(
self,
log_level: LogLevel,
stream: Optional[TextIO] = None,
native_filename: Optional[str] = None,
):
if stream is not None and native_filename is not None:
raise RuntimeError("NativeHandler must output to either a stream or a file, not both")
super().__init__(stream)
self.native = Native()
self.native_filename = native_filename
self.setLevel(log_level.level)
if stream:
try:
self.native.setup_stderr_logger(log_level.level)
except Exception as e:
print(f"Error setting up pantsd logger: {e!r}", file=sys.stderr)
raise e
