def _init_graph_session(
cls,
options_bootstrapper: OptionsBootstrapper,
build_config: BuildConfiguration,
options: Options,
scheduler: Optional[GraphScheduler] = None,
) -> GraphSession:
native = Native()
native.set_panic_handler()
graph_scheduler_helper = scheduler or EngineInitializer.setup_graph(
options_bootstrapper, build_config)
try:
global_scope = options.for_global_scope()
except UnknownFlagsError as err:
cls._handle_unknown_flags(err, options_bootstrapper)
raise
stream_workunits = len(
options.for_global_scope().streaming_workunits_handlers) != 0
return graph_scheduler_helper.new_session(
RunTracker.global_instance().run_id,
dynamic_ui=global_scope.dynamic_ui,
use_colors=global_scope.get("colors", True),
should_report_workunits=stream_workunits,
session_values=SessionValues({
OptionsBootstrapper:
options_bootstrapper,
PantsEnvironment:
PantsEnvironment(os.environ),
}),
)
def _init_graph_session(
cls,
options_bootstrapper: OptionsBootstrapper,
build_config: BuildConfiguration,
run_id: str,
options: Options,
scheduler: Optional[GraphScheduler] = None,
cancellation_latch: Optional[PySessionCancellationLatch] = None,
) -> GraphSession:
native = Native()
native.set_panic_handler()
graph_scheduler_helper = scheduler or EngineInitializer.setup_graph(
options_bootstrapper, build_config
)
try:
global_scope = options.for_global_scope()
except UnknownFlagsError as err:
cls._handle_unknown_flags(err, options_bootstrapper)
raise
return graph_scheduler_helper.new_session(
run_id,
dynamic_ui=global_scope.dynamic_ui,
use_colors=global_scope.get("colors", True),
session_values=SessionValues(
{
OptionsBootstrapper: options_bootstrapper,
PantsEnvironment: PantsEnvironment(os.environ),
}
),
cancellation_latch=cancellation_latch,
)
def _init_graph_session(
cls,
options_initializer: OptionsInitializer,
options_bootstrapper: OptionsBootstrapper,
build_config: BuildConfiguration,
env: CompleteEnvironment,
run_id: str,
options: Options,
scheduler: Optional[GraphScheduler] = None,
cancellation_latch: Optional[PySessionCancellationLatch] = None,
) -> GraphSession:
native = Native()
native.set_panic_handler()
graph_scheduler_helper = scheduler or EngineInitializer.setup_graph(
options_bootstrapper, build_config, env
)
with options_initializer.handle_unknown_flags(options_bootstrapper, env, raise_=True):
global_options = options.for_global_scope()
return graph_scheduler_helper.new_session(
run_id,
dynamic_ui=global_options.dynamic_ui,
use_colors=global_options.get("colors", True),
session_values=SessionValues(
{
OptionsBootstrapper: options_bootstrapper,
CompleteEnvironment: env,
}
),
cancellation_latch=cancellation_latch,
)
def _init_graph_session(
cls,
options_initializer: OptionsInitializer,
options_bootstrapper: OptionsBootstrapper,
build_config: BuildConfiguration,
env: CompleteEnvironment,
run_id: str,
options: Options,
scheduler: Optional[GraphScheduler] = None,
cancellation_latch: Optional[PySessionCancellationLatch] = None,
) -> GraphSession:
native_engine.maybe_set_panic_handler()
if scheduler is None:
dynamic_remote_options, _ = DynamicRemoteOptions.from_options(options, env)
bootstrap_options = options.bootstrap_option_values()
assert bootstrap_options is not None
scheduler = EngineInitializer.setup_graph(
bootstrap_options, build_config, dynamic_remote_options
)
with options_initializer.handle_unknown_flags(options_bootstrapper, env, raise_=True):
global_options = options.for_global_scope()
return scheduler.new_session(
run_id,
dynamic_ui=global_options.dynamic_ui,
use_colors=global_options.get("colors", True),
session_values=SessionValues(
{
OptionsBootstrapper: options_bootstrapper,
CompleteEnvironment: env,
}
),
cancellation_latch=cancellation_latch,
)
def __init__(
self,
*,
rules: Optional[Iterable] = None,
target_types: Optional[Iterable[Type[Target]]] = None,
objects: Optional[Dict[str, Any]] = None,
context_aware_object_factories: Optional[Dict[str, Any]] = None,
) -> None:
self.build_root = os.path.realpath(mkdtemp(suffix="_BUILD_ROOT"))
safe_mkdir(self.build_root, clean=True)
safe_mkdir(self.pants_workdir)
BuildRoot().path = self.build_root
# TODO: Redesign rule registration for tests to be more ergonomic and to make this less
# special-cased.
all_rules = (
*(rules or ()),
*source_root.rules(),
*pants_environment.rules(),
QueryRule(WrappedTarget, (Address,)),
)
build_config_builder = BuildConfiguration.Builder()
build_config_builder.register_aliases(
BuildFileAliases(
objects=objects, context_aware_object_factories=context_aware_object_factories
)
)
build_config_builder.register_rules(all_rules)
build_config_builder.register_target_types(target_types or ())
self.build_config = build_config_builder.create()
options_bootstrapper = create_options_bootstrapper()
global_options = options_bootstrapper.bootstrap_options.for_global_scope()
local_store_dir = global_options.local_store_dir
local_execution_root_dir = global_options.local_execution_root_dir
named_caches_dir = global_options.named_caches_dir
graph_session = EngineInitializer.setup_graph_extended(
pants_ignore_patterns=[],
use_gitignore=False,
local_store_dir=local_store_dir,
local_execution_root_dir=local_execution_root_dir,
named_caches_dir=named_caches_dir,
native=Native(),
options_bootstrapper=options_bootstrapper,
build_root=self.build_root,
build_configuration=self.build_config,
execution_options=ExecutionOptions.from_bootstrap_options(global_options),
).new_session(
build_id="buildid_for_test",
session_values=SessionValues(
{OptionsBootstrapper: options_bootstrapper, PantsEnvironment: PantsEnvironment()}
),
should_report_workunits=True,
)
self.scheduler = graph_session.scheduler_session
def _set_new_session(self, scheduler: Scheduler) -> None:
self.scheduler = scheduler.new_session(
build_id="buildid_for_test",
session_values=SessionValues({
OptionsBootstrapper: self.options_bootstrapper,
CompleteEnvironment: self.environment,
**self.extra_session_values,
}),
max_workunit_level=self.max_workunit_verbosity,
)
def _resolve_plugins(
self, options_bootstrapper: OptionsBootstrapper
) -> ResolvedPluginDistributions:
session = self._scheduler.scheduler.new_session(
"plugin_resolver",
session_values=SessionValues({
OptionsBootstrapper:
options_bootstrapper,
PantsEnvironment:
PantsEnvironment(dict(options_bootstrapper.env_tuples)),
}),
)
return cast(
ResolvedPluginDistributions,
session.product_request(ResolvedPluginDistributions,
[self._interpreter_constraints])[0],
)
def _resolve_plugins(
self,
options_bootstrapper: OptionsBootstrapper,
env: CompleteEnvironment,
) -> ResolvedPluginDistributions:
session = self._scheduler.scheduler.new_session(
"plugin_resolver",
session_values=SessionValues({
OptionsBootstrapper: options_bootstrapper,
CompleteEnvironment: env,
}),
)
return cast(
ResolvedPluginDistributions,
session.product_request(ResolvedPluginDistributions,
[self._request])[0],
)
def run_goal_rule(
self,
goal: Type[Goal],
*,
global_args: Optional[Iterable[str]] = None,
args: Optional[Iterable[str]] = None,
env: Optional[Mapping[str, str]] = None,
) -> GoalRuleResult:
options_bootstrapper = create_options_bootstrapper(
args=(*(global_args or []), goal.name, *(args or [])),
env=env,
)
raw_specs = options_bootstrapper.get_full_options([
*GlobalOptions.known_scope_infos(),
*goal.subsystem_cls.known_scope_infos()
]).specs
specs = SpecsParser(self.build_root).parse_specs(raw_specs)
stdout, stderr = StringIO(), StringIO()
console = Console(stdout=stdout, stderr=stderr)
session = self.scheduler.scheduler.new_session(
build_id="buildid_for_test",
should_report_workunits=True,
session_values=SessionValues({
OptionsBootstrapper:
options_bootstrapper,
PantsEnvironment:
PantsEnvironment(env)
}),
)
exit_code = session.run_goal_rule(
goal,
Params(
specs,
console,
Workspace(self.scheduler),
InteractiveRunner(self.scheduler),
),
)
console.flush()
return GoalRuleResult(exit_code, stdout.getvalue(), stderr.getvalue())
def set_options(self, args: Iterable[str], *, env: Optional[Mapping[str, str]] = None) -> None:
"""Update the engine session with new options and/or environment variables.
The environment variables will be used to set the `PantsEnvironment`, which is the
environment variables captured by the parent Pants process. Some rules use this to be able
to read arbitrary env vars. Any options that start with `PANTS_` will also be used to set
options.
This will override any previously configured values.
"""
options_bootstrapper = create_options_bootstrapper(args=args, env=env)
self.scheduler = self.scheduler.scheduler.new_session(
build_id="buildid_for_test",
should_report_workunits=True,
session_values=SessionValues(
{OptionsBootstrapper: options_bootstrapper, PantsEnvironment: PantsEnvironment(env)}
),
)
def new_session(
self,
build_id,
dynamic_ui: bool = False,
should_report_workunits: bool = False,
session_values: Optional[SessionValues] = None,
) -> "SchedulerSession":
"""Creates a new SchedulerSession for this Scheduler."""
return SchedulerSession(
self,
self._native.new_session(
self._scheduler,
dynamic_ui,
build_id,
should_report_workunits,
session_values or SessionValues(),
),
)
def new_session(
self,
build_id,
dynamic_ui: bool = False,
session_values: Optional[SessionValues] = None,
cancellation_latch: Optional[PySessionCancellationLatch] = None,
) -> SchedulerSession:
"""Creates a new SchedulerSession for this Scheduler."""
return SchedulerSession(
self,
self._native.new_session(
self._scheduler,
dynamic_ui,
build_id,
session_values or SessionValues(),
cancellation_latch or PySessionCancellationLatch(),
),
)
def new_session(
self,
build_id: str,
dynamic_ui: bool = False,
session_values: SessionValues | None = None,
cancellation_latch: PySessionCancellationLatch | None = None,
) -> SchedulerSession:
"""Creates a new SchedulerSession for this Scheduler."""
return SchedulerSession(
self,
PySession(
scheduler=self.py_scheduler,
should_render_ui=dynamic_ui,
build_id=build_id,
session_values=session_values or SessionValues(),
cancellation_latch=cancellation_latch
or PySessionCancellationLatch(),
),
)
def _init_graph_session(
cls,
options_initializer: OptionsInitializer,
options_bootstrapper: OptionsBootstrapper,
build_config: BuildConfiguration,
env: CompleteEnvironment,
run_id: str,
options: Options,
scheduler: GraphScheduler | None = None,
cancellation_latch: PySessionCancellationLatch | None = None,
) -> GraphSession:
native_engine.maybe_set_panic_handler()
if scheduler is None:
dynamic_remote_options, _ = DynamicRemoteOptions.from_options(options, env)
bootstrap_options = options.bootstrap_option_values()
assert bootstrap_options is not None
scheduler = EngineInitializer.setup_graph(
bootstrap_options, build_config, dynamic_remote_options
)
with options_initializer.handle_unknown_flags(options_bootstrapper, env, raise_=True):
global_options = options.for_global_scope()
return scheduler.new_session(
run_id,
dynamic_ui=global_options.dynamic_ui,
ui_use_prodash=global_options.dynamic_ui_renderer
== DynamicUIRenderer.experimental_prodash,
use_colors=global_options.get("colors", True),
max_workunit_level=max(
global_options.streaming_workunits_level,
global_options.level,
*(
LogLevel[level.upper()]
for level in global_options.log_levels_by_target.values()
),
),
session_values=SessionValues(
{
OptionsBootstrapper: options_bootstrapper,
CompleteEnvironment: env,
}
),
cancellation_latch=cancellation_latch,
)
def request(self, output_type: Type["TestBase._O"],
inputs: Iterable[Any]) -> "TestBase._O":
# TODO: Update all callsites to pass this explicitly via session values.
session = self.scheduler
for value in inputs:
if type(value) == OptionsBootstrapper:
session = self.scheduler.scheduler.new_session(
build_id="buildid_for_test",
should_report_workunits=True,
session_values=SessionValues({
OptionsBootstrapper:
value,
PantsEnvironment:
PantsEnvironment()
}),
)
result = assert_single_element(
session.product_request(output_type, [Params(*inputs)]))
return cast(TestBase._O, result)
def _run_server(
self,
*,
graph_session: GraphSession,
union_membership: UnionMembership,
) -> ExitCode:
"""Run the BSP server."""
current_session_values = graph_session.scheduler_session.py_session.session_values
context = BSPContext()
session_values = SessionValues(
{
**current_session_values,
BSPContext: context,
}
)
scheduler_session = graph_session.scheduler_session.scheduler.new_session(
build_id="bsp", dynamic_ui=False, session_values=session_values
)
saved_stdout = sys.stdout
saved_stdin = sys.stdin
try:
sys.stdout = os.fdopen(sys.stdout.fileno(), "wb", buffering=0) # type: ignore[assignment]
sys.stdin = os.fdopen(sys.stdin.fileno(), "rb", buffering=0) # type: ignore[assignment]
conn = BSPConnection(
scheduler_session,
union_membership,
context,
sys.stdin, # type: ignore[arg-type]
sys.stdout, # type: ignore[arg-type]
)
conn.run()
finally:
sys.stdout = saved_stdout
sys.stdin = saved_stdin
return ExitCode(0)
def new_session(
self,
build_id: str,
dynamic_ui: bool = False,
ui_use_prodash: bool = False,
max_workunit_level: LogLevel = LogLevel.DEBUG,
session_values: SessionValues | None = None,
cancellation_latch: PySessionCancellationLatch | None = None,
) -> SchedulerSession:
"""Creates a new SchedulerSession for this Scheduler."""
return SchedulerSession(
self,
PySession(
scheduler=self.py_scheduler,
dynamic_ui=dynamic_ui,
ui_use_prodash=ui_use_prodash,
max_workunit_level=max_workunit_level.level,
build_id=build_id,
session_values=session_values or SessionValues(),
cancellation_latch=cancellation_latch
or PySessionCancellationLatch(),
),
)
async def check_default_tools(
console: Console,
real_opts: _Options,
) -> CheckDefaultTools:
# The real options know about all the registered tools.
for scope, si in real_opts.options.known_scope_to_info.items():
if si.subsystem_cls and issubclass(si.subsystem_cls, ExternalTool):
tool_cls = si.subsystem_cls
console.print_stdout(f"Checking {console.cyan(tool_cls.name)}:")
for known_version in tool_cls.default_known_versions:
ver, plat_val, sha256, length = tool_cls.split_known_version_str(
known_version)
# Note that we don't want to use the real option values here - we want to
# verify that the *defaults* aren't broken. However the get_request_for() method
# requires an instance (since it can consult option values, including custom
# options for specific tools, that we don't know about), so we construct a
# default one, but we force the --version to the one we're checking (which will
# typically be the same as the default version, but doesn't have to be, if the
# tool provides default_known_versions for versions other than default_version).
args = ("./pants", f"--{scope}-version={ver}")
blank_opts = await Get(
_Options,
SessionValues({
OptionsBootstrapper:
OptionsBootstrapper(tuple(), ("./pants", ), args,
_ChainedConfig(tuple()),
CliAlias())
}),
)
instance = tool_cls(blank_opts.options.for_scope(scope))
req = instance.get_request_for(plat_val, sha256, length)
console.write_stdout(f" version {ver} for {plat_val}... ")
# TODO: We'd like to run all the requests concurrently, but since we can't catch
# engine exceptions, we wouldn't have an easy way to output which one failed.
await Get(DownloadedExternalTool, ExternalToolRequest, req)
console.print_stdout(console.sigil_succeeded())
return CheckDefaultTools(exit_code=0)
def set_options(
self,
args: Iterable[str],
*,
env: Mapping[str, str] | None = None,
env_inherit: set[str] | None = None,
) -> None:
"""Update the engine session with new options and/or environment variables.
The environment variables will be used to set the `CompleteEnvironment`, which is the
environment variables captured by the parent Pants process. Some rules use this to be able
to read arbitrary env vars. Any options that start with `PANTS_` will also be used to set
options.
Environment variables listed in `env_inherit` and not in `env` will be inherited from the test
runner's environment (os.environ)
This will override any previously configured values.
"""
env = {
**{
k: os.environ[k]
for k in (env_inherit or set()) if k in os.environ
},
**(env or {}),
}
self.options_bootstrapper = create_options_bootstrapper(args=args,
env=env)
self.environment = CompleteEnvironment(env)
self.scheduler = self.scheduler.scheduler.new_session(
build_id="buildid_for_test",
session_values=SessionValues({
OptionsBootstrapper: self.options_bootstrapper,
CompleteEnvironment: self.environment,
}),
)
def __init__(
self,
*,
rules: Iterable | None = None,
target_types: Iterable[type[Target]] | None = None,
objects: dict[str, Any] | None = None,
context_aware_object_factories: dict[str, Any] | None = None,
isolated_local_store: bool = False,
preserve_tmpdirs: bool = False,
ca_certs_path: str | None = None,
bootstrap_args: Iterable[str] = (),
) -> None:
bootstrap_args = [*bootstrap_args]
root_dir: Path | None = None
if preserve_tmpdirs:
root_dir = Path(mkdtemp(prefix="RuleRunner."))
print(
f"Preserving rule runner temporary directories at {root_dir}.",
file=sys.stderr)
bootstrap_args.extend([
"--no-process-execution-local-cleanup",
f"--local-execution-root-dir={root_dir}",
])
build_root = (root_dir / "BUILD_ROOT").resolve()
build_root.mkdir()
self.build_root = str(build_root)
else:
self.build_root = os.path.realpath(
safe_mkdtemp(prefix="_BUILD_ROOT"))
safe_mkdir(self.pants_workdir)
BuildRoot().path = self.build_root
# TODO: Redesign rule registration for tests to be more ergonomic and to make this less
# special-cased.
all_rules = (
*(rules or ()),
*source_root.rules(),
QueryRule(WrappedTarget, [Address]),
QueryRule(UnionMembership, []),
)
build_config_builder = BuildConfiguration.Builder()
build_config_builder.register_aliases(
BuildFileAliases(
objects=objects,
context_aware_object_factories=context_aware_object_factories))
build_config_builder.register_rules("_dummy_for_test_", all_rules)
build_config_builder.register_target_types("_dummy_for_test_",
target_types or ())
self.build_config = build_config_builder.create()
self.environment = CompleteEnvironment({})
self.options_bootstrapper = create_options_bootstrapper(
args=bootstrap_args)
options = self.options_bootstrapper.full_options(self.build_config)
global_options = self.options_bootstrapper.bootstrap_options.for_global_scope(
)
dynamic_remote_options, _ = DynamicRemoteOptions.from_options(
options, self.environment)
local_store_options = LocalStoreOptions.from_options(global_options)
if isolated_local_store:
if root_dir:
lmdb_store_dir = root_dir / "lmdb_store"
lmdb_store_dir.mkdir()
store_dir = str(lmdb_store_dir)
else:
store_dir = safe_mkdtemp(prefix="lmdb_store.")
local_store_options = dataclasses.replace(local_store_options,
store_dir=store_dir)
local_execution_root_dir = global_options.local_execution_root_dir
named_caches_dir = global_options.named_caches_dir
graph_session = EngineInitializer.setup_graph_extended(
pants_ignore_patterns=GlobalOptions.compute_pants_ignore(
self.build_root, global_options),
use_gitignore=False,
local_store_options=local_store_options,
local_execution_root_dir=local_execution_root_dir,
named_caches_dir=named_caches_dir,
build_root=self.build_root,
build_configuration=self.build_config,
executor=_EXECUTOR,
execution_options=ExecutionOptions.from_options(
global_options, dynamic_remote_options),
ca_certs_path=ca_certs_path,
engine_visualize_to=None,
).new_session(
build_id="buildid_for_test",
session_values=SessionValues({
OptionsBootstrapper: self.options_bootstrapper,
CompleteEnvironment: self.environment,
}),
)
self.scheduler = graph_session.scheduler_session
def __init__(
self,
*,
rules: Iterable | None = None,
target_types: Iterable[type[Target]] | None = None,
objects: dict[str, Any] | None = None,
context_aware_object_factories: dict[str, Any] | None = None,
isolated_local_store: bool = False,
ca_certs_path: str | None = None,
) -> None:
self.build_root = os.path.realpath(mkdtemp(suffix="_BUILD_ROOT"))
safe_mkdir(self.build_root, clean=True)
safe_mkdir(self.pants_workdir)
BuildRoot().path = self.build_root
# TODO: Redesign rule registration for tests to be more ergonomic and to make this less
# special-cased.
all_rules = (
*(rules or ()),
*source_root.rules(),
*pants_environment.rules(),
QueryRule(WrappedTarget, [Address]),
QueryRule(UnionMembership, []),
)
build_config_builder = BuildConfiguration.Builder()
build_config_builder.register_aliases(
BuildFileAliases(
objects=objects,
context_aware_object_factories=context_aware_object_factories))
build_config_builder.register_rules(all_rules)
build_config_builder.register_target_types(target_types or ())
self.build_config = build_config_builder.create()
self.options_bootstrapper = create_options_bootstrapper()
options = self.options_bootstrapper.full_options(self.build_config)
global_options = self.options_bootstrapper.bootstrap_options.for_global_scope(
)
local_store_dir = (os.path.realpath(safe_mkdtemp())
if isolated_local_store else
global_options.local_store_dir)
local_execution_root_dir = global_options.local_execution_root_dir
named_caches_dir = global_options.named_caches_dir
graph_session = EngineInitializer.setup_graph_extended(
pants_ignore_patterns=GlobalOptions.compute_pants_ignore(
self.build_root, global_options),
use_gitignore=False,
local_store_dir=local_store_dir,
local_execution_root_dir=local_execution_root_dir,
named_caches_dir=named_caches_dir,
native=Native(),
build_root=self.build_root,
build_configuration=self.build_config,
executor=_EXECUTOR,
execution_options=ExecutionOptions.from_options(options),
ca_certs_path=ca_certs_path,
native_engine_visualize_to=None,
).new_session(
build_id="buildid_for_test",
session_values=SessionValues({
OptionsBootstrapper: self.options_bootstrapper,
PantsEnvironment: PantsEnvironment(),
}),
)
self.scheduler = graph_session.scheduler_session