def setup_graph(
bootstrap_options: OptionValueContainer,
build_configuration: BuildConfiguration,
dynamic_remote_options: DynamicRemoteOptions,
executor: PyExecutor | None = None,
) -> GraphScheduler:
build_root = get_buildroot()
executor = executor or GlobalOptions.create_py_executor(
bootstrap_options)
execution_options = ExecutionOptions.from_options(
bootstrap_options, dynamic_remote_options)
local_store_options = LocalStoreOptions.from_options(bootstrap_options)
return EngineInitializer.setup_graph_extended(
build_configuration,
execution_options,
executor=executor,
pants_ignore_patterns=GlobalOptions.compute_pants_ignore(
build_root, bootstrap_options),
use_gitignore=bootstrap_options.pants_ignore_use_gitignore,
local_store_options=local_store_options,
local_execution_root_dir=bootstrap_options.
local_execution_root_dir,
named_caches_dir=bootstrap_options.named_caches_dir,
ca_certs_path=bootstrap_options.ca_certs_path,
build_root=build_root,
include_trace_on_error=bootstrap_options.print_stacktrace,
engine_visualize_to=bootstrap_options.engine_visualize_to,
watch_filesystem=bootstrap_options.watch_filesystem,
use_deprecated_python_macros=bootstrap_options.
use_deprecated_python_macros,
)
def setup_graph(
options_bootstrapper: OptionsBootstrapper,
build_configuration: BuildConfiguration,
env: CompleteEnvironment,
executor: Optional[PyExecutor] = None,
local_only: bool = False,
) -> GraphScheduler:
native = Native()
build_root = get_buildroot()
bootstrap_options = options_bootstrapper.bootstrap_options.for_global_scope(
)
options = options_bootstrapper.full_options(build_configuration)
assert bootstrap_options is not None
executor = executor or PyExecutor(
*GlobalOptions.compute_executor_arguments(bootstrap_options))
execution_options = ExecutionOptions.from_options(
options, env, local_only=local_only)
return EngineInitializer.setup_graph_extended(
build_configuration,
execution_options,
native=native,
executor=executor,
pants_ignore_patterns=GlobalOptions.compute_pants_ignore(
build_root, bootstrap_options),
use_gitignore=bootstrap_options.pants_ignore_use_gitignore,
local_store_dir=bootstrap_options.local_store_dir,
local_execution_root_dir=bootstrap_options.
local_execution_root_dir,
named_caches_dir=bootstrap_options.named_caches_dir,
ca_certs_path=bootstrap_options.ca_certs_path,
build_root=build_root,
include_trace_on_error=bootstrap_options.print_stacktrace,
native_engine_visualize_to=bootstrap_options.
native_engine_visualize_to,
)
def create_options_for_optionables(optionables,
options=None,
options_fingerprintable=None,
passthru_args=None):
"""Create a fake Options object for testing with appropriate defaults for the given optionables.
Any scoped `options` provided will override defaults, behaving as-if set on the command line.
:param iterable optionables: A series of `Optionable` types to register default options for.
:param dict options: A dict of scope -> (dict of option name -> value) representing option values
explicitly set via the command line.
:param dict options_fingerprintable: A dict of scope -> (dict of option name -> option type)
representing the fingerprintable options
and the scopes they are registered for.
:param list passthru_args: A list of passthrough args (specified after `--` on the command line).
:returns: A fake `Options` object with defaults populated for the given `optionables` and any
explicitly set `options` overlayed.
"""
all_options = defaultdict(dict)
fingerprintable_options = defaultdict(dict)
bootstrap_option_values = None
if options_fingerprintable:
for scope, opts in options_fingerprintable.items():
fingerprintable_options[scope].update(opts)
def register_func(on_scope):
scoped_options = all_options[on_scope]
scoped_fingerprintables = fingerprintable_options[on_scope]
register = _options_registration_function(scoped_options,
scoped_fingerprintables)
register.bootstrap = bootstrap_option_values
register.scope = on_scope
return register
# TODO: This sequence is a bit repetitive of the real registration sequence.
# Register bootstrap options and grab their default values for use in subsequent registration.
GlobalOptions.register_bootstrap_options(register_func(GLOBAL_SCOPE))
bootstrap_option_values = _FakeOptionValues(
all_options[GLOBAL_SCOPE].copy())
# Now register the full global scope options.
GlobalOptions.register_options(register_func(GLOBAL_SCOPE))
for optionable in optionables:
optionable.register_options(register_func(optionable.options_scope))
if options:
for scope, opts in options.items():
all_options[scope].update(opts)
return create_options(all_options,
passthru_args=passthru_args,
fingerprintable_options=fingerprintable_options)
def parse_bootstrap_options(
env: Mapping[str, str], args: Sequence[str], config: Config
) -> Options:
bootstrap_options = Options.create(
env=env, config=config, known_scope_infos=[GlobalOptions.get_scope_info()], args=args,
)
def register_global(*args, **kwargs):
# Only use of Options.register?
bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs)
GlobalOptions.register_bootstrap_options(register_global)
return bootstrap_options
def create(cls, options_bootstrapper: OptionsBootstrapper,
env: CompleteEnvironment) -> PantsDaemon:
# Any warnings that would be triggered here are re-triggered later per-run of Pants, so we
# silence them.
with warnings.catch_warnings(record=True):
bootstrap_options = options_bootstrapper.bootstrap_options
bootstrap_options_values = bootstrap_options.for_global_scope()
executor = GlobalOptions.create_py_executor(bootstrap_options_values)
core = PantsDaemonCore(options_bootstrapper, env, executor,
cls._setup_services)
server = native_engine.nailgun_server_create(
executor,
bootstrap_options_values.pantsd_pailgun_port,
DaemonPantsRunner(core),
)
return PantsDaemon(
work_dir=bootstrap_options_values.pants_workdir,
log_level=bootstrap_options_values.level,
server=server,
core=core,
metadata_base_dir=bootstrap_options_values.pants_subprocessdir,
bootstrap_options=bootstrap_options,
)
def _setup_services(
bootstrap_options: OptionValueContainer,
graph_scheduler: GraphScheduler,
):
"""Initialize pantsd services.
:returns: A PantsServices instance.
"""
build_root = get_buildroot()
invalidation_globs = GlobalOptions.compute_pantsd_invalidation_globs(
build_root,
bootstrap_options,
)
scheduler_service = SchedulerService(
graph_scheduler=graph_scheduler,
build_root=build_root,
invalidation_globs=invalidation_globs,
pidfile=PantsDaemon.metadata_file_path(
"pantsd", "pid", bootstrap_options.pants_subprocessdir),
pid=os.getpid(),
max_memory_usage_in_bytes=bootstrap_options.
pantsd_max_memory_usage,
)
store_gc_service = StoreGCService(graph_scheduler.scheduler)
return PantsServices(services=(scheduler_service, store_gc_service))
def run_goal_rule(
self,
goal: Type[Goal],
*,
global_args: Optional[Iterable[str]] = None,
args: Optional[Iterable[str]] = None,
env: Optional[Mapping[str, str]] = None,
) -> GoalRuleResult:
options_bootstrapper = create_options_bootstrapper(
args=(*(global_args or []), goal.name, *(args or [])),
env=env,
)
raw_specs = options_bootstrapper.get_full_options([
*GlobalOptions.known_scope_infos(),
*goal.subsystem_cls.known_scope_infos()
]).specs
specs = SpecsParser(self.build_root).parse_specs(raw_specs)
stdout, stderr = StringIO(), StringIO()
console = Console(stdout=stdout, stderr=stderr)
exit_code = self.scheduler.run_goal_rule(
goal,
Params(
specs,
console,
options_bootstrapper,
Workspace(self.scheduler),
InteractiveRunner(self.scheduler),
),
)
console.flush()
return GoalRuleResult(exit_code, stdout.getvalue(), stderr.getvalue())
def run_goal_rule(
self,
goal: Type[Goal],
*,
global_args: Iterable[str] | None = None,
args: Iterable[str] | None = None,
env: Mapping[str, str] | None = None,
env_inherit: set[str] | None = None,
) -> GoalRuleResult:
merged_args = (*(global_args or []), goal.name, *(args or []))
self.set_options(merged_args, env=env, env_inherit=env_inherit)
raw_specs = self.options_bootstrapper.full_options_for_scopes([
GlobalOptions.get_scope_info(),
goal.subsystem_cls.get_scope_info()
]).specs
specs = SpecsParser(self.build_root).parse_specs(raw_specs)
stdout, stderr = StringIO(), StringIO()
console = Console(stdout=stdout, stderr=stderr)
exit_code = self.scheduler.run_goal_rule(
goal,
Params(
specs,
console,
Workspace(self.scheduler),
InteractiveRunner(self.scheduler),
),
)
console.flush()
return GoalRuleResult(exit_code, stdout.getvalue(), stderr.getvalue())
def single_target_run(
self,
*,
console: MockConsole,
program_text: bytes,
address_spec: str,
) -> Run:
workspace = Workspace(self.scheduler)
interactive_runner = InteractiveRunner(self.scheduler)
BuildRoot().path = self.build_root
res = run_rule(
run,
rule_args=[
console,
workspace,
interactive_runner,
BuildRoot(),
Addresses([Address.parse(address_spec)]),
MockOptions(args=[]),
GlobalOptions.global_instance(),
],
mock_gets=[
MockGet(
product_type=CreatedBinary,
subject_type=Address,
mock=lambda _: self.create_mock_binary(program_text),
),
],
)
return cast(Run, res)
def full_options(self, build_configuration: BuildConfiguration) -> Options:
global_bootstrap_options = self.get_bootstrap_options().for_global_scope()
if global_bootstrap_options.pants_version != pants_version():
raise BuildConfigurationError(
f"Version mismatch: Requested version was {global_bootstrap_options.pants_version}, "
f"our version is {pants_version()}."
)
# Parse and register options.
known_scope_infos = [
subsystem.get_scope_info() for subsystem in build_configuration.all_subsystems
]
options = self.full_options_for_scopes(
known_scope_infos, allow_unknown_options=build_configuration.allow_unknown_options
)
GlobalOptions.validate_instance(options.for_global_scope())
return options
def create(cls, options_bootstrapper, build_configuration, init_subsystems=True):
global_bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope()
if global_bootstrap_options.pants_version != pants_version():
raise BuildConfigurationError(
f"Version mismatch: Requested version was {global_bootstrap_options.pants_version}, "
f"our version is {pants_version()}."
)
# Parse and register options.
options = cls._construct_options(options_bootstrapper, build_configuration)
GlobalOptions.validate_instance(options.for_global_scope())
if init_subsystems:
Subsystem.set_options(options)
return options
def parse_bootstrap_options(env: Mapping[str, str], args: Sequence[str],
config: Config) -> Options:
bootstrap_options = Options.create(
env=env,
config=config,
known_scope_infos=[GlobalOptions.get_scope_info()],
args=args,
)
def register_global(*args, **kwargs):
## Only use of Options.register?
bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs)
GlobalOptions.register_bootstrap_options(register_global)
opts = bootstrap_options.for_global_scope()
if opts.v2 and not opts.v1 and opts.backend_packages == []:
is_v2_exclusive.set()
return bootstrap_options
def test_invalidation_globs() -> None:
# Confirm that an un-normalized relative path in the pythonpath is filtered out.
suffix = "something-ridiculous"
ob = OptionsBootstrapper.create(env={}, args=[f"--pythonpath=../{suffix}"], allow_pantsrc=False)
globs = GlobalOptions.compute_pantsd_invalidation_globs(
get_buildroot(), ob.bootstrap_options.for_global_scope()
)
for glob in globs:
assert suffix not in glob
def _connect_and_execute(
self, pantsd_handle: PantsDaemonClient.Handle) -> ExitCode:
global_options = self._bootstrap_options.for_global_scope()
executor = GlobalOptions.create_py_executor(global_options)
# Merge the nailgun TTY capability environment variables with the passed environment dict.
ng_env = NailgunProtocol.ttynames_to_env(sys.stdin, sys.stdout,
sys.stderr)
modified_env = {
**self._env,
**ng_env,
"PANTSD_RUNTRACKER_CLIENT_START_TIME":
str(self._start_time),
"PANTSD_REQUEST_TIMEOUT_LIMIT":
str(global_options.pantsd_timeout_when_multiple_invocations),
}
command = self._args[0]
args = self._args[1:]
retries = 3
attempt = 1
while True:
port = pantsd_handle.port
logger.debug(
f"Connecting to pantsd on port {port} attempt {attempt}/{retries}"
)
# We preserve TTY settings since the server might write directly to the TTY, and we'd like
# to clean up any side effects before exiting.
#
# We ignore keyboard interrupts because the nailgun client will handle them.
with STTYSettings.preserved(), interrupts_ignored():
try:
return native_engine.nailgun_client_create(
executor, port).execute(command, args, modified_env)
# NailgunConnectionException represents a failure connecting to pantsd, so we retry
# up to the retry limit.
except NailgunConnectionException as e:
if attempt > retries:
raise self.Fallback(e)
# Wait one second before retrying
logger.warning(
f"Pantsd was unresponsive on port {port}, retrying.")
time.sleep(1)
# One possible cause of the daemon being non-responsive during an attempt might be if a
# another lifecycle operation is happening concurrently (incl teardown). To account for
# this, we won't begin attempting restarts until at least 1 attempt has passed.
if attempt > 1:
pantsd_handle = self._client.restart()
attempt += 1
def create(
cls,
options_bootstrapper: OptionsBootstrapper,
build_configuration: BuildConfiguration,
) -> Options:
global_bootstrap_options = options_bootstrapper.get_bootstrap_options(
).for_global_scope()
if global_bootstrap_options.pants_version != pants_version():
raise BuildConfigurationError(
f"Version mismatch: Requested version was {global_bootstrap_options.pants_version}, "
f"our version is {pants_version()}.")
# Parse and register options.
known_scope_infos = [
si for optionable in build_configuration.all_optionables
for si in optionable.known_scope_infos()
]
options = options_bootstrapper.get_full_options(known_scope_infos)
GlobalOptions.validate_instance(options.for_global_scope())
return options
def test_invalidation_globs() -> None:
# Confirm that an un-normalized relative path in the pythonpath is filtered out, and that an
# empty entry (i.e.: a relative path for the current directory) doesn't cause an error.
suffix = "something-ridiculous"
ob = OptionsBootstrapper.create(
env={},
args=[f"--pythonpath=../{suffix}", "--pythonpath="],
allow_pantsrc=False)
globs = GlobalOptions.compute_pantsd_invalidation_globs(
get_buildroot(), ob.bootstrap_options.for_global_scope())
for glob in globs:
assert suffix not in glob
def do_test(kwargs, expected_basic=False, expected_advanced=False):
def exp_to_len(exp):
return int(exp) # True -> 1, False -> 0.
parser = Parser(
env={},
config=Config.load([]),
scope_info=GlobalOptions.get_scope_info(),
)
parser.register("--foo", **kwargs)
oshi = HelpInfoExtracter("").get_option_scope_help_info(
"", parser, False)
assert exp_to_len(expected_basic) == len(oshi.basic)
assert exp_to_len(expected_advanced) == len(oshi.advanced)
def _format_for_global_scope(show_advanced, show_deprecated, args, kwargs):
parser = Parser(
env={},
config=Config.load([]),
scope_info=GlobalOptions.get_scope_info(),
parent_parser=None,
)
parser.register(*args, **kwargs)
# Force a parse to generate the derivation history.
parser.parse_args(Parser.ParseArgsRequest((), OptionValueContainerBuilder(), [], False))
oshi = HelpInfoExtracter("").get_option_scope_help_info("", parser, False)
return HelpFormatter(
show_advanced=show_advanced, show_deprecated=show_deprecated, color=False
).format_options(oshi)
def do_test(args, kwargs, expected_default_str):
# Defaults are computed in the parser and added into the kwargs, so we
# must jump through this hoop in this test.
parser = Parser(
env={},
config=Config.load([]),
scope_info=GlobalOptions.get_scope_info(),
)
parser.register(*args, **kwargs)
oshi = HelpInfoExtracter(parser.scope).get_option_scope_help_info(
"description", parser, False)
assert oshi.description == "description"
assert len(oshi.basic) == 1
ohi = oshi.basic[0]
assert to_help_str(ohi.default) == expected_default_str
def parse_bootstrap_options(env: Mapping[str, str], args: Sequence[str],
config: Config) -> Options:
bootstrap_options = Options.create(
env=env,
config=config,
known_scope_infos=[GlobalOptions.get_scope_info()],
args=args,
)
for options_info in collect_options_info(BootstrapOptions):
# Only use of Options.register?
bootstrap_options.register(GLOBAL_SCOPE, *options_info.flag_names,
**options_info.flag_options)
return bootstrap_options
def execute_rule(
self,
args: Optional[Iterable[str]] = None,
global_args: Optional[Iterable[str]] = None,
env: Optional[Dict[str, str]] = None,
exit_code: int = 0,
additional_params: Optional[Iterable[Any]] = None,
) -> str:
"""Executes the @goal_rule for this test class.
:API: public
Returns the text output of the task.
"""
# Create an OptionsBootstrapper for these args/env, and a captured Console instance.
options_bootstrapper = create_options_bootstrapper(
args=(*(global_args or []), self.goal_cls.name, *(args or [])), env=env,
)
BuildConfigInitializer.get(options_bootstrapper)
full_options = options_bootstrapper.get_full_options(
[*GlobalOptions.known_scope_infos(), *self.goal_cls.subsystem_cls.known_scope_infos()]
)
stdout, stderr = StringIO(), StringIO()
console = Console(stdout=stdout, stderr=stderr)
scheduler = self.scheduler
workspace = Workspace(scheduler)
# Run for the target specs parsed from the args.
specs = SpecsCalculator.parse_specs(full_options.specs, self.build_root)
params = Params(
specs.provided_specs,
console,
options_bootstrapper,
workspace,
*(additional_params or []),
)
actual_exit_code = self.scheduler.run_goal_rule(self.goal_cls, params)
# Flush and capture console output.
console.flush()
stdout_val = stdout.getvalue()
stderr_val = stderr.getvalue()
assert (
exit_code == actual_exit_code
), f"Exited with {actual_exit_code} (expected {exit_code}):\nstdout:\n{stdout_val}\nstderr:\n{stderr_val}"
return stdout_val
def do_test(args, kwargs, expected_default):
# Defaults are computed in the parser and added into the kwargs, so we
# must jump through this hoop in this test.
parser = Parser(
env={},
config=Config.load([]),
scope_info=GlobalOptions.get_scope_info(),
parent_parser=None,
option_tracker=OptionTracker(),
)
parser.register(*args, **kwargs)
oshi = HelpInfoExtracter.get_option_scope_help_info_from_parser(
parser).basic
assert len(oshi) == 1
ohi = oshi[0]
assert ohi.default == expected_default
def run_goal_rule(
self,
goal: Type[Goal],
*,
global_args: Optional[Iterable[str]] = None,
args: Optional[Iterable[str]] = None,
env: Optional[Mapping[str, str]] = None,
) -> GoalRuleResult:
options_bootstrapper = create_options_bootstrapper(
args=(*(global_args or []), goal.name, *(args or [])),
env=env,
)
raw_specs = options_bootstrapper.get_full_options([
*GlobalOptions.known_scope_infos(),
*goal.subsystem_cls.known_scope_infos()
]).specs
specs = SpecsParser(self.build_root).parse_specs(raw_specs)
stdout, stderr = StringIO(), StringIO()
console = Console(stdout=stdout, stderr=stderr)
session = self.scheduler.scheduler.new_session(
build_id="buildid_for_test",
should_report_workunits=True,
session_values=SessionValues({
OptionsBootstrapper:
options_bootstrapper,
PantsEnvironment:
PantsEnvironment(env)
}),
)
exit_code = session.run_goal_rule(
goal,
Params(
specs,
console,
Workspace(self.scheduler),
InteractiveRunner(self.scheduler),
),
)
console.flush()
return GoalRuleResult(exit_code, stdout.getvalue(), stderr.getvalue())
def _get_bootstrap_args(cls, args: Sequence[str]) -> tuple[str, ...]:
# TODO(13244): there is a typing issue with `memoized_classmethod`.
options = GlobalOptions.get_options_flags() # type: ignore[call-arg]
def is_bootstrap_option(arg: str) -> bool:
components = arg.split("=", 1)
if components[0] in options.flags:
return True
for flag in options.short_flags:
if arg.startswith(flag):
return True
return False
# Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
# Stop before '--' since args after that are pass-through and may have duplicate names to our
# bootstrap options.
bargs = ("./pants", ) + tuple(
filter(is_bootstrap_option,
itertools.takewhile(lambda arg: arg != "--", args)))
return bargs
def mock_console(
options_bootstrapper: OptionsBootstrapper,
*,
stdin_content: bytes | str | None = None,
) -> Iterator[tuple[Console, StdioReader]]:
global_bootstrap_options = options_bootstrapper.bootstrap_options.for_global_scope(
)
colors = (options_bootstrapper.full_options_for_scopes(
[GlobalOptions.get_scope_info()],
allow_unknown_options=True).for_global_scope().colors)
with initialize_stdio(global_bootstrap_options), stdin_context(
stdin_content) as stdin, temporary_file(
binary_mode=False) as stdout, temporary_file(
binary_mode=False) as stderr, stdio_destination(
stdin_fileno=stdin.fileno(),
stdout_fileno=stdout.fileno(),
stderr_fileno=stderr.fileno(),
):
# NB: We yield a Console without overriding the destination argument, because we have
# already done a sys.std* level replacement. The replacement is necessary in order for
# InteractiveProcess to have native file handles to interact with.
yield Console(use_colors=colors), StdioReader(
_stdout=Path(stdout.name), _stderr=Path(stderr.name))
def __init__(
self,
*,
rules: Iterable | None = None,
target_types: Iterable[type[Target]] | None = None,
objects: dict[str, Any] | None = None,
context_aware_object_factories: dict[str, Any] | None = None,
isolated_local_store: bool = False,
preserve_tmpdirs: bool = False,
ca_certs_path: str | None = None,
bootstrap_args: Iterable[str] = (),
) -> None:
bootstrap_args = [*bootstrap_args]
root_dir: Path | None = None
if preserve_tmpdirs:
root_dir = Path(mkdtemp(prefix="RuleRunner."))
print(
f"Preserving rule runner temporary directories at {root_dir}.",
file=sys.stderr)
bootstrap_args.extend([
"--no-process-execution-local-cleanup",
f"--local-execution-root-dir={root_dir}",
])
build_root = (root_dir / "BUILD_ROOT").resolve()
build_root.mkdir()
self.build_root = str(build_root)
else:
self.build_root = os.path.realpath(
safe_mkdtemp(prefix="_BUILD_ROOT"))
safe_mkdir(self.pants_workdir)
BuildRoot().path = self.build_root
# TODO: Redesign rule registration for tests to be more ergonomic and to make this less
# special-cased.
all_rules = (
*(rules or ()),
*source_root.rules(),
QueryRule(WrappedTarget, [Address]),
QueryRule(UnionMembership, []),
)
build_config_builder = BuildConfiguration.Builder()
build_config_builder.register_aliases(
BuildFileAliases(
objects=objects,
context_aware_object_factories=context_aware_object_factories))
build_config_builder.register_rules("_dummy_for_test_", all_rules)
build_config_builder.register_target_types("_dummy_for_test_",
target_types or ())
self.build_config = build_config_builder.create()
self.environment = CompleteEnvironment({})
self.options_bootstrapper = create_options_bootstrapper(
args=bootstrap_args)
options = self.options_bootstrapper.full_options(self.build_config)
global_options = self.options_bootstrapper.bootstrap_options.for_global_scope(
)
dynamic_remote_options, _ = DynamicRemoteOptions.from_options(
options, self.environment)
local_store_options = LocalStoreOptions.from_options(global_options)
if isolated_local_store:
if root_dir:
lmdb_store_dir = root_dir / "lmdb_store"
lmdb_store_dir.mkdir()
store_dir = str(lmdb_store_dir)
else:
store_dir = safe_mkdtemp(prefix="lmdb_store.")
local_store_options = dataclasses.replace(local_store_options,
store_dir=store_dir)
local_execution_root_dir = global_options.local_execution_root_dir
named_caches_dir = global_options.named_caches_dir
graph_session = EngineInitializer.setup_graph_extended(
pants_ignore_patterns=GlobalOptions.compute_pants_ignore(
self.build_root, global_options),
use_gitignore=False,
local_store_options=local_store_options,
local_execution_root_dir=local_execution_root_dir,
named_caches_dir=named_caches_dir,
build_root=self.build_root,
build_configuration=self.build_config,
executor=_EXECUTOR,
execution_options=ExecutionOptions.from_options(
global_options, dynamic_remote_options),
ca_certs_path=ca_certs_path,
engine_visualize_to=None,
).new_session(
build_id="buildid_for_test",
session_values=SessionValues({
OptionsBootstrapper: self.options_bootstrapper,
CompleteEnvironment: self.environment,
}),
)
self.scheduler = graph_session.scheduler_session
def log_level(global_options: GlobalOptions) -> LogLevel:
log_level: LogLevel = global_options.get_options().level
return log_level
def __init__(
self,
*,
rules: Iterable | None = None,
target_types: Iterable[type[Target]] | None = None,
objects: dict[str, Any] | None = None,
context_aware_object_factories: dict[str, Any] | None = None,
isolated_local_store: bool = False,
ca_certs_path: str | None = None,
) -> None:
self.build_root = os.path.realpath(mkdtemp(suffix="_BUILD_ROOT"))
safe_mkdir(self.build_root, clean=True)
safe_mkdir(self.pants_workdir)
BuildRoot().path = self.build_root
# TODO: Redesign rule registration for tests to be more ergonomic and to make this less
# special-cased.
all_rules = (
*(rules or ()),
*source_root.rules(),
*pants_environment.rules(),
QueryRule(WrappedTarget, [Address]),
QueryRule(UnionMembership, []),
)
build_config_builder = BuildConfiguration.Builder()
build_config_builder.register_aliases(
BuildFileAliases(
objects=objects,
context_aware_object_factories=context_aware_object_factories))
build_config_builder.register_rules(all_rules)
build_config_builder.register_target_types(target_types or ())
self.build_config = build_config_builder.create()
self.options_bootstrapper = create_options_bootstrapper()
options = self.options_bootstrapper.full_options(self.build_config)
global_options = self.options_bootstrapper.bootstrap_options.for_global_scope(
)
local_store_dir = (os.path.realpath(safe_mkdtemp())
if isolated_local_store else
global_options.local_store_dir)
local_execution_root_dir = global_options.local_execution_root_dir
named_caches_dir = global_options.named_caches_dir
graph_session = EngineInitializer.setup_graph_extended(
pants_ignore_patterns=GlobalOptions.compute_pants_ignore(
self.build_root, global_options),
use_gitignore=False,
local_store_dir=local_store_dir,
local_execution_root_dir=local_execution_root_dir,
named_caches_dir=named_caches_dir,
native=Native(),
build_root=self.build_root,
build_configuration=self.build_config,
executor=_EXECUTOR,
execution_options=ExecutionOptions.from_options(options),
ca_certs_path=ca_certs_path,
native_engine_visualize_to=None,
).new_session(
build_id="buildid_for_test",
session_values=SessionValues({
OptionsBootstrapper: self.options_bootstrapper,
PantsEnvironment: PantsEnvironment(),
}),
)
self.scheduler = graph_session.scheduler_session
def create(
cls, env: Mapping[str, str], args: Sequence[str], *, allow_pantsrc: bool
) -> OptionsBootstrapper:
"""Parses the minimum amount of configuration necessary to create an OptionsBootstrapper.
:param env: An environment dictionary, or None to use `os.environ`.
:param args: An args array, or None to use `sys.argv`.
:param allow_pantsrc: True to allow pantsrc files to be used. Unless tests are expecting to
consume pantsrc files, they should pass False in order to avoid reading files from
absolute paths. Production usecases should pass True to allow options values to make the
decision of whether to respect pantsrc files.
"""
with warnings.catch_warnings(record=True):
env = {k: v for k, v in env.items() if k.startswith("PANTS_")}
args = tuple(args)
flags = set()
short_flags = set()
# We can't use pants.engine.fs.FileContent here because it would cause a circular dep.
@dataclass(frozen=True)
class FileContent:
path: str
content: bytes
def filecontent_for(path: str) -> FileContent:
return FileContent(
ensure_text(path),
read_file(path, binary_mode=True),
)
def capture_the_flags(*args: str, **kwargs) -> None:
for arg in args:
flags.add(arg)
if len(arg) == 2:
short_flags.add(arg)
elif kwargs.get("type") == bool:
flags.add(f"--no-{arg[2:]}")
GlobalOptions.register_bootstrap_options(capture_the_flags)
def is_bootstrap_option(arg: str) -> bool:
components = arg.split("=", 1)
if components[0] in flags:
return True
for flag in short_flags:
if arg.startswith(flag):
return True
return False
# Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
# Stop before '--' since args after that are pass-through and may have duplicate names to our
# bootstrap options.
bargs = ("./pants",) + tuple(
filter(is_bootstrap_option, itertools.takewhile(lambda arg: arg != "--", args))
)
config_file_paths = cls.get_config_file_paths(env=env, args=args)
config_files_products = [filecontent_for(p) for p in config_file_paths]
pre_bootstrap_config = Config.load_file_contents(config_files_products)
initial_bootstrap_options = cls.parse_bootstrap_options(
env, bargs, pre_bootstrap_config
)
bootstrap_option_values = initial_bootstrap_options.for_global_scope()
# Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped
# from (typically pants.toml), then config override, then rcfiles.
full_config_paths = pre_bootstrap_config.sources()
if allow_pantsrc and bootstrap_option_values.pantsrc:
rcfiles = [
os.path.expanduser(str(rcfile))
for rcfile in bootstrap_option_values.pantsrc_files
]
existing_rcfiles = list(filter(os.path.exists, rcfiles))
full_config_paths.extend(existing_rcfiles)
full_config_files_products = [filecontent_for(p) for p in full_config_paths]
post_bootstrap_config = Config.load_file_contents(
full_config_files_products,
seed_values=bootstrap_option_values.as_dict(),
)
env_tuples = tuple(sorted(env.items(), key=lambda x: x[0]))
return cls(
env_tuples=env_tuples, bootstrap_args=bargs, args=args, config=post_bootstrap_config
)
def global_options(options_bootstrapper: OptionsBootstrapper) -> GlobalOptions: global_options = options_bootstrapper.bootstrap_options.for_global_scope() return GlobalOptions(_inner=global_options)
