def create_execution_options(
*,
initial_headers: dict[str, str],
token_path: str | None = None,
plugin: str | None = None,
remote_store_address: str = "grpc://fake.url:10",
remote_execution_address: str = "grpc://fake.url:10",
local_only: bool = False,
) -> ExecutionOptions:
args = [
"--remote-cache-read",
f"--remote-execution-address={remote_execution_address}",
f"--remote-store-address={remote_store_address}",
f"--remote-store-headers={initial_headers}",
f"--remote-execution-headers={initial_headers}",
"--remote-instance-name=main",
]
if token_path:
args.append(f"--remote-oauth-bearer-token-path={token_path}")
if plugin:
args.append(f"--remote-auth-plugin={plugin}")
ob = create_options_bootstrapper(args)
env = CompleteEnvironment({})
_build_config, options = OptionsInitializer(
ob, env).build_config_and_options(ob, env, raise_=False)
return ExecutionOptions.from_options(options, env, local_only=local_only)
def create_dynamic_remote_options(
*,
initial_headers: dict[str, str] | None = None,
address: str | None = "grpc://fake.url:10",
token_path: str | None = None,
plugin: str | None = None,
) -> DynamicRemoteOptions:
if initial_headers is None:
initial_headers = {}
args = [
"--remote-cache-read",
f"--remote-execution-address={address}",
f"--remote-store-address={address}",
f"--remote-store-headers={initial_headers}",
f"--remote-execution-headers={initial_headers}",
"--remote-instance-name=main",
]
if token_path:
args.append(f"--remote-oauth-bearer-token-path={token_path}")
if plugin:
args.append(f"--remote-auth-plugin={plugin}")
ob = create_options_bootstrapper(args)
env = CompleteEnvironment({})
_build_config, options = OptionsInitializer(ob).build_config_and_options(ob, env, raise_=False)
return DynamicRemoteOptions.from_options(options, env)[0]
def runner_factory(sock, arguments, environment):
exiter = self._exiter_class(sock)
graph_helper = None
deferred_exc = None
# Capture the size of the graph prior to any warming, for stats.
preceding_graph_size = self._scheduler_service.product_graph_len()
self._logger.debug('resident graph size: %s', preceding_graph_size)
self._logger.debug('execution commandline: %s', arguments)
options, _ = OptionsInitializer(
OptionsBootstrapper(args=arguments)).setup(init_logging=False)
target_roots = self._target_roots_calculator.create(
options,
change_calculator=self._scheduler_service.change_calculator)
try:
self._logger.debug('warming the product graph via %s',
self._scheduler_service)
# N.B. This call is made in the pre-fork daemon context for reach and reuse of the
# resident scheduler.
graph_helper = self._scheduler_service.warm_product_graph(
target_roots)
except Exception:
deferred_exc = sys.exc_info()
self._logger.warning(
'encountered exception during SchedulerService.warm_product_graph(), deferring:\n%s',
''.join(traceback.format_exception(*deferred_exc)))
return self._runner_class(sock, exiter, arguments, environment,
target_roots, graph_helper,
self.fork_lock, preceding_graph_size,
deferred_exc)
def test_global_options_validation(self):
# Specify an invalid combination of options.
ob = OptionsBootstrapper.create(
env={}, args=["--backend-packages=[]", "--remote-execution"], allow_pantsrc=False
)
env = CompleteEnvironment({})
with self.assertRaises(OptionsError) as exc:
OptionsInitializer(ob, env).build_config_and_options(ob, env, raise_=True)
self.assertIn("The `--remote-execution` option requires", str(exc.exception))
def _run(self):
# Bootstrap options and logging.
options_bootstrapper = self._options_bootstrapper or OptionsBootstrapper(
env=self._env, args=self._args)
options, build_config = OptionsInitializer(
options_bootstrapper, exiter=self._exiter).setup()
global_options = options.for_global_scope()
# Apply exiter options.
self._exiter.apply_options(options)
# Option values are usually computed lazily on demand,
# but command line options are eagerly computed for validation.
for scope in options.scope_to_flags.keys():
options.for_scope(scope)
# Verify the configs here.
if global_options.verify_config:
options_bootstrapper.verify_configs_against_options(options)
# Launch RunTracker as early as possible (just after Subsystem options are initialized).
run_tracker = RunTracker.global_instance()
reporting = Reporting.global_instance()
reporting.initialize(run_tracker, self._run_start_time)
try:
# Determine the build root dir.
root_dir = get_buildroot()
# Capture a repro of the 'before' state for this build, if needed.
repro = Reproducer.global_instance().create_repro()
if repro:
repro.capture(run_tracker.run_info.get_as_dict())
# Record the preceding product graph size.
run_tracker.pantsd_stats.set_preceding_graph_size(
self._preceding_graph_size)
# Setup and run GoalRunner.
goal_runner = GoalRunner.Factory(root_dir, options, build_config,
run_tracker, reporting,
self._target_roots,
self._daemon_build_graph,
self._exiter).setup()
goal_runner_result = goal_runner.run()
if repro:
# TODO: Have Repro capture the 'after' state (as a diff) as well?
repro.log_location_of_repro_file()
finally:
run_tracker_result = run_tracker.end()
# Take the exit code with higher abs value in case of negative values.
final_exit_code = goal_runner_result if abs(goal_runner_result) > abs(
run_tracker_result) else run_tracker_result
self._exiter.exit(final_exit_code)
def test_invalid_version(self):
options_bootstrapper = OptionsBootstrapper.create(
env={},
args=["--backend-packages=[]", "--pants-version=99.99.9999"],
allow_pantsrc=False,
)
with self.assertRaises(BuildConfigurationError):
OptionsInitializer(options_bootstrapper).build_config_and_options(
options_bootstrapper, raise_=True)
def test_invalid_version(self) -> None:
options_bootstrapper = OptionsBootstrapper.create(
env={},
args=["--backend-packages=[]", "--pants-version=99.99.9999"],
allow_pantsrc=False,
)
env = CompleteEnvironment({})
with self.assertRaises(ExecutionError):
OptionsInitializer(options_bootstrapper).build_config_and_options(
options_bootstrapper, env, raise_=True)
def __init__(
self,
options_bootstrapper: OptionsBootstrapper,
executor: PyExecutor,
services_constructor: PantsServicesConstructor,
):
self._options_initializer = OptionsInitializer(options_bootstrapper,
executor=executor)
self._executor = executor
self._services_constructor = services_constructor
self._lifecycle_lock = threading.RLock()
# N.B. This Event is used as nothing more than an atomic flag - nothing waits on it.
self._kill_switch = threading.Event()
self._scheduler: Optional[GraphScheduler] = None
self._services: Optional[PantsServices] = None
self._fingerprint: Optional[str] = None
def create(cls,
options=None,
args=None,
build_root=None,
change_calculator=None):
"""
:param Options options: An `Options` instance to use, if available.
:param string args: Raw cli args to use for parsing if an `Options` instance isn't available.
:param string build_root: The build root.
:param ChangeCalculator change_calculator: A `ChangeCalculator` for calculating changes.
"""
if not options:
assert args is not None, 'must pass `args` if not passing `options`'
options, _ = OptionsInitializer(
OptionsBootstrapper(args=args)).setup(init_logging=False)
# Determine the literal target roots.
spec_roots = cls.parse_specs(options.target_specs, build_root)
# Determine `Changed` arguments directly from options to support pre-`Subsystem`
# initialization paths.
changed_options = options.for_scope('changed')
changed_request = ChangedRequest.from_options(changed_options)
logger.debug('args are: %s', args)
logger.debug('spec_roots are: %s', spec_roots)
logger.debug('changed_request is: %s', changed_request)
if change_calculator and changed_request.is_actionable():
if spec_roots:
# We've been provided spec roots (e.g. `./pants list ::`) AND a changed request. Error out.
raise InvalidSpecConstraint(
'cannot provide changed parameters and target specs!')
# We've been provided no spec roots (e.g. `./pants list`) AND a changed request. Compute
# alternate target roots.
changed_addresses = change_calculator.changed_target_addresses(
changed_request)
logger.debug('changed addresses: %s', changed_addresses)
return ChangedTargetRoots(
tuple(
SingleAddress(a.spec_path, a.target_name)
for a in changed_addresses))
return LiteralTargetRoots(spec_roots)
def __init__(
self,
options_bootstrapper: OptionsBootstrapper,
executor: PyExecutor,
services_constructor: PantsServicesConstructor,
):
self._options_initializer = OptionsInitializer(options_bootstrapper,
executor)
self._executor = executor
self._services_constructor = services_constructor
self._lifecycle_lock = threading.RLock()
# N.B. This Event is used as nothing more than an atomic flag - nothing waits on it.
self._kill_switch = threading.Event()
self._scheduler: GraphScheduler | None = None
self._services: PantsServices | None = None
self._fingerprint: str | None = None
self._prior_dynamic_remote_options: DynamicRemoteOptions | None = None
self._prior_auth_plugin_result: AuthPluginResult | None = None
def create(cls, bootstrap_options=None, full_init=True):
"""
:param Options bootstrap_options: The bootstrap options, if available.
:param bool full_init: Whether or not to fully initialize an engine et al for the purposes
of spawning a new daemon. `full_init=False` is intended primarily
for lightweight lifecycle checks (since there is a ~1s overhead to
initialize the engine). See the impl of `maybe_launch` for an example
of the intended usage.
"""
bootstrap_options = bootstrap_options or cls._parse_bootstrap_options(
)
bootstrap_options_values = bootstrap_options.for_global_scope()
# TODO: https://github.com/pantsbuild/pants/issues/3479
watchman = WatchmanLauncher.create(
bootstrap_options_values).watchman
native = None
build_root = None
services = None
port_map = None
if full_init:
build_root = get_buildroot()
native = Native.create(bootstrap_options_values)
_, build_config = OptionsInitializer(
OptionsBootstrapper()).setup(init_logging=False)
legacy_graph_scheduler = EngineInitializer.setup_legacy_graph(
native, bootstrap_options_values, build_config)
services, port_map = cls._setup_services(
build_root, bootstrap_options_values,
legacy_graph_scheduler, watchman)
return PantsDaemon(
native=native,
build_root=build_root,
work_dir=bootstrap_options_values.pants_workdir,
log_level=bootstrap_options_values.level.upper(),
services=services,
socket_map=port_map,
metadata_base_dir=bootstrap_options_values.pants_subprocessdir,
bootstrap_options=bootstrap_options)
def aliases(cls):
"""TODO: This is a nasty escape hatch to pass aliases to LegacyPythonCallbacksParser."""
_, build_config = OptionsInitializer(
OptionsBootstrapper()).setup(init_logging=False)
return build_config.registered_aliases()
def _default_build_file_aliases(self):
# TODO: Get default BuildFileAliases by extending BaseTest post
# https://github.com/pantsbuild/pants/issues/4401
_, build_config = OptionsInitializer(
OptionsBootstrapper()).setup(init_logging=False)
return build_config.registered_aliases()
def get_default_build_file_aliases(): _, build_config = OptionsInitializer(OptionsBootstrapper()).setup(init_logging=False) return build_config.registered_aliases()
def setup_legacy_graph(pants_ignore_patterns,
workdir,
build_file_imports_behavior,
build_root=None,
native=None,
build_file_aliases=None,
build_ignore_patterns=None,
exclude_target_regexps=None,
subproject_roots=None,
include_trace_on_error=True):
"""Construct and return the components necessary for LegacyBuildGraph construction.
:param list pants_ignore_patterns: A list of path ignore patterns for FileSystemProjectTree,
usually taken from the '--pants-ignore' global option.
:param str workdir: The pants workdir.
:param build_file_imports_behavior: How to behave if a BUILD file being parsed tries to use
import statements. Valid values: "allow", "warn", "error".
:type build_file_imports_behavior: string
:param str build_root: A path to be used as the build root. If None, then default is used.
:param Native native: An instance of the native-engine subsystem.
:param build_file_aliases: BuildFileAliases to register.
:type build_file_aliases: :class:`pants.build_graph.build_file_aliases.BuildFileAliases`
:param list build_ignore_patterns: A list of paths ignore patterns used when searching for BUILD
files, usually taken from the '--build-ignore' global option.
:param list exclude_target_regexps: A list of regular expressions for excluding targets.
:param list subproject_roots: Paths that correspond with embedded build roots
under the current build root.
:param bool include_trace_on_error: If True, when an error occurs, the error message will
include the graph trace.
:returns: A tuple of (scheduler, engine, symbol_table, build_graph_cls).
"""
build_root = build_root or get_buildroot()
scm = get_scm()
if not build_file_aliases:
_, build_config = OptionsInitializer(
OptionsBootstrapper()).setup(init_logging=False)
build_file_aliases = build_config.registered_aliases()
symbol_table = LegacySymbolTable(build_file_aliases)
project_tree = FileSystemProjectTree(build_root, pants_ignore_patterns)
# Register "literal" subjects required for these tasks.
parser = LegacyPythonCallbacksParser(symbol_table, build_file_aliases,
build_file_imports_behavior)
address_mapper = AddressMapper(
parser=parser,
build_ignore_patterns=build_ignore_patterns,
exclude_target_regexps=exclude_target_regexps,
subproject_roots=subproject_roots)
# Load the native backend.
native = native or Native.create()
# Create a Scheduler containing graph and filesystem tasks, with no installed goals. The
# LegacyBuildGraph will explicitly request the products it needs.
tasks = (create_legacy_graph_tasks(symbol_table) + create_fs_rules() +
create_graph_rules(address_mapper, symbol_table) +
create_process_rules())
scheduler = LocalScheduler(
workdir,
dict(),
tasks,
project_tree,
native,
include_trace_on_error=include_trace_on_error)
change_calculator = EngineChangeCalculator(scheduler, symbol_table,
scm) if scm else None
return LegacyGraphHelper(scheduler, symbol_table, change_calculator)
def create(
cls,
env: CompleteEnvironment,
options_bootstrapper: OptionsBootstrapper,
options_initializer: Optional[OptionsInitializer] = None,
scheduler: Optional[GraphScheduler] = None,
cancellation_latch: Optional[PySessionCancellationLatch] = None,
) -> LocalPantsRunner:
"""Creates a new LocalPantsRunner instance by parsing options.
By the time this method runs, logging will already have been initialized in either
PantsRunner or DaemonPantsRunner.
:param env: The environment for this run.
:param options_bootstrapper: The OptionsBootstrapper instance to reuse.
:param scheduler: If being called from the daemon, a warmed scheduler to use.
"""
options_initializer = options_initializer or OptionsInitializer(options_bootstrapper, env)
build_config, options = options_initializer.build_config_and_options(
options_bootstrapper, env, raise_=True
)
run_tracker = RunTracker(options)
union_membership = UnionMembership.from_rules(build_config.union_rules)
# If we're running with the daemon, we'll be handed a warmed Scheduler, which we use
# to initialize a session here.
graph_session = cls._init_graph_session(
options_initializer,
options_bootstrapper,
build_config,
env,
run_tracker.run_id,
options,
scheduler,
cancellation_latch,
)
# Option values are usually computed lazily on demand, but command line options are
# eagerly computed for validation.
with options_initializer.handle_unknown_flags(options_bootstrapper, env, raise_=True):
for scope in options.scope_to_flags.keys():
options.for_scope(scope)
# Verify configs.
global_bootstrap_options = options_bootstrapper.bootstrap_options.for_global_scope()
if global_bootstrap_options.verify_config:
options.verify_configs(options_bootstrapper.config)
specs = calculate_specs(
options_bootstrapper=options_bootstrapper,
options=options,
build_root=get_buildroot(),
session=graph_session.scheduler_session,
)
profile_path = env.get("PANTS_PROFILE")
return cls(
options=options,
options_bootstrapper=options_bootstrapper,
build_config=build_config,
run_tracker=run_tracker,
specs=specs,
graph_session=graph_session,
union_membership=union_membership,
profile_path=profile_path,
)
def test_invalid_version(self):
options_bootstrapper = OptionsBootstrapper(args=['--pants-version=99.99.9999'])
initializer = OptionsInitializer(options_bootstrapper, WorkingSet())
with self.assertRaises(BuildConfigurationError):
initializer.setup()