def test_invalid_version(self):
options_bootstrapper = OptionsBootstrapper(
args=['--pants-version=99.99.9999'])
build_config = BuildConfigInitializer.get(options_bootstrapper)
with self.assertRaises(BuildConfigurationError):
OptionsInitializer.create(options_bootstrapper, build_config)
def test_global_options_validation(self):
# Specify an invalid combination of options.
ob = OptionsBootstrapper.create(args=['--loop', '--v1'])
build_config = BuildConfigInitializer.get(ob)
with self.assertRaises(OptionsError) as exc:
OptionsInitializer.create(ob, build_config)
self.assertIn('loop option only works with', str(exc.exception))
def test_global_options_validation(self):
# Specify an invalid combination of options.
ob = OptionsBootstrapper(args=['--loop', '--v1'])
build_config = BuildConfigInitializer.get(ob)
with self.assertRaises(OptionsError) as exc:
OptionsInitializer.create(ob, build_config)
self.assertIn('loop option only works with', exc.exception.message)
def test_global_options_validation(self):
# Specify an invalid combination of options.
ob = OptionsBootstrapper.create(
args=['--backend-packages=[]', '--backend-packages2=[]',
'--v2', '--no-v1', '--loop', '--no-enable-pantsd'])
build_config = BuildConfigInitializer.get(ob)
with self.assertRaises(OptionsError) as exc:
OptionsInitializer.create(ob, build_config)
self.assertIn('The `--loop` option requires `--enable-pantsd`', str(exc.exception))
def test_invalid_version(self):
options_bootstrapper = OptionsBootstrapper.create(
env={},
args=["--backend-packages=[]", "--pants-version=99.99.9999"],
allow_pantsrc=False,
)
build_config = BuildConfigInitializer.get(options_bootstrapper)
with self.assertRaises(BuildConfigurationError):
OptionsInitializer.create(options_bootstrapper, build_config)
def test_global_options_validation(self):
# Specify an invalid combination of options.
ob = OptionsBootstrapper.create(
env={},
args=["--backend-packages=[]", "--remote-execution"],
allow_pantsrc=False)
build_config = BuildConfigInitializer.get(ob)
with self.assertRaises(OptionsError) as exc:
OptionsInitializer.create(ob, build_config)
self.assertIn("The `--remote-execution` option requires",
str(exc.exception))
def options(self) -> Options:
return cast(
Options,
OptionsInitializer.create(self.options_bootstrapper,
self.build_config,
init_subsystems=False),
)
def _init_scheduler(self, options_fingerprint: str,
options_bootstrapper: OptionsBootstrapper) -> None:
"""(Re-)Initialize the scheduler.
Must be called under the lifecycle lock.
"""
try:
if self._scheduler:
logger.info(
"initialization options changed: reinitializing pantsd...")
else:
logger.info("initializing pantsd...")
if self._services:
self._services.shutdown()
with OptionsInitializer.handle_unknown_flags(options_bootstrapper,
raise_=True):
build_config = BuildConfigInitializer.get(options_bootstrapper)
options = OptionsInitializer.create(options_bootstrapper,
build_config)
self._scheduler = EngineInitializer.setup_graph(
options, build_config, executor=self._executor)
bootstrap_options_values = options.bootstrap_option_values()
assert bootstrap_options_values is not None
self._services = self._services_constructor(
bootstrap_options_values, self._scheduler)
self._fingerprint = options_fingerprint
logger.info("pantsd initialized.")
except Exception as e:
self._kill_switch.set()
self._scheduler = None
raise e
def _handle_unknown_flags(err: UnknownFlagsError, options_bootstrapper: OptionsBootstrapper):
build_config = BuildConfigInitializer.get(options_bootstrapper)
# We need an options instance in order to get "did you mean" suggestions, but we know
# there are bad flags in the args, so we generate options with no flags.
no_arg_bootstrapper = replace(options_bootstrapper, args=("dummy_first_arg",))
options = OptionsInitializer.create(no_arg_bootstrapper, build_config)
FlagErrorHelpPrinter(options).handle_unknown_flags(err)
def runner_factory(sock, arguments, environment):
exiter = self._exiter_class(sock)
graph_helper = None
deferred_exc = None
self._logger.debug('execution commandline: %s', arguments)
options_bootstrapper = OptionsBootstrapper(args=arguments)
build_config = BuildConfigInitializer.get(options_bootstrapper)
options = OptionsInitializer.create(options_bootstrapper,
build_config)
graph_helper, target_roots = None, None
try:
self._logger.debug('warming the product graph via %s',
self._scheduler_service)
# N.B. This call is made in the pre-fork daemon context for reach and reuse of the
# resident scheduler.
graph_helper, target_roots = self._scheduler_service.warm_product_graph(
options, self._target_roots_calculator)
except Exception:
deferred_exc = sys.exc_info()
self._logger.warning(
'encountered exception during SchedulerService.warm_product_graph(), deferring:\n%s',
''.join(traceback.format_exception(*deferred_exc)))
return self._runner_class(sock, exiter, arguments, environment,
target_roots, graph_helper,
self.fork_lock, deferred_exc)
def parse_options(options_bootstrapper: OptionsBootstrapper,
build_config: BuildConfiguration) -> _Options:
# TODO: Because _OptionsBootstapper is currently provided as a Param, this @rule relies on options
# remaining relatively stable in order to be efficient. See #6845 for a discussion of how to make
# minimize the size of that value.
return _Options(
OptionsInitializer.create(options_bootstrapper,
build_config,
init_subsystems=False))
def _run(self):
# Bootstrap options and logging.
options_bootstrapper = self._options_bootstrapper or OptionsBootstrapper(env=self._env,
args=self._args)
bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope()
setup_logging_from_options(bootstrap_options)
build_config = BuildConfigInitializer.get(options_bootstrapper)
options = OptionsInitializer.create(options_bootstrapper, build_config)
global_options = options.for_global_scope()
# Apply exiter options.
self._exiter.apply_options(options)
# Option values are usually computed lazily on demand,
# but command line options are eagerly computed for validation.
for scope in options.scope_to_flags.keys():
options.for_scope(scope)
# Verify the configs here.
if global_options.verify_config:
options_bootstrapper.verify_configs_against_options(options)
# Launch RunTracker as early as possible (just after Subsystem options are initialized).
run_tracker = RunTracker.global_instance()
reporting = Reporting.global_instance()
reporting.initialize(run_tracker, self._run_start_time)
try:
# Determine the build root dir.
root_dir = get_buildroot()
# Capture a repro of the 'before' state for this build, if needed.
repro = Reproducer.global_instance().create_repro()
if repro:
repro.capture(run_tracker.run_info.get_as_dict())
# Setup and run GoalRunner.
goal_runner = GoalRunner.Factory(root_dir,
options,
build_config,
run_tracker,
reporting,
self._target_roots,
self._daemon_build_graph,
self._exiter).setup()
goal_runner_result = goal_runner.run()
if repro:
# TODO: Have Repro capture the 'after' state (as a diff) as well?
repro.log_location_of_repro_file()
finally:
run_tracker_result = run_tracker.end()
# Take the exit code with higher abs value in case of negative values.
final_exit_code = goal_runner_result if abs(goal_runner_result) > abs(run_tracker_result) else run_tracker_result
self._exiter.exit(final_exit_code)
def parse_options( args: List[str], env: Dict[str, str], options_bootstrapper: Optional[OptionsBootstrapper] = None, ) -> Tuple[Options, BuildConfiguration, OptionsBootstrapper]: options_bootstrapper = options_bootstrapper or OptionsBootstrapper.create(args=args, env=env) build_config = BuildConfigInitializer.get(options_bootstrapper) options = OptionsInitializer.create(options_bootstrapper, build_config) return options, build_config, options_bootstrapper
def parse_options(args, env, setup_logging=False, options_bootstrapper=None):
options_bootstrapper = options_bootstrapper or OptionsBootstrapper(args=args, env=env)
bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope()
if setup_logging:
# Bootstrap logging and then fully initialize options.
setup_logging_from_options(bootstrap_options)
build_config = BuildConfigInitializer.get(options_bootstrapper)
options = OptionsInitializer.create(options_bootstrapper, build_config)
return options, build_config, options_bootstrapper
def _run(self):
# Bootstrap options and logging.
options_bootstrapper = self._options_bootstrapper or OptionsBootstrapper(
env=self._env, args=self._args)
bootstrap_options = options_bootstrapper.get_bootstrap_options(
).for_global_scope()
setup_logging_from_options(bootstrap_options)
build_config = BuildConfigInitializer.get(options_bootstrapper)
options = OptionsInitializer.create(options_bootstrapper, build_config)
global_options = options.for_global_scope()
# Apply exiter options.
self._exiter.apply_options(options)
# Option values are usually computed lazily on demand,
# but command line options are eagerly computed for validation.
for scope in options.scope_to_flags.keys():
options.for_scope(scope)
# Verify the configs here.
if global_options.verify_config:
options_bootstrapper.verify_configs_against_options(options)
# Launch RunTracker as early as possible (just after Subsystem options are initialized).
run_tracker = RunTracker.global_instance()
reporting = Reporting.global_instance()
reporting.initialize(run_tracker, self._run_start_time)
try:
# Determine the build root dir.
root_dir = get_buildroot()
# Capture a repro of the 'before' state for this build, if needed.
repro = Reproducer.global_instance().create_repro()
if repro:
repro.capture(run_tracker.run_info.get_as_dict())
# Setup and run GoalRunner.
goal_runner = GoalRunner.Factory(root_dir, options, build_config,
run_tracker, reporting,
self._target_roots,
self._daemon_build_graph,
self._exiter).setup()
goal_runner_result = goal_runner.run()
if repro:
# TODO: Have Repro capture the 'after' state (as a diff) as well?
repro.log_location_of_repro_file()
finally:
run_tracker_result = run_tracker.end()
# Take the exit code with higher abs value in case of negative values.
final_exit_code = goal_runner_result if abs(goal_runner_result) > abs(
run_tracker_result) else run_tracker_result
self._exiter.exit(final_exit_code)
def parse_options(
cls,
options_bootstrapper: OptionsBootstrapper,
) -> Tuple[Options, BuildConfiguration]:
build_config = BuildConfigInitializer.get(options_bootstrapper)
try:
options = OptionsInitializer.create(options_bootstrapper, build_config)
except UnknownFlagsError as err:
cls._handle_unknown_flags(err, options_bootstrapper)
raise
return options, build_config
def runner_factory(sock, arguments, environment):
exiter = self._exiter_class(sock)
graph_helper = None
deferred_exc = None
self._logger.debug('execution commandline: %s', arguments)
options_bootstrapper = OptionsBootstrapper(args=arguments)
build_config = BuildConfigInitializer.get(options_bootstrapper)
options = OptionsInitializer.create(options_bootstrapper, build_config)
graph_helper, target_roots = None, None
try:
self._logger.debug('warming the product graph via %s', self._scheduler_service)
# N.B. This call is made in the pre-fork daemon context for reach and reuse of the
# resident scheduler.
graph_helper, target_roots = self._scheduler_service.warm_product_graph(
options,
self._target_roots_calculator
)
except Exception:
deferred_exc = sys.exc_info()
self._logger.warning(
'encountered exception during SchedulerService.warm_product_graph(), deferring:\n%s',
''.join(traceback.format_exception(*deferred_exc))
)
return self._runner_class(
sock,
exiter,
arguments,
environment,
target_roots,
graph_helper,
self.fork_lock,
deferred_exc
)
def options(self) -> Options:
return OptionsInitializer.create(self.options_bootstrapper, self.build_config)
def create(
cls,
env: Mapping[str, str],
options_bootstrapper: OptionsBootstrapper,
scheduler: Optional[GraphScheduler] = None,
cancellation_latch: Optional[PySessionCancellationLatch] = None,
) -> LocalPantsRunner:
"""Creates a new LocalPantsRunner instance by parsing options.
By the time this method runs, logging will already have been initialized in either
PantsRunner or DaemonPantsRunner.
:param env: The environment (e.g. os.environ) for this run.
:param options_bootstrapper: The OptionsBootstrapper instance to reuse.
:param scheduler: If being called from the daemon, a warmed scheduler to use.
"""
build_root = get_buildroot()
global_bootstrap_options = options_bootstrapper.bootstrap_options.for_global_scope()
build_config = BuildConfigInitializer.get(options_bootstrapper)
try:
options = OptionsInitializer.create(options_bootstrapper, build_config)
except UnknownFlagsError as err:
cls._handle_unknown_flags(err, options_bootstrapper)
raise
run_tracker = RunTracker(options)
union_membership = UnionMembership.from_rules(build_config.union_rules)
# If we're running with the daemon, we'll be handed a warmed Scheduler, which we use
# to initialize a session here.
graph_session = cls._init_graph_session(
options_bootstrapper,
build_config,
run_tracker.run_id,
options,
scheduler,
cancellation_latch,
)
# Option values are usually computed lazily on demand,
# but command line options are eagerly computed for validation.
for scope in options.scope_to_flags.keys():
try:
options.for_scope(scope)
except UnknownFlagsError as err:
cls._handle_unknown_flags(err, options_bootstrapper)
raise
# Verify configs.
if global_bootstrap_options.verify_config:
options.verify_configs(options_bootstrapper.config)
specs = calculate_specs(
options_bootstrapper=options_bootstrapper,
options=options,
build_root=build_root,
session=graph_session.scheduler_session,
)
profile_path = env.get("PANTS_PROFILE")
return cls(
build_root=build_root,
options=options,
build_config=build_config,
run_tracker=run_tracker,
specs=specs,
graph_session=graph_session,
union_membership=union_membership,
profile_path=profile_path,
)
def parse_options(args, env, options_bootstrapper=None): options_bootstrapper = options_bootstrapper or OptionsBootstrapper.create(args=args, env=env) build_config = BuildConfigInitializer.get(options_bootstrapper) options = OptionsInitializer.create(options_bootstrapper, build_config) return options, build_config, options_bootstrapper
def parse_options(options_bootstrapper): # TODO: Because _OptionsBootstapper is currently provided as a Param, this @rule relies on options # remaining relatively stable in order to be efficient. See #6845 for a discussion of how to make # minimize the size of that value. build_config = BuildConfigInitializer.get(options_bootstrapper) return _Options(OptionsInitializer.create(options_bootstrapper, build_config, init_subsystems=False))
def __init__(
self,
*,
rules: Optional[Iterable] = None,
target_types: Optional[Iterable[Type[Target]]] = None,
objects: Optional[Dict[str, Any]] = None,
context_aware_object_factories: Optional[Dict[str, Any]] = None,
isolated_local_store: bool = False,
ca_certs_path: Optional[str] = None,
) -> None:
self.build_root = os.path.realpath(mkdtemp(suffix="_BUILD_ROOT"))
safe_mkdir(self.build_root, clean=True)
safe_mkdir(self.pants_workdir)
BuildRoot().path = self.build_root
# TODO: Redesign rule registration for tests to be more ergonomic and to make this less
# special-cased.
all_rules = (
*(rules or ()),
*source_root.rules(),
*pants_environment.rules(),
QueryRule(WrappedTarget, [Address]),
QueryRule(UnionMembership, []),
)
build_config_builder = BuildConfiguration.Builder()
build_config_builder.register_aliases(
BuildFileAliases(
objects=objects,
context_aware_object_factories=context_aware_object_factories))
build_config_builder.register_rules(all_rules)
build_config_builder.register_target_types(target_types or ())
self.build_config = build_config_builder.create()
options_bootstrapper = create_options_bootstrapper()
options = OptionsInitializer.create(options_bootstrapper,
self.build_config)
global_options = options_bootstrapper.bootstrap_options.for_global_scope(
)
local_store_dir = (os.path.realpath(safe_mkdtemp())
if isolated_local_store else
global_options.local_store_dir)
local_execution_root_dir = global_options.local_execution_root_dir
named_caches_dir = global_options.named_caches_dir
graph_session = EngineInitializer.setup_graph_extended(
pants_ignore_patterns=OptionsInitializer.compute_pants_ignore(
self.build_root, global_options),
use_gitignore=False,
local_store_dir=local_store_dir,
local_execution_root_dir=local_execution_root_dir,
named_caches_dir=named_caches_dir,
native=Native(),
build_root=self.build_root,
build_configuration=self.build_config,
executor=_EXECUTOR,
execution_options=ExecutionOptions.from_options(options),
ca_certs_path=ca_certs_path,
native_engine_visualize_to=None,
).new_session(
build_id="buildid_for_test",
session_values=SessionValues({
OptionsBootstrapper: options_bootstrapper,
PantsEnvironment: PantsEnvironment()
}),
)
self.scheduler = graph_session.scheduler_session
def parse_options(args, env, options_bootstrapper=None): options_bootstrapper = options_bootstrapper or OptionsBootstrapper.create(args=args, env=env) build_config = BuildConfigInitializer.get(options_bootstrapper) options = OptionsInitializer.create(options_bootstrapper, build_config) return options, build_config, options_bootstrapper
def test_invalid_version(self):
options_bootstrapper = OptionsBootstrapper(args=['--pants-version=99.99.9999'])
build_config = BuildConfigInitializer(options_bootstrapper)
with self.assertRaises(BuildConfigurationError):
OptionsInitializer.create(options_bootstrapper, build_config)
def parse_options(
options_bootstrapper: OptionsBootstrapper,
) -> Tuple[Options, BuildConfiguration]:
build_config = BuildConfigInitializer.get(options_bootstrapper)
options = OptionsInitializer.create(options_bootstrapper, build_config)
return options, build_config
def parse_options(options_bootstrapper, build_config): options = OptionsInitializer.create(options_bootstrapper, build_config) options.freeze() return Options(options, build_config)