def test_complete_scopes(self):
_global = GlobalOptionsRegistrar.get_scope_info()
self.assertEquals(
{_global, intermediate("foo"), intermediate("foo.bar"), task("foo.bar.baz")},
Options.complete_scopes({task("foo.bar.baz")}),
)
self.assertEquals(
{_global, intermediate("foo"), intermediate("foo.bar"), task("foo.bar.baz")},
Options.complete_scopes({GlobalOptionsRegistrar.get_scope_info(), task("foo.bar.baz")}),
)
self.assertEquals(
{_global, intermediate("foo"), intermediate("foo.bar"), task("foo.bar.baz")},
Options.complete_scopes({intermediate("foo"), task("foo.bar.baz")}),
)
self.assertEquals(
{
_global,
intermediate("foo"),
intermediate("foo.bar"),
task("foo.bar.baz"),
intermediate("qux"),
task("qux.quux"),
},
Options.complete_scopes({task("foo.bar.baz"), task("qux.quux")}),
)
def test_complete_scopes(self):
self.assertEquals({'', 'foo', 'foo.bar', 'foo.bar.baz'},
Options.complete_scopes({'foo.bar.baz'}))
self.assertEquals({'', 'foo', 'foo.bar', 'foo.bar.baz'},
Options.complete_scopes({'', 'foo.bar.baz'}))
self.assertEquals({'', 'foo', 'foo.bar', 'foo.bar.baz'},
Options.complete_scopes({'foo', 'foo.bar.baz'}))
self.assertEquals({'', 'foo', 'foo.bar', 'foo.bar.baz', 'qux', 'qux.quux'},
Options.complete_scopes({'foo.bar.baz', 'qux.quux'}))
def test_complete_scopes(self):
_global = ScopeInfo.for_global_scope()
self.assertEquals({_global, intermediate('foo'), intermediate('foo.bar'), task('foo.bar.baz')},
Options.complete_scopes({task('foo.bar.baz')}))
self.assertEquals({_global, intermediate('foo'), intermediate('foo.bar'), task('foo.bar.baz')},
Options.complete_scopes({ScopeInfo.for_global_scope(), task('foo.bar.baz')}))
self.assertEquals({_global, goal('foo'), intermediate('foo.bar'), task('foo.bar.baz')},
Options.complete_scopes({goal('foo'), task('foo.bar.baz')}))
self.assertEquals({_global, intermediate('foo'), intermediate('foo.bar'), task('foo.bar.baz'),
intermediate('qux'), task('qux.quux')},
Options.complete_scopes({task('foo.bar.baz'), task('qux.quux')}))
def test_complete_scopes(self):
_global = GlobalOptionsRegistrar.get_scope_info()
self.assertEquals({_global, intermediate('foo'), intermediate('foo.bar'), task('foo.bar.baz')},
Options.complete_scopes({task('foo.bar.baz')}))
self.assertEquals({_global, intermediate('foo'), intermediate('foo.bar'), task('foo.bar.baz')},
Options.complete_scopes({GlobalOptionsRegistrar.get_scope_info(),
task('foo.bar.baz')}))
self.assertEquals({_global, intermediate('foo'), intermediate('foo.bar'), task('foo.bar.baz')},
Options.complete_scopes({intermediate('foo'), task('foo.bar.baz')}))
self.assertEquals({_global, intermediate('foo'), intermediate('foo.bar'), task('foo.bar.baz'),
intermediate('qux'), task('qux.quux')},
Options.complete_scopes({task('foo.bar.baz'), task('qux.quux')}))
def get_bootstrap_option_values(env=None, config=None, args=None, buildroot=None):
"""Get the values of just the bootstrap options."""
# Filter just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
flags = set()
def capture_the_flags(*args, **kwargs):
flags.update(args)
register_bootstrap_options(capture_the_flags, buildroot=buildroot)
bargs = filter(lambda x: x.partition('=')[0] in flags, args)
bootstrap_options = Options(env=env, config=config, known_scopes=[GLOBAL_SCOPE], args=bargs)
register_bootstrap_options(bootstrap_options.register_global, buildroot=buildroot)
return bootstrap_options.for_global_scope()
def get_bootstrap_options(self):
"""Returns an Options instance that only knows about the bootstrap options."""
if not self._bootstrap_options:
flags = set()
def capture_the_flags(*args, **kwargs):
flags.update(args)
register_bootstrap_options(capture_the_flags, buildroot=self._buildroot)
# Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
bargs = filter(lambda x: x.partition('=')[0] in flags, self._args or [])
self._bootstrap_options = Options(env=self._env, config=self._pre_bootstrap_config,
known_scopes=[GLOBAL_SCOPE], args=bargs)
register_bootstrap_options(self._bootstrap_options.register_global, buildroot=self._buildroot)
bootstrap_option_values = self._bootstrap_options.for_global_scope()
Config.reset_default_bootstrap_option_values(values=bootstrap_option_values)
# Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped
# from (typically pants.ini), then config override, then rcfiles.
configpaths = list(self._pre_bootstrap_config.sources())
if bootstrap_option_values.config_override:
configpaths.append(bootstrap_option_values.config_override)
if bootstrap_option_values.pantsrc:
rcfiles = [os.path.expanduser(rcfile) for rcfile in bootstrap_option_values.pantsrc_files]
existing_rcfiles = filter(os.path.exists, rcfiles)
configpaths.extend(existing_rcfiles)
self._post_bootstrap_config = Config.load(configpaths)
Config.cache(self._post_bootstrap_config)
return self._bootstrap_options
def test_env_type_int(self):
options = Options.create(env={'PANTS_FOO_BAR': "['123','456']"},
config=self._create_config({}),
known_scope_infos=OptionsTest._known_scope_infos,
args=shlex.split('./pants'),
option_tracker=OptionTracker())
options.register(GLOBAL_SCOPE, '--foo-bar', type=list, member_type=int)
self.assertEqual([123, 456], options.for_global_scope().foo_bar)
options = Options.create(env={'PANTS_FOO_BAR': '123'},
config=self._create_config({}),
known_scope_infos=OptionsTest._known_scope_infos,
args=shlex.split('./pants'),
option_tracker=OptionTracker())
options.register(GLOBAL_SCOPE, '--foo-bar', type=int)
self.assertEqual(123, options.for_global_scope().foo_bar)
def __init__(self, bootstrap_options: Options, daemon_entrypoint: str):
super().__init__(
name="pantsd",
metadata_base_dir=bootstrap_options.for_global_scope().pants_subprocessdir,
)
self._bootstrap_options = bootstrap_options
self._daemon_entrypoint = daemon_entrypoint
def bootstrap_options_from_config(config):
bootstrap_options = Options.create(env=self._env, config=config,
known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()], args=bargs)
def register_global(*args, **kwargs):
bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs)
GlobalOptionsRegistrar.register_bootstrap_options(register_global)
return bootstrap_options
def test_double_registration(self):
options = Options.create(env={},
config=self._create_config({}),
known_scope_infos=OptionsTest._known_scope_infos,
args=shlex.split('./pants'),
option_tracker=OptionTracker())
options.register(GLOBAL_SCOPE, '--foo-bar')
self.assertRaises(OptionAlreadyRegistered, lambda: options.register(GLOBAL_SCOPE, '--foo-bar'))
def test_double_registration(self):
options = Options.create(env={},
config=self._create_config({}),
known_scope_infos=OptionsTest._known_scope_infos,
args=shlex.split('./pants'),
option_tracker=OptionTracker())
options.register(GLOBAL_SCOPE, '--foo-bar')
self.assertRaises(OptionAlreadyRegistered, lambda: options.register(GLOBAL_SCOPE, '--foo-bar'))
def test_deprecated_option_past_removal(self):
with self.assertRaises(PastRemovalVersionError):
options = Options.create(env={},
config=self._create_config({}),
known_scope_infos=OptionsTest._known_scope_infos,
args="./pants")
options.register(GLOBAL_SCOPE, '--too-old-option', deprecated_version='0.0.24',
deprecated_hint='The semver for this option has already passed.')
def _full_options(self, known_scope_infos):
bootstrap_option_values = self.get_bootstrap_options(
).for_global_scope()
return Options.create(self.env,
self.config,
known_scope_infos,
args=self.args,
bootstrap_option_values=bootstrap_option_values)
def create(
cls,
options_bootstrapper: OptionsBootstrapper,
options: Options,
session: SchedulerSession,
build_root: Optional[str] = None,
) -> Specs:
specs = cls.parse_specs(raw_specs=options.specs, build_root=build_root)
changed_options = ChangedOptions.from_options(
options.for_scope("changed"))
logger.debug("specs are: %s", specs)
logger.debug("changed_options are: %s", changed_options)
if specs.provided and changed_options.provided:
changed_name = "--changed-since" if changed_options.since else "--changed-diffspec"
if specs.filesystem_specs and specs.address_specs:
specs_description = "target and file arguments"
elif specs.filesystem_specs:
specs_description = "file arguments"
else:
specs_description = "target arguments"
raise InvalidSpecConstraint(
f"You used `{changed_name}` at the same time as using {specs_description}. Please "
"use only one.")
if not changed_options.provided:
return specs
scm = get_scm()
if not scm:
raise InvalidSpecConstraint(
"The `--changed-*` options are not available without a recognized SCM (usually "
"Git).")
changed_request = ChangedRequest(
sources=tuple(changed_options.changed_files(scm=scm)),
dependees=changed_options.dependees,
)
(changed_addresses, ) = session.product_request(
ChangedAddresses, [Params(changed_request, options_bootstrapper)])
logger.debug("changed addresses: %s", changed_addresses)
address_specs = []
filesystem_specs = []
for address in cast(ChangedAddresses, changed_addresses):
if not address.is_base_target:
# TODO: Should adjust Specs parsing to support parsing the disambiguated file
# Address, which would bypass-rediscovering owners.
filesystem_specs.append(FilesystemLiteralSpec(
address.filename))
else:
address_specs.append(
SingleAddress(address.spec_path, address.target_name))
return Specs(
AddressSpecs(address_specs, filter_by_global_options=True),
FilesystemSpecs(filesystem_specs),
)
def __init__(
self,
root_dir: str,
options_bootstrapper: OptionsBootstrapper,
options: Options,
build_config: BuildConfiguration,
run_tracker: RunTracker,
reporting: Reporting,
graph_session: LegacyGraphSession,
specs: Specs,
exiter=sys.exit,
) -> None:
"""
:param root_dir: The root directory of the pants workspace (aka the "build root").
:param options: The global, pre-initialized Options instance.
:param build_config: A pre-initialized BuildConfiguration instance.
:param run_tracker: The global, pre-initialized/running RunTracker instance.
:param reporting: The global, pre-initialized Reporting instance.
:param graph_session: The graph session for this run.
:param specs: The specs for this run, i.e. either the address or filesystem specs.
:param func exiter: A function that accepts an exit code value and exits. (for tests, Optional)
"""
self._root_dir = root_dir
self._options_bootstrapper = options_bootstrapper
self._options = options
self._build_config = build_config
self._run_tracker = run_tracker
self._reporting = reporting
self._graph_session = graph_session
self._specs = specs
self._exiter = exiter
self._global_options = options.for_global_scope()
self._fail_fast = self._global_options.fail_fast
self._explain = self._global_options.explain
self._kill_nailguns = self._global_options.kill_nailguns
# V1 tasks do not understand FilesystemSpecs, so we eagerly convert them into AddressSpecs.
if self._specs.filesystem_specs.dependencies:
(owned_addresses,
) = self._graph_session.scheduler_session.product_request(
Addresses, [
Params(self._specs.filesystem_specs,
self._options_bootstrapper)
])
updated_address_specs = AddressSpecs(
dependencies=tuple(
SingleAddress(a.spec_path, a.target_name)
for a in owned_addresses),
tags=self._specs.address_specs.matcher.tags,
exclude_patterns=self._specs.address_specs.matcher.
exclude_patterns,
)
self._specs = Specs(
address_specs=updated_address_specs,
filesystem_specs=FilesystemSpecs([]),
)
def run_goal_rules(
self,
*,
options_bootstrapper: OptionsBootstrapper,
union_membership: UnionMembership,
options: Options,
goals: Iterable[str],
specs: Specs,
):
"""Runs @goal_rules sequentially and interactively by requesting their implicit Goal
products.
For retryable failures, raises scheduler.ExecutionError.
:returns: An exit code.
"""
global_options = options.for_global_scope()
console = Console(
use_colors=global_options.colors,
session=self.scheduler_session
if global_options.get("v2_ui") else None,
)
workspace = Workspace(self.scheduler_session)
interactive_runner = InteractiveRunner(self.scheduler_session)
for goal in goals:
goal_product = self.goal_map[goal]
# NB: We no-op for goals that have no V2 implementation because no relevant backends are
# registered. This allows us to safely set `--v1 --v2`, even if no V2 backends are registered.
# Once V1 is removed, we might want to reconsider the behavior to instead warn or error when
# trying to run something like `./pants run` without any backends registered.
is_implemented = union_membership.has_members_for_all(
goal_product.subsystem_cls.required_union_implementations)
if not is_implemented:
continue
params = Params(
specs.provided_specs,
options_bootstrapper,
console,
workspace,
interactive_runner,
)
logger.debug(
f"requesting {goal_product} to satisfy execution of `{goal}` goal"
)
try:
exit_code = self.scheduler_session.run_goal_rule(
goal_product, params)
finally:
console.flush()
if exit_code != PANTS_SUCCEEDED_EXIT_CODE:
return exit_code
return PANTS_SUCCEEDED_EXIT_CODE
def _parse(self, args_str, env=None, config=None, bootstrap_option_values=None):
args = shlex.split(str(args_str))
options = Options.create(env=env or {},
config=self._create_config(config or {}),
known_scope_infos=OptionsTest._known_scope_infos,
args=args,
bootstrap_option_values=bootstrap_option_values)
self._register(options)
return options
def bootstrap_options_from_config(config):
bootstrap_options = Options.create(env=self._env, config=config,
known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()], args=bargs,
option_tracker=self._option_tracker)
def register_global(*args, **kwargs):
bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs)
GlobalOptionsRegistrar.register_bootstrap_options(register_global)
return bootstrap_options
def test_frozen_registration(self):
options = Options.create(args=[],
env={},
config=self._create_config({}),
known_scope_infos=[task('foo')],
option_tracker=OptionTracker())
options.register('foo', '--arg1')
with self.assertRaises(FrozenRegistration):
options.register(GLOBAL_SCOPE, '--arg2')
def assertError(expected_error, *args, **kwargs):
with self.assertRaises(expected_error):
options = Options.create(args=[],
env={},
config=self._create_config({}),
known_scope_infos=[],
option_tracker=OptionTracker())
options.register(GLOBAL_SCOPE, *args, **kwargs)
options.for_global_scope()
def get_all_help_info(
cls,
options: Options,
union_membership: UnionMembership,
consumed_scopes_mapper: ConsumedScopesMapper,
) -> AllHelpInfo:
scope_to_help_info = {}
name_to_goal_info = {}
for scope_info in sorted(options.known_scope_to_info.values(), key=lambda x: x.scope):
options.for_scope(scope_info.scope) # Force parsing.
optionable_cls = scope_info.optionable_cls
if not scope_info.description:
cls_name = (
f"{optionable_cls.__module__}.{optionable_cls.__qualname__}"
if optionable_cls
else ""
)
raise ValueError(
f"Subsystem {cls_name} with scope `{scope_info.scope}` has no description. "
f"Add a docstring or implement get_description()."
)
is_goal = optionable_cls is not None and issubclass(optionable_cls, GoalSubsystem)
oshi: OptionScopeHelpInfo = HelpInfoExtracter(
scope_info.scope
).get_option_scope_help_info(
scope_info.description, options.get_parser(scope_info.scope), is_goal
)
scope_to_help_info[oshi.scope] = oshi
if is_goal:
goal_subsystem_cls = cast(Type[GoalSubsystem], optionable_cls)
is_implemented = union_membership.has_members_for_all(
goal_subsystem_cls.required_union_implementations
)
name_to_goal_info[scope_info.scope] = GoalHelpInfo(
goal_subsystem_cls.name,
scope_info.description,
is_implemented,
consumed_scopes_mapper(scope_info.scope),
)
return AllHelpInfo(
scope_to_help_info=scope_to_help_info, name_to_goal_info=name_to_goal_info
)
def calculate_specs(
options_bootstrapper: OptionsBootstrapper,
options: Options,
session: SchedulerSession,
*,
build_root: Optional[str] = None,
) -> Specs:
"""Determine the specs for a given Pants run."""
build_root = build_root or get_buildroot()
specs = SpecsParser(build_root).parse_specs(options.specs)
changed_options = ChangedOptions.from_options(options.for_scope("changed"))
logger.debug("specs are: %s", specs)
logger.debug("changed_options are: %s", changed_options)
if specs.provided and changed_options.provided:
changed_name = "--changed-since" if changed_options.since else "--changed-diffspec"
if specs.filesystem_specs and specs.address_specs:
specs_description = "target and file arguments"
elif specs.filesystem_specs:
specs_description = "file arguments"
else:
specs_description = "target arguments"
raise InvalidSpecConstraint(
f"You used `{changed_name}` at the same time as using {specs_description}. Please "
"use only one.")
if not changed_options.provided:
return specs
git = get_git()
if not git:
raise InvalidSpecConstraint(
"The `--changed-*` options are only available if Git is used for the repository."
)
changed_request = ChangedRequest(
sources=tuple(changed_options.changed_files(git)),
dependees=changed_options.dependees,
)
(changed_addresses, ) = session.product_request(
ChangedAddresses, [Params(changed_request, options_bootstrapper)])
logger.debug("changed addresses: %s", changed_addresses)
address_specs = []
for address in cast(ChangedAddresses, changed_addresses):
address_input = AddressInput.parse(address.spec)
address_specs.append(
AddressLiteralSpec(
path_component=address_input.path_component,
# NB: AddressInput.target_component may be None, but AddressLiteralSpec expects a
# string.
target_component=address_input.target_component
or address.target_name,
))
return Specs(AddressSpecs(address_specs, filter_by_global_options=True),
FilesystemSpecs([]))
def __init__(self, options: Options):
super().__init__(options.for_global_scope().colors)
self._all_help_info = HelpInfoExtracter.get_all_help_info(
options,
# We only care about the options-related help info, so we pass in
# dummy values for the other arguments.
UnionMembership({}),
lambda x: tuple(),
RegisteredTargetTypes({}),
)
def _parse_type_int(self, args_str, env=None, config=None, bootstrap_option_values=None,
action=None):
args = shlex.split(str(args_str))
options = Options.create(env=env or {},
config=self._create_config(config or {}),
known_scope_infos=OptionsTest._known_scope_infos,
args=args,
bootstrap_option_values=bootstrap_option_values)
options.register(GLOBAL_SCOPE, '--config-override', action=action, type=int)
return options
def __init__(self, options: Options):
super().__init__(options.for_global_scope().colors)
self._bin_name = options.for_global_scope().pants_bin_name
self._all_help_info = HelpInfoExtracter.get_all_help_info(
options,
# We only care about the options-related help info, so we pass in
# dummy values for union_membership and consumed_scopes_mapper.
UnionMembership({}),
lambda x: tuple(),
)
def test_deprecated_option_past_removal(self):
with self.assertRaises(PastRemovalVersionError):
options = Options.create({}, FakeConfig({}),
OptionsTest._known_scope_infos, "./pants")
options.register(
GLOBAL_SCOPE,
'--too-old-option',
deprecated_version='0.0.24',
deprecated_hint='The semver for this option has already passed.'
)
def _parse(self, args_str, env=None, config=None, bootstrap_option_values=None):
args = shlex.split(str(args_str))
options = Options.create(env=env or {},
config=self._create_config(config or {}),
known_scope_infos=OptionsTest._known_scope_infos,
args=args,
bootstrap_option_values=bootstrap_option_values,
option_tracker=OptionTracker())
self._register(options)
return options
def test_deprecated_option_past_removal(self):
with self.assertRaises(PastRemovalVersionError):
options = Options.create(env={},
config=self._create_config({}),
known_scope_infos=OptionsTest._known_scope_infos,
args='./pants',
option_tracker=OptionTracker())
options.register(GLOBAL_SCOPE, '--too-old-option', deprecated_version='0.0.24',
deprecated_hint='The semver for this option has already passed.')
options.for_global_scope()
def _init_graph_session(
cls,
options_initializer: OptionsInitializer,
options_bootstrapper: OptionsBootstrapper,
build_config: BuildConfiguration,
env: CompleteEnvironment,
run_id: str,
options: Options,
scheduler: GraphScheduler | None = None,
cancellation_latch: PySessionCancellationLatch | None = None,
) -> GraphSession:
native_engine.maybe_set_panic_handler()
if scheduler is None:
dynamic_remote_options, _ = DynamicRemoteOptions.from_options(options, env)
bootstrap_options = options.bootstrap_option_values()
assert bootstrap_options is not None
scheduler = EngineInitializer.setup_graph(
bootstrap_options, build_config, dynamic_remote_options
)
with options_initializer.handle_unknown_flags(options_bootstrapper, env, raise_=True):
global_options = options.for_global_scope()
return scheduler.new_session(
run_id,
dynamic_ui=global_options.dynamic_ui,
ui_use_prodash=global_options.dynamic_ui_renderer
== DynamicUIRenderer.experimental_prodash,
use_colors=global_options.get("colors", True),
max_workunit_level=max(
global_options.streaming_workunits_level,
global_options.level,
*(
LogLevel[level.upper()]
for level in global_options.log_levels_by_target.values()
),
),
session_values=SessionValues(
{
OptionsBootstrapper: options_bootstrapper,
CompleteEnvironment: env,
}
),
cancellation_latch=cancellation_latch,
)
def test_shadowing(self):
options = Options.create(
env={},
config=self._create_config({}),
known_scope_infos=[task("foo")],
args="./pants",
option_tracker=OptionTracker(),
)
options.register("", "--bar")
with self.assertRaises(RegistrationError):
options.register("foo", "--bar")
def create(
cls,
options: Options,
session: SchedulerSession,
build_root: Optional[str] = None,
exclude_patterns: Optional[Iterable[str]] = None,
tags: Optional[Iterable[str]] = None,
) -> Specs:
specs = cls.parse_specs(
raw_specs=options.specs,
build_root=build_root,
exclude_patterns=exclude_patterns,
tags=tags,
)
changed_options = ChangedOptions.from_options(
options.for_scope("changed"))
logger.debug("specs are: %s", specs)
logger.debug("changed_options are: %s", changed_options)
if changed_options.is_actionable(
) and specs.provided_specs.dependencies:
# We've been provided both a change request and specs.
raise InvalidSpecConstraint(
"Multiple target selection methods provided. Please use only one of "
"`--changed-*`, address specs, or filesystem specs.")
if changed_options.is_actionable():
scm = get_scm()
if not scm:
raise InvalidSpecConstraint(
"The `--changed-*` options are not available without a recognized SCM (usually Git)."
)
changed_request = ChangedRequest(
sources=tuple(changed_options.changed_files(scm=scm)),
include_dependees=changed_options.include_dependees,
)
(changed_addresses, ) = session.product_request(
ChangedAddresses, [changed_request])
logger.debug("changed addresses: %s", changed_addresses.addresses)
dependencies = tuple(
SingleAddress(a.spec_path, a.target_name)
for a in changed_addresses.addresses)
return Specs(
address_specs=AddressSpecs(
dependencies=dependencies,
exclude_patterns=exclude_patterns,
tags=tags,
),
filesystem_specs=FilesystemSpecs([]),
)
return specs
def _parse_type_int(self, args_str, env=None, config=None, bootstrap_option_values=None,
action=None):
args = shlex.split(str(args_str))
options = Options.create(env=env or {},
config=self._create_config(config or {}),
known_scope_infos=OptionsTest._known_scope_infos,
args=args,
bootstrap_option_values=bootstrap_option_values,
option_tracker=OptionTracker())
options.register(GLOBAL_SCOPE, '--config-override', action=action, type=int)
return options
def _init_graph_session(
options_bootstrapper: OptionsBootstrapper,
build_config: BuildConfiguration,
options: Options,
scheduler: Optional[LegacyGraphScheduler] = None,
) -> LegacyGraphSession:
native = Native()
native.set_panic_handler()
graph_scheduler_helper = scheduler or EngineInitializer.setup_legacy_graph(
options_bootstrapper, build_config)
global_scope = options.for_global_scope()
if global_scope.v2:
dynamic_ui = resolve_conflicting_options(
old_option="v2_ui",
new_option="dynamic_ui",
old_scope=GLOBAL_SCOPE,
new_scope=GLOBAL_SCOPE,
old_container=global_scope,
new_container=global_scope,
)
else:
dynamic_ui = False
use_colors = global_scope.get("colors", True)
zipkin_trace_v2 = options.for_scope("reporting").zipkin_trace_v2
# TODO(#8658) This should_report_workunits flag must be set to True for
# StreamingWorkunitHandler to receive WorkUnits. It should eventually
# be merged with the zipkin_trace_v2 flag, since they both involve most
# of the same engine functionality, but for now is separate to avoid
# breaking functionality associated with zipkin tracing while iterating on streaming workunit reporting.
stream_workunits = len(
options.for_global_scope().streaming_workunits_handlers) != 0
return graph_scheduler_helper.new_session(
zipkin_trace_v2,
RunTracker.global_instance().run_id,
dynamic_ui=dynamic_ui,
use_colors=use_colors,
should_report_workunits=stream_workunits,
)
def get_bootstrap_option_values(env=None,
config=None,
args=None,
buildroot=None):
"""Get the values of just the bootstrap options."""
# Filter just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
flags = set()
def capture_the_flags(*args, **kwargs):
flags.update(args)
register_bootstrap_options(capture_the_flags, buildroot=buildroot)
bargs = filter(lambda x: x.partition('=')[0] in flags, args)
bootstrap_options = Options(env=env,
config=config,
known_scopes=[GLOBAL_SCOPE],
args=bargs)
register_bootstrap_options(bootstrap_options.register_global,
buildroot=buildroot)
return bootstrap_options.for_global_scope()
def prepare_task(self,
config=None,
args=None,
targets=None,
build_graph=None,
build_file_parser=None,
address_mapper=None,
console_outstream=None,
workspace=None):
"""Prepares a Task for execution.
task_type: The class of the Task to create.
config: An optional string representing the contents of a pants.ini config.
args: optional list of command line flags, these should be prefixed with '--test-'.
targets: optional list of Target objects passed on the command line.
Returns a new Task ready to execute.
"""
task_type = self.task_type()
assert issubclass(task_type, Task), 'task_type must be a Task subclass, got %s' % task_type
config = create_config(config or '')
workdir = os.path.join(config.getdefault('pants_workdir'), 'test', task_type.__name__)
bootstrap_options = OptionsBootstrapper().get_bootstrap_options()
options = Options(env={}, config=config, known_scopes=['', 'test'], args=args or [])
# A lot of basic code uses these options, so always register them.
register_bootstrap_options(options.register_global)
# We need to wrap register_global (can't set .bootstrap attr on the bound instancemethod).
def register_global_wrapper(*args, **kwargs):
return options.register_global(*args, **kwargs)
register_global_wrapper.bootstrap = bootstrap_options.for_global_scope()
register_global_options(register_global_wrapper)
task_type.options_scope = 'test'
task_type.register_options_on_scope(options)
run_tracker = create_run_tracker()
context = Context(config,
options,
run_tracker,
targets or [],
build_graph=build_graph,
build_file_parser=build_file_parser,
address_mapper=address_mapper,
console_outstream=console_outstream,
workspace=workspace)
return task_type(context, workdir)
def get_bootstrap_options(self):
""":returns: an Options instance that only knows about the bootstrap options.
:rtype: Options
"""
if not self._bootstrap_options:
flags = set()
short_flags = set()
def capture_the_flags(*args, **kwargs):
for flag in Parser.expand_flags(*args, **kwargs):
flags.add(flag.name)
if len(flag.name) == 2:
short_flags.add(flag.name)
if flag.inverse_name:
flags.add(flag.inverse_name)
register_bootstrap_options(capture_the_flags, buildroot=self._buildroot)
def is_bootstrap_option(arg):
components = arg.split('=', 1)
if components[0] in flags:
return True
for flag in short_flags:
if arg.startswith(flag):
return True
return False
# Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
# Stop before '--' since args after that are pass-through and may have duplicate names to our
# bootstrap options.
bargs = filter(is_bootstrap_option, itertools.takewhile(lambda arg: arg != '--', self._args))
self._bootstrap_options = Options(env=self._env, config=self._pre_bootstrap_config,
known_scopes=[GLOBAL_SCOPE], args=bargs)
register_bootstrap_options(self._bootstrap_options.register_global, buildroot=self._buildroot)
bootstrap_option_values = self._bootstrap_options.for_global_scope()
Config.reset_default_bootstrap_option_values(values=bootstrap_option_values)
# Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped
# from (typically pants.ini), then config override, then rcfiles.
configpaths = list(self._pre_bootstrap_config.sources())
if bootstrap_option_values.config_override:
configpaths.append(bootstrap_option_values.config_override)
if bootstrap_option_values.pantsrc:
rcfiles = [os.path.expanduser(rcfile) for rcfile in bootstrap_option_values.pantsrc_files]
existing_rcfiles = filter(os.path.exists, rcfiles)
configpaths.extend(existing_rcfiles)
self._post_bootstrap_config = Config.load(configpaths)
Config.cache(self._post_bootstrap_config)
return self._bootstrap_options
def bootstrap_options_from_config(config):
bootstrap_options = Options(
env=self._env,
config=config,
known_scope_infos=[ScopeInfo.for_global_scope()],
args=bargs)
def register_global(*args, **kwargs):
bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs)
GlobalOptionsRegistrar.register_bootstrap_options(
register_global)
return bootstrap_options
def parse_bootstrap_options(
env: Mapping[str, str], args: Sequence[str], config: Config
) -> Options:
bootstrap_options = Options.create(
env=env, config=config, known_scope_infos=[GlobalOptions.get_scope_info()], args=args,
)
def register_global(*args, **kwargs):
# Only use of Options.register?
bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs)
GlobalOptions.register_bootstrap_options(register_global)
return bootstrap_options
def _parse(self,
args_str,
env=None,
config=None,
bootstrap_option_values=None):
args = shlex.split(str(args_str))
options = Options(env or {},
FakeConfig(config or {}),
OptionsTest._known_scope_infos,
args,
bootstrap_option_values=bootstrap_option_values)
self._register(options)
return options
def test_complete_scopes(self):
_global = ScopeInfo.for_global_scope()
self.assertEquals(
{
_global,
intermediate('foo'),
intermediate('foo.bar'),
task('foo.bar.baz')
}, Options.complete_scopes({task('foo.bar.baz')}))
self.assertEquals(
{
_global,
intermediate('foo'),
intermediate('foo.bar'),
task('foo.bar.baz')
},
Options.complete_scopes(
{ScopeInfo.for_global_scope(),
task('foo.bar.baz')}))
self.assertEquals(
{
_global,
intermediate('foo'),
intermediate('foo.bar'),
task('foo.bar.baz')
},
Options.complete_scopes({intermediate('foo'),
task('foo.bar.baz')}))
self.assertEquals(
{
_global,
intermediate('foo'),
intermediate('foo.bar'),
task('foo.bar.baz'),
intermediate('qux'),
task('qux.quux')
}, Options.complete_scopes({task('foo.bar.baz'),
task('qux.quux')}))
def test_complete_scopes(self):
_global = GlobalOptionsRegistrar.get_scope_info()
self.assertEquals(
{
_global,
intermediate('foo'),
intermediate('foo.bar'),
task('foo.bar.baz')
}, Options.complete_scopes({task('foo.bar.baz')}))
self.assertEquals(
{
_global,
intermediate('foo'),
intermediate('foo.bar'),
task('foo.bar.baz')
},
Options.complete_scopes(
{GlobalOptionsRegistrar.get_scope_info(),
task('foo.bar.baz')}))
self.assertEquals(
{
_global,
intermediate('foo'),
intermediate('foo.bar'),
task('foo.bar.baz')
},
Options.complete_scopes({intermediate('foo'),
task('foo.bar.baz')}))
self.assertEquals(
{
_global,
intermediate('foo'),
intermediate('foo.bar'),
task('foo.bar.baz'),
intermediate('qux'),
task('qux.quux')
}, Options.complete_scopes({task('foo.bar.baz'),
task('qux.quux')}))
def parse_bootstrap_options(env, args, config):
bootstrap_options = Options.create(
env=env,
config=config,
known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()],
args=args,
)
def register_global(*args, **kwargs):
## Only use of Options.register?
bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs)
GlobalOptionsRegistrar.register_bootstrap_options(register_global)
return bootstrap_options
def parse_bootstrap_options(env, args, config):
bootstrap_options = Options.create(
env=env,
config=config,
known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()],
args=args,
)
def register_global(*args, **kwargs):
## Only use of Options.register?
bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs)
GlobalOptionsRegistrar.register_bootstrap_options(register_global)
return bootstrap_options
def _init_graph_session(
options_bootstrapper: OptionsBootstrapper,
build_config: BuildConfiguration,
options: Options,
scheduler: Optional[LegacyGraphScheduler] = None,
) -> LegacyGraphSession:
native = Native()
native.set_panic_handler()
graph_scheduler_helper = scheduler or EngineInitializer.setup_legacy_graph(
options_bootstrapper, build_config)
global_scope = options.for_global_scope()
dynamic_ui = global_scope.dynamic_ui if global_scope.v2 else False
use_colors = global_scope.get("colors", True)
stream_workunits = len(
options.for_global_scope().streaming_workunits_handlers) != 0
return graph_scheduler_helper.new_session(
RunTracker.global_instance().run_id,
dynamic_ui=dynamic_ui,
use_colors=use_colors,
should_report_workunits=stream_workunits,
)
def test_register_options_blessed(caplog) -> None:
class GoodToGo(Subsystem):
options_scope = "good-to-go"
options = Options.create(
env={},
config=Config.load([]),
known_scope_infos=[GoodToGo.get_scope_info()],
args=["./pants"],
bootstrap_option_values=None,
)
GoodToGo.register_options_on_scope(options)
assert not caplog.records, "The current blessed means of registering options should never warn."
def test_no_recursive_subsystem_options(self):
options = Options.create(env={},
config=self._create_config({}),
known_scope_infos=[subsystem('foo')],
args='./pants',
option_tracker=OptionTracker())
# All subsystem options are implicitly recursive (a subscope of subsystem scope represents
# a separate instance of the subsystem, so it needs all the options).
# We disallow explicit specification of recursive (even if set to True), to avoid confusion.
with self.assertRaises(RecursiveSubsystemOption):
options.register('foo', '--bar', recursive=False)
options.for_scope('foo')
with self.assertRaises(RecursiveSubsystemOption):
options.register('foo', '--baz', recursive=True)
options.for_scope('foo')
def _full_options(self, known_scope_infos):
bootstrap_option_values = self.get_bootstrap_options().for_global_scope()
options = Options.create(self.env,
self.config,
known_scope_infos,
args=self.args,
bootstrap_option_values=bootstrap_option_values)
distinct_optionable_classes = set()
for ksi in sorted(known_scope_infos, key=lambda si: si.scope):
if not ksi.optionable_cls or ksi.optionable_cls in distinct_optionable_classes:
continue
distinct_optionable_classes.add(ksi.optionable_cls)
ksi.optionable_cls.register_options_on_scope(options)
return options
def __init__(self, *args, **kwargs):
self.targets = []
self.config = Config.load()
known_scopes = ['']
for goal in Goal.all():
# Note that enclosing scopes will appear before scopes they enclose.
known_scopes.extend(filter(None, goal.known_scopes()))
# Annoying but temporary hack to get the parser. We can't use self.parser because
# that only gets set up in the superclass ctor, and we can't call that until we have
# self.new_options set up because the superclass ctor calls our register_options().
# Fortunately this will all go away once we're fully off the old "Command" mechanism.
legacy_parser = args[2] if len(args) > 2 else kwargs['parser']
self.new_options = Options(os.environ.copy(), self.config, known_scopes, args=sys.argv,
legacy_parser=legacy_parser)
super(GoalRunner, self).__init__(*args, **kwargs)
def test_shadowing(self):
options = Options.create(env={},
config=self._create_config({}),
known_scope_infos=[task('bar'), intermediate('foo'), task('foo.bar')],
args='./pants',
option_tracker=OptionTracker())
options.register('', '--opt1')
options.register('foo', '-o', '--opt2')
with self.assertRaises(Shadowing):
options.register('bar', '--opt1')
with self.assertRaises(Shadowing):
options.register('foo.bar', '--opt1')
with self.assertRaises(Shadowing):
options.register('foo.bar', '--opt2')
with self.assertRaises(Shadowing):
options.register('foo.bar', '--opt1', '--opt3')
with self.assertRaises(Shadowing):
options.register('foo.bar', '--opt3', '--opt2')
def get_full_options(self, known_scope_infos):
"""Get the full Options instance bootstrapped by this object for the given known scopes.
:param known_scope_infos: ScopeInfos for all scopes that may be encountered.
:returns: A bootrapped Options instance that also carries options for all the supplied known
scopes.
:rtype: :class:`Options`
"""
key = frozenset(sorted(known_scope_infos))
if key not in self._full_options:
# Note: Don't inline this into the Options() call, as this populates
# self._post_bootstrap_config, which is another argument to that call.
bootstrap_option_values = self.get_bootstrap_options().for_global_scope()
self._full_options[key] = Options.create(self._env,
self._post_bootstrap_config,
known_scope_infos,
args=self._args,
bootstrap_option_values=bootstrap_option_values)
return self._full_options[key]
def test_option_tracker_required(self):
with self.assertRaises(Options.OptionTrackerRequiredError):
Options.create(None, None, [])
class GoalRunner(Command):
"""Lists installed goals or else executes a named goal."""
class IntermixedArgumentsError(GoalError):
pass
__command__ = 'goal'
output = None
@staticmethod
def parse_args(args):
goals = OrderedSet()
specs = OrderedSet()
explicit_multi = False
fail_fast = False
logger = logging.getLogger(__name__)
has_double_dash = u'--' in args
goal_names = [goal.name for goal in Goal.all()]
if not goal_names:
raise GoalError(
'Arguments cannot be parsed before the list of goals from Goal.all() is populated.')
def is_spec(spec):
if os.sep in spec or ':' in spec:
return True # Definitely not a goal.
if not (spec in goal_names):
return True # Definitely not a (known) goal.
if has_double_dash:
# This means that we're parsing the half of the expression before a --, so assume it's a
# goal without warning.
return False
# Here, it's possible we have a goal and target with the same name. For now, always give
# priority to the goal, but give a warning if they might have meant the target (if the BUILD
# file exists).
if BuildFile.from_cache(get_buildroot(), spec, must_exist=False).exists():
logger.warning(' Command-line argument "{spec}" is ambiguous, and was assumed to be a '
'goal. If this is incorrect, disambiguate it with the "--" argument to '
'separate goals from targets.'.format(spec=spec))
return False
for i, arg in enumerate(args):
if not arg.startswith('-'):
specs.add(arg) if is_spec(arg) else goals.add(arg)
elif '--' == arg:
if specs:
raise GoalRunner.IntermixedArgumentsError(
'Cannot intermix targets with goals when using --. Targets should appear on the right')
explicit_multi = True
del args[i]
break
elif '--fail-fast' == arg.lower():
fail_fast = True
if explicit_multi:
specs.update(arg for arg in args[len(goals):] if not arg.startswith('-'))
return goals, specs, fail_fast
# TODO(John Sirois): revisit wholesale locking when we move py support into pants new
@classmethod
def serialized(cls):
# Goal serialization is now handled in goal execution during group processing.
# The goal command doesn't need to hold the serialization lock; individual goals will
# acquire the lock if they need to be serialized.
return False
def __init__(self, *args, **kwargs):
self.targets = []
self.config = Config.load()
known_scopes = ['']
for goal in Goal.all():
# Note that enclosing scopes will appear before scopes they enclose.
known_scopes.extend(filter(None, goal.known_scopes()))
# Annoying but temporary hack to get the parser. We can't use self.parser because
# that only gets set up in the superclass ctor, and we can't call that until we have
# self.new_options set up because the superclass ctor calls our register_options().
# Fortunately this will all go away once we're fully off the old "Command" mechanism.
legacy_parser = args[2] if len(args) > 2 else kwargs['parser']
self.new_options = Options(os.environ.copy(), self.config, known_scopes, args=sys.argv,
legacy_parser=legacy_parser)
super(GoalRunner, self).__init__(*args, **kwargs)
def get_spec_excludes(self):
spec_excludes = self.config.getlist(Config.DEFAULT_SECTION, 'spec_excludes',
default=None)
if spec_excludes is None:
return [self.config.getdefault('pants_workdir')]
return [os.path.join(self.root_dir, spec_exclude) for spec_exclude in spec_excludes]
@contextmanager
def check_errors(self, banner):
errors = {}
def error(key, include_traceback=False):
exc_type, exc_value, _ = sys.exc_info()
msg = StringIO()
if include_traceback:
frame = inspect.trace()[-2]
filename = frame[1]
lineno = frame[2]
funcname = frame[3]
code = ''.join(frame[4]) if frame[4] else None
traceback.print_list([(filename, lineno, funcname, code)], file=msg)
if exc_type:
msg.write(''.join(traceback.format_exception_only(exc_type, exc_value)))
errors[key] = msg.getvalue()
sys.exc_clear()
yield error
if errors:
msg = StringIO()
msg.write(banner)
invalid_keys = [key for key, exc in errors.items() if not exc]
if invalid_keys:
msg.write('\n %s' % '\n '.join(invalid_keys))
for key, exc in errors.items():
if exc:
msg.write('\n %s =>\n %s' % (key, '\n '.join(exc.splitlines())))
# The help message for goal is extremely verbose, and will obscure the
# actual error message, so we don't show it in this case.
self.error(msg.getvalue(), show_help=False)
def register_options(self):
register_global_options(self.new_options.register_global)
for goal in Goal.all():
goal.register_options(self.new_options)
def setup_parser(self, parser, args):
# We support attempting zero or more goals. Multiple goals must be delimited from further
# options and non goal args with a '--'. The key permutations we need to support:
# ./pants goal => goals
# ./pants goal goals => goals
# ./pants goal compile src/java/... => compile
# ./pants goal compile -x src/java/... => compile
# ./pants goal compile src/java/... -x => compile
# ./pants goal compile run -- src/java/... => compile, run
# ./pants goal compile run -- src/java/... -x => compile, run
# ./pants goal compile run -- -x src/java/... => compile, run
if not args:
args.append('help')
help_flags = set(['-h', '--help', 'help'])
show_help = len(help_flags.intersection(args)) > 0
non_help_args = filter(lambda f: f not in help_flags, args)
goals, specs, fail_fast = GoalRunner.parse_args(non_help_args)
if show_help:
self.new_options.print_help(goals=goals, legacy=True)
sys.exit(0)
self.requested_goals = goals
with self.run_tracker.new_workunit(name='setup', labels=[WorkUnit.SETUP]):
spec_parser = CmdLineSpecParser(self.root_dir, self.address_mapper, spec_excludes=self.get_spec_excludes())
with self.run_tracker.new_workunit(name='parse', labels=[WorkUnit.SETUP]):
for spec in specs:
for address in spec_parser.parse_addresses(spec, fail_fast):
self.build_graph.inject_address_closure(address)
self.targets.append(self.build_graph.get_target(address))
self.goals = [Goal.by_name(goal) for goal in goals]
rcfiles = self.config.getdefault('rcfiles', type=list,
default=['/etc/pantsrc', '~/.pants.rc'])
if rcfiles:
rcfile = RcFile(rcfiles, default_prepend=False, process_default=True)
# Break down the goals specified on the command line to the full set that will be run so we
# can apply default flags to inner goal nodes. Also break down goals by Task subclass and
# register the task class hierarchy fully qualified names so we can apply defaults to
# baseclasses.
sections = OrderedSet()
for goal in Engine.execution_order(self.goals):
for task_name in goal.ordered_task_names():
sections.add(task_name)
task_type = goal.task_type_by_name(task_name)
for clazz in task_type.mro():
if clazz == Task:
break
sections.add('%s.%s' % (clazz.__module__, clazz.__name__))
augmented_args = rcfile.apply_defaults(sections, args)
if augmented_args != args:
# TODO(John Sirois): Cleanup this currently important mutation of the passed in args
# once the 2-layer of command -> goal is squashed into one.
del args[:]
args.extend(augmented_args)
sys.stderr.write("(using pantsrc expansion: pants goal %s)\n" % ' '.join(augmented_args))
Goal.setup_parser(parser, args, self.goals)
def run(self, lock):
# TODO(John Sirois): Consider moving to straight python logging. The divide between the
# context/work-unit logging and standard python logging doesn't buy us anything.
# Enable standard python logging for code with no handle to a context/work-unit.
if self.old_options.log_level:
LogOptions.set_stderr_log_level((self.old_options.log_level or 'info').upper())
logdir = self.old_options.logdir or self.config.get('goals', 'logdir', default=None)
if logdir:
safe_mkdir(logdir)
LogOptions.set_log_dir(logdir)
log.init('goals')
else:
log.init()
# Update the reporting settings, now that we have flags etc.
def is_quiet_task():
for goal in self.goals:
if goal.has_task_of_type(QuietTaskMixin):
return True
return False
# Target specs are mapped to the patterns which match them, if any. This variable is a key for
# specs which don't match any exclusion regexes. We know it won't already be in the list of
# patterns, because the asterisks in its name make it an invalid regex.
_UNMATCHED_KEY = '** unmatched **'
def targets_by_pattern(targets, patterns):
mapping = defaultdict(list)
for target in targets:
matched_pattern = None
for pattern in patterns:
if re.search(pattern, target.address.spec) is not None:
matched_pattern = pattern
break
if matched_pattern is None:
mapping[_UNMATCHED_KEY].append(target)
else:
mapping[matched_pattern].append(target)
return mapping
is_explain = self.old_options.explain
update_reporting(self.old_options, is_quiet_task() or is_explain, self.run_tracker)
if self.old_options.target_excludes:
excludes = self.old_options.target_excludes
log.debug('excludes:\n {excludes}'.format(excludes='\n '.join(excludes)))
by_pattern = targets_by_pattern(self.targets, excludes)
self.targets = by_pattern[_UNMATCHED_KEY]
# The rest of this if-statement is just for debug logging.
log.debug('Targets after excludes: {targets}'.format(
targets=', '.join(t.address.spec for t in self.targets)))
excluded_count = sum(len(by_pattern[p]) for p in excludes)
log.debug('Excluded {count} target{plural}.'.format(count=excluded_count,
plural=('s' if excluded_count != 1 else '')))
for pattern in excludes:
log.debug('Targets excluded by pattern {pattern}\n {targets}'.format(pattern=pattern,
targets='\n '.join(t.address.spec for t in by_pattern[pattern])))
context = Context(
config=self.config,
old_options=self.old_options,
new_options=self.new_options,
run_tracker=self.run_tracker,
target_roots=self.targets,
requested_goals=self.requested_goals,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
address_mapper=self.address_mapper,
spec_excludes=self.get_spec_excludes(),
lock=lock)
unknown = []
for goal in self.goals:
if not goal.ordered_task_names():
unknown.append(goal)
if unknown:
context.log.error('Unknown goal(s): %s\n' % ' '.join(goal.name for goal in unknown))
return 1
engine = RoundEngine()
return engine.execute(context, self.goals)
def cleanup(self):
# TODO: This is JVM-specific and really doesn't belong here.
# TODO: Make this more selective? Only kill nailguns that affect state? E.g., checkstyle
# may not need to be killed.
NailgunTask.killall(log.info)
sys.exit(1)
def test_frozen_registration(self):
options = Options.create(args=[], env={}, config=self._create_config({}),
known_scope_infos=[task('foo')], option_tracker=OptionTracker())
options.register('foo', '--arg1')
with self.assertRaises(FrozenRegistration):
options.register(GLOBAL_SCOPE, '--arg2')
def assertError(expected_error, *args, **kwargs):
with self.assertRaises(expected_error):
options = Options.create(args=[], env={}, config=self._create_config({}),
known_scope_infos=[], option_tracker=OptionTracker())
options.register(GLOBAL_SCOPE, *args, **kwargs)
options.for_global_scope()
def test_deprecated_option_past_removal(self):
with self.assertRaises(PastRemovalVersionError):
options = Options({}, FakeConfig({}), OptionsTest._known_scopes, "./pants")
options.register_global('--too-old-option',
deprecated_version='0.0.24',
deprecated_hint='The semver for this option has already passed.')
class OptionsBootstrapper(object):
"""An object that knows how to create options in two stages: bootstrap, and then full options."""
def __init__(self, env=None, configpath=None, args=None, buildroot=None):
self._buildroot = buildroot or get_buildroot()
self._env = env or os.environ.copy()
Config.reset_default_bootstrap_option_values(buildroot=self._buildroot)
self._pre_bootstrap_config = Config.load([configpath] if configpath else None)
self._post_bootstrap_config = None # Will be set later.
self._args = args or sys.argv
self._bootstrap_options = None # We memoize the bootstrap options here.
self._full_options = None # We memoize the full options here.
# So other startup code has config to work with. This will go away once we replace direct
# config accesses with options, and plumb those through everywhere that needs them.
Config.cache(self._pre_bootstrap_config)
def get_bootstrap_options(self):
"""Returns an Options instance that only knows about the bootstrap options."""
if not self._bootstrap_options:
flags = set()
short_flags = set()
def capture_the_flags(*args, **kwargs):
for flag in Parser.expand_flags(*args, **kwargs):
flags.add(flag.name)
if len(flag.name) == 2:
short_flags.add(flag.name)
if flag.inverse_name:
flags.add(flag.inverse_name)
register_bootstrap_options(capture_the_flags, buildroot=self._buildroot)
def is_bootstrap_option(arg):
components = arg.split('=', 1)
if components[0] in flags:
return True
for flag in short_flags:
if arg.startswith(flag):
return True
return False
# Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
# Stop before '--' since args after that are pass-through and may have duplicate names to our
# bootstrap options.
bargs = filter(is_bootstrap_option, itertools.takewhile(lambda arg: arg != '--', self._args))
self._bootstrap_options = Options(env=self._env, config=self._pre_bootstrap_config,
known_scopes=[GLOBAL_SCOPE], args=bargs)
register_bootstrap_options(self._bootstrap_options.register_global, buildroot=self._buildroot)
bootstrap_option_values = self._bootstrap_options.for_global_scope()
Config.reset_default_bootstrap_option_values(values=bootstrap_option_values)
# Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped
# from (typically pants.ini), then config override, then rcfiles.
configpaths = list(self._pre_bootstrap_config.sources())
if bootstrap_option_values.config_override:
configpaths.append(bootstrap_option_values.config_override)
if bootstrap_option_values.pantsrc:
rcfiles = [os.path.expanduser(rcfile) for rcfile in bootstrap_option_values.pantsrc_files]
existing_rcfiles = filter(os.path.exists, rcfiles)
configpaths.extend(existing_rcfiles)
self._post_bootstrap_config = Config.load(configpaths)
Config.cache(self._post_bootstrap_config)
return self._bootstrap_options
def get_full_options(self, known_scopes):
if not self._full_options:
# Note: Don't inline this into the Options() call, as this populates
# self._post_bootstrap_config, which is another argument to that call.
bootstrap_options = self.get_bootstrap_options()
self._full_options = Options(self._env,
self._post_bootstrap_config,
known_scopes,
args=self._args,
bootstrap_option_values=bootstrap_options.for_global_scope())
# The bootstrap options need to be registered on the post-bootstrap Options instance, so it
# won't choke on them on the command line, and also so we can access their values as regular
# global-scope options, for convenience.
register_bootstrap_options(self._full_options.register_global, buildroot=self._buildroot)
return self._full_options
def test_scope_deprecation(self):
# Note: This test demonstrates that two different new scopes can deprecate the same
# old scope. I.e., it's possible to split an old scope's options among multiple new scopes.
class DummyOptionable1(Optionable):
options_scope = 'new-scope1'
options_scope_category = ScopeInfo.SUBSYSTEM
deprecated_options_scope = 'deprecated-scope'
deprecated_options_scope_removal_version = '9999.9.9'
class DummyOptionable2(Optionable):
options_scope = 'new-scope2'
options_scope_category = ScopeInfo.SUBSYSTEM
deprecated_options_scope = 'deprecated-scope'
deprecated_options_scope_removal_version = '9999.9.9'
options = Options.create(env={},
config=self._create_config({
DummyOptionable1.options_scope: {
'foo': 'xx'
},
DummyOptionable1.deprecated_options_scope: {
'foo': 'yy',
'bar': 'zz',
'baz': 'ww',
'qux': 'uu'
},
}),
known_scope_infos=[
DummyOptionable1.get_scope_info(),
DummyOptionable2.get_scope_info()
],
args=shlex.split('./pants --new-scope1-baz=vv'),
option_tracker=OptionTracker())
options.register(DummyOptionable1.options_scope, '--foo')
options.register(DummyOptionable1.options_scope, '--bar')
options.register(DummyOptionable1.options_scope, '--baz')
options.register(DummyOptionable2.options_scope, '--qux')
with self.warnings_catcher() as w:
vals1 = options.for_scope(DummyOptionable1.options_scope)
# Check that we got a warning.
self.assertEquals(1, len(w))
self.assertTrue(isinstance(w[0].message, DeprecationWarning))
# Check values.
# Deprecated scope takes precedence at equal rank.
self.assertEquals('yy', vals1.foo)
self.assertEquals('zz', vals1.bar)
# New scope takes precedence at higher rank.
self.assertEquals('vv', vals1.baz)
with self.warnings_catcher() as w:
vals2 = options.for_scope(DummyOptionable2.options_scope)
# Check that we got a warning.
self.assertEquals(1, len(w))
self.assertTrue(isinstance(w[0].message, DeprecationWarning))
# Check values.
self.assertEquals('uu', vals2.qux)
