def test_valid_invalid_scope(self) -> None:
"""Scopes with dashes or underscores are treated as a single component, and scopes with
empty components raise an InvalidScopeError."""
base_dashed_scope = "base-scope"
self.assertEqual(enclosing_scope(base_dashed_scope), GLOBAL_SCOPE)
subscope_underscore = "sub_scope"
self.assertEqual(enclosing_scope(subscope_underscore), GLOBAL_SCOPE)
compound_scope = f"{base_dashed_scope}.{subscope_underscore}"
self.assertEqual(enclosing_scope(compound_scope), base_dashed_scope)
self.assertEqual(
list(all_enclosing_scopes(compound_scope)),
[
compound_scope,
base_dashed_scope,
GLOBAL_SCOPE,
],
)
invalid_scope = "a.b..c.d"
with self.assertRaises(InvalidScopeError):
enclosing_scope(invalid_scope)
with self.assertRaises(InvalidScopeError):
# need to bounce to list to get it to error since this is a generator
list(all_enclosing_scopes(invalid_scope))
def test_enclosing_scope(self):
"""The enclosing scope of any non-nested scope should be the global scope,
and the enclosing scope of a nested scope should be the scope without its
final component."""
self.assertEqual(GLOBAL_SCOPE, enclosing_scope(GLOBAL_SCOPE))
self.assertEqual(GLOBAL_SCOPE, enclosing_scope('scope'))
self.assertEqual('base', enclosing_scope('base.subscope'))
def test_enclosing_scope(self):
"""The enclosing scope of any non-nested scope should be the global scope,
and the enclosing scope of a nested scope should be the scope without its
final component."""
self.assertEqual(GLOBAL_SCOPE, enclosing_scope(GLOBAL_SCOPE))
self.assertEqual(GLOBAL_SCOPE, enclosing_scope('scope'))
self.assertEqual('base', enclosing_scope('base.subscope'))
def _print_options_help(self):
"""Print a help screen.
Assumes that self._help_request is an instance of OptionsHelp.
Note: Ony useful if called after options have been registered.
"""
show_all_help = self._help_request.all_scopes
if show_all_help:
help_scopes = self._known_scope_to_info.keys()
else:
# The scopes explicitly mentioned by the user on the cmd line.
help_scopes = set(self._options.scope_to_flags.keys()) - set([GLOBAL_SCOPE])
# As a user-friendly heuristic, add all task scopes under requested scopes, so that e.g.,
# `./pants help compile` will show help for compile.java, compile.scala etc.
# Note that we don't do anything similar for subsystems - that would just create noise by
# repeating options for every task-specific subsystem instance.
for scope, info in self._known_scope_to_info.items():
if info.category == ScopeInfo.TASK:
outer = enclosing_scope(scope)
while outer != GLOBAL_SCOPE:
if outer in help_scopes:
help_scopes.add(scope)
break
outer = enclosing_scope(outer)
help_scope_infos = [self._known_scope_to_info[s] for s in sorted(help_scopes)]
if help_scope_infos:
for scope_info in self._help_subscopes_iter(help_scope_infos):
description = (scope_info.optionable_cls.get_description() if scope_info.optionable_cls
else None)
help_str = self._format_help(scope_info, description)
if help_str:
print(help_str)
return
else:
print(pants_release())
print('\nUsage:')
print(' ./pants [option ...] [goal ...] [target...] Attempt the specified goals.')
print(' ./pants help Get help.')
print(' ./pants help [goal] Get help for a goal.')
print(' ./pants help-advanced [goal] Get help for a goal\'s advanced options.')
print(' ./pants help-all Get help for all goals.')
print(' ./pants goals List all installed goals.')
print('')
print(' [target] accepts two special forms:')
print(' dir: to include all targets in the specified directory.')
print(' dir:: to include all targets found recursively under the directory.')
print('\nFriendly docs:\n http://pantsbuild.github.io/')
print(self._format_help(ScopeInfo(GLOBAL_SCOPE, ScopeInfo.GLOBAL), ''))
def _expand_tasks(self, scopes):
"""Add all tasks in any requested goals.
Returns the requested scopes, plus the added tasks, sorted by scope name.
"""
expanded_scopes = set(scopes)
for scope, info in self._scope_to_info.items():
if info.category == ScopeInfo.TASK:
outer = enclosing_scope(scope)
while outer != GLOBAL_SCOPE:
if outer in expanded_scopes:
expanded_scopes.add(scope)
break
outer = enclosing_scope(outer)
return sorted(expanded_scopes)
def _expand_tasks(self, scopes):
"""Add all tasks in any requested goals.
Returns the requested scopes, plus the added tasks, sorted by scope name.
"""
expanded_scopes = set(scopes)
for scope, info in self._scope_to_info.items():
if info.category == ScopeInfo.TASK:
outer = enclosing_scope(scope)
while outer != GLOBAL_SCOPE:
if outer in expanded_scopes:
expanded_scopes.add(scope)
break
outer = enclosing_scope(outer)
return sorted(expanded_scopes)
def get_fingerprintable_for_scope(self, scope):
"""Returns a list of fingerprintable (option type, option value) pairs for the given scope.
Fingerprintable options are options registered via a "fingerprint=True" kwarg.
"""
pairs = []
# Note that we iterate over options registered at `scope` and at all enclosing scopes, since
# option-using code can read those values indirectly via its own OptionValueContainer, so
# they can affect that code's output.
registration_scope = scope
while registration_scope is not None:
# This iterator will have already sorted the options, so their order is deterministic.
for (name, _, kwargs) in self.registration_args_iter_for_scope(
registration_scope):
if kwargs.get(
'recursive') and not kwargs.get('recursive_root'):
continue # We only need to fprint recursive options once.
if kwargs.get('fingerprint') is not True:
continue
# Note that we read the value from scope, even if the registration was on an enclosing
# scope, to get the right value for recursive options (and because this mirrors what
# option-using code does).
val = self.for_scope(scope)[name]
val_type = kwargs.get('type', '')
pairs.append((val_type, val))
registration_scope = (None if registration_scope == '' else
enclosing_scope(registration_scope))
return pairs
def complete_scopes(cls, scope_infos):
"""Expand a set of scopes to include all enclosing scopes.
E.g., if the set contains `foo.bar.baz`, ensure that it also contains `foo.bar` and `foo`.
Also adds any deprecated scopes.
"""
ret = {GlobalOptionsRegistrar.get_scope_info()}
original_scopes = set()
for si in scope_infos:
ret.add(si)
original_scopes.add(si.scope)
if si.deprecated_scope:
ret.add(
ScopeInfo(si.deprecated_scope, si.category,
si.optionable_cls))
original_scopes.add(si.deprecated_scope)
# TODO: Once scope name validation is enforced (so there can be no dots in scope name
# components) we can replace this line with `for si in scope_infos:`, because it will
# not be possible for a deprecated_scope to introduce any new intermediate scopes.
for si in copy.copy(ret):
scope = si.scope
while scope != '':
if scope not in original_scopes:
ret.add(ScopeInfo(scope, ScopeInfo.INTERMEDIATE))
scope = enclosing_scope(scope)
return ret
def complete_scopes(cls, scope_infos):
"""Expand a set of scopes to include all enclosing scopes.
E.g., if the set contains `foo.bar.baz`, ensure that it also contains `foo.bar` and `foo`.
Also adds any deprecated scopes.
"""
ret = {GlobalOptionsRegistrar.get_scope_info()}
original_scopes = set()
for si in scope_infos:
ret.add(si)
original_scopes.add(si.scope)
if si.deprecated_scope:
ret.add(ScopeInfo(si.deprecated_scope, si.category, si.optionable_cls))
original_scopes.add(si.deprecated_scope)
# TODO: Once scope name validation is enforced (so there can be no dots in scope name
# components) we can replace this line with `for si in scope_infos:`, because it will
# not be possible for a deprecated_scope to introduce any new intermediate scopes.
for si in copy.copy(ret):
scope = si.scope
while scope != '':
if scope not in original_scopes:
ret.add(ScopeInfo(scope, ScopeInfo.INTERMEDIATE))
scope = enclosing_scope(scope)
return ret
def for_scope(self, scope):
"""Return the option values for the given scope.
Values are attributes of the returned object, e.g., options.foo.
Computed lazily per scope.
:API: public
"""
# Short-circuit, if already computed.
if scope in self._values_by_scope:
return self._values_by_scope[scope]
# First get enclosing scope's option values, if any.
if scope == GLOBAL_SCOPE:
values = OptionValueContainer()
else:
values = copy.copy(self.for_scope(enclosing_scope(scope)))
# Now add our values.
flags_in_scope = self._scope_to_flags.get(scope, [])
self._parser_hierarchy.get_parser_by_scope(scope).parse_args(
flags_in_scope, values)
self._values_by_scope[scope] = values
for option in values:
self._option_tracker.record_option(scope=scope,
option=option,
value=values[option],
rank=values.get_rank(option))
return values
def for_scope(self, scope):
# TODO(John Sirois): Some users pass in A dict of scope -> _FakeOptionValues instead of a
# dict of scope -> (dict of option name -> value). Clean up these usages and kill this
# accommodation.
options_for_this_scope = options.get(scope) or {}
if isinstance(options_for_this_scope, _FakeOptionValues):
options_for_this_scope = options_for_this_scope.option_values
if passthru_args:
# TODO: This is _very_ partial support for passthrough args: this should be
# inspecting the kwargs of option registrations to decide which arguments to
# extend: this explicit `passthrough_args` argument is only passthrough because
# it is marked as such.
pa = options_for_this_scope.get("passthrough_args", [])
if isinstance(pa, RankedValue):
pa = pa.value
options_for_this_scope["passthrough_args"] = [
*pa, *passthru_args
]
scoped_options = {}
if scope:
scoped_options.update(
self.for_scope(enclosing_scope(scope)).option_values)
scoped_options.update(options_for_this_scope)
return _FakeOptionValues(scoped_options)
def _expand_subsystems(self, scope_infos):
"""Add all subsystems tied to a scope, right after that scope."""
# Get non-global subsystem dependencies of the specified subsystem client.
def subsys_deps(subsystem_client_cls):
for dep in subsystem_client_cls.subsystem_dependencies_iter():
if dep.scope != GLOBAL_SCOPE:
yield self._scope_to_info[dep.options_scope]
for x in subsys_deps(dep.subsystem_cls):
yield x
for scope_info in scope_infos:
yield scope_info
if scope_info.optionable_cls is not None:
# We don't currently subclass GlobalOptionsRegistrar, and I can't think of any reason why
# we would, but might as well be robust.
if issubclass(scope_info.optionable_cls,
GlobalOptionsRegistrar):
# We were asked for global help, so also yield for all global subsystems.
for scope, info in self._scope_to_info.items():
if info.category == ScopeInfo.SUBSYSTEM and enclosing_scope(
scope) == GLOBAL_SCOPE:
yield info
for subsys_dep in subsys_deps(info.optionable_cls):
yield subsys_dep
elif issubclass(scope_info.optionable_cls,
SubsystemClientMixin):
for subsys_dep in subsys_deps(scope_info.optionable_cls):
yield subsys_dep
def get_fingerprintable_for_scope(self, scope):
"""Returns a list of fingerprintable (option type, option value) pairs for the given scope.
Fingerprintable options are options registered via a "fingerprint=True" kwarg.
"""
pairs = []
# Note that we iterate over options registered at `scope` and at all enclosing scopes, since
# option-using code can read those values indirectly via its own OptionValueContainer, so
# they can affect that code's output.
registration_scope = scope
while registration_scope is not None:
parser = self._parser_hierarchy.get_parser_by_scope(registration_scope)
# Sort the arguments, so that the fingerprint is consistent.
for (_, kwargs) in sorted(parser.option_registrations_iter()):
if kwargs.get('recursive') and not kwargs.get('recursive_root'):
continue # We only need to fprint recursive options once.
if kwargs.get('fingerprint') is not True:
continue
# Note that we read the value from scope, even if the registration was on an enclosing
# scope, to get the right value for recursive options (and because this mirrors what
# option-using code does).
val = self.for_scope(scope)[kwargs['dest']]
val_type = kwargs.get('type', '')
pairs.append((val_type, val))
registration_scope = (None if registration_scope == ''
else enclosing_scope(registration_scope))
return pairs
def get_fingerprintable_for_scope(self, scope):
"""Returns a list of fingerprintable (option type, option value) pairs for the given scope.
Fingerprintable options are options registered via a "fingerprint=True" kwarg.
"""
pairs = []
# Note that we iterate over options registered at `scope` and at all enclosing scopes, since
# option-using code can read those values indirectly via its own OptionValueContainer, so
# they can affect that code's output.
registration_scope = scope
while registration_scope is not None:
# This iterator will have already sorted the options, so their order is deterministic.
for (name, _, kwargs) in self.registration_args_iter_for_scope(registration_scope):
if kwargs.get("recursive") and not kwargs.get("recursive_root"):
continue # We only need to fprint recursive options once.
if kwargs.get("fingerprint") is not True:
continue
# Note that we read the value from scope, even if the registration was on an enclosing
# scope, to get the right value for recursive options (and because this mirrors what
# option-using code does).
val = self.for_scope(scope)[name]
val_type = kwargs.get("type", "")
pairs.append((val_type, val))
registration_scope = None if registration_scope == "" else enclosing_scope(registration_scope)
return pairs
def for_scope(self, scope):
"""Return the option values for the given scope.
Values are attributes of the returned object, e.g., options.foo.
Computed lazily per scope.
"""
# Short-circuit, if already computed.
if scope in self._values_by_scope:
return self._values_by_scope[scope]
# First get enclosing scope's option values, if any.
if scope == GLOBAL_SCOPE:
values = OptionValueContainer()
else:
values = copy.deepcopy(self.for_scope(enclosing_scope(scope)))
# Now add our values.
flags_in_scope = self._scope_to_flags.get(scope, [])
self._parser_hierarchy.get_parser_by_scope(scope).parse_args(flags_in_scope, values)
self._values_by_scope[scope] = values
for option in values:
self._option_tracker.record_option(
scope=scope, option=option, value=values[option], rank=values.get_rank(option)
)
return values
def for_scope(self, scope, inherit_from_enclosing_scope=True):
"""Return the option values for the given scope.
Values are attributes of the returned object, e.g., options.foo.
Computed lazily per scope.
:API: public
"""
# First get enclosing scope's option values, if any.
if scope == GLOBAL_SCOPE or not inherit_from_enclosing_scope:
values = OptionValueContainer()
else:
values = copy.copy(self.for_scope(enclosing_scope(scope)))
# Now add our values.
flags_in_scope = self._scope_to_flags.get(scope, [])
parse_args_request = self._make_parse_args_request(flags_in_scope, values)
self._parser_hierarchy.get_parser_by_scope(scope).parse_args(parse_args_request)
# Check for any deprecation conditions, which are evaluated using `self._flag_matchers`.
if inherit_from_enclosing_scope:
self._check_and_apply_deprecations(scope, values)
return values
def _expand_subsystems(self, scope_infos):
"""Add all subsystems tied to a scope, right after that scope."""
# Get non-global subsystem dependencies of the specified subsystem client.
def subsys_deps(subsystem_client_cls):
for dep in subsystem_client_cls.subsystem_dependencies_iter():
if dep.scope != GLOBAL_SCOPE:
yield self._scope_to_info[dep.options_scope()]
for x in subsys_deps(dep.subsystem_cls):
yield x
for scope_info in scope_infos:
yield scope_info
if scope_info.optionable_cls is not None:
# We don't currently subclass GlobalOptionsRegistrar, and I can't think of any reason why
# we would, but might as well be robust.
if issubclass(scope_info.optionable_cls, GlobalOptionsRegistrar):
# We were asked for global help, so also yield for all global subsystems.
for scope, info in self._scope_to_info.items():
if info.category == ScopeInfo.SUBSYSTEM and enclosing_scope(scope) == GLOBAL_SCOPE:
yield info
for subsys_dep in subsys_deps(info.optionable_cls):
yield subsys_dep
elif issubclass(scope_info.optionable_cls, SubsystemClientMixin):
for subsys_dep in subsys_deps(scope_info.optionable_cls):
yield subsys_dep
def get_fingerprintable_for_scope(self, scope):
"""Returns a list of fingerprintable (option type, option value) pairs for the given scope.
Fingerprintable options are options registered via a "fingerprint=True" kwarg.
:API: public
"""
pairs = []
# Note that we iterate over options registered at `scope` and at all enclosing scopes, since
# option-using code can read those values indirectly via its own OptionValueContainer, so
# they can affect that code's output.
registration_scope = scope
while registration_scope is not None:
parser = self._parser_hierarchy.get_parser_by_scope(registration_scope)
# Sort the arguments, so that the fingerprint is consistent.
for (_, kwargs) in sorted(parser.option_registrations_iter()):
if kwargs.get('recursive') and not kwargs.get('recursive_root'):
continue # We only need to fprint recursive options once.
if kwargs.get('fingerprint') is not True:
continue
# Note that we read the value from scope, even if the registration was on an enclosing
# scope, to get the right value for recursive options (and because this mirrors what
# option-using code does).
val = self.for_scope(scope)[kwargs['dest']]
# If we have a list then we delegate to the fingerprinting implementation of the members.
if is_list_option(kwargs):
val_type = kwargs.get('member_type', str)
else:
val_type = kwargs.get('type', str)
pairs.append((val_type, val))
registration_scope = (None if registration_scope == ''
else enclosing_scope(registration_scope))
return pairs
def for_scope(self, scope, inherit_from_enclosing_scope=True):
"""Return the option values for the given scope.
Values are attributes of the returned object, e.g., options.foo.
Computed lazily per scope.
:API: public
"""
# Short-circuit, if already computed.
if scope in self._values_by_scope:
return self._values_by_scope[scope]
# First get enclosing scope's option values, if any.
if scope == GLOBAL_SCOPE or not inherit_from_enclosing_scope:
values = OptionValueContainer()
else:
values = copy.copy(self.for_scope(enclosing_scope(scope)))
# Now add our values.
flags_in_scope = self._scope_to_flags.get(scope, [])
self._parser_hierarchy.get_parser_by_scope(scope).parse_args(
flags_in_scope, values)
# Check for any deprecation conditions, which are evaluated using `self._flag_matchers`.
self._check_deprecations(scope, flags_in_scope, values)
# Cache the values.
self._values_by_scope[scope] = values
return values
def _help_subscopes_iter(self, scope_infos):
"""Yields the scopes to actually show help for when the user asks for help for scope_info."""
for scope_info in scope_infos:
yield scope_info
# We don't currently subclass GlobalOptionsRegistrar, and I can't think of any reason why
# we would, but might as well be robust.
if scope_info.optionable_cls is not None:
if issubclass(scope_info.optionable_cls,
GlobalOptionsRegistrar):
for scope, info in self._known_scope_to_info.items():
if info.category == ScopeInfo.SUBSYSTEM and enclosing_scope(
scope) == GLOBAL_SCOPE:
# This is a global subsystem, so show it when asked for global help.
yield info
elif issubclass(scope_info.optionable_cls,
SubsystemClientMixin):
def yield_deps(subsystem_client_cls):
for dep in subsystem_client_cls.subsystem_dependencies_iter(
):
if dep.scope != GLOBAL_SCOPE:
yield self._known_scope_to_info[
dep.options_scope()]
for info in yield_deps(dep.subsystem_cls):
yield info
for info in yield_deps(scope_info.optionable_cls):
yield info
def for_scope(self, scope, inherit_from_enclosing_scope=True):
"""Return the option values for the given scope.
Values are attributes of the returned object, e.g., options.foo.
Computed lazily per scope.
:API: public
"""
# Short-circuit, if already computed.
if scope in self._values_by_scope:
return self._values_by_scope[scope]
# First get enclosing scope's option values, if any.
if scope == GLOBAL_SCOPE or not inherit_from_enclosing_scope:
values = OptionValueContainer()
else:
values = copy.copy(self.for_scope(enclosing_scope(scope)))
# Now add our values.
flags_in_scope = self._scope_to_flags.get(scope, [])
self._parser_hierarchy.get_parser_by_scope(scope).parse_args(
flags_in_scope, values)
# If we're the new name of a deprecated scope, also get values from that scope.
deprecated_scope = self.known_scope_to_info[scope].deprecated_scope
# Note that deprecated_scope and scope share the same Optionable class, so deprecated_scope's
# Optionable has a deprecated_options_scope equal to deprecated_scope. Therefore we must
# check that scope != deprecated_scope to prevent infinite recursion.
if deprecated_scope is not None and scope != deprecated_scope:
# Do the deprecation check only on keys that were explicitly set on the deprecated scope
# (and not on its enclosing scopes).
explicit_keys = self.for_scope(
deprecated_scope,
inherit_from_enclosing_scope=False).get_explicit_keys()
if explicit_keys:
warn_or_error(
self.known_scope_to_info[scope].
deprecated_scope_removal_version,
'scope {}'.format(deprecated_scope),
'Use scope {} instead (options: {})'.format(
scope, ', '.join(explicit_keys)))
# Update our values with those of the deprecated scope (now including values inherited
# from its enclosing scope).
# Note that a deprecated val will take precedence over a val of equal rank.
# This makes the code a bit neater.
values.update(self.for_scope(deprecated_scope))
# Record the value derivation.
for option in values:
self._option_tracker.record_option(scope=scope,
option=option,
value=values[option],
rank=values.get_rank(option))
# Cache the values.
self._values_by_scope[scope] = values
return values
def get_fingerprintable_for_scope(self,
scope,
include_passthru=False,
fingerprint_key=None,
invert=False):
"""Returns a list of fingerprintable (option type, option value) pairs for the given scope.
Fingerprintable options are options registered via a "fingerprint=True" kwarg. This flag
can be parameterized with `fingerprint_key` for special cases.
:param str scope: The scope to gather fingerprintable options for.
:param bool include_passthru: Whether to include passthru args captured by `scope` in the
fingerprintable options.
:param string fingerprint_key: The option kwarg to match against (defaults to 'fingerprint').
:param bool invert: Whether or not to invert the boolean check for the fingerprint_key value.
:API: public
"""
fingerprint_key = fingerprint_key or 'fingerprint'
fingerprint_default = False if invert else None
pairs = []
if include_passthru:
# Passthru args can only be sent to outermost scopes so we gather them once here up-front.
passthru_args = self.passthru_args_for_scope(scope)
# NB: We can't sort passthru args, the underlying consumer may be order-sensitive.
pairs.extend((str, passthru_arg) for passthru_arg in passthru_args)
# Note that we iterate over options registered at `scope` and at all enclosing scopes, since
# option-using code can read those values indirectly via its own OptionValueContainer, so
# they can affect that code's output.
registration_scope = scope
while registration_scope is not None:
parser = self._parser_hierarchy.get_parser_by_scope(
registration_scope)
# Sort the arguments, so that the fingerprint is consistent.
for (_, kwargs) in sorted(parser.option_registrations_iter()):
if kwargs.get(
'recursive') and not kwargs.get('recursive_root'):
continue # We only need to fprint recursive options once.
if kwargs.get(fingerprint_key,
fingerprint_default) is not (False if invert else
True):
continue
# Note that we read the value from scope, even if the registration was on an enclosing
# scope, to get the right value for recursive options (and because this mirrors what
# option-using code does).
val = self.for_scope(scope)[kwargs['dest']]
# If we have a list then we delegate to the fingerprinting implementation of the members.
if is_list_option(kwargs):
val_type = kwargs.get('member_type', str)
else:
val_type = kwargs.get('type', str)
pairs.append((val_type, val))
registration_scope = (None if registration_scope == '' else
enclosing_scope(registration_scope))
return pairs
def complete_scopes(scopes):
"""Return all enclosing scopes.
This is similar to what `complete_scopes` does in `pants.option.options.Options` without
creating `ScopeInfo`s.
"""
completed_scopes = set(scopes)
for scope in scopes:
while scope != '':
if scope not in completed_scopes:
completed_scopes.add(scope)
scope = enclosing_scope(scope)
return completed_scopes
def for_scope(self, scope):
# TODO(John Sirois): Some users pass in A dict of scope -> _FakeOptionValues instead of a
# dict of scope -> (dict of option name -> value). Clean up these usages and kill this
# accommodation.
options_for_this_scope = options.get(scope) or {}
if isinstance(options_for_this_scope, _FakeOptionValues):
options_for_this_scope = options_for_this_scope.option_values
scoped_options = {}
if scope:
scoped_options.update(self.for_scope(enclosing_scope(scope)).option_values)
scoped_options.update(options_for_this_scope)
return _FakeOptionValues(scoped_options)
def complete_scopes(scopes):
"""Return all enclosing scopes.
This is similar to what `complete_scopes` does in `pants.option.options.Options` without
creating `ScopeInfo`s.
"""
completed_scopes = set(scopes)
for scope in scopes:
while scope != '':
if scope not in completed_scopes:
completed_scopes.add(scope)
scope = enclosing_scope(scope)
return completed_scopes
def for_scope(self, scope, inherit_from_enclosing_scope=True):
"""Return the option values for the given scope.
Values are attributes of the returned object, e.g., options.foo.
Computed lazily per scope.
:API: public
"""
# Short-circuit, if already computed.
if scope in self._values_by_scope:
return self._values_by_scope[scope]
# First get enclosing scope's option values, if any.
if scope == GLOBAL_SCOPE or not inherit_from_enclosing_scope:
values = OptionValueContainer()
else:
values = copy.copy(self.for_scope(enclosing_scope(scope)))
# Now add our values.
flags_in_scope = self._scope_to_flags.get(scope, [])
self._parser_hierarchy.get_parser_by_scope(scope).parse_args(flags_in_scope, values)
# If we're the new name of a deprecated scope, also get values from that scope.
deprecated_scope = self.known_scope_to_info[scope].deprecated_scope
# Note that deprecated_scope and scope share the same Optionable class, so deprecated_scope's
# Optionable has a deprecated_options_scope equal to deprecated_scope. Therefore we must
# check that scope != deprecated_scope to prevent infinite recursion.
if deprecated_scope is not None and scope != deprecated_scope:
# Do the deprecation check only on keys that were explicitly set on the deprecated scope
# (and not on its enclosing scopes).
explicit_keys = self.for_scope(deprecated_scope,
inherit_from_enclosing_scope=False).get_explicit_keys()
if explicit_keys:
warn_or_error(self.known_scope_to_info[scope].deprecated_scope_removal_version,
'scope {}'.format(deprecated_scope),
'Use scope {} instead (options: {})'.format(scope, ', '.join(explicit_keys)))
# Update our values with those of the deprecated scope (now including values inherited
# from its enclosing scope).
# Note that a deprecated val will take precedence over a val of equal rank.
# This makes the code a bit neater.
values.update(self.for_scope(deprecated_scope))
# Record the value derivation.
for option in values:
self._option_tracker.record_option(scope=scope, option=option, value=values[option],
rank=values.get_rank(option))
# Cache the values.
self._values_by_scope[scope] = values
return values
def test_valid_invalid_scope(self):
"""Scopes with dashes or underscores are treated as a single component, and
scopes with empty components raise an InvalidScopeError."""
base_dashed_scope = 'base-scope'
self.assertEqual(enclosing_scope(base_dashed_scope), GLOBAL_SCOPE)
subscope_underscore = 'sub_scope'
self.assertEqual(enclosing_scope(subscope_underscore), GLOBAL_SCOPE)
compound_scope = '{}.{}'.format(base_dashed_scope, subscope_underscore)
self.assertEqual(enclosing_scope(compound_scope), base_dashed_scope)
self.assertEqual(list(all_enclosing_scopes(compound_scope)), [
compound_scope,
base_dashed_scope,
GLOBAL_SCOPE,
])
invalid_scope = 'a.b..c.d'
with self.assertRaises(InvalidScopeError):
enclosing_scope(invalid_scope)
with self.assertRaises(InvalidScopeError):
# need to bounce to list to get it to error since this is a generator
list(all_enclosing_scopes(invalid_scope))
def for_scope(self, scope):
# TODO(John Sirois): Some users pass in A dict of scope -> _FakeOptionValues instead of a
# dict of scope -> (dict of option name -> value). Clean up these usages and kill this
# accommodation.
options_for_this_scope = options.get(scope) or {}
if isinstance(options_for_this_scope, _FakeOptionValues):
options_for_this_scope = options_for_this_scope.option_values
scoped_options = {}
if scope:
scoped_options.update(
self.for_scope(enclosing_scope(scope)).option_values)
scoped_options.update(options_for_this_scope)
return _FakeOptionValues(scoped_options)
def complete_scopes(cls, scope_infos):
"""Expand a set of scopes to include all enclosing scopes.
E.g., if the set contains `foo.bar.baz`, ensure that it also contains `foo.bar` and `foo`.
"""
ret = {GlobalOptionsRegistrar.get_scope_info()}
for scope_info in scope_infos:
ret.add(scope_info)
original_scopes = {si.scope for si in scope_infos}
for scope_info in scope_infos:
scope = scope_info.scope
while scope != "":
if scope not in original_scopes:
ret.add(ScopeInfo(scope, ScopeInfo.INTERMEDIATE))
scope = enclosing_scope(scope)
return ret
def complete_scopes(cls, scope_infos):
"""Expand a set of scopes to include all enclosing scopes.
E.g., if the set contains `foo.bar.baz`, ensure that it also contains `foo.bar` and `foo`.
"""
ret = {GlobalOptionsRegistrar.get_scope_info()}
for scope_info in scope_infos:
ret.add(scope_info)
original_scopes = {si.scope for si in scope_infos}
for scope_info in scope_infos:
scope = scope_info.scope
while scope != '':
if scope not in original_scopes:
ret.add(ScopeInfo(scope, ScopeInfo.INTERMEDIATE))
scope = enclosing_scope(scope)
return ret
def get_fingerprintable_for_scope(self, scope, include_passthru=False):
"""Returns a list of fingerprintable (option type, option value) pairs for the given scope.
Fingerprintable options are options registered via a "fingerprint=True" kwarg.
:param str scope: The scope to gather fingerprintable options for.
:param bool include_passthru: Whether to include passthru args captured by `scope` in the
fingerprintable options.
:API: public
"""
pairs = []
if include_passthru:
# Passthru args can only be sent to outermost scopes so we gather them once here up-front.
passthru_args = self.passthru_args_for_scope(scope)
# NB: We can't sort passthru args, the underlying consumer may be order-sensitive.
pairs.extend((str, passthru_arg) for passthru_arg in passthru_args)
# Note that we iterate over options registered at `scope` and at all enclosing scopes, since
# option-using code can read those values indirectly via its own OptionValueContainer, so
# they can affect that code's output.
registration_scope = scope
while registration_scope is not None:
parser = self._parser_hierarchy.get_parser_by_scope(registration_scope)
# Sort the arguments, so that the fingerprint is consistent.
for (_, kwargs) in sorted(parser.option_registrations_iter()):
if kwargs.get('recursive') and not kwargs.get('recursive_root'):
continue # We only need to fprint recursive options once.
if kwargs.get('fingerprint') is not True:
continue
# Note that we read the value from scope, even if the registration was on an enclosing
# scope, to get the right value for recursive options (and because this mirrors what
# option-using code does).
val = self.for_scope(scope)[kwargs['dest']]
# If we have a list then we delegate to the fingerprinting implementation of the members.
if is_list_option(kwargs):
val_type = kwargs.get('member_type', str)
else:
val_type = kwargs.get('type', str)
pairs.append((val_type, val))
registration_scope = (None if registration_scope == ''
else enclosing_scope(registration_scope))
return pairs
def _help_subscopes_iter(self, scope_infos):
"""Yields the scopes to actually show help for when the user asks for help for scope_info."""
for scope_info in scope_infos:
yield scope_info
# We don't currently subclass GlobalOptionsRegistrar, and I can't think of any reason why
# we would, but might as well be robust.
if scope_info.optionable_cls is not None:
if issubclass(scope_info.optionable_cls, GlobalOptionsRegistrar):
for scope, info in self._known_scope_to_info.items():
if info.category == ScopeInfo.SUBSYSTEM and enclosing_scope(scope) == GLOBAL_SCOPE:
# This is a global subsystem, so show it when asked for global help.
yield info
elif issubclass(scope_info.optionable_cls, SubsystemClientMixin):
def yield_deps(subsystem_client_cls):
for dep in subsystem_client_cls.subsystem_dependencies_iter():
if dep.scope != GLOBAL_SCOPE:
yield self._known_scope_to_info[dep.options_scope()]
for info in yield_deps(dep.subsystem_cls):
yield info
for info in yield_deps(scope_info.optionable_cls):
yield info
def create_options_for_optionables(optionables, extra_scopes=None, options=None):
"""Create a fake Options object for testing with appropriate defaults for the given optionables.
Any scoped `options` provided will override defaults, behaving as-if set on the command line.
:param iterable optionables: A series of `Optionable` types to register default options for.
:param iterable extra_scopes: An optional series of extra known scopes in play.
:param dict options: A dict of scope -> (dict of option name -> value) representing option values
explicitly set via the command line.
:returns: A fake `Options` object with defaults populated for the given `optionables` and any
explicitly set `options` overlayed.
"""
all_options = defaultdict(dict)
bootstrap_option_values = None
def complete_scopes(scopes):
"""Return all enclosing scopes.
This is similar to what `complete_scopes` does in `pants.option.options.Options` without
creating `ScopeInfo`s.
"""
completed_scopes = set(scopes)
for scope in scopes:
while scope != '':
if scope not in completed_scopes:
completed_scopes.add(scope)
scope = enclosing_scope(scope)
return completed_scopes
def register_func(on_scope):
scoped_options = all_options[on_scope]
register = _options_registration_function(scoped_options)
register.bootstrap = bootstrap_option_values
register.scope = on_scope
return register
# TODO: This sequence is a bit repetitive of the real registration sequence.
# Register bootstrap options and grab their default values for use in subsequent registration.
GlobalOptionsRegistrar.register_bootstrap_options(register_func(GLOBAL_SCOPE))
bootstrap_option_values = create_option_values(all_options[GLOBAL_SCOPE].copy())
# Now register the full global scope options.
GlobalOptionsRegistrar.register_options(register_func(GLOBAL_SCOPE))
for optionable in optionables:
optionable.register_options(register_func(optionable.options_scope))
# Make inner scopes inherit option values from their enclosing scopes.
all_scopes = set(all_options.keys())
# TODO(John Sirois): Kill extra scopes one this goes in:
# https://github.com/pantsbuild/pants/issues/1957
# For now we need a way for users of this utility to provide extra derived scopes out of band.
# With #1957 resolved, the extra scopes will be embedded in the Optionable's option_scope
# directly.
if extra_scopes:
all_scopes.update(extra_scopes)
all_scopes = complete_scopes(all_scopes)
# Iterating in sorted order guarantees that we see outer scopes before inner scopes,
# and therefore only have to inherit from our immediately enclosing scope.
for s in sorted(all_scopes):
if s != GLOBAL_SCOPE:
scope = enclosing_scope(s)
opts = all_options[s]
for key, val in all_options.get(scope, {}).items():
if key not in opts: # Inner scope values override the inherited ones.
opts[key] = val
if options:
for scope, opts in options.items():
all_options[scope].update(opts)
return create_options(all_options)
def _print_options_help(self):
"""Print a help screen.
Assumes that self._help_request is an instance of OptionsHelp.
Note: Ony useful if called after options have been registered.
"""
show_all_help = self._help_request.all_scopes
if show_all_help:
help_scopes = self._known_scope_to_info.keys()
else:
# The scopes explicitly mentioned by the user on the cmd line.
help_scopes = set(self._options.scope_to_flags.keys()) - set(
[GLOBAL_SCOPE])
# As a user-friendly heuristic, add all task scopes under requested scopes, so that e.g.,
# `./pants help compile` will show help for compile.java, compile.scala etc.
# Note that we don't do anything similar for subsystems - that would just create noise by
# repeating options for every task-specific subsystem instance.
for scope, info in self._known_scope_to_info.items():
if info.category == ScopeInfo.TASK:
outer = enclosing_scope(scope)
while outer != GLOBAL_SCOPE:
if outer in help_scopes:
help_scopes.add(scope)
break
outer = enclosing_scope(outer)
help_scope_infos = [
self._known_scope_to_info[s] for s in sorted(help_scopes)
]
if help_scope_infos:
for scope_info in self._help_subscopes_iter(help_scope_infos):
description = (scope_info.optionable_cls.get_description()
if scope_info.optionable_cls else None)
help_str = self._format_help(scope_info, description)
if help_str:
print(help_str)
return
else:
print(pants_release())
print('\nUsage:')
print(
' ./pants [option ...] [goal ...] [target...] Attempt the specified goals.'
)
print(' ./pants help Get help.')
print(
' ./pants help [goal] Get help for a goal.'
)
print(
' ./pants help-advanced [goal] Get help for a goal\'s advanced options.'
)
print(
' ./pants help-all Get help for all goals.'
)
print(
' ./pants goals List all installed goals.'
)
print('')
print(' [target] accepts two special forms:')
print(
' dir: to include all targets in the specified directory.')
print(
' dir:: to include all targets found recursively under the directory.'
)
print('\nFriendly docs:\n http://pantsbuild.github.io/')
print(
self._format_help(ScopeInfo(GLOBAL_SCOPE, ScopeInfo.GLOBAL),
''))
def create_options_for_optionables(optionables,
extra_scopes=None,
options=None):
"""Create a fake Options object for testing with appropriate defaults for the given optionables.
Any scoped `options` provided will override defaults, behaving as-if set on the command line.
:param iterable optionables: A series of `Optionable` types to register default options for.
:param iterable extra_scopes: An optional series of extra known scopes in play.
:param dict options: A dict of scope -> (dict of option name -> value) representing option values
explicitly set via the command line.
:returns: A fake `Options` object with defaults populated for the given `optionables` and any
explicitly set `options` overlayed.
"""
all_options = defaultdict(dict)
bootstrap_option_values = None
def complete_scopes(scopes):
"""Return all enclosing scopes.
This is similar to what `complete_scopes` does in `pants.option.options.Options` without
creating `ScopeInfo`s.
"""
completed_scopes = set(scopes)
for scope in scopes:
while scope != '':
if scope not in completed_scopes:
completed_scopes.add(scope)
scope = enclosing_scope(scope)
return completed_scopes
def register_func(on_scope):
scoped_options = all_options[on_scope]
register = _options_registration_function(scoped_options)
register.bootstrap = bootstrap_option_values
register.scope = on_scope
return register
# TODO: This sequence is a bit repetitive of the real registration sequence.
# Register bootstrap options and grab their default values for use in subsequent registration.
GlobalOptionsRegistrar.register_bootstrap_options(
register_func(GLOBAL_SCOPE))
bootstrap_option_values = create_option_values(
all_options[GLOBAL_SCOPE].copy())
# Now register the full global scope options.
GlobalOptionsRegistrar.register_options(register_func(GLOBAL_SCOPE))
for optionable in optionables:
optionable.register_options(register_func(optionable.options_scope))
# Make inner scopes inherit option values from their enclosing scopes.
all_scopes = set(all_options.keys())
# TODO(John Sirois): Kill extra scopes one this goes in:
# https://github.com/pantsbuild/pants/issues/1957
# For now we need a way for users of this utility to provide extra derived scopes out of band.
# With #1957 resolved, the extra scopes will be embedded in the Optionable's option_scope
# directly.
if extra_scopes:
all_scopes.update(extra_scopes)
all_scopes = complete_scopes(all_scopes)
# Iterating in sorted order guarantees that we see outer scopes before inner scopes,
# and therefore only have to inherit from our immediately enclosing scope.
for s in sorted(all_scopes):
if s != GLOBAL_SCOPE:
scope = enclosing_scope(s)
opts = all_options[s]
for key, val in all_options.get(scope, {}).items():
if key not in opts: # Inner scope values override the inherited ones.
opts[key] = val
if options:
for scope, opts in options.items():
all_options[scope].update(opts)
return create_options(all_options)