def test_complete_scopes(self):
_global = GlobalOptionsRegistrar.get_scope_info()
self.assertEquals(
{_global, intermediate("foo"), intermediate("foo.bar"), task("foo.bar.baz")},
Options.complete_scopes({task("foo.bar.baz")}),
)
self.assertEquals(
{_global, intermediate("foo"), intermediate("foo.bar"), task("foo.bar.baz")},
Options.complete_scopes({GlobalOptionsRegistrar.get_scope_info(), task("foo.bar.baz")}),
)
self.assertEquals(
{_global, intermediate("foo"), intermediate("foo.bar"), task("foo.bar.baz")},
Options.complete_scopes({intermediate("foo"), task("foo.bar.baz")}),
)
self.assertEquals(
{
_global,
intermediate("foo"),
intermediate("foo.bar"),
task("foo.bar.baz"),
intermediate("qux"),
task("qux.quux"),
},
Options.complete_scopes({task("foo.bar.baz"), task("qux.quux")}),
)
def create(cls, options_bootstrapper, build_configuration, init_subsystems=True):
global_bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope()
if global_bootstrap_options.pants_version != pants_version():
raise BuildConfigurationError(
'Version mismatch: Requested version was {}, our version is {}.'
.format(global_bootstrap_options.pants_version, pants_version())
)
pants_runtime_python_version = global_bootstrap_options.pants_runtime_python_version
current_python_version = '.'.join(map(str, sys.version_info[0:2]))
if pants_runtime_python_version and pants_runtime_python_version != current_python_version:
raise BuildConfigurationError(
'Running Pants with a different Python interpreter version than requested. '
'You requested {}, but are running with {}.\n\n'
'Note that Pants cannot use the value you give for `--pants-runtime-python-version` to '
'dynamically change the interpreter it uses, as it is too late for it to change once the program '
'is already running. Instead, your setup script (e.g. `./pants`) must configure which Python '
'interpreter and virtualenv to use. For example, the setup script we distribute '
'at https://www.pantsbuild.org/install.html#recommended-installation will read the '
'`pants_runtime_python_version` defined in your pants.ini to determine which Python '
'version to run with.'.format(pants_runtime_python_version, current_python_version)
)
# Parse and register options.
options = cls._construct_options(options_bootstrapper, build_configuration)
GlobalOptionsRegistrar.validate_instance(options.for_global_scope())
if init_subsystems:
Subsystem.set_options(options)
return options
def create(cls, options_bootstrapper, build_configuration, init_subsystems=True):
global_bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope()
if global_bootstrap_options.pants_version != pants_version():
raise BuildConfigurationError(
'Version mismatch: Requested version was {}, our version is {}.'
.format(global_bootstrap_options.pants_version, pants_version())
)
pants_runtime_python_version = global_bootstrap_options.pants_runtime_python_version
current_python_version = '.'.join(map(str, sys.version_info[0:2]))
if pants_runtime_python_version and pants_runtime_python_version != current_python_version:
raise BuildConfigurationError(
'Running Pants with a different Python interpreter version than requested. '
'You requested {}, but are running with {}.\n\n'
'Note that Pants cannot use the value you give for `--pants-runtime-python-version` to '
'dynamically change the interpreter it uses, as it is too late for it to change once the program '
'is already running. Instead, your setup script (e.g. `./pants`) must configure which Python '
'interpreter and virtualenv to use. For example, the setup script we distribute '
'at https://www.pantsbuild.org/install.html#recommended-installation will read the '
'`pants_runtime_python_version` defined in your pants.ini to determine which Python '
'version to run with.'.format(pants_runtime_python_version, current_python_version)
)
# Parse and register options.
options = cls._construct_options(options_bootstrapper, build_configuration)
GlobalOptionsRegistrar.validate_instance(options.for_global_scope())
if init_subsystems:
Subsystem.set_options(options)
return options
def bootstrap_options_from_config(config):
bootstrap_options = Options.create(env=self._env, config=config,
known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()], args=bargs)
def register_global(*args, **kwargs):
bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs)
GlobalOptionsRegistrar.register_bootstrap_options(register_global)
return bootstrap_options
def create_options_for_optionables(optionables, extra_scopes=None, options=None):
"""Create a fake Options object for testing with appropriate defaults for the given optionables.
Any scoped `options` provided will override defaults, behaving as-if set on the command line.
:param iterable optionables: A series of `Optionable` types to register default options for.
:param iterable extra_scopes: An optional series of extra known scopes in play.
:param dict options: A dict of scope -> (dict of option name -> value) representing option values
explicitly set via the command line.
:returns: A fake `Options` object with defaults populated for the given `optionables` and any
explicitly set `options` overlayed.
"""
all_options = defaultdict(dict)
bootstrap_option_values = None
def register_func(on_scope):
scoped_options = all_options[on_scope]
register = _options_registration_function(scoped_options)
register.bootstrap = bootstrap_option_values
register.scope = on_scope
return register
# TODO: This sequence is a bit repetitive of the real registration sequence.
# Register bootstrap options and grab their default values for use in subsequent registration.
GlobalOptionsRegistrar.register_bootstrap_options(register_func(GLOBAL_SCOPE))
bootstrap_option_values = create_option_values(all_options[GLOBAL_SCOPE].copy())
# Now register the full global scope options.
GlobalOptionsRegistrar.register_options(register_func(GLOBAL_SCOPE))
for optionable in optionables:
optionable.register_options(register_func(optionable.options_scope))
# Make inner scopes inherit option values from their enclosing scopes.
all_scopes = set(all_options.keys())
# TODO(John Sirois): Kill extra scopes one this goes in:
# https://github.com/pantsbuild/pants/issues/1957
# For now we need a way for users of this utility to provide extra derived scopes out of band.
# With #1957 resolved, the extra scopes will be embedded in the Optionable's option_scope
# directly.
if extra_scopes:
all_scopes.update(extra_scopes)
# Iterating in sorted order guarantees that we see outer scopes before inner scopes,
# and therefore only have to inherit from our immediately enclosing scope.
for s in sorted(all_scopes):
if s != GLOBAL_SCOPE:
enclosing_scope = s.rpartition(".")[0]
opts = all_options[s]
for key, val in all_options.get(enclosing_scope, {}).items():
if key not in opts: # Inner scope values override the inherited ones.
opts[key] = val
if options:
for scope, opts in options.items():
all_options[scope].update(opts)
return create_options(all_options)
def create_options_for_optionables(optionables,
options=None,
options_fingerprintable=None,
passthru_args=None):
"""Create a fake Options object for testing with appropriate defaults for the given optionables.
Any scoped `options` provided will override defaults, behaving as-if set on the command line.
:param iterable optionables: A series of `Optionable` types to register default options for.
:param dict options: A dict of scope -> (dict of option name -> value) representing option values
explicitly set via the command line.
:param dict options_fingerprintable: A dict of scope -> (dict of option name -> option type)
representing the fingerprintable options
and the scopes they are registered for.
:param list passthru_args: A list of passthrough args (specified after `--` on the command line).
:returns: A fake `Options` object with defaults populated for the given `optionables` and any
explicitly set `options` overlayed.
"""
all_options = defaultdict(dict)
fingerprintable_options = defaultdict(dict)
bootstrap_option_values = None
if options_fingerprintable:
for scope, opts in options_fingerprintable.items():
fingerprintable_options[scope].update(opts)
def register_func(on_scope):
scoped_options = all_options[on_scope]
scoped_fingerprintables = fingerprintable_options[on_scope]
register = _options_registration_function(scoped_options,
scoped_fingerprintables)
register.bootstrap = bootstrap_option_values
register.scope = on_scope
return register
# TODO: This sequence is a bit repetitive of the real registration sequence.
# Register bootstrap options and grab their default values for use in subsequent registration.
GlobalOptionsRegistrar.register_bootstrap_options(
register_func(GLOBAL_SCOPE))
bootstrap_option_values = _FakeOptionValues(
all_options[GLOBAL_SCOPE].copy())
# Now register the full global scope options.
GlobalOptionsRegistrar.register_options(register_func(GLOBAL_SCOPE))
for optionable in optionables:
optionable.register_options(register_func(optionable.options_scope))
if options:
for scope, opts in options.items():
all_options[scope].update(opts)
return create_options(all_options,
passthru_args=passthru_args,
fingerprintable_options=fingerprintable_options)
def test_complete_scopes(self):
_global = GlobalOptionsRegistrar.get_scope_info()
self.assertEquals({_global, intermediate('foo'), intermediate('foo.bar'), task('foo.bar.baz')},
Options.complete_scopes({task('foo.bar.baz')}))
self.assertEquals({_global, intermediate('foo'), intermediate('foo.bar'), task('foo.bar.baz')},
Options.complete_scopes({GlobalOptionsRegistrar.get_scope_info(),
task('foo.bar.baz')}))
self.assertEquals({_global, intermediate('foo'), intermediate('foo.bar'), task('foo.bar.baz')},
Options.complete_scopes({intermediate('foo'), task('foo.bar.baz')}))
self.assertEquals({_global, intermediate('foo'), intermediate('foo.bar'), task('foo.bar.baz'),
intermediate('qux'), task('qux.quux')},
Options.complete_scopes({task('foo.bar.baz'), task('qux.quux')}))
def create_options_for_optionables(optionables,
options=None,
options_fingerprintable=None,
passthru_args=None):
"""Create a fake Options object for testing with appropriate defaults for the given optionables.
Any scoped `options` provided will override defaults, behaving as-if set on the command line.
:param iterable optionables: A series of `Optionable` types to register default options for.
:param dict options: A dict of scope -> (dict of option name -> value) representing option values
explicitly set via the command line.
:param dict options_fingerprintable: A dict of scope -> (dict of option name -> option type)
representing the fingerprintable options
and the scopes they are registered for.
:param list passthru_args: A list of passthrough args (specified after `--` on the command line).
:returns: A fake `Options` object with defaults populated for the given `optionables` and any
explicitly set `options` overlayed.
"""
all_options = defaultdict(dict)
fingerprintable_options = defaultdict(dict)
bootstrap_option_values = None
if options_fingerprintable:
for scope, opts in options_fingerprintable.items():
fingerprintable_options[scope].update(opts)
def register_func(on_scope):
scoped_options = all_options[on_scope]
scoped_fingerprintables = fingerprintable_options[on_scope]
register = _options_registration_function(scoped_options, scoped_fingerprintables)
register.bootstrap = bootstrap_option_values
register.scope = on_scope
return register
# TODO: This sequence is a bit repetitive of the real registration sequence.
# Register bootstrap options and grab their default values for use in subsequent registration.
GlobalOptionsRegistrar.register_bootstrap_options(register_func(GLOBAL_SCOPE))
bootstrap_option_values = _FakeOptionValues(all_options[GLOBAL_SCOPE].copy())
# Now register the full global scope options.
GlobalOptionsRegistrar.register_options(register_func(GLOBAL_SCOPE))
for optionable in optionables:
optionable.register_options(register_func(optionable.options_scope))
if options:
for scope, opts in options.items():
all_options[scope].update(opts)
return create_options(all_options,
passthru_args=passthru_args,
fingerprintable_options=fingerprintable_options)
def test_complete_scopes(self):
_global = GlobalOptionsRegistrar.get_scope_info()
self.assertEquals({_global, intermediate('foo'), intermediate('foo.bar'), task('foo.bar.baz')},
Options.complete_scopes({task('foo.bar.baz')}))
self.assertEquals({_global, intermediate('foo'), intermediate('foo.bar'), task('foo.bar.baz')},
Options.complete_scopes({GlobalOptionsRegistrar.get_scope_info(),
task('foo.bar.baz')}))
self.assertEquals({_global, intermediate('foo'), intermediate('foo.bar'), task('foo.bar.baz')},
Options.complete_scopes({intermediate('foo'), task('foo.bar.baz')}))
self.assertEquals({_global, intermediate('foo'), intermediate('foo.bar'), task('foo.bar.baz'),
intermediate('qux'), task('qux.quux')},
Options.complete_scopes({task('foo.bar.baz'), task('qux.quux')}))
def register_options(self, subsystems):
# Standalone global options.
GlobalOptionsRegistrar.register_options_on_scope(self.options)
# Options for subsystems.
for subsystem in subsystems:
subsystem.register_options_on_scope(self.options)
# TODO(benjy): Should Goals be subsystems? Or should the entire goal-running mechanism
# be a subsystem?
for goal in Goal.all():
# Register task options.
goal.register_options(self.options)
def _register_options(self, subsystems, options):
"""Registers global options."""
# Standalone global options.
GlobalOptionsRegistrar.register_options_on_scope(options)
# Options for subsystems.
for subsystem in subsystems:
subsystem.register_options_on_scope(options)
# TODO(benjy): Should Goals or the entire goal-running mechanism be a Subsystem?
for goal in Goal.all():
# Register task options.
goal.register_options(options)
def _register_options(self, subsystems, options):
"""Registers global options."""
# Standalone global options.
GlobalOptionsRegistrar.register_options_on_scope(options)
# Options for subsystems.
for subsystem in subsystems:
subsystem.register_options_on_scope(options)
# TODO(benjy): Should Goals or the entire goal-running mechanism be a Subsystem?
for goal in Goal.all():
# Register task options.
goal.register_options(options)
def parse_bootstrap_options(env, args, config):
bootstrap_options = Options.create(
env=env,
config=config,
known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()],
args=args,
)
def register_global(*args, **kwargs):
## Only use of Options.register?
bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs)
GlobalOptionsRegistrar.register_bootstrap_options(register_global)
return bootstrap_options
def parse_bootstrap_options(env, args, config):
bootstrap_options = Options.create(
env=env,
config=config,
known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()],
args=args,
)
def register_global(*args, **kwargs):
## Only use of Options.register?
bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs)
GlobalOptionsRegistrar.register_bootstrap_options(register_global)
return bootstrap_options
def complete_scopes(cls, scope_infos):
"""Expand a set of scopes to include all enclosing scopes.
E.g., if the set contains `foo.bar.baz`, ensure that it also contains `foo.bar` and `foo`.
Also adds any deprecated scopes.
"""
ret = {GlobalOptionsRegistrar.get_scope_info()}
original_scopes = set()
for si in scope_infos:
ret.add(si)
original_scopes.add(si.scope)
if si.deprecated_scope:
ret.add(
ScopeInfo(si.deprecated_scope, si.category,
si.optionable_cls))
original_scopes.add(si.deprecated_scope)
# TODO: Once scope name validation is enforced (so there can be no dots in scope name
# components) we can replace this line with `for si in scope_infos:`, because it will
# not be possible for a deprecated_scope to introduce any new intermediate scopes.
for si in copy.copy(ret):
scope = si.scope
while scope != '':
if scope not in original_scopes:
ret.add(ScopeInfo(scope, ScopeInfo.INTERMEDIATE))
scope = enclosing_scope(scope)
return ret
def complete_scopes(cls, scope_infos):
"""Expand a set of scopes to include all enclosing scopes.
E.g., if the set contains `foo.bar.baz`, ensure that it also contains `foo.bar` and `foo`.
Also adds any deprecated scopes.
"""
ret = {GlobalOptionsRegistrar.get_scope_info()}
original_scopes = set()
for si in scope_infos:
ret.add(si)
original_scopes.add(si.scope)
if si.deprecated_scope:
ret.add(ScopeInfo(si.deprecated_scope, si.category, si.optionable_cls))
original_scopes.add(si.deprecated_scope)
# TODO: Once scope name validation is enforced (so there can be no dots in scope name
# components) we can replace this line with `for si in scope_infos:`, because it will
# not be possible for a deprecated_scope to introduce any new intermediate scopes.
for si in copy.copy(ret):
scope = si.scope
while scope != '':
if scope not in original_scopes:
ret.add(ScopeInfo(scope, ScopeInfo.INTERMEDIATE))
scope = enclosing_scope(scope)
return ret
def complete_scopes(cls, scope_infos):
"""Expand a set of scopes to include all enclosing scopes.
E.g., if the set contains `foo.bar.baz`, ensure that it also contains `foo.bar` and `foo`.
Also adds any deprecated scopes.
"""
ret = {GlobalOptionsRegistrar.get_scope_info()}
original_scopes = dict()
for si in scope_infos:
ret.add(si)
if si.scope in original_scopes:
raise cls.DuplicateScopeError('Scope `{}` claimed by {}, was also claimed by {}.'.format(
si.scope, si, original_scopes[si.scope]
))
original_scopes[si.scope] = si
if si.deprecated_scope:
ret.add(ScopeInfo(si.deprecated_scope, si.category, si.optionable_cls))
original_scopes[si.deprecated_scope] = si
# TODO: Once scope name validation is enforced (so there can be no dots in scope name
# components) we can replace this line with `for si in scope_infos:`, because it will
# not be possible for a deprecated_scope to introduce any new intermediate scopes.
for si in copy.copy(ret):
for scope in all_enclosing_scopes(si.scope, allow_global=False):
if scope not in original_scopes:
ret.add(ScopeInfo(scope, ScopeInfo.INTERMEDIATE))
return ret
def _install_options(self, options_bootstrapper, build_configuration):
"""Parse and register options.
:returns: An Options object representing the full set of runtime options.
"""
# TODO: This inline import is currently necessary to resolve a ~legitimate cycle between
# `GoalRunner`->`EngineInitializer`->`OptionsInitializer`->`GoalRunner`.
from pants.bin.goal_runner import GoalRunner
# Now that plugins and backends are loaded, we can gather the known scopes.
known_scope_infos = [GlobalOptionsRegistrar.get_scope_info()]
# Add scopes for all needed subsystems via a union of all known subsystem sets.
subsystems = Subsystem.closure(
GoalRunner.subsystems() | Goal.subsystems() | build_configuration.subsystems()
)
for subsystem in subsystems:
known_scope_infos.append(subsystem.get_scope_info())
# Add scopes for all tasks in all goals.
for goal in Goal.all():
known_scope_infos.extend(filter(None, goal.known_scope_infos()))
# Now that we have the known scopes we can get the full options.
options = options_bootstrapper.get_full_options(known_scope_infos)
self._register_options(subsystems, options)
# Make the options values available to all subsystems.
Subsystem.set_options(options)
return options
def create(cls, options_bootstrapper, build_configuration, init_subsystems=True):
global_bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope()
if global_bootstrap_options.pants_version != pants_version():
raise BuildConfigurationError(
'Version mismatch: Requested version was {}, our version is {}.'
.format(global_bootstrap_options.pants_version, pants_version())
)
# Parse and register options.
options = cls._construct_options(options_bootstrapper, build_configuration)
GlobalOptionsRegistrar.validate_instance(options.for_global_scope())
if init_subsystems:
Subsystem.set_options(options)
return options
def _options(self):
# NB: The PluginResolver runs very early in the pants startup sequence before the standard
# Subsystem facility is wired up. As a result PluginResolver is not itself a Subsystem with
# (PythonRepos, PythonSetup) returned as `dependencies()`. Instead it does the minimum possible
# work to hand-roll bootstrapping of the Subsystems it needs.
subsystems = Subsystem.closure([PythonRepos, PythonSetup])
known_scope_infos = [subsystem.get_scope_info() for subsystem in subsystems]
options = self._options_bootstrapper.get_full_options(known_scope_infos)
# Ignore command line flags since we'd blow up on any we don't understand (most of them).
# If someone wants to bootstrap plugins in a one-off custom way they'll need to use env vars
# or a --config-override pointing to a custom pants.ini snippet.
defaulted_only_options = options.drop_flag_values()
GlobalOptionsRegistrar.register_options_on_scope(defaulted_only_options)
for subsystem in subsystems:
subsystem.register_options_on_scope(defaulted_only_options)
return defaulted_only_options
def _options(self):
# NB: The PluginResolver runs very early in the pants startup sequence before the standard
# Subsystem facility is wired up. As a result PluginResolver is not itself a Subsystem with
# PythonRepos as a dependency. Instead it does the minimum possible work to hand-roll
# bootstrapping of the Subsystem it needs.
subsystems = Subsystem.closure([PythonRepos])
known_scope_infos = [subsystem.get_scope_info() for subsystem in subsystems]
options = self._options_bootstrapper.get_full_options(known_scope_infos)
# Ignore command line flags since we'd blow up on any we don't understand (most of them).
# If someone wants to bootstrap plugins in a one-off custom way they'll need to use env vars
# or a --pants-config-files pointing to a custom pants.ini snippet.
defaulted_only_options = options.drop_flag_values()
GlobalOptionsRegistrar.register_options_on_scope(defaulted_only_options)
for subsystem in subsystems:
subsystem.register_options_on_scope(defaulted_only_options)
return defaulted_only_options
def do_test(args, kwargs, expected_default):
# Defaults are computed in the parser and added into the kwargs, so we
# must jump through this hoop in this test.
parser = Parser(env={}, config=Config.load([]),
scope_info=GlobalOptionsRegistrar.get_scope_info(),
parent_parser=None, option_tracker=OptionTracker())
parser.register(*args, **kwargs)
oshi = HelpInfoExtracter.get_option_scope_help_info_from_parser(parser).basic
self.assertEquals(1, len(oshi))
ohi = oshi[0]
self.assertEqual(expected_default, ohi.default)
def do_test(args, kwargs, expected_default):
# Defaults are computed in the parser and added into the kwargs, so we
# must jump through this hoop in this test.
parser = Parser(env={}, config=Config.load([]),
scope_info=GlobalOptionsRegistrar.get_scope_info(),
parent_parser=None, option_tracker=OptionTracker())
parser.register(*args, **kwargs)
oshi = HelpInfoExtracter.get_option_scope_help_info_from_parser(parser).basic
self.assertEquals(1, len(oshi))
ohi = oshi[0]
self.assertEqual(expected_default, ohi.default)
def _setup_options(self, options_bootstrapper, working_set):
# TODO: This inline import is currently necessary to resolve a ~legitimate cycle between
# `GoalRunner`->`EngineInitializer`->`OptionsInitializer`->`GoalRunner`.
from pants.bin.goal_runner import GoalRunner
bootstrap_options = options_bootstrapper.get_bootstrap_options()
global_bootstrap_options = bootstrap_options.for_global_scope()
if global_bootstrap_options.pants_version != pants_version():
raise BuildConfigurationError(
'Version mismatch: Requested version was {}, our version is {}.'.format(
global_bootstrap_options.pants_version, pants_version()
)
)
# Get logging setup prior to loading backends so that they can log as needed.
if self._init_logging:
self._setup_logging(global_bootstrap_options)
# Add any extra paths to python path (e.g., for loading extra source backends).
for path in global_bootstrap_options.pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
plugins = global_bootstrap_options.plugins
backend_packages = global_bootstrap_options.backend_packages
build_configuration = load_plugins_and_backends(plugins, working_set, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
known_scope_infos = [GlobalOptionsRegistrar.get_scope_info()]
# Add scopes for all needed subsystems via a union of all known subsystem sets.
subsystems = Subsystem.closure(
GoalRunner.subsystems() | Goal.subsystems() | build_configuration.subsystems()
)
for subsystem in subsystems:
known_scope_infos.append(subsystem.get_scope_info())
# Add scopes for all tasks in all goals.
for goal in Goal.all():
known_scope_infos.extend(filter(None, goal.known_scope_infos()))
# Now that we have the known scopes we can get the full options.
options = options_bootstrapper.get_full_options(known_scope_infos)
self._register_options(subsystems, options)
# Make the options values available to all subsystems.
Subsystem.set_options(options)
return options, build_configuration
def _setup_options(self, options_bootstrapper, working_set):
# TODO: This inline import is currently necessary to resolve a ~legitimate cycle between
# `GoalRunner`->`EngineInitializer`->`OptionsInitializer`->`GoalRunner`.
from pants.bin.goal_runner import GoalRunner
bootstrap_options = options_bootstrapper.get_bootstrap_options()
global_bootstrap_options = bootstrap_options.for_global_scope()
if global_bootstrap_options.pants_version != pants_version():
raise BuildConfigurationError(
'Version mismatch: Requested version was {}, our version is {}.'
.format(global_bootstrap_options.pants_version,
pants_version()))
# Get logging setup prior to loading backends so that they can log as needed.
if self._init_logging:
self._setup_logging(global_bootstrap_options)
# Add any extra paths to python path (e.g., for loading extra source backends).
for path in global_bootstrap_options.pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
plugins = global_bootstrap_options.plugins
backend_packages = global_bootstrap_options.backend_packages
build_configuration = load_backends_and_plugins(
plugins, working_set, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
known_scope_infos = [GlobalOptionsRegistrar.get_scope_info()]
# Add scopes for all needed subsystems via a union of all known subsystem sets.
subsystems = Subsystem.closure(GoalRunner.subsystems()
| Goal.subsystems()
| build_configuration.subsystems())
for subsystem in subsystems:
known_scope_infos.append(subsystem.get_scope_info())
# Add scopes for all tasks in all goals.
for goal in Goal.all():
known_scope_infos.extend(filter(None, goal.known_scope_infos()))
# Now that we have the known scopes we can get the full options.
options = options_bootstrapper.get_full_options(known_scope_infos)
self._register_options(subsystems, options)
# Make the options values available to all subsystems.
Subsystem.set_options(options)
return options, build_configuration
def _options(self):
# NB: The PluginResolver runs very early in the pants startup sequence before the standard
# Subsystem facility is wired up. As a result PluginResolver is not itself a Subsystem with
# PythonRepos as a dependency. Instead it does the minimum possible work to hand-roll
# bootstrapping of the Subsystems it needs.
known_scope_infos = PythonRepos.known_scope_infos()
options = self._options_bootstrapper.get_full_options(
known_scope_infos)
# Ignore command line flags since we'd blow up on any we don't understand (most of them).
# If someone wants to bootstrap plugins in a one-off custom way they'll need to use env vars
# or a --pants-config-files pointing to a custom pants.ini snippet.
defaulted_only_options = options.drop_flag_values()
GlobalOptionsRegistrar.register_options_on_scope(
defaulted_only_options)
distinct_optionable_classes = sorted(
{si.optionable_cls
for si in known_scope_infos},
key=lambda o: o.options_scope)
for optionable_cls in distinct_optionable_classes:
optionable_cls.register_options_on_scope(defaulted_only_options)
return defaulted_only_options
def complete_scopes(cls, scope_infos):
"""Expand a set of scopes to include all enclosing scopes.
E.g., if the set contains `foo.bar.baz`, ensure that it also contains `foo.bar` and `foo`.
"""
ret = {GlobalOptionsRegistrar.get_scope_info()}
for scope_info in scope_infos:
ret.add(scope_info)
original_scopes = {si.scope for si in scope_infos}
for scope_info in scope_infos:
scope = scope_info.scope
while scope != '':
if scope not in original_scopes:
ret.add(ScopeInfo(scope, ScopeInfo.INTERMEDIATE))
scope = enclosing_scope(scope)
return ret
def _setup_options(self, options_bootstrapper, working_set):
bootstrap_options = options_bootstrapper.get_bootstrap_options()
global_bootstrap_options = bootstrap_options.for_global_scope()
# The pants_version may be set in pants.ini for bootstrapping, so we make sure the user actually
# requested the version on the command line before deciding to print the version and exit.
if global_bootstrap_options.is_flagged('pants_version'):
print(global_bootstrap_options.pants_version)
self._exiter(0)
# Get logging setup prior to loading backends so that they can log as needed.
self._setup_logging(global_bootstrap_options)
# Add any extra paths to python path (e.g., for loading extra source backends).
for path in global_bootstrap_options.pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
plugins = global_bootstrap_options.plugins
backend_packages = global_bootstrap_options.backend_packages
build_configuration = load_plugins_and_backends(
plugins, working_set, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
known_scope_infos = [GlobalOptionsRegistrar.get_scope_info()]
# Add scopes for all needed subsystems via a union of all known subsystem sets.
subsystems = Subsystem.closure(GoalRunner.subsystems()
| Goal.subsystems()
| build_configuration.subsystems())
for subsystem in subsystems:
known_scope_infos.append(subsystem.get_scope_info())
# Add scopes for all tasks in all goals.
for goal in Goal.all():
known_scope_infos.extend(filter(None, goal.known_scope_infos()))
# Now that we have the known scopes we can get the full options.
options = options_bootstrapper.get_full_options(known_scope_infos)
self._register_options(subsystems, options)
# Make the options values available to all subsystems.
Subsystem.set_options(options)
return options, build_configuration
def complete_scopes(cls, scope_infos):
"""Expand a set of scopes to include all enclosing scopes.
E.g., if the set contains `foo.bar.baz`, ensure that it also contains `foo.bar` and `foo`.
"""
ret = {GlobalOptionsRegistrar.get_scope_info()}
for scope_info in scope_infos:
ret.add(scope_info)
original_scopes = {si.scope for si in scope_infos}
for scope_info in scope_infos:
scope = scope_info.scope
while scope != "":
if scope not in original_scopes:
ret.add(ScopeInfo(scope, ScopeInfo.INTERMEDIATE))
scope = enclosing_scope(scope)
return ret
def _setup_options(self, options_bootstrapper, working_set):
bootstrap_options = options_bootstrapper.get_bootstrap_options()
global_bootstrap_options = bootstrap_options.for_global_scope()
# The pants_version may be set in pants.ini for bootstrapping, so we make sure the user actually
# requested the version on the command line before deciding to print the version and exit.
if global_bootstrap_options.is_flagged('pants_version'):
print(global_bootstrap_options.pants_version)
self._exiter(0)
# Get logging setup prior to loading backends so that they can log as needed.
self._setup_logging(global_bootstrap_options)
# Add any extra paths to python path (e.g., for loading extra source backends).
for path in global_bootstrap_options.pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
plugins = global_bootstrap_options.plugins
backend_packages = global_bootstrap_options.backend_packages
build_configuration = load_plugins_and_backends(plugins, working_set, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
known_scope_infos = [GlobalOptionsRegistrar.get_scope_info()]
# Add scopes for all needed subsystems via a union of all known subsystem sets.
subsystems = Subsystem.closure(
GoalRunner.subsystems() | Goal.subsystems() | build_configuration.subsystems()
)
for subsystem in subsystems:
known_scope_infos.append(subsystem.get_scope_info())
# Add scopes for all tasks in all goals.
for goal in Goal.all():
known_scope_infos.extend(filter(None, goal.known_scope_infos()))
# Now that we have the known scopes we can get the full options.
options = options_bootstrapper.get_full_options(known_scope_infos)
self._register_options(subsystems, options)
# Make the options values available to all subsystems.
Subsystem.set_options(options)
return options, build_configuration
def create(cls, env=None, args=None):
"""Parses the minimum amount of configuration necessary to create an OptionsBootstrapper.
:param env: An environment dictionary, or None to use `os.environ`.
:param args: An args array, or None to use `sys.argv`.
"""
env = {
k: v
for k, v in (os.environ if env is None else env).items()
if k.startswith('PANTS_')
}
args = tuple(sys.argv if args is None else args)
flags = set()
short_flags = set()
# TODO: This codepath probably shouldn't be using FileContent, which is a very v2 engine thing.
def filecontent_for(path):
is_executable = os.stat(
path).st_mode & stat.S_IXUSR == stat.S_IXUSR
return FileContent(
ensure_text(path),
read_file(path, binary_mode=True),
is_executable=is_executable,
)
def capture_the_flags(*args, **kwargs):
for arg in args:
flags.add(arg)
if len(arg) == 2:
short_flags.add(arg)
elif kwargs.get('type') == bool:
flags.add(f'--no-{arg[2:]}')
GlobalOptionsRegistrar.register_bootstrap_options(capture_the_flags)
def is_bootstrap_option(arg):
components = arg.split('=', 1)
if components[0] in flags:
return True
for flag in short_flags:
if arg.startswith(flag):
return True
return False
# Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
# Stop before '--' since args after that are pass-through and may have duplicate names to our
# bootstrap options.
bargs = tuple(
filter(is_bootstrap_option,
itertools.takewhile(lambda arg: arg != '--', args)))
config_file_paths = cls.get_config_file_paths(env=env, args=args)
config_files_products = [filecontent_for(p) for p in config_file_paths]
pre_bootstrap_config = Config.load_file_contents(config_files_products)
initial_bootstrap_options = cls.parse_bootstrap_options(
env, bargs, pre_bootstrap_config)
bootstrap_option_values = initial_bootstrap_options.for_global_scope()
# Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped
# from (typically pants.ini), then config override, then rcfiles.
full_configpaths = pre_bootstrap_config.sources()
if bootstrap_option_values.pantsrc:
rcfiles = [
os.path.expanduser(str(rcfile))
for rcfile in bootstrap_option_values.pantsrc_files
]
existing_rcfiles = list(filter(os.path.exists, rcfiles))
full_configpaths.extend(existing_rcfiles)
full_config_files_products = [
filecontent_for(p) for p in full_configpaths
]
post_bootstrap_config = Config.load_file_contents(
full_config_files_products, seed_values=bootstrap_option_values)
env_tuples = tuple(sorted(env.items(), key=lambda x: x[0]))
return cls(env_tuples=env_tuples,
bootstrap_args=bargs,
args=args,
config=post_bootstrap_config)
def get_bootstrap_options(self):
""":returns: an Options instance that only knows about the bootstrap options.
:rtype: :class:`Options`
"""
if not self._bootstrap_options:
flags = set()
short_flags = set()
def capture_the_flags(*args, **kwargs):
for arg in args:
flags.add(arg)
if len(arg) == 2:
short_flags.add(arg)
elif is_boolean_flag(kwargs):
flags.add('--no-{}'.format(arg[2:]))
GlobalOptionsRegistrar.register_bootstrap_options(
capture_the_flags)
def is_bootstrap_option(arg):
components = arg.split('=', 1)
if components[0] in flags:
return True
for flag in short_flags:
if arg.startswith(flag):
return True
return False
# Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
# Stop before '--' since args after that are pass-through and may have duplicate names to our
# bootstrap options.
bargs = filter(
is_bootstrap_option,
itertools.takewhile(lambda arg: arg != '--', self._args))
configpaths = [self._configpath] if self._configpath else None
pre_bootstrap_config = Config.load(configpaths)
def bootstrap_options_from_config(config):
bootstrap_options = Options.create(
env=self._env,
config=config,
known_scope_infos=[
GlobalOptionsRegistrar.get_scope_info()
],
args=bargs,
option_tracker=self._option_tracker)
def register_global(*args, **kwargs):
bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs)
GlobalOptionsRegistrar.register_bootstrap_options(
register_global)
return bootstrap_options
initial_bootstrap_options = bootstrap_options_from_config(
pre_bootstrap_config)
bootstrap_option_values = initial_bootstrap_options.for_global_scope(
)
# Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped
# from (typically pants.ini), then config override, then rcfiles.
full_configpaths = pre_bootstrap_config.sources()
if bootstrap_option_values.config_override:
full_configpaths.extend(
bootstrap_option_values.config_override)
if bootstrap_option_values.pantsrc:
rcfiles = [
os.path.expanduser(rcfile)
for rcfile in bootstrap_option_values.pantsrc_files
]
existing_rcfiles = filter(os.path.exists, rcfiles)
full_configpaths.extend(existing_rcfiles)
self._post_bootstrap_config = Config.load(
full_configpaths, seed_values=bootstrap_option_values)
# Now recompute the bootstrap options with the full config. This allows us to pick up
# bootstrap values (such as backends) from a config override file, for example.
self._bootstrap_options = bootstrap_options_from_config(
self._post_bootstrap_config)
return self._bootstrap_options
def context(self,
for_task_types=None,
options=None,
target_roots=None,
console_outstream=None,
workspace=None):
for_task_types = for_task_types or []
options = options or {}
option_values = defaultdict(dict)
registered_subsystems = set()
bootstrap_option_values = None # We fill these in after registering bootstrap options.
# We provide our own test-only registration implementation, bypassing argparse.
# When testing we set option values directly, so we don't care about cmd-line flags, config,
# env vars etc. In fact, for test isolation we explicitly don't want to look at those.
# All this does is make the names available in code, with the default values.
# Individual tests can then override the option values they care about.
def register_func(on_scope):
def register(*rargs, **rkwargs):
scoped_options = option_values[on_scope]
default = rkwargs.get('default')
if default is None and rkwargs.get('action') == 'append':
default = []
for flag_name in rargs:
option_name = flag_name.lstrip('-').replace('-', '_')
scoped_options[option_name] = default
register.bootstrap = bootstrap_option_values
register.scope = on_scope
return register
# TODO: This sequence is a bit repetitive of the real registration sequence.
# Register bootstrap options and grab their default values for use in subsequent registration.
GlobalOptionsRegistrar.register_bootstrap_options(
register_func(Options.GLOBAL_SCOPE))
bootstrap_option_values = create_option_values(
copy.copy(option_values[Options.GLOBAL_SCOPE]))
# Now register the full global scope options.
GlobalOptionsRegistrar.register_options(
register_func(Options.GLOBAL_SCOPE))
# Now register task and subsystem options for relevant tasks.
for task_type in for_task_types:
scope = task_type.options_scope
if scope is None:
raise TaskError(
'You must set a scope on your task type before using it in tests.'
)
task_type.register_options(register_func(scope))
for subsystem in (set(task_type.global_subsystems())
| set(task_type.task_subsystems())
| self._build_configuration.subsystems()):
if subsystem not in registered_subsystems:
subsystem.register_options(
register_func(subsystem.options_scope))
registered_subsystems.add(subsystem)
# Now default option values override with any caller-specified values.
# TODO(benjy): Get rid of the options arg, and require tests to call set_options.
for scope, opts in options.items():
for key, val in opts.items():
option_values[scope][key] = val
for scope, opts in self.options.items():
for key, val in opts.items():
option_values[scope][key] = val
# Make inner scopes inherit option values from their enclosing scopes.
all_scopes = set(option_values.keys())
for task_type in for_task_types: # Make sure we know about pre-task subsystem scopes.
all_scopes.update(
[si.scope for si in task_type.known_scope_infos()])
# Iterating in sorted order guarantees that we see outer scopes before inner scopes,
# and therefore only have to inherit from our immediately enclosing scope.
for scope in sorted(all_scopes):
if scope != Options.GLOBAL_SCOPE:
enclosing_scope = scope.rpartition('.')[0]
opts = option_values[scope]
for key, val in option_values.get(enclosing_scope, {}).items():
if key not in opts: # Inner scope values override the inherited ones.
opts[key] = val
context = create_context(options=option_values,
target_roots=target_roots,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
address_mapper=self.address_mapper,
console_outstream=console_outstream,
workspace=workspace)
Subsystem._options = context.options
return context
def produce_and_set_bootstrap_options(self):
"""Cooperatively populates the internal bootstrap_options cache with
a producer of `FileContent`."""
flags = set()
short_flags = set()
def capture_the_flags(*args, **kwargs):
for arg in args:
flags.add(arg)
if len(arg) == 2:
short_flags.add(arg)
elif kwargs.get('type') == bool:
flags.add('--no-{}'.format(arg[2:]))
GlobalOptionsRegistrar.register_bootstrap_options(capture_the_flags)
def is_bootstrap_option(arg):
components = arg.split('=', 1)
if components[0] in flags:
return True
for flag in short_flags:
if arg.startswith(flag):
return True
return False
# Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
# Stop before '--' since args after that are pass-through and may have duplicate names to our
# bootstrap options.
bargs = list(filter(is_bootstrap_option, itertools.takewhile(lambda arg: arg != '--', self._args)))
config_file_paths = self.get_config_file_paths(env=self._env, args=self._args)
config_files_products = yield config_file_paths
pre_bootstrap_config = Config.load_file_contents(config_files_products)
def bootstrap_options_from_config(config):
bootstrap_options = Options.create(
env=self._env,
config=config,
known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()],
args=bargs,
option_tracker=self._option_tracker
)
def register_global(*args, **kwargs):
## Only use of Options.register?
bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs)
GlobalOptionsRegistrar.register_bootstrap_options(register_global)
return bootstrap_options
initial_bootstrap_options = bootstrap_options_from_config(pre_bootstrap_config)
bootstrap_option_values = initial_bootstrap_options.for_global_scope()
# Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped
# from (typically pants.ini), then config override, then rcfiles.
full_configpaths = pre_bootstrap_config.sources()
if bootstrap_option_values.pantsrc:
rcfiles = [os.path.expanduser(rcfile) for rcfile in bootstrap_option_values.pantsrc_files]
existing_rcfiles = list(filter(os.path.exists, rcfiles))
full_configpaths.extend(existing_rcfiles)
full_config_files_products = yield full_configpaths
self._post_bootstrap_config = Config.load_file_contents(
full_config_files_products,
seed_values=bootstrap_option_values
)
# Now recompute the bootstrap options with the full config. This allows us to pick up
# bootstrap values (such as backends) from a config override file, for example.
self._bootstrap_options = bootstrap_options_from_config(self._post_bootstrap_config)
def produce_and_set_bootstrap_options(self):
"""Cooperatively populates the internal bootstrap_options cache with
a producer of `FileContent`."""
flags = set()
short_flags = set()
def capture_the_flags(*args, **kwargs):
for arg in args:
flags.add(arg)
if len(arg) == 2:
short_flags.add(arg)
elif kwargs.get('type') == bool:
flags.add('--no-{}'.format(arg[2:]))
GlobalOptionsRegistrar.register_bootstrap_options(capture_the_flags)
def is_bootstrap_option(arg):
components = arg.split('=', 1)
if components[0] in flags:
return True
for flag in short_flags:
if arg.startswith(flag):
return True
return False
# Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
# Stop before '--' since args after that are pass-through and may have duplicate names to our
# bootstrap options.
bargs = list(
filter(is_bootstrap_option,
itertools.takewhile(lambda arg: arg != '--', self._args)))
config_file_paths = self.get_config_file_paths(env=self._env,
args=self._args)
config_files_products = yield config_file_paths
pre_bootstrap_config = Config.load_file_contents(config_files_products)
def bootstrap_options_from_config(config):
bootstrap_options = Options.create(
env=self._env,
config=config,
known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()],
args=bargs,
option_tracker=self._option_tracker)
def register_global(*args, **kwargs):
## Only use of Options.register?
bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs)
GlobalOptionsRegistrar.register_bootstrap_options(register_global)
return bootstrap_options
initial_bootstrap_options = bootstrap_options_from_config(
pre_bootstrap_config)
bootstrap_option_values = initial_bootstrap_options.for_global_scope()
# Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped
# from (typically pants.ini), then config override, then rcfiles.
full_configpaths = pre_bootstrap_config.sources()
if bootstrap_option_values.pantsrc:
rcfiles = [
os.path.expanduser(rcfile)
for rcfile in bootstrap_option_values.pantsrc_files
]
existing_rcfiles = list(filter(os.path.exists, rcfiles))
full_configpaths.extend(existing_rcfiles)
full_config_files_products = yield full_configpaths
self._post_bootstrap_config = Config.load_file_contents(
full_config_files_products, seed_values=bootstrap_option_values)
# Now recompute the bootstrap options with the full config. This allows us to pick up
# bootstrap values (such as backends) from a config override file, for example.
self._bootstrap_options = bootstrap_options_from_config(
self._post_bootstrap_config)
def create_options_for_optionables(optionables,
extra_scopes=None,
options=None,
options_fingerprintable=None,
passthru_args=None):
"""Create a fake Options object for testing with appropriate defaults for the given optionables.
Any scoped `options` provided will override defaults, behaving as-if set on the command line.
:param iterable optionables: A series of `Optionable` types to register default options for.
:param iterable extra_scopes: An optional series of extra known scopes in play.
:param dict options: A dict of scope -> (dict of option name -> value) representing option values
explicitly set via the command line.
:param dict options_fingerprintable: A dict of scope -> (dict of option name -> option type)
representing the fingerprintable options
and the scopes they are registered for.
:param list passthru_args: A list of passthrough args (specified after `--` on the command line).
:returns: A fake `Options` object with defaults populated for the given `optionables` and any
explicitly set `options` overlayed.
"""
all_options = defaultdict(dict)
fingerprintable_options = defaultdict(dict)
bootstrap_option_values = None
# NB(cosmicexplorer): we do this again for all_options after calling
# register_func below, this is a hack
if options:
for scope, opts in options.items():
all_options[scope].update(opts)
if options_fingerprintable:
for scope, opts in options_fingerprintable.items():
fingerprintable_options[scope].update(opts)
def complete_scopes(scopes):
"""Return all enclosing scopes.
This is similar to what `complete_scopes` does in `pants.option.options.Options` without
creating `ScopeInfo`s.
"""
completed_scopes = set(scopes)
for scope in scopes:
while scope != GLOBAL_SCOPE:
if scope not in completed_scopes:
completed_scopes.add(scope)
scope = enclosing_scope(scope)
return completed_scopes
def register_func(on_scope):
scoped_options = all_options[on_scope]
scoped_fingerprintables = fingerprintable_options[on_scope]
register = _options_registration_function(scoped_options, scoped_fingerprintables)
register.bootstrap = bootstrap_option_values
register.scope = on_scope
return register
# TODO: This sequence is a bit repetitive of the real registration sequence.
# Register bootstrap options and grab their default values for use in subsequent registration.
GlobalOptionsRegistrar.register_bootstrap_options(register_func(GLOBAL_SCOPE))
bootstrap_option_values = _FakeOptionValues(all_options[GLOBAL_SCOPE].copy())
# Now register the full global scope options.
GlobalOptionsRegistrar.register_options(register_func(GLOBAL_SCOPE))
for optionable in optionables:
optionable.register_options(register_func(optionable.options_scope))
# Make inner scopes inherit option values from their enclosing scopes.
all_scopes = set(all_options.keys())
# TODO(John Sirois): Kill extra scopes one this goes in:
# https://github.com/pantsbuild/pants/issues/1957
# For now we need a way for users of this utility to provide extra derived scopes out of band.
# With #1957 resolved, the extra scopes will be embedded in the Optionable's option_scope
# directly.
if extra_scopes:
all_scopes.update(extra_scopes)
all_scopes = complete_scopes(all_scopes)
# We need to update options before completing them based on inner/outer relation.
if options:
for scope, opts in options.items():
all_options[scope].update(opts)
# Iterating in sorted order guarantees that we see outer scopes before inner scopes,
# and therefore only have to inherit from our immediately enclosing scope.
for s in sorted(all_scopes):
if s != GLOBAL_SCOPE:
scope = enclosing_scope(s)
opts = all_options[s]
for key, val in all_options.get(scope, {}).items():
if key not in opts: # Inner scope values override the inherited ones.
opts[key] = val
return create_options(all_options,
passthru_args=passthru_args,
fingerprintable_options=fingerprintable_options)
def setup(self, options_bootstrapper, working_set):
bootstrap_options = options_bootstrapper.get_bootstrap_options()
global_bootstrap_options = bootstrap_options.for_global_scope()
# The pants_version may be set in pants.ini for bootstrapping, so we make sure the user actually
# requested the version on the command line before deciding to print the version and exit.
if global_bootstrap_options.is_flagged('pants_version'):
print(global_bootstrap_options.pants_version)
self._exiter(0)
# Get logging setup prior to loading backends so that they can log as needed.
self._setup_logging(global_bootstrap_options)
# Add any extra paths to python path (e.g., for loading extra source backends).
for path in global_bootstrap_options.pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
plugins = global_bootstrap_options.plugins
backend_packages = global_bootstrap_options.backend_packages
build_configuration = load_plugins_and_backends(plugins, working_set, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
self.targets = []
known_scope_infos = [GlobalOptionsRegistrar.get_scope_info()]
# Add scopes for all needed subsystems.
subsystems = Subsystem.closure(set(self.subsystems) |
Goal.subsystems() |
build_configuration.subsystems())
for subsystem in subsystems:
known_scope_infos.append(subsystem.get_scope_info())
# Add scopes for all tasks in all goals.
for goal in Goal.all():
known_scope_infos.extend(filter(None, goal.known_scope_infos()))
# Now that we have the known scopes we can get the full options.
self.options = options_bootstrapper.get_full_options(known_scope_infos)
self.register_options(subsystems)
# Make the options values available to all subsystems.
Subsystem._options = self.options
# Now that we have options we can instantiate subsystems.
self.run_tracker = RunTracker.global_instance()
self.reporting = Reporting.global_instance()
report = self.reporting.initial_reporting(self.run_tracker)
self.run_tracker.start(report)
url = self.run_tracker.run_info.get_info('report_url')
if url:
self.run_tracker.log(Report.INFO, 'See a report at: {}'.format(url))
else:
self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')
self.build_file_parser = BuildFileParser(build_configuration=build_configuration,
root_dir=self.root_dir,
run_tracker=self.run_tracker)
rev = self.options.for_global_scope().build_file_rev
if rev:
ScmBuildFile.set_rev(rev)
ScmBuildFile.set_scm(get_scm())
build_file_type = ScmBuildFile
else:
build_file_type = FilesystemBuildFile
self.address_mapper = BuildFileAddressMapper(self.build_file_parser, build_file_type)
self.build_graph = BuildGraph(run_tracker=self.run_tracker,
address_mapper=self.address_mapper)
# TODO(John Sirois): Kill when source root registration is lifted out of BUILD files.
with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnitLabel.SETUP]):
source_root_bootstrapper = SourceRootBootstrapper.global_instance()
source_root_bootstrapper.bootstrap(self.address_mapper, self.build_file_parser)
self._expand_goals_and_specs()
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
def create_options_for_optionables(optionables,
extra_scopes=None,
options=None):
"""Create a fake Options object for testing with appropriate defaults for the given optionables.
Any scoped `options` provided will override defaults, behaving as-if set on the command line.
:param iterable optionables: A series of `Optionable` types to register default options for.
:param iterable extra_scopes: An optional series of extra known scopes in play.
:param dict options: A dict of scope -> (dict of option name -> value) representing option values
explicitly set via the command line.
:returns: A fake `Options` object with defaults populated for the given `optionables` and any
explicitly set `options` overlayed.
"""
all_options = defaultdict(dict)
bootstrap_option_values = None
def complete_scopes(scopes):
"""Return all enclosing scopes.
This is similar to what `complete_scopes` does in `pants.option.options.Options` without
creating `ScopeInfo`s.
"""
completed_scopes = set(scopes)
for scope in scopes:
while scope != '':
if scope not in completed_scopes:
completed_scopes.add(scope)
scope = enclosing_scope(scope)
return completed_scopes
def register_func(on_scope):
scoped_options = all_options[on_scope]
register = _options_registration_function(scoped_options)
register.bootstrap = bootstrap_option_values
register.scope = on_scope
return register
# TODO: This sequence is a bit repetitive of the real registration sequence.
# Register bootstrap options and grab their default values for use in subsequent registration.
GlobalOptionsRegistrar.register_bootstrap_options(
register_func(GLOBAL_SCOPE))
bootstrap_option_values = create_option_values(
all_options[GLOBAL_SCOPE].copy())
# Now register the full global scope options.
GlobalOptionsRegistrar.register_options(register_func(GLOBAL_SCOPE))
for optionable in optionables:
optionable.register_options(register_func(optionable.options_scope))
# Make inner scopes inherit option values from their enclosing scopes.
all_scopes = set(all_options.keys())
# TODO(John Sirois): Kill extra scopes one this goes in:
# https://github.com/pantsbuild/pants/issues/1957
# For now we need a way for users of this utility to provide extra derived scopes out of band.
# With #1957 resolved, the extra scopes will be embedded in the Optionable's option_scope
# directly.
if extra_scopes:
all_scopes.update(extra_scopes)
all_scopes = complete_scopes(all_scopes)
# Iterating in sorted order guarantees that we see outer scopes before inner scopes,
# and therefore only have to inherit from our immediately enclosing scope.
for s in sorted(all_scopes):
if s != GLOBAL_SCOPE:
scope = enclosing_scope(s)
opts = all_options[s]
for key, val in all_options.get(scope, {}).items():
if key not in opts: # Inner scope values override the inherited ones.
opts[key] = val
if options:
for scope, opts in options.items():
all_options[scope].update(opts)
return create_options(all_options)
def context(self, for_task_types=None, options=None, target_roots=None,
console_outstream=None, workspace=None):
for_task_types = for_task_types or []
options = options or {}
option_values = defaultdict(dict)
registered_subsystems = set()
bootstrap_option_values = None # We fill these in after registering bootstrap options.
# We provide our own test-only registration implementation, bypassing argparse.
# When testing we set option values directly, so we don't care about cmd-line flags, config,
# env vars etc. In fact, for test isolation we explicitly don't want to look at those.
# All this does is make the names available in code, with the default values.
# Individual tests can then override the option values they care about.
def register_func(on_scope):
def register(*rargs, **rkwargs):
scoped_options = option_values[on_scope]
default = rkwargs.get('default')
if default is None and rkwargs.get('action') == 'append':
default = []
for flag_name in rargs:
option_name = flag_name.lstrip('-').replace('-', '_')
scoped_options[option_name] = default
register.bootstrap = bootstrap_option_values
register.scope = on_scope
return register
# TODO: This sequence is a bit repetitive of the real registration sequence.
# Register bootstrap options and grab their default values for use in subsequent registration.
GlobalOptionsRegistrar.register_bootstrap_options(register_func(Options.GLOBAL_SCOPE))
bootstrap_option_values = create_option_values(copy.copy(option_values[Options.GLOBAL_SCOPE]))
# Now register the full global scope options.
GlobalOptionsRegistrar.register_options(register_func(Options.GLOBAL_SCOPE))
# Now register task and subsystem options for relevant tasks.
for task_type in for_task_types:
scope = task_type.options_scope
if scope is None:
raise TaskError('You must set a scope on your task type before using it in tests.')
task_type.register_options(register_func(scope))
for subsystem in (set(task_type.global_subsystems()) |
set(task_type.task_subsystems()) |
self._build_configuration.subsystems()):
if subsystem not in registered_subsystems:
subsystem.register_options(register_func(subsystem.options_scope))
registered_subsystems.add(subsystem)
# Now default option values override with any caller-specified values.
# TODO(benjy): Get rid of the options arg, and require tests to call set_options.
for scope, opts in options.items():
for key, val in opts.items():
option_values[scope][key] = val
for scope, opts in self.options.items():
for key, val in opts.items():
option_values[scope][key] = val
# Make inner scopes inherit option values from their enclosing scopes.
all_scopes = set(option_values.keys())
for task_type in for_task_types: # Make sure we know about pre-task subsystem scopes.
all_scopes.update([si.scope for si in task_type.known_scope_infos()])
# Iterating in sorted order guarantees that we see outer scopes before inner scopes,
# and therefore only have to inherit from our immediately enclosing scope.
for scope in sorted(all_scopes):
if scope != Options.GLOBAL_SCOPE:
enclosing_scope = scope.rpartition('.')[0]
opts = option_values[scope]
for key, val in option_values.get(enclosing_scope, {}).items():
if key not in opts: # Inner scope values override the inherited ones.
opts[key] = val
context = create_context(options=option_values,
target_roots=target_roots,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
address_mapper=self.address_mapper,
console_outstream=console_outstream,
workspace=workspace)
Subsystem._options = context.options
return context
def create(cls, env=None, args=None):
"""Parses the minimum amount of configuration necessary to create an OptionsBootstrapper.
:param env: An environment dictionary, or None to use `os.environ`.
:param args: An args array, or None to use `sys.argv`.
"""
env = {k: v for k, v in (os.environ if env is None else env).items()
if k.startswith('PANTS_')}
args = tuple(sys.argv if args is None else args)
flags = set()
short_flags = set()
def filecontent_for(path):
return FileContent(ensure_text(path), read_file(path, binary_mode=True))
def capture_the_flags(*args, **kwargs):
for arg in args:
flags.add(arg)
if len(arg) == 2:
short_flags.add(arg)
elif kwargs.get('type') == bool:
flags.add('--no-{}'.format(arg[2:]))
GlobalOptionsRegistrar.register_bootstrap_options(capture_the_flags)
def is_bootstrap_option(arg):
components = arg.split('=', 1)
if components[0] in flags:
return True
for flag in short_flags:
if arg.startswith(flag):
return True
return False
# Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
# Stop before '--' since args after that are pass-through and may have duplicate names to our
# bootstrap options.
bargs = tuple(filter(is_bootstrap_option, itertools.takewhile(lambda arg: arg != '--', args)))
config_file_paths = cls.get_config_file_paths(env=env, args=args)
config_files_products = [filecontent_for(p) for p in config_file_paths]
pre_bootstrap_config = Config.load_file_contents(config_files_products)
initial_bootstrap_options = cls.parse_bootstrap_options(env, bargs, pre_bootstrap_config)
bootstrap_option_values = initial_bootstrap_options.for_global_scope()
# Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped
# from (typically pants.ini), then config override, then rcfiles.
full_configpaths = pre_bootstrap_config.sources()
if bootstrap_option_values.pantsrc:
rcfiles = [os.path.expanduser(str(rcfile)) for rcfile in bootstrap_option_values.pantsrc_files]
existing_rcfiles = list(filter(os.path.exists, rcfiles))
full_configpaths.extend(existing_rcfiles)
full_config_files_products = [filecontent_for(p) for p in full_configpaths]
post_bootstrap_config = Config.load_file_contents(
full_config_files_products,
seed_values=bootstrap_option_values
)
env_tuples = tuple(sorted(iteritems(env), key=lambda x: x[0]))
return cls(env_tuples=env_tuples, bootstrap_args=bargs, args=args, config=post_bootstrap_config)
def setup(self, options_bootstrapper, working_set):
bootstrap_options = options_bootstrapper.get_bootstrap_options()
global_bootstrap_options = bootstrap_options.for_global_scope()
# The pants_version may be set in pants.ini for bootstrapping, so we make sure the user actually
# requested the version on the command line before deciding to print the version and exit.
if global_bootstrap_options.is_flagged('pants_version'):
print(global_bootstrap_options.pants_version)
self._exiter(0)
# Get logging setup prior to loading backends so that they can log as needed.
self._setup_logging(global_bootstrap_options)
# Add any extra paths to python path (eg for loading extra source backends)
for path in global_bootstrap_options.pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
plugins = global_bootstrap_options.plugins
backend_packages = global_bootstrap_options.backend_packages
build_configuration = load_plugins_and_backends(
plugins, working_set, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
self.targets = []
known_scope_infos = [GlobalOptionsRegistrar.get_scope_info()]
# Add scopes for all needed subsystems.
subsystems = Subsystem.closure(
set(self.subsystems) | Goal.subsystems()
| build_configuration.subsystems())
for subsystem in subsystems:
known_scope_infos.append(subsystem.get_scope_info())
# Add scopes for all tasks in all goals.
for goal in Goal.all():
known_scope_infos.extend(filter(None, goal.known_scope_infos()))
# Now that we have the known scopes we can get the full options.
self.options = options_bootstrapper.get_full_options(known_scope_infos)
self.register_options(subsystems)
# Make the options values available to all subsystems.
Subsystem._options = self.options
# Now that we have options we can instantiate subsystems.
self.run_tracker = RunTracker.global_instance()
self.reporting = Reporting.global_instance()
report = self.reporting.initial_reporting(self.run_tracker)
self.run_tracker.start(report)
url = self.run_tracker.run_info.get_info('report_url')
if url:
self.run_tracker.log(Report.INFO,
'See a report at: {}'.format(url))
else:
self.run_tracker.log(
Report.INFO, '(To run a reporting server: ./pants server)')
self.build_file_parser = BuildFileParser(
build_configuration=build_configuration,
root_dir=self.root_dir,
run_tracker=self.run_tracker)
rev = self.options.for_global_scope().build_file_rev
if rev:
ScmBuildFile.set_rev(rev)
ScmBuildFile.set_scm(get_scm())
build_file_type = ScmBuildFile
else:
build_file_type = FilesystemBuildFile
self.address_mapper = BuildFileAddressMapper(self.build_file_parser,
build_file_type)
self.build_graph = BuildGraph(run_tracker=self.run_tracker,
address_mapper=self.address_mapper)
# TODO(John Sirois): Kill when source root registration is lifted out of BUILD files.
with self.run_tracker.new_workunit(name='bootstrap',
labels=[WorkUnitLabel.SETUP]):
source_root_bootstrapper = SourceRootBootstrapper.global_instance()
source_root_bootstrapper.bootstrap(self.address_mapper,
self.build_file_parser)
self._expand_goals_and_specs()
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
def get_bootstrap_options(self):
""":returns: an Options instance that only knows about the bootstrap options.
:rtype: :class:`Options`
"""
if not self._bootstrap_options:
flags = set()
short_flags = set()
def capture_the_flags(*args, **kwargs):
for arg in args:
flags.add(arg)
if len(arg) == 2:
short_flags.add(arg)
elif is_boolean_flag(kwargs):
flags.add('--no-{}'.format(arg[2:]))
GlobalOptionsRegistrar.register_bootstrap_options(capture_the_flags)
def is_bootstrap_option(arg):
components = arg.split('=', 1)
if components[0] in flags:
return True
for flag in short_flags:
if arg.startswith(flag):
return True
return False
# Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
# Stop before '--' since args after that are pass-through and may have duplicate names to our
# bootstrap options.
bargs = filter(is_bootstrap_option, itertools.takewhile(lambda arg: arg != '--', self._args))
configpaths = [self._configpath] if self._configpath else None
pre_bootstrap_config = Config.load(configpaths)
def bootstrap_options_from_config(config):
bootstrap_options = Options.create(env=self._env, config=config,
known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()], args=bargs)
def register_global(*args, **kwargs):
bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs)
GlobalOptionsRegistrar.register_bootstrap_options(register_global)
return bootstrap_options
initial_bootstrap_options = bootstrap_options_from_config(pre_bootstrap_config)
bootstrap_option_values = initial_bootstrap_options.for_global_scope()
# Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped
# from (typically pants.ini), then config override, then rcfiles.
full_configpaths = pre_bootstrap_config.sources()
if bootstrap_option_values.config_override:
full_configpaths.append(bootstrap_option_values.config_override)
if bootstrap_option_values.pantsrc:
rcfiles = [os.path.expanduser(rcfile) for rcfile in bootstrap_option_values.pantsrc_files]
existing_rcfiles = filter(os.path.exists, rcfiles)
full_configpaths.extend(existing_rcfiles)
self._post_bootstrap_config = Config.load(full_configpaths,
seed_values=bootstrap_option_values)
# Now recompute the bootstrap options with the full config. This allows us to pick up
# bootstrap values (such as backends) from a config override file, for example.
self._bootstrap_options = bootstrap_options_from_config(self._post_bootstrap_config)
return self._bootstrap_options
