def test_complete_scopes(self): _global = GlobalOptionsRegistrar.get_scope_info() self.assertEquals( {_global, intermediate("foo"), intermediate("foo.bar"), task("foo.bar.baz")}, Options.complete_scopes({task("foo.bar.baz")}), ) self.assertEquals( {_global, intermediate("foo"), intermediate("foo.bar"), task("foo.bar.baz")}, Options.complete_scopes({GlobalOptionsRegistrar.get_scope_info(), task("foo.bar.baz")}), ) self.assertEquals( {_global, intermediate("foo"), intermediate("foo.bar"), task("foo.bar.baz")}, Options.complete_scopes({intermediate("foo"), task("foo.bar.baz")}), ) self.assertEquals( { _global, intermediate("foo"), intermediate("foo.bar"), task("foo.bar.baz"), intermediate("qux"), task("qux.quux"), }, Options.complete_scopes({task("foo.bar.baz"), task("qux.quux")}), )
def create(cls, options_bootstrapper, build_configuration, init_subsystems=True): global_bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope() if global_bootstrap_options.pants_version != pants_version(): raise BuildConfigurationError( 'Version mismatch: Requested version was {}, our version is {}.' .format(global_bootstrap_options.pants_version, pants_version()) ) pants_runtime_python_version = global_bootstrap_options.pants_runtime_python_version current_python_version = '.'.join(map(str, sys.version_info[0:2])) if pants_runtime_python_version and pants_runtime_python_version != current_python_version: raise BuildConfigurationError( 'Running Pants with a different Python interpreter version than requested. ' 'You requested {}, but are running with {}.\n\n' 'Note that Pants cannot use the value you give for `--pants-runtime-python-version` to ' 'dynamically change the interpreter it uses, as it is too late for it to change once the program ' 'is already running. Instead, your setup script (e.g. `./pants`) must configure which Python ' 'interpreter and virtualenv to use. For example, the setup script we distribute ' 'at https://www.pantsbuild.org/install.html#recommended-installation will read the ' '`pants_runtime_python_version` defined in your pants.ini to determine which Python ' 'version to run with.'.format(pants_runtime_python_version, current_python_version) ) # Parse and register options. options = cls._construct_options(options_bootstrapper, build_configuration) GlobalOptionsRegistrar.validate_instance(options.for_global_scope()) if init_subsystems: Subsystem.set_options(options) return options
def bootstrap_options_from_config(config): bootstrap_options = Options.create(env=self._env, config=config, known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()], args=bargs) def register_global(*args, **kwargs): bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs) GlobalOptionsRegistrar.register_bootstrap_options(register_global) return bootstrap_options
def create_options_for_optionables(optionables, extra_scopes=None, options=None): """Create a fake Options object for testing with appropriate defaults for the given optionables. Any scoped `options` provided will override defaults, behaving as-if set on the command line. :param iterable optionables: A series of `Optionable` types to register default options for. :param iterable extra_scopes: An optional series of extra known scopes in play. :param dict options: A dict of scope -> (dict of option name -> value) representing option values explicitly set via the command line. :returns: A fake `Options` object with defaults populated for the given `optionables` and any explicitly set `options` overlayed. """ all_options = defaultdict(dict) bootstrap_option_values = None def register_func(on_scope): scoped_options = all_options[on_scope] register = _options_registration_function(scoped_options) register.bootstrap = bootstrap_option_values register.scope = on_scope return register # TODO: This sequence is a bit repetitive of the real registration sequence. # Register bootstrap options and grab their default values for use in subsequent registration. GlobalOptionsRegistrar.register_bootstrap_options(register_func(GLOBAL_SCOPE)) bootstrap_option_values = create_option_values(all_options[GLOBAL_SCOPE].copy()) # Now register the full global scope options. GlobalOptionsRegistrar.register_options(register_func(GLOBAL_SCOPE)) for optionable in optionables: optionable.register_options(register_func(optionable.options_scope)) # Make inner scopes inherit option values from their enclosing scopes. all_scopes = set(all_options.keys()) # TODO(John Sirois): Kill extra scopes one this goes in: # https://github.com/pantsbuild/pants/issues/1957 # For now we need a way for users of this utility to provide extra derived scopes out of band. # With #1957 resolved, the extra scopes will be embedded in the Optionable's option_scope # directly. if extra_scopes: all_scopes.update(extra_scopes) # Iterating in sorted order guarantees that we see outer scopes before inner scopes, # and therefore only have to inherit from our immediately enclosing scope. for s in sorted(all_scopes): if s != GLOBAL_SCOPE: enclosing_scope = s.rpartition(".")[0] opts = all_options[s] for key, val in all_options.get(enclosing_scope, {}).items(): if key not in opts: # Inner scope values override the inherited ones. opts[key] = val if options: for scope, opts in options.items(): all_options[scope].update(opts) return create_options(all_options)
def create_options_for_optionables(optionables, options=None, options_fingerprintable=None, passthru_args=None): """Create a fake Options object for testing with appropriate defaults for the given optionables. Any scoped `options` provided will override defaults, behaving as-if set on the command line. :param iterable optionables: A series of `Optionable` types to register default options for. :param dict options: A dict of scope -> (dict of option name -> value) representing option values explicitly set via the command line. :param dict options_fingerprintable: A dict of scope -> (dict of option name -> option type) representing the fingerprintable options and the scopes they are registered for. :param list passthru_args: A list of passthrough args (specified after `--` on the command line). :returns: A fake `Options` object with defaults populated for the given `optionables` and any explicitly set `options` overlayed. """ all_options = defaultdict(dict) fingerprintable_options = defaultdict(dict) bootstrap_option_values = None if options_fingerprintable: for scope, opts in options_fingerprintable.items(): fingerprintable_options[scope].update(opts) def register_func(on_scope): scoped_options = all_options[on_scope] scoped_fingerprintables = fingerprintable_options[on_scope] register = _options_registration_function(scoped_options, scoped_fingerprintables) register.bootstrap = bootstrap_option_values register.scope = on_scope return register # TODO: This sequence is a bit repetitive of the real registration sequence. # Register bootstrap options and grab their default values for use in subsequent registration. GlobalOptionsRegistrar.register_bootstrap_options( register_func(GLOBAL_SCOPE)) bootstrap_option_values = _FakeOptionValues( all_options[GLOBAL_SCOPE].copy()) # Now register the full global scope options. GlobalOptionsRegistrar.register_options(register_func(GLOBAL_SCOPE)) for optionable in optionables: optionable.register_options(register_func(optionable.options_scope)) if options: for scope, opts in options.items(): all_options[scope].update(opts) return create_options(all_options, passthru_args=passthru_args, fingerprintable_options=fingerprintable_options)
def test_complete_scopes(self): _global = GlobalOptionsRegistrar.get_scope_info() self.assertEquals({_global, intermediate('foo'), intermediate('foo.bar'), task('foo.bar.baz')}, Options.complete_scopes({task('foo.bar.baz')})) self.assertEquals({_global, intermediate('foo'), intermediate('foo.bar'), task('foo.bar.baz')}, Options.complete_scopes({GlobalOptionsRegistrar.get_scope_info(), task('foo.bar.baz')})) self.assertEquals({_global, intermediate('foo'), intermediate('foo.bar'), task('foo.bar.baz')}, Options.complete_scopes({intermediate('foo'), task('foo.bar.baz')})) self.assertEquals({_global, intermediate('foo'), intermediate('foo.bar'), task('foo.bar.baz'), intermediate('qux'), task('qux.quux')}, Options.complete_scopes({task('foo.bar.baz'), task('qux.quux')}))
def create_options_for_optionables(optionables, options=None, options_fingerprintable=None, passthru_args=None): """Create a fake Options object for testing with appropriate defaults for the given optionables. Any scoped `options` provided will override defaults, behaving as-if set on the command line. :param iterable optionables: A series of `Optionable` types to register default options for. :param dict options: A dict of scope -> (dict of option name -> value) representing option values explicitly set via the command line. :param dict options_fingerprintable: A dict of scope -> (dict of option name -> option type) representing the fingerprintable options and the scopes they are registered for. :param list passthru_args: A list of passthrough args (specified after `--` on the command line). :returns: A fake `Options` object with defaults populated for the given `optionables` and any explicitly set `options` overlayed. """ all_options = defaultdict(dict) fingerprintable_options = defaultdict(dict) bootstrap_option_values = None if options_fingerprintable: for scope, opts in options_fingerprintable.items(): fingerprintable_options[scope].update(opts) def register_func(on_scope): scoped_options = all_options[on_scope] scoped_fingerprintables = fingerprintable_options[on_scope] register = _options_registration_function(scoped_options, scoped_fingerprintables) register.bootstrap = bootstrap_option_values register.scope = on_scope return register # TODO: This sequence is a bit repetitive of the real registration sequence. # Register bootstrap options and grab their default values for use in subsequent registration. GlobalOptionsRegistrar.register_bootstrap_options(register_func(GLOBAL_SCOPE)) bootstrap_option_values = _FakeOptionValues(all_options[GLOBAL_SCOPE].copy()) # Now register the full global scope options. GlobalOptionsRegistrar.register_options(register_func(GLOBAL_SCOPE)) for optionable in optionables: optionable.register_options(register_func(optionable.options_scope)) if options: for scope, opts in options.items(): all_options[scope].update(opts) return create_options(all_options, passthru_args=passthru_args, fingerprintable_options=fingerprintable_options)
def register_options(self, subsystems): # Standalone global options. GlobalOptionsRegistrar.register_options_on_scope(self.options) # Options for subsystems. for subsystem in subsystems: subsystem.register_options_on_scope(self.options) # TODO(benjy): Should Goals be subsystems? Or should the entire goal-running mechanism # be a subsystem? for goal in Goal.all(): # Register task options. goal.register_options(self.options)
def _register_options(self, subsystems, options): """Registers global options.""" # Standalone global options. GlobalOptionsRegistrar.register_options_on_scope(options) # Options for subsystems. for subsystem in subsystems: subsystem.register_options_on_scope(options) # TODO(benjy): Should Goals or the entire goal-running mechanism be a Subsystem? for goal in Goal.all(): # Register task options. goal.register_options(options)
def parse_bootstrap_options(env, args, config): bootstrap_options = Options.create( env=env, config=config, known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()], args=args, ) def register_global(*args, **kwargs): ## Only use of Options.register? bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs) GlobalOptionsRegistrar.register_bootstrap_options(register_global) return bootstrap_options
def complete_scopes(cls, scope_infos): """Expand a set of scopes to include all enclosing scopes. E.g., if the set contains `foo.bar.baz`, ensure that it also contains `foo.bar` and `foo`. Also adds any deprecated scopes. """ ret = {GlobalOptionsRegistrar.get_scope_info()} original_scopes = set() for si in scope_infos: ret.add(si) original_scopes.add(si.scope) if si.deprecated_scope: ret.add( ScopeInfo(si.deprecated_scope, si.category, si.optionable_cls)) original_scopes.add(si.deprecated_scope) # TODO: Once scope name validation is enforced (so there can be no dots in scope name # components) we can replace this line with `for si in scope_infos:`, because it will # not be possible for a deprecated_scope to introduce any new intermediate scopes. for si in copy.copy(ret): scope = si.scope while scope != '': if scope not in original_scopes: ret.add(ScopeInfo(scope, ScopeInfo.INTERMEDIATE)) scope = enclosing_scope(scope) return ret
def complete_scopes(cls, scope_infos): """Expand a set of scopes to include all enclosing scopes. E.g., if the set contains `foo.bar.baz`, ensure that it also contains `foo.bar` and `foo`. Also adds any deprecated scopes. """ ret = {GlobalOptionsRegistrar.get_scope_info()} original_scopes = set() for si in scope_infos: ret.add(si) original_scopes.add(si.scope) if si.deprecated_scope: ret.add(ScopeInfo(si.deprecated_scope, si.category, si.optionable_cls)) original_scopes.add(si.deprecated_scope) # TODO: Once scope name validation is enforced (so there can be no dots in scope name # components) we can replace this line with `for si in scope_infos:`, because it will # not be possible for a deprecated_scope to introduce any new intermediate scopes. for si in copy.copy(ret): scope = si.scope while scope != '': if scope not in original_scopes: ret.add(ScopeInfo(scope, ScopeInfo.INTERMEDIATE)) scope = enclosing_scope(scope) return ret
def complete_scopes(cls, scope_infos): """Expand a set of scopes to include all enclosing scopes. E.g., if the set contains `foo.bar.baz`, ensure that it also contains `foo.bar` and `foo`. Also adds any deprecated scopes. """ ret = {GlobalOptionsRegistrar.get_scope_info()} original_scopes = dict() for si in scope_infos: ret.add(si) if si.scope in original_scopes: raise cls.DuplicateScopeError('Scope `{}` claimed by {}, was also claimed by {}.'.format( si.scope, si, original_scopes[si.scope] )) original_scopes[si.scope] = si if si.deprecated_scope: ret.add(ScopeInfo(si.deprecated_scope, si.category, si.optionable_cls)) original_scopes[si.deprecated_scope] = si # TODO: Once scope name validation is enforced (so there can be no dots in scope name # components) we can replace this line with `for si in scope_infos:`, because it will # not be possible for a deprecated_scope to introduce any new intermediate scopes. for si in copy.copy(ret): for scope in all_enclosing_scopes(si.scope, allow_global=False): if scope not in original_scopes: ret.add(ScopeInfo(scope, ScopeInfo.INTERMEDIATE)) return ret
def _install_options(self, options_bootstrapper, build_configuration): """Parse and register options. :returns: An Options object representing the full set of runtime options. """ # TODO: This inline import is currently necessary to resolve a ~legitimate cycle between # `GoalRunner`->`EngineInitializer`->`OptionsInitializer`->`GoalRunner`. from pants.bin.goal_runner import GoalRunner # Now that plugins and backends are loaded, we can gather the known scopes. known_scope_infos = [GlobalOptionsRegistrar.get_scope_info()] # Add scopes for all needed subsystems via a union of all known subsystem sets. subsystems = Subsystem.closure( GoalRunner.subsystems() | Goal.subsystems() | build_configuration.subsystems() ) for subsystem in subsystems: known_scope_infos.append(subsystem.get_scope_info()) # Add scopes for all tasks in all goals. for goal in Goal.all(): known_scope_infos.extend(filter(None, goal.known_scope_infos())) # Now that we have the known scopes we can get the full options. options = options_bootstrapper.get_full_options(known_scope_infos) self._register_options(subsystems, options) # Make the options values available to all subsystems. Subsystem.set_options(options) return options
def create(cls, options_bootstrapper, build_configuration, init_subsystems=True): global_bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope() if global_bootstrap_options.pants_version != pants_version(): raise BuildConfigurationError( 'Version mismatch: Requested version was {}, our version is {}.' .format(global_bootstrap_options.pants_version, pants_version()) ) # Parse and register options. options = cls._construct_options(options_bootstrapper, build_configuration) GlobalOptionsRegistrar.validate_instance(options.for_global_scope()) if init_subsystems: Subsystem.set_options(options) return options
def _options(self): # NB: The PluginResolver runs very early in the pants startup sequence before the standard # Subsystem facility is wired up. As a result PluginResolver is not itself a Subsystem with # (PythonRepos, PythonSetup) returned as `dependencies()`. Instead it does the minimum possible # work to hand-roll bootstrapping of the Subsystems it needs. subsystems = Subsystem.closure([PythonRepos, PythonSetup]) known_scope_infos = [subsystem.get_scope_info() for subsystem in subsystems] options = self._options_bootstrapper.get_full_options(known_scope_infos) # Ignore command line flags since we'd blow up on any we don't understand (most of them). # If someone wants to bootstrap plugins in a one-off custom way they'll need to use env vars # or a --config-override pointing to a custom pants.ini snippet. defaulted_only_options = options.drop_flag_values() GlobalOptionsRegistrar.register_options_on_scope(defaulted_only_options) for subsystem in subsystems: subsystem.register_options_on_scope(defaulted_only_options) return defaulted_only_options
def _options(self): # NB: The PluginResolver runs very early in the pants startup sequence before the standard # Subsystem facility is wired up. As a result PluginResolver is not itself a Subsystem with # PythonRepos as a dependency. Instead it does the minimum possible work to hand-roll # bootstrapping of the Subsystem it needs. subsystems = Subsystem.closure([PythonRepos]) known_scope_infos = [subsystem.get_scope_info() for subsystem in subsystems] options = self._options_bootstrapper.get_full_options(known_scope_infos) # Ignore command line flags since we'd blow up on any we don't understand (most of them). # If someone wants to bootstrap plugins in a one-off custom way they'll need to use env vars # or a --pants-config-files pointing to a custom pants.ini snippet. defaulted_only_options = options.drop_flag_values() GlobalOptionsRegistrar.register_options_on_scope(defaulted_only_options) for subsystem in subsystems: subsystem.register_options_on_scope(defaulted_only_options) return defaulted_only_options
def do_test(args, kwargs, expected_default): # Defaults are computed in the parser and added into the kwargs, so we # must jump through this hoop in this test. parser = Parser(env={}, config=Config.load([]), scope_info=GlobalOptionsRegistrar.get_scope_info(), parent_parser=None, option_tracker=OptionTracker()) parser.register(*args, **kwargs) oshi = HelpInfoExtracter.get_option_scope_help_info_from_parser(parser).basic self.assertEquals(1, len(oshi)) ohi = oshi[0] self.assertEqual(expected_default, ohi.default)
def _setup_options(self, options_bootstrapper, working_set): # TODO: This inline import is currently necessary to resolve a ~legitimate cycle between # `GoalRunner`->`EngineInitializer`->`OptionsInitializer`->`GoalRunner`. from pants.bin.goal_runner import GoalRunner bootstrap_options = options_bootstrapper.get_bootstrap_options() global_bootstrap_options = bootstrap_options.for_global_scope() if global_bootstrap_options.pants_version != pants_version(): raise BuildConfigurationError( 'Version mismatch: Requested version was {}, our version is {}.'.format( global_bootstrap_options.pants_version, pants_version() ) ) # Get logging setup prior to loading backends so that they can log as needed. if self._init_logging: self._setup_logging(global_bootstrap_options) # Add any extra paths to python path (e.g., for loading extra source backends). for path in global_bootstrap_options.pythonpath: sys.path.append(path) pkg_resources.fixup_namespace_packages(path) # Load plugins and backends. plugins = global_bootstrap_options.plugins backend_packages = global_bootstrap_options.backend_packages build_configuration = load_plugins_and_backends(plugins, working_set, backend_packages) # Now that plugins and backends are loaded, we can gather the known scopes. known_scope_infos = [GlobalOptionsRegistrar.get_scope_info()] # Add scopes for all needed subsystems via a union of all known subsystem sets. subsystems = Subsystem.closure( GoalRunner.subsystems() | Goal.subsystems() | build_configuration.subsystems() ) for subsystem in subsystems: known_scope_infos.append(subsystem.get_scope_info()) # Add scopes for all tasks in all goals. for goal in Goal.all(): known_scope_infos.extend(filter(None, goal.known_scope_infos())) # Now that we have the known scopes we can get the full options. options = options_bootstrapper.get_full_options(known_scope_infos) self._register_options(subsystems, options) # Make the options values available to all subsystems. Subsystem.set_options(options) return options, build_configuration
def _setup_options(self, options_bootstrapper, working_set): # TODO: This inline import is currently necessary to resolve a ~legitimate cycle between # `GoalRunner`->`EngineInitializer`->`OptionsInitializer`->`GoalRunner`. from pants.bin.goal_runner import GoalRunner bootstrap_options = options_bootstrapper.get_bootstrap_options() global_bootstrap_options = bootstrap_options.for_global_scope() if global_bootstrap_options.pants_version != pants_version(): raise BuildConfigurationError( 'Version mismatch: Requested version was {}, our version is {}.' .format(global_bootstrap_options.pants_version, pants_version())) # Get logging setup prior to loading backends so that they can log as needed. if self._init_logging: self._setup_logging(global_bootstrap_options) # Add any extra paths to python path (e.g., for loading extra source backends). for path in global_bootstrap_options.pythonpath: sys.path.append(path) pkg_resources.fixup_namespace_packages(path) # Load plugins and backends. plugins = global_bootstrap_options.plugins backend_packages = global_bootstrap_options.backend_packages build_configuration = load_backends_and_plugins( plugins, working_set, backend_packages) # Now that plugins and backends are loaded, we can gather the known scopes. known_scope_infos = [GlobalOptionsRegistrar.get_scope_info()] # Add scopes for all needed subsystems via a union of all known subsystem sets. subsystems = Subsystem.closure(GoalRunner.subsystems() | Goal.subsystems() | build_configuration.subsystems()) for subsystem in subsystems: known_scope_infos.append(subsystem.get_scope_info()) # Add scopes for all tasks in all goals. for goal in Goal.all(): known_scope_infos.extend(filter(None, goal.known_scope_infos())) # Now that we have the known scopes we can get the full options. options = options_bootstrapper.get_full_options(known_scope_infos) self._register_options(subsystems, options) # Make the options values available to all subsystems. Subsystem.set_options(options) return options, build_configuration
def _options(self): # NB: The PluginResolver runs very early in the pants startup sequence before the standard # Subsystem facility is wired up. As a result PluginResolver is not itself a Subsystem with # PythonRepos as a dependency. Instead it does the minimum possible work to hand-roll # bootstrapping of the Subsystems it needs. known_scope_infos = PythonRepos.known_scope_infos() options = self._options_bootstrapper.get_full_options( known_scope_infos) # Ignore command line flags since we'd blow up on any we don't understand (most of them). # If someone wants to bootstrap plugins in a one-off custom way they'll need to use env vars # or a --pants-config-files pointing to a custom pants.ini snippet. defaulted_only_options = options.drop_flag_values() GlobalOptionsRegistrar.register_options_on_scope( defaulted_only_options) distinct_optionable_classes = sorted( {si.optionable_cls for si in known_scope_infos}, key=lambda o: o.options_scope) for optionable_cls in distinct_optionable_classes: optionable_cls.register_options_on_scope(defaulted_only_options) return defaulted_only_options
def complete_scopes(cls, scope_infos): """Expand a set of scopes to include all enclosing scopes. E.g., if the set contains `foo.bar.baz`, ensure that it also contains `foo.bar` and `foo`. """ ret = {GlobalOptionsRegistrar.get_scope_info()} for scope_info in scope_infos: ret.add(scope_info) original_scopes = {si.scope for si in scope_infos} for scope_info in scope_infos: scope = scope_info.scope while scope != '': if scope not in original_scopes: ret.add(ScopeInfo(scope, ScopeInfo.INTERMEDIATE)) scope = enclosing_scope(scope) return ret
def _setup_options(self, options_bootstrapper, working_set): bootstrap_options = options_bootstrapper.get_bootstrap_options() global_bootstrap_options = bootstrap_options.for_global_scope() # The pants_version may be set in pants.ini for bootstrapping, so we make sure the user actually # requested the version on the command line before deciding to print the version and exit. if global_bootstrap_options.is_flagged('pants_version'): print(global_bootstrap_options.pants_version) self._exiter(0) # Get logging setup prior to loading backends so that they can log as needed. self._setup_logging(global_bootstrap_options) # Add any extra paths to python path (e.g., for loading extra source backends). for path in global_bootstrap_options.pythonpath: sys.path.append(path) pkg_resources.fixup_namespace_packages(path) # Load plugins and backends. plugins = global_bootstrap_options.plugins backend_packages = global_bootstrap_options.backend_packages build_configuration = load_plugins_and_backends( plugins, working_set, backend_packages) # Now that plugins and backends are loaded, we can gather the known scopes. known_scope_infos = [GlobalOptionsRegistrar.get_scope_info()] # Add scopes for all needed subsystems via a union of all known subsystem sets. subsystems = Subsystem.closure(GoalRunner.subsystems() | Goal.subsystems() | build_configuration.subsystems()) for subsystem in subsystems: known_scope_infos.append(subsystem.get_scope_info()) # Add scopes for all tasks in all goals. for goal in Goal.all(): known_scope_infos.extend(filter(None, goal.known_scope_infos())) # Now that we have the known scopes we can get the full options. options = options_bootstrapper.get_full_options(known_scope_infos) self._register_options(subsystems, options) # Make the options values available to all subsystems. Subsystem.set_options(options) return options, build_configuration
def complete_scopes(cls, scope_infos): """Expand a set of scopes to include all enclosing scopes. E.g., if the set contains `foo.bar.baz`, ensure that it also contains `foo.bar` and `foo`. """ ret = {GlobalOptionsRegistrar.get_scope_info()} for scope_info in scope_infos: ret.add(scope_info) original_scopes = {si.scope for si in scope_infos} for scope_info in scope_infos: scope = scope_info.scope while scope != "": if scope not in original_scopes: ret.add(ScopeInfo(scope, ScopeInfo.INTERMEDIATE)) scope = enclosing_scope(scope) return ret
def _setup_options(self, options_bootstrapper, working_set): bootstrap_options = options_bootstrapper.get_bootstrap_options() global_bootstrap_options = bootstrap_options.for_global_scope() # The pants_version may be set in pants.ini for bootstrapping, so we make sure the user actually # requested the version on the command line before deciding to print the version and exit. if global_bootstrap_options.is_flagged('pants_version'): print(global_bootstrap_options.pants_version) self._exiter(0) # Get logging setup prior to loading backends so that they can log as needed. self._setup_logging(global_bootstrap_options) # Add any extra paths to python path (e.g., for loading extra source backends). for path in global_bootstrap_options.pythonpath: sys.path.append(path) pkg_resources.fixup_namespace_packages(path) # Load plugins and backends. plugins = global_bootstrap_options.plugins backend_packages = global_bootstrap_options.backend_packages build_configuration = load_plugins_and_backends(plugins, working_set, backend_packages) # Now that plugins and backends are loaded, we can gather the known scopes. known_scope_infos = [GlobalOptionsRegistrar.get_scope_info()] # Add scopes for all needed subsystems via a union of all known subsystem sets. subsystems = Subsystem.closure( GoalRunner.subsystems() | Goal.subsystems() | build_configuration.subsystems() ) for subsystem in subsystems: known_scope_infos.append(subsystem.get_scope_info()) # Add scopes for all tasks in all goals. for goal in Goal.all(): known_scope_infos.extend(filter(None, goal.known_scope_infos())) # Now that we have the known scopes we can get the full options. options = options_bootstrapper.get_full_options(known_scope_infos) self._register_options(subsystems, options) # Make the options values available to all subsystems. Subsystem.set_options(options) return options, build_configuration
def create(cls, env=None, args=None): """Parses the minimum amount of configuration necessary to create an OptionsBootstrapper. :param env: An environment dictionary, or None to use `os.environ`. :param args: An args array, or None to use `sys.argv`. """ env = { k: v for k, v in (os.environ if env is None else env).items() if k.startswith('PANTS_') } args = tuple(sys.argv if args is None else args) flags = set() short_flags = set() # TODO: This codepath probably shouldn't be using FileContent, which is a very v2 engine thing. def filecontent_for(path): is_executable = os.stat( path).st_mode & stat.S_IXUSR == stat.S_IXUSR return FileContent( ensure_text(path), read_file(path, binary_mode=True), is_executable=is_executable, ) def capture_the_flags(*args, **kwargs): for arg in args: flags.add(arg) if len(arg) == 2: short_flags.add(arg) elif kwargs.get('type') == bool: flags.add(f'--no-{arg[2:]}') GlobalOptionsRegistrar.register_bootstrap_options(capture_the_flags) def is_bootstrap_option(arg): components = arg.split('=', 1) if components[0] in flags: return True for flag in short_flags: if arg.startswith(flag): return True return False # Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line. # Stop before '--' since args after that are pass-through and may have duplicate names to our # bootstrap options. bargs = tuple( filter(is_bootstrap_option, itertools.takewhile(lambda arg: arg != '--', args))) config_file_paths = cls.get_config_file_paths(env=env, args=args) config_files_products = [filecontent_for(p) for p in config_file_paths] pre_bootstrap_config = Config.load_file_contents(config_files_products) initial_bootstrap_options = cls.parse_bootstrap_options( env, bargs, pre_bootstrap_config) bootstrap_option_values = initial_bootstrap_options.for_global_scope() # Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped # from (typically pants.ini), then config override, then rcfiles. full_configpaths = pre_bootstrap_config.sources() if bootstrap_option_values.pantsrc: rcfiles = [ os.path.expanduser(str(rcfile)) for rcfile in bootstrap_option_values.pantsrc_files ] existing_rcfiles = list(filter(os.path.exists, rcfiles)) full_configpaths.extend(existing_rcfiles) full_config_files_products = [ filecontent_for(p) for p in full_configpaths ] post_bootstrap_config = Config.load_file_contents( full_config_files_products, seed_values=bootstrap_option_values) env_tuples = tuple(sorted(env.items(), key=lambda x: x[0])) return cls(env_tuples=env_tuples, bootstrap_args=bargs, args=args, config=post_bootstrap_config)
def get_bootstrap_options(self): """:returns: an Options instance that only knows about the bootstrap options. :rtype: :class:`Options` """ if not self._bootstrap_options: flags = set() short_flags = set() def capture_the_flags(*args, **kwargs): for arg in args: flags.add(arg) if len(arg) == 2: short_flags.add(arg) elif is_boolean_flag(kwargs): flags.add('--no-{}'.format(arg[2:])) GlobalOptionsRegistrar.register_bootstrap_options( capture_the_flags) def is_bootstrap_option(arg): components = arg.split('=', 1) if components[0] in flags: return True for flag in short_flags: if arg.startswith(flag): return True return False # Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line. # Stop before '--' since args after that are pass-through and may have duplicate names to our # bootstrap options. bargs = filter( is_bootstrap_option, itertools.takewhile(lambda arg: arg != '--', self._args)) configpaths = [self._configpath] if self._configpath else None pre_bootstrap_config = Config.load(configpaths) def bootstrap_options_from_config(config): bootstrap_options = Options.create( env=self._env, config=config, known_scope_infos=[ GlobalOptionsRegistrar.get_scope_info() ], args=bargs, option_tracker=self._option_tracker) def register_global(*args, **kwargs): bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs) GlobalOptionsRegistrar.register_bootstrap_options( register_global) return bootstrap_options initial_bootstrap_options = bootstrap_options_from_config( pre_bootstrap_config) bootstrap_option_values = initial_bootstrap_options.for_global_scope( ) # Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped # from (typically pants.ini), then config override, then rcfiles. full_configpaths = pre_bootstrap_config.sources() if bootstrap_option_values.config_override: full_configpaths.extend( bootstrap_option_values.config_override) if bootstrap_option_values.pantsrc: rcfiles = [ os.path.expanduser(rcfile) for rcfile in bootstrap_option_values.pantsrc_files ] existing_rcfiles = filter(os.path.exists, rcfiles) full_configpaths.extend(existing_rcfiles) self._post_bootstrap_config = Config.load( full_configpaths, seed_values=bootstrap_option_values) # Now recompute the bootstrap options with the full config. This allows us to pick up # bootstrap values (such as backends) from a config override file, for example. self._bootstrap_options = bootstrap_options_from_config( self._post_bootstrap_config) return self._bootstrap_options
def context(self, for_task_types=None, options=None, target_roots=None, console_outstream=None, workspace=None): for_task_types = for_task_types or [] options = options or {} option_values = defaultdict(dict) registered_subsystems = set() bootstrap_option_values = None # We fill these in after registering bootstrap options. # We provide our own test-only registration implementation, bypassing argparse. # When testing we set option values directly, so we don't care about cmd-line flags, config, # env vars etc. In fact, for test isolation we explicitly don't want to look at those. # All this does is make the names available in code, with the default values. # Individual tests can then override the option values they care about. def register_func(on_scope): def register(*rargs, **rkwargs): scoped_options = option_values[on_scope] default = rkwargs.get('default') if default is None and rkwargs.get('action') == 'append': default = [] for flag_name in rargs: option_name = flag_name.lstrip('-').replace('-', '_') scoped_options[option_name] = default register.bootstrap = bootstrap_option_values register.scope = on_scope return register # TODO: This sequence is a bit repetitive of the real registration sequence. # Register bootstrap options and grab their default values for use in subsequent registration. GlobalOptionsRegistrar.register_bootstrap_options( register_func(Options.GLOBAL_SCOPE)) bootstrap_option_values = create_option_values( copy.copy(option_values[Options.GLOBAL_SCOPE])) # Now register the full global scope options. GlobalOptionsRegistrar.register_options( register_func(Options.GLOBAL_SCOPE)) # Now register task and subsystem options for relevant tasks. for task_type in for_task_types: scope = task_type.options_scope if scope is None: raise TaskError( 'You must set a scope on your task type before using it in tests.' ) task_type.register_options(register_func(scope)) for subsystem in (set(task_type.global_subsystems()) | set(task_type.task_subsystems()) | self._build_configuration.subsystems()): if subsystem not in registered_subsystems: subsystem.register_options( register_func(subsystem.options_scope)) registered_subsystems.add(subsystem) # Now default option values override with any caller-specified values. # TODO(benjy): Get rid of the options arg, and require tests to call set_options. for scope, opts in options.items(): for key, val in opts.items(): option_values[scope][key] = val for scope, opts in self.options.items(): for key, val in opts.items(): option_values[scope][key] = val # Make inner scopes inherit option values from their enclosing scopes. all_scopes = set(option_values.keys()) for task_type in for_task_types: # Make sure we know about pre-task subsystem scopes. all_scopes.update( [si.scope for si in task_type.known_scope_infos()]) # Iterating in sorted order guarantees that we see outer scopes before inner scopes, # and therefore only have to inherit from our immediately enclosing scope. for scope in sorted(all_scopes): if scope != Options.GLOBAL_SCOPE: enclosing_scope = scope.rpartition('.')[0] opts = option_values[scope] for key, val in option_values.get(enclosing_scope, {}).items(): if key not in opts: # Inner scope values override the inherited ones. opts[key] = val context = create_context(options=option_values, target_roots=target_roots, build_graph=self.build_graph, build_file_parser=self.build_file_parser, address_mapper=self.address_mapper, console_outstream=console_outstream, workspace=workspace) Subsystem._options = context.options return context
def produce_and_set_bootstrap_options(self): """Cooperatively populates the internal bootstrap_options cache with a producer of `FileContent`.""" flags = set() short_flags = set() def capture_the_flags(*args, **kwargs): for arg in args: flags.add(arg) if len(arg) == 2: short_flags.add(arg) elif kwargs.get('type') == bool: flags.add('--no-{}'.format(arg[2:])) GlobalOptionsRegistrar.register_bootstrap_options(capture_the_flags) def is_bootstrap_option(arg): components = arg.split('=', 1) if components[0] in flags: return True for flag in short_flags: if arg.startswith(flag): return True return False # Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line. # Stop before '--' since args after that are pass-through and may have duplicate names to our # bootstrap options. bargs = list(filter(is_bootstrap_option, itertools.takewhile(lambda arg: arg != '--', self._args))) config_file_paths = self.get_config_file_paths(env=self._env, args=self._args) config_files_products = yield config_file_paths pre_bootstrap_config = Config.load_file_contents(config_files_products) def bootstrap_options_from_config(config): bootstrap_options = Options.create( env=self._env, config=config, known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()], args=bargs, option_tracker=self._option_tracker ) def register_global(*args, **kwargs): ## Only use of Options.register? bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs) GlobalOptionsRegistrar.register_bootstrap_options(register_global) return bootstrap_options initial_bootstrap_options = bootstrap_options_from_config(pre_bootstrap_config) bootstrap_option_values = initial_bootstrap_options.for_global_scope() # Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped # from (typically pants.ini), then config override, then rcfiles. full_configpaths = pre_bootstrap_config.sources() if bootstrap_option_values.pantsrc: rcfiles = [os.path.expanduser(rcfile) for rcfile in bootstrap_option_values.pantsrc_files] existing_rcfiles = list(filter(os.path.exists, rcfiles)) full_configpaths.extend(existing_rcfiles) full_config_files_products = yield full_configpaths self._post_bootstrap_config = Config.load_file_contents( full_config_files_products, seed_values=bootstrap_option_values ) # Now recompute the bootstrap options with the full config. This allows us to pick up # bootstrap values (such as backends) from a config override file, for example. self._bootstrap_options = bootstrap_options_from_config(self._post_bootstrap_config)
def produce_and_set_bootstrap_options(self): """Cooperatively populates the internal bootstrap_options cache with a producer of `FileContent`.""" flags = set() short_flags = set() def capture_the_flags(*args, **kwargs): for arg in args: flags.add(arg) if len(arg) == 2: short_flags.add(arg) elif kwargs.get('type') == bool: flags.add('--no-{}'.format(arg[2:])) GlobalOptionsRegistrar.register_bootstrap_options(capture_the_flags) def is_bootstrap_option(arg): components = arg.split('=', 1) if components[0] in flags: return True for flag in short_flags: if arg.startswith(flag): return True return False # Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line. # Stop before '--' since args after that are pass-through and may have duplicate names to our # bootstrap options. bargs = list( filter(is_bootstrap_option, itertools.takewhile(lambda arg: arg != '--', self._args))) config_file_paths = self.get_config_file_paths(env=self._env, args=self._args) config_files_products = yield config_file_paths pre_bootstrap_config = Config.load_file_contents(config_files_products) def bootstrap_options_from_config(config): bootstrap_options = Options.create( env=self._env, config=config, known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()], args=bargs, option_tracker=self._option_tracker) def register_global(*args, **kwargs): ## Only use of Options.register? bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs) GlobalOptionsRegistrar.register_bootstrap_options(register_global) return bootstrap_options initial_bootstrap_options = bootstrap_options_from_config( pre_bootstrap_config) bootstrap_option_values = initial_bootstrap_options.for_global_scope() # Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped # from (typically pants.ini), then config override, then rcfiles. full_configpaths = pre_bootstrap_config.sources() if bootstrap_option_values.pantsrc: rcfiles = [ os.path.expanduser(rcfile) for rcfile in bootstrap_option_values.pantsrc_files ] existing_rcfiles = list(filter(os.path.exists, rcfiles)) full_configpaths.extend(existing_rcfiles) full_config_files_products = yield full_configpaths self._post_bootstrap_config = Config.load_file_contents( full_config_files_products, seed_values=bootstrap_option_values) # Now recompute the bootstrap options with the full config. This allows us to pick up # bootstrap values (such as backends) from a config override file, for example. self._bootstrap_options = bootstrap_options_from_config( self._post_bootstrap_config)
def create_options_for_optionables(optionables, extra_scopes=None, options=None, options_fingerprintable=None, passthru_args=None): """Create a fake Options object for testing with appropriate defaults for the given optionables. Any scoped `options` provided will override defaults, behaving as-if set on the command line. :param iterable optionables: A series of `Optionable` types to register default options for. :param iterable extra_scopes: An optional series of extra known scopes in play. :param dict options: A dict of scope -> (dict of option name -> value) representing option values explicitly set via the command line. :param dict options_fingerprintable: A dict of scope -> (dict of option name -> option type) representing the fingerprintable options and the scopes they are registered for. :param list passthru_args: A list of passthrough args (specified after `--` on the command line). :returns: A fake `Options` object with defaults populated for the given `optionables` and any explicitly set `options` overlayed. """ all_options = defaultdict(dict) fingerprintable_options = defaultdict(dict) bootstrap_option_values = None # NB(cosmicexplorer): we do this again for all_options after calling # register_func below, this is a hack if options: for scope, opts in options.items(): all_options[scope].update(opts) if options_fingerprintable: for scope, opts in options_fingerprintable.items(): fingerprintable_options[scope].update(opts) def complete_scopes(scopes): """Return all enclosing scopes. This is similar to what `complete_scopes` does in `pants.option.options.Options` without creating `ScopeInfo`s. """ completed_scopes = set(scopes) for scope in scopes: while scope != GLOBAL_SCOPE: if scope not in completed_scopes: completed_scopes.add(scope) scope = enclosing_scope(scope) return completed_scopes def register_func(on_scope): scoped_options = all_options[on_scope] scoped_fingerprintables = fingerprintable_options[on_scope] register = _options_registration_function(scoped_options, scoped_fingerprintables) register.bootstrap = bootstrap_option_values register.scope = on_scope return register # TODO: This sequence is a bit repetitive of the real registration sequence. # Register bootstrap options and grab their default values for use in subsequent registration. GlobalOptionsRegistrar.register_bootstrap_options(register_func(GLOBAL_SCOPE)) bootstrap_option_values = _FakeOptionValues(all_options[GLOBAL_SCOPE].copy()) # Now register the full global scope options. GlobalOptionsRegistrar.register_options(register_func(GLOBAL_SCOPE)) for optionable in optionables: optionable.register_options(register_func(optionable.options_scope)) # Make inner scopes inherit option values from their enclosing scopes. all_scopes = set(all_options.keys()) # TODO(John Sirois): Kill extra scopes one this goes in: # https://github.com/pantsbuild/pants/issues/1957 # For now we need a way for users of this utility to provide extra derived scopes out of band. # With #1957 resolved, the extra scopes will be embedded in the Optionable's option_scope # directly. if extra_scopes: all_scopes.update(extra_scopes) all_scopes = complete_scopes(all_scopes) # We need to update options before completing them based on inner/outer relation. if options: for scope, opts in options.items(): all_options[scope].update(opts) # Iterating in sorted order guarantees that we see outer scopes before inner scopes, # and therefore only have to inherit from our immediately enclosing scope. for s in sorted(all_scopes): if s != GLOBAL_SCOPE: scope = enclosing_scope(s) opts = all_options[s] for key, val in all_options.get(scope, {}).items(): if key not in opts: # Inner scope values override the inherited ones. opts[key] = val return create_options(all_options, passthru_args=passthru_args, fingerprintable_options=fingerprintable_options)
def setup(self, options_bootstrapper, working_set): bootstrap_options = options_bootstrapper.get_bootstrap_options() global_bootstrap_options = bootstrap_options.for_global_scope() # The pants_version may be set in pants.ini for bootstrapping, so we make sure the user actually # requested the version on the command line before deciding to print the version and exit. if global_bootstrap_options.is_flagged('pants_version'): print(global_bootstrap_options.pants_version) self._exiter(0) # Get logging setup prior to loading backends so that they can log as needed. self._setup_logging(global_bootstrap_options) # Add any extra paths to python path (e.g., for loading extra source backends). for path in global_bootstrap_options.pythonpath: sys.path.append(path) pkg_resources.fixup_namespace_packages(path) # Load plugins and backends. plugins = global_bootstrap_options.plugins backend_packages = global_bootstrap_options.backend_packages build_configuration = load_plugins_and_backends(plugins, working_set, backend_packages) # Now that plugins and backends are loaded, we can gather the known scopes. self.targets = [] known_scope_infos = [GlobalOptionsRegistrar.get_scope_info()] # Add scopes for all needed subsystems. subsystems = Subsystem.closure(set(self.subsystems) | Goal.subsystems() | build_configuration.subsystems()) for subsystem in subsystems: known_scope_infos.append(subsystem.get_scope_info()) # Add scopes for all tasks in all goals. for goal in Goal.all(): known_scope_infos.extend(filter(None, goal.known_scope_infos())) # Now that we have the known scopes we can get the full options. self.options = options_bootstrapper.get_full_options(known_scope_infos) self.register_options(subsystems) # Make the options values available to all subsystems. Subsystem._options = self.options # Now that we have options we can instantiate subsystems. self.run_tracker = RunTracker.global_instance() self.reporting = Reporting.global_instance() report = self.reporting.initial_reporting(self.run_tracker) self.run_tracker.start(report) url = self.run_tracker.run_info.get_info('report_url') if url: self.run_tracker.log(Report.INFO, 'See a report at: {}'.format(url)) else: self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)') self.build_file_parser = BuildFileParser(build_configuration=build_configuration, root_dir=self.root_dir, run_tracker=self.run_tracker) rev = self.options.for_global_scope().build_file_rev if rev: ScmBuildFile.set_rev(rev) ScmBuildFile.set_scm(get_scm()) build_file_type = ScmBuildFile else: build_file_type = FilesystemBuildFile self.address_mapper = BuildFileAddressMapper(self.build_file_parser, build_file_type) self.build_graph = BuildGraph(run_tracker=self.run_tracker, address_mapper=self.address_mapper) # TODO(John Sirois): Kill when source root registration is lifted out of BUILD files. with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnitLabel.SETUP]): source_root_bootstrapper = SourceRootBootstrapper.global_instance() source_root_bootstrapper.bootstrap(self.address_mapper, self.build_file_parser) self._expand_goals_and_specs() # Now that we've parsed the bootstrap BUILD files, and know about the SCM system. self.run_tracker.run_info.add_scm_info()
def create_options_for_optionables(optionables, extra_scopes=None, options=None): """Create a fake Options object for testing with appropriate defaults for the given optionables. Any scoped `options` provided will override defaults, behaving as-if set on the command line. :param iterable optionables: A series of `Optionable` types to register default options for. :param iterable extra_scopes: An optional series of extra known scopes in play. :param dict options: A dict of scope -> (dict of option name -> value) representing option values explicitly set via the command line. :returns: A fake `Options` object with defaults populated for the given `optionables` and any explicitly set `options` overlayed. """ all_options = defaultdict(dict) bootstrap_option_values = None def complete_scopes(scopes): """Return all enclosing scopes. This is similar to what `complete_scopes` does in `pants.option.options.Options` without creating `ScopeInfo`s. """ completed_scopes = set(scopes) for scope in scopes: while scope != '': if scope not in completed_scopes: completed_scopes.add(scope) scope = enclosing_scope(scope) return completed_scopes def register_func(on_scope): scoped_options = all_options[on_scope] register = _options_registration_function(scoped_options) register.bootstrap = bootstrap_option_values register.scope = on_scope return register # TODO: This sequence is a bit repetitive of the real registration sequence. # Register bootstrap options and grab their default values for use in subsequent registration. GlobalOptionsRegistrar.register_bootstrap_options( register_func(GLOBAL_SCOPE)) bootstrap_option_values = create_option_values( all_options[GLOBAL_SCOPE].copy()) # Now register the full global scope options. GlobalOptionsRegistrar.register_options(register_func(GLOBAL_SCOPE)) for optionable in optionables: optionable.register_options(register_func(optionable.options_scope)) # Make inner scopes inherit option values from their enclosing scopes. all_scopes = set(all_options.keys()) # TODO(John Sirois): Kill extra scopes one this goes in: # https://github.com/pantsbuild/pants/issues/1957 # For now we need a way for users of this utility to provide extra derived scopes out of band. # With #1957 resolved, the extra scopes will be embedded in the Optionable's option_scope # directly. if extra_scopes: all_scopes.update(extra_scopes) all_scopes = complete_scopes(all_scopes) # Iterating in sorted order guarantees that we see outer scopes before inner scopes, # and therefore only have to inherit from our immediately enclosing scope. for s in sorted(all_scopes): if s != GLOBAL_SCOPE: scope = enclosing_scope(s) opts = all_options[s] for key, val in all_options.get(scope, {}).items(): if key not in opts: # Inner scope values override the inherited ones. opts[key] = val if options: for scope, opts in options.items(): all_options[scope].update(opts) return create_options(all_options)
def context(self, for_task_types=None, options=None, target_roots=None, console_outstream=None, workspace=None): for_task_types = for_task_types or [] options = options or {} option_values = defaultdict(dict) registered_subsystems = set() bootstrap_option_values = None # We fill these in after registering bootstrap options. # We provide our own test-only registration implementation, bypassing argparse. # When testing we set option values directly, so we don't care about cmd-line flags, config, # env vars etc. In fact, for test isolation we explicitly don't want to look at those. # All this does is make the names available in code, with the default values. # Individual tests can then override the option values they care about. def register_func(on_scope): def register(*rargs, **rkwargs): scoped_options = option_values[on_scope] default = rkwargs.get('default') if default is None and rkwargs.get('action') == 'append': default = [] for flag_name in rargs: option_name = flag_name.lstrip('-').replace('-', '_') scoped_options[option_name] = default register.bootstrap = bootstrap_option_values register.scope = on_scope return register # TODO: This sequence is a bit repetitive of the real registration sequence. # Register bootstrap options and grab their default values for use in subsequent registration. GlobalOptionsRegistrar.register_bootstrap_options(register_func(Options.GLOBAL_SCOPE)) bootstrap_option_values = create_option_values(copy.copy(option_values[Options.GLOBAL_SCOPE])) # Now register the full global scope options. GlobalOptionsRegistrar.register_options(register_func(Options.GLOBAL_SCOPE)) # Now register task and subsystem options for relevant tasks. for task_type in for_task_types: scope = task_type.options_scope if scope is None: raise TaskError('You must set a scope on your task type before using it in tests.') task_type.register_options(register_func(scope)) for subsystem in (set(task_type.global_subsystems()) | set(task_type.task_subsystems()) | self._build_configuration.subsystems()): if subsystem not in registered_subsystems: subsystem.register_options(register_func(subsystem.options_scope)) registered_subsystems.add(subsystem) # Now default option values override with any caller-specified values. # TODO(benjy): Get rid of the options arg, and require tests to call set_options. for scope, opts in options.items(): for key, val in opts.items(): option_values[scope][key] = val for scope, opts in self.options.items(): for key, val in opts.items(): option_values[scope][key] = val # Make inner scopes inherit option values from their enclosing scopes. all_scopes = set(option_values.keys()) for task_type in for_task_types: # Make sure we know about pre-task subsystem scopes. all_scopes.update([si.scope for si in task_type.known_scope_infos()]) # Iterating in sorted order guarantees that we see outer scopes before inner scopes, # and therefore only have to inherit from our immediately enclosing scope. for scope in sorted(all_scopes): if scope != Options.GLOBAL_SCOPE: enclosing_scope = scope.rpartition('.')[0] opts = option_values[scope] for key, val in option_values.get(enclosing_scope, {}).items(): if key not in opts: # Inner scope values override the inherited ones. opts[key] = val context = create_context(options=option_values, target_roots=target_roots, build_graph=self.build_graph, build_file_parser=self.build_file_parser, address_mapper=self.address_mapper, console_outstream=console_outstream, workspace=workspace) Subsystem._options = context.options return context
def create(cls, env=None, args=None): """Parses the minimum amount of configuration necessary to create an OptionsBootstrapper. :param env: An environment dictionary, or None to use `os.environ`. :param args: An args array, or None to use `sys.argv`. """ env = {k: v for k, v in (os.environ if env is None else env).items() if k.startswith('PANTS_')} args = tuple(sys.argv if args is None else args) flags = set() short_flags = set() def filecontent_for(path): return FileContent(ensure_text(path), read_file(path, binary_mode=True)) def capture_the_flags(*args, **kwargs): for arg in args: flags.add(arg) if len(arg) == 2: short_flags.add(arg) elif kwargs.get('type') == bool: flags.add('--no-{}'.format(arg[2:])) GlobalOptionsRegistrar.register_bootstrap_options(capture_the_flags) def is_bootstrap_option(arg): components = arg.split('=', 1) if components[0] in flags: return True for flag in short_flags: if arg.startswith(flag): return True return False # Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line. # Stop before '--' since args after that are pass-through and may have duplicate names to our # bootstrap options. bargs = tuple(filter(is_bootstrap_option, itertools.takewhile(lambda arg: arg != '--', args))) config_file_paths = cls.get_config_file_paths(env=env, args=args) config_files_products = [filecontent_for(p) for p in config_file_paths] pre_bootstrap_config = Config.load_file_contents(config_files_products) initial_bootstrap_options = cls.parse_bootstrap_options(env, bargs, pre_bootstrap_config) bootstrap_option_values = initial_bootstrap_options.for_global_scope() # Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped # from (typically pants.ini), then config override, then rcfiles. full_configpaths = pre_bootstrap_config.sources() if bootstrap_option_values.pantsrc: rcfiles = [os.path.expanduser(str(rcfile)) for rcfile in bootstrap_option_values.pantsrc_files] existing_rcfiles = list(filter(os.path.exists, rcfiles)) full_configpaths.extend(existing_rcfiles) full_config_files_products = [filecontent_for(p) for p in full_configpaths] post_bootstrap_config = Config.load_file_contents( full_config_files_products, seed_values=bootstrap_option_values ) env_tuples = tuple(sorted(iteritems(env), key=lambda x: x[0])) return cls(env_tuples=env_tuples, bootstrap_args=bargs, args=args, config=post_bootstrap_config)
def setup(self, options_bootstrapper, working_set): bootstrap_options = options_bootstrapper.get_bootstrap_options() global_bootstrap_options = bootstrap_options.for_global_scope() # The pants_version may be set in pants.ini for bootstrapping, so we make sure the user actually # requested the version on the command line before deciding to print the version and exit. if global_bootstrap_options.is_flagged('pants_version'): print(global_bootstrap_options.pants_version) self._exiter(0) # Get logging setup prior to loading backends so that they can log as needed. self._setup_logging(global_bootstrap_options) # Add any extra paths to python path (eg for loading extra source backends) for path in global_bootstrap_options.pythonpath: sys.path.append(path) pkg_resources.fixup_namespace_packages(path) # Load plugins and backends. plugins = global_bootstrap_options.plugins backend_packages = global_bootstrap_options.backend_packages build_configuration = load_plugins_and_backends( plugins, working_set, backend_packages) # Now that plugins and backends are loaded, we can gather the known scopes. self.targets = [] known_scope_infos = [GlobalOptionsRegistrar.get_scope_info()] # Add scopes for all needed subsystems. subsystems = Subsystem.closure( set(self.subsystems) | Goal.subsystems() | build_configuration.subsystems()) for subsystem in subsystems: known_scope_infos.append(subsystem.get_scope_info()) # Add scopes for all tasks in all goals. for goal in Goal.all(): known_scope_infos.extend(filter(None, goal.known_scope_infos())) # Now that we have the known scopes we can get the full options. self.options = options_bootstrapper.get_full_options(known_scope_infos) self.register_options(subsystems) # Make the options values available to all subsystems. Subsystem._options = self.options # Now that we have options we can instantiate subsystems. self.run_tracker = RunTracker.global_instance() self.reporting = Reporting.global_instance() report = self.reporting.initial_reporting(self.run_tracker) self.run_tracker.start(report) url = self.run_tracker.run_info.get_info('report_url') if url: self.run_tracker.log(Report.INFO, 'See a report at: {}'.format(url)) else: self.run_tracker.log( Report.INFO, '(To run a reporting server: ./pants server)') self.build_file_parser = BuildFileParser( build_configuration=build_configuration, root_dir=self.root_dir, run_tracker=self.run_tracker) rev = self.options.for_global_scope().build_file_rev if rev: ScmBuildFile.set_rev(rev) ScmBuildFile.set_scm(get_scm()) build_file_type = ScmBuildFile else: build_file_type = FilesystemBuildFile self.address_mapper = BuildFileAddressMapper(self.build_file_parser, build_file_type) self.build_graph = BuildGraph(run_tracker=self.run_tracker, address_mapper=self.address_mapper) # TODO(John Sirois): Kill when source root registration is lifted out of BUILD files. with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnitLabel.SETUP]): source_root_bootstrapper = SourceRootBootstrapper.global_instance() source_root_bootstrapper.bootstrap(self.address_mapper, self.build_file_parser) self._expand_goals_and_specs() # Now that we've parsed the bootstrap BUILD files, and know about the SCM system. self.run_tracker.run_info.add_scm_info()
def get_bootstrap_options(self): """:returns: an Options instance that only knows about the bootstrap options. :rtype: :class:`Options` """ if not self._bootstrap_options: flags = set() short_flags = set() def capture_the_flags(*args, **kwargs): for arg in args: flags.add(arg) if len(arg) == 2: short_flags.add(arg) elif is_boolean_flag(kwargs): flags.add('--no-{}'.format(arg[2:])) GlobalOptionsRegistrar.register_bootstrap_options(capture_the_flags) def is_bootstrap_option(arg): components = arg.split('=', 1) if components[0] in flags: return True for flag in short_flags: if arg.startswith(flag): return True return False # Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line. # Stop before '--' since args after that are pass-through and may have duplicate names to our # bootstrap options. bargs = filter(is_bootstrap_option, itertools.takewhile(lambda arg: arg != '--', self._args)) configpaths = [self._configpath] if self._configpath else None pre_bootstrap_config = Config.load(configpaths) def bootstrap_options_from_config(config): bootstrap_options = Options.create(env=self._env, config=config, known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()], args=bargs) def register_global(*args, **kwargs): bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs) GlobalOptionsRegistrar.register_bootstrap_options(register_global) return bootstrap_options initial_bootstrap_options = bootstrap_options_from_config(pre_bootstrap_config) bootstrap_option_values = initial_bootstrap_options.for_global_scope() # Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped # from (typically pants.ini), then config override, then rcfiles. full_configpaths = pre_bootstrap_config.sources() if bootstrap_option_values.config_override: full_configpaths.append(bootstrap_option_values.config_override) if bootstrap_option_values.pantsrc: rcfiles = [os.path.expanduser(rcfile) for rcfile in bootstrap_option_values.pantsrc_files] existing_rcfiles = filter(os.path.exists, rcfiles) full_configpaths.extend(existing_rcfiles) self._post_bootstrap_config = Config.load(full_configpaths, seed_values=bootstrap_option_values) # Now recompute the bootstrap options with the full config. This allows us to pick up # bootstrap values (such as backends) from a config override file, for example. self._bootstrap_options = bootstrap_options_from_config(self._post_bootstrap_config) return self._bootstrap_options