def bootstrap_options_from_config(config): bootstrap_options = Options.create(env=self._env, config=config, known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()], args=bargs) def register_global(*args, **kwargs): bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs) GlobalOptionsRegistrar.register_bootstrap_options(register_global) return bootstrap_options
def create_options_for_optionables(optionables, extra_scopes=None, options=None): """Create a fake Options object for testing with appropriate defaults for the given optionables. Any scoped `options` provided will override defaults, behaving as-if set on the command line. :param iterable optionables: A series of `Optionable` types to register default options for. :param iterable extra_scopes: An optional series of extra known scopes in play. :param dict options: A dict of scope -> (dict of option name -> value) representing option values explicitly set via the command line. :returns: A fake `Options` object with defaults populated for the given `optionables` and any explicitly set `options` overlayed. """ all_options = defaultdict(dict) bootstrap_option_values = None def register_func(on_scope): scoped_options = all_options[on_scope] register = _options_registration_function(scoped_options) register.bootstrap = bootstrap_option_values register.scope = on_scope return register # TODO: This sequence is a bit repetitive of the real registration sequence. # Register bootstrap options and grab their default values for use in subsequent registration. GlobalOptionsRegistrar.register_bootstrap_options(register_func(GLOBAL_SCOPE)) bootstrap_option_values = create_option_values(all_options[GLOBAL_SCOPE].copy()) # Now register the full global scope options. GlobalOptionsRegistrar.register_options(register_func(GLOBAL_SCOPE)) for optionable in optionables: optionable.register_options(register_func(optionable.options_scope)) # Make inner scopes inherit option values from their enclosing scopes. all_scopes = set(all_options.keys()) # TODO(John Sirois): Kill extra scopes one this goes in: # https://github.com/pantsbuild/pants/issues/1957 # For now we need a way for users of this utility to provide extra derived scopes out of band. # With #1957 resolved, the extra scopes will be embedded in the Optionable's option_scope # directly. if extra_scopes: all_scopes.update(extra_scopes) # Iterating in sorted order guarantees that we see outer scopes before inner scopes, # and therefore only have to inherit from our immediately enclosing scope. for s in sorted(all_scopes): if s != GLOBAL_SCOPE: enclosing_scope = s.rpartition(".")[0] opts = all_options[s] for key, val in all_options.get(enclosing_scope, {}).items(): if key not in opts: # Inner scope values override the inherited ones. opts[key] = val if options: for scope, opts in options.items(): all_options[scope].update(opts) return create_options(all_options)
def create_options_for_optionables(optionables, options=None, options_fingerprintable=None, passthru_args=None): """Create a fake Options object for testing with appropriate defaults for the given optionables. Any scoped `options` provided will override defaults, behaving as-if set on the command line. :param iterable optionables: A series of `Optionable` types to register default options for. :param dict options: A dict of scope -> (dict of option name -> value) representing option values explicitly set via the command line. :param dict options_fingerprintable: A dict of scope -> (dict of option name -> option type) representing the fingerprintable options and the scopes they are registered for. :param list passthru_args: A list of passthrough args (specified after `--` on the command line). :returns: A fake `Options` object with defaults populated for the given `optionables` and any explicitly set `options` overlayed. """ all_options = defaultdict(dict) fingerprintable_options = defaultdict(dict) bootstrap_option_values = None if options_fingerprintable: for scope, opts in options_fingerprintable.items(): fingerprintable_options[scope].update(opts) def register_func(on_scope): scoped_options = all_options[on_scope] scoped_fingerprintables = fingerprintable_options[on_scope] register = _options_registration_function(scoped_options, scoped_fingerprintables) register.bootstrap = bootstrap_option_values register.scope = on_scope return register # TODO: This sequence is a bit repetitive of the real registration sequence. # Register bootstrap options and grab their default values for use in subsequent registration. GlobalOptionsRegistrar.register_bootstrap_options( register_func(GLOBAL_SCOPE)) bootstrap_option_values = _FakeOptionValues( all_options[GLOBAL_SCOPE].copy()) # Now register the full global scope options. GlobalOptionsRegistrar.register_options(register_func(GLOBAL_SCOPE)) for optionable in optionables: optionable.register_options(register_func(optionable.options_scope)) if options: for scope, opts in options.items(): all_options[scope].update(opts) return create_options(all_options, passthru_args=passthru_args, fingerprintable_options=fingerprintable_options)
def create_options_for_optionables(optionables, options=None, options_fingerprintable=None, passthru_args=None): """Create a fake Options object for testing with appropriate defaults for the given optionables. Any scoped `options` provided will override defaults, behaving as-if set on the command line. :param iterable optionables: A series of `Optionable` types to register default options for. :param dict options: A dict of scope -> (dict of option name -> value) representing option values explicitly set via the command line. :param dict options_fingerprintable: A dict of scope -> (dict of option name -> option type) representing the fingerprintable options and the scopes they are registered for. :param list passthru_args: A list of passthrough args (specified after `--` on the command line). :returns: A fake `Options` object with defaults populated for the given `optionables` and any explicitly set `options` overlayed. """ all_options = defaultdict(dict) fingerprintable_options = defaultdict(dict) bootstrap_option_values = None if options_fingerprintable: for scope, opts in options_fingerprintable.items(): fingerprintable_options[scope].update(opts) def register_func(on_scope): scoped_options = all_options[on_scope] scoped_fingerprintables = fingerprintable_options[on_scope] register = _options_registration_function(scoped_options, scoped_fingerprintables) register.bootstrap = bootstrap_option_values register.scope = on_scope return register # TODO: This sequence is a bit repetitive of the real registration sequence. # Register bootstrap options and grab their default values for use in subsequent registration. GlobalOptionsRegistrar.register_bootstrap_options(register_func(GLOBAL_SCOPE)) bootstrap_option_values = _FakeOptionValues(all_options[GLOBAL_SCOPE].copy()) # Now register the full global scope options. GlobalOptionsRegistrar.register_options(register_func(GLOBAL_SCOPE)) for optionable in optionables: optionable.register_options(register_func(optionable.options_scope)) if options: for scope, opts in options.items(): all_options[scope].update(opts) return create_options(all_options, passthru_args=passthru_args, fingerprintable_options=fingerprintable_options)
def parse_bootstrap_options(env, args, config): bootstrap_options = Options.create( env=env, config=config, known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()], args=args, ) def register_global(*args, **kwargs): ## Only use of Options.register? bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs) GlobalOptionsRegistrar.register_bootstrap_options(register_global) return bootstrap_options
def produce_and_set_bootstrap_options(self): """Cooperatively populates the internal bootstrap_options cache with a producer of `FileContent`.""" flags = set() short_flags = set() def capture_the_flags(*args, **kwargs): for arg in args: flags.add(arg) if len(arg) == 2: short_flags.add(arg) elif kwargs.get('type') == bool: flags.add('--no-{}'.format(arg[2:])) GlobalOptionsRegistrar.register_bootstrap_options(capture_the_flags) def is_bootstrap_option(arg): components = arg.split('=', 1) if components[0] in flags: return True for flag in short_flags: if arg.startswith(flag): return True return False # Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line. # Stop before '--' since args after that are pass-through and may have duplicate names to our # bootstrap options. bargs = list( filter(is_bootstrap_option, itertools.takewhile(lambda arg: arg != '--', self._args))) config_file_paths = self.get_config_file_paths(env=self._env, args=self._args) config_files_products = yield config_file_paths pre_bootstrap_config = Config.load_file_contents(config_files_products) def bootstrap_options_from_config(config): bootstrap_options = Options.create( env=self._env, config=config, known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()], args=bargs, option_tracker=self._option_tracker) def register_global(*args, **kwargs): ## Only use of Options.register? bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs) GlobalOptionsRegistrar.register_bootstrap_options(register_global) return bootstrap_options initial_bootstrap_options = bootstrap_options_from_config( pre_bootstrap_config) bootstrap_option_values = initial_bootstrap_options.for_global_scope() # Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped # from (typically pants.ini), then config override, then rcfiles. full_configpaths = pre_bootstrap_config.sources() if bootstrap_option_values.pantsrc: rcfiles = [ os.path.expanduser(rcfile) for rcfile in bootstrap_option_values.pantsrc_files ] existing_rcfiles = list(filter(os.path.exists, rcfiles)) full_configpaths.extend(existing_rcfiles) full_config_files_products = yield full_configpaths self._post_bootstrap_config = Config.load_file_contents( full_config_files_products, seed_values=bootstrap_option_values) # Now recompute the bootstrap options with the full config. This allows us to pick up # bootstrap values (such as backends) from a config override file, for example. self._bootstrap_options = bootstrap_options_from_config( self._post_bootstrap_config)
def create_options_for_optionables(optionables, extra_scopes=None, options=None): """Create a fake Options object for testing with appropriate defaults for the given optionables. Any scoped `options` provided will override defaults, behaving as-if set on the command line. :param iterable optionables: A series of `Optionable` types to register default options for. :param iterable extra_scopes: An optional series of extra known scopes in play. :param dict options: A dict of scope -> (dict of option name -> value) representing option values explicitly set via the command line. :returns: A fake `Options` object with defaults populated for the given `optionables` and any explicitly set `options` overlayed. """ all_options = defaultdict(dict) bootstrap_option_values = None def complete_scopes(scopes): """Return all enclosing scopes. This is similar to what `complete_scopes` does in `pants.option.options.Options` without creating `ScopeInfo`s. """ completed_scopes = set(scopes) for scope in scopes: while scope != '': if scope not in completed_scopes: completed_scopes.add(scope) scope = enclosing_scope(scope) return completed_scopes def register_func(on_scope): scoped_options = all_options[on_scope] register = _options_registration_function(scoped_options) register.bootstrap = bootstrap_option_values register.scope = on_scope return register # TODO: This sequence is a bit repetitive of the real registration sequence. # Register bootstrap options and grab their default values for use in subsequent registration. GlobalOptionsRegistrar.register_bootstrap_options( register_func(GLOBAL_SCOPE)) bootstrap_option_values = create_option_values( all_options[GLOBAL_SCOPE].copy()) # Now register the full global scope options. GlobalOptionsRegistrar.register_options(register_func(GLOBAL_SCOPE)) for optionable in optionables: optionable.register_options(register_func(optionable.options_scope)) # Make inner scopes inherit option values from their enclosing scopes. all_scopes = set(all_options.keys()) # TODO(John Sirois): Kill extra scopes one this goes in: # https://github.com/pantsbuild/pants/issues/1957 # For now we need a way for users of this utility to provide extra derived scopes out of band. # With #1957 resolved, the extra scopes will be embedded in the Optionable's option_scope # directly. if extra_scopes: all_scopes.update(extra_scopes) all_scopes = complete_scopes(all_scopes) # Iterating in sorted order guarantees that we see outer scopes before inner scopes, # and therefore only have to inherit from our immediately enclosing scope. for s in sorted(all_scopes): if s != GLOBAL_SCOPE: scope = enclosing_scope(s) opts = all_options[s] for key, val in all_options.get(scope, {}).items(): if key not in opts: # Inner scope values override the inherited ones. opts[key] = val if options: for scope, opts in options.items(): all_options[scope].update(opts) return create_options(all_options)
def get_bootstrap_options(self): """:returns: an Options instance that only knows about the bootstrap options. :rtype: :class:`Options` """ if not self._bootstrap_options: flags = set() short_flags = set() def capture_the_flags(*args, **kwargs): for arg in args: flags.add(arg) if len(arg) == 2: short_flags.add(arg) elif is_boolean_flag(kwargs): flags.add('--no-{}'.format(arg[2:])) GlobalOptionsRegistrar.register_bootstrap_options(capture_the_flags) def is_bootstrap_option(arg): components = arg.split('=', 1) if components[0] in flags: return True for flag in short_flags: if arg.startswith(flag): return True return False # Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line. # Stop before '--' since args after that are pass-through and may have duplicate names to our # bootstrap options. bargs = filter(is_bootstrap_option, itertools.takewhile(lambda arg: arg != '--', self._args)) configpaths = [self._configpath] if self._configpath else None pre_bootstrap_config = Config.load(configpaths) def bootstrap_options_from_config(config): bootstrap_options = Options.create(env=self._env, config=config, known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()], args=bargs) def register_global(*args, **kwargs): bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs) GlobalOptionsRegistrar.register_bootstrap_options(register_global) return bootstrap_options initial_bootstrap_options = bootstrap_options_from_config(pre_bootstrap_config) bootstrap_option_values = initial_bootstrap_options.for_global_scope() # Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped # from (typically pants.ini), then config override, then rcfiles. full_configpaths = pre_bootstrap_config.sources() if bootstrap_option_values.config_override: full_configpaths.append(bootstrap_option_values.config_override) if bootstrap_option_values.pantsrc: rcfiles = [os.path.expanduser(rcfile) for rcfile in bootstrap_option_values.pantsrc_files] existing_rcfiles = filter(os.path.exists, rcfiles) full_configpaths.extend(existing_rcfiles) self._post_bootstrap_config = Config.load(full_configpaths, seed_values=bootstrap_option_values) # Now recompute the bootstrap options with the full config. This allows us to pick up # bootstrap values (such as backends) from a config override file, for example. self._bootstrap_options = bootstrap_options_from_config(self._post_bootstrap_config) return self._bootstrap_options
def get_bootstrap_options(self): """:returns: an Options instance that only knows about the bootstrap options. :rtype: :class:`Options` """ if not self._bootstrap_options: flags = set() short_flags = set() def capture_the_flags(*args, **kwargs): for arg in args: flags.add(arg) if len(arg) == 2: short_flags.add(arg) elif is_boolean_flag(kwargs): flags.add('--no-{}'.format(arg[2:])) GlobalOptionsRegistrar.register_bootstrap_options( capture_the_flags) def is_bootstrap_option(arg): components = arg.split('=', 1) if components[0] in flags: return True for flag in short_flags: if arg.startswith(flag): return True return False # Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line. # Stop before '--' since args after that are pass-through and may have duplicate names to our # bootstrap options. bargs = filter( is_bootstrap_option, itertools.takewhile(lambda arg: arg != '--', self._args)) configpaths = [self._configpath] if self._configpath else None pre_bootstrap_config = Config.load(configpaths) def bootstrap_options_from_config(config): bootstrap_options = Options.create( env=self._env, config=config, known_scope_infos=[ GlobalOptionsRegistrar.get_scope_info() ], args=bargs, option_tracker=self._option_tracker) def register_global(*args, **kwargs): bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs) GlobalOptionsRegistrar.register_bootstrap_options( register_global) return bootstrap_options initial_bootstrap_options = bootstrap_options_from_config( pre_bootstrap_config) bootstrap_option_values = initial_bootstrap_options.for_global_scope( ) # Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped # from (typically pants.ini), then config override, then rcfiles. full_configpaths = pre_bootstrap_config.sources() if bootstrap_option_values.config_override: full_configpaths.extend( bootstrap_option_values.config_override) if bootstrap_option_values.pantsrc: rcfiles = [ os.path.expanduser(rcfile) for rcfile in bootstrap_option_values.pantsrc_files ] existing_rcfiles = filter(os.path.exists, rcfiles) full_configpaths.extend(existing_rcfiles) self._post_bootstrap_config = Config.load( full_configpaths, seed_values=bootstrap_option_values) # Now recompute the bootstrap options with the full config. This allows us to pick up # bootstrap values (such as backends) from a config override file, for example. self._bootstrap_options = bootstrap_options_from_config( self._post_bootstrap_config) return self._bootstrap_options
def create(cls, env=None, args=None): """Parses the minimum amount of configuration necessary to create an OptionsBootstrapper. :param env: An environment dictionary, or None to use `os.environ`. :param args: An args array, or None to use `sys.argv`. """ env = { k: v for k, v in (os.environ if env is None else env).items() if k.startswith('PANTS_') } args = tuple(sys.argv if args is None else args) flags = set() short_flags = set() # TODO: This codepath probably shouldn't be using FileContent, which is a very v2 engine thing. def filecontent_for(path): is_executable = os.stat( path).st_mode & stat.S_IXUSR == stat.S_IXUSR return FileContent( ensure_text(path), read_file(path, binary_mode=True), is_executable=is_executable, ) def capture_the_flags(*args, **kwargs): for arg in args: flags.add(arg) if len(arg) == 2: short_flags.add(arg) elif kwargs.get('type') == bool: flags.add(f'--no-{arg[2:]}') GlobalOptionsRegistrar.register_bootstrap_options(capture_the_flags) def is_bootstrap_option(arg): components = arg.split('=', 1) if components[0] in flags: return True for flag in short_flags: if arg.startswith(flag): return True return False # Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line. # Stop before '--' since args after that are pass-through and may have duplicate names to our # bootstrap options. bargs = tuple( filter(is_bootstrap_option, itertools.takewhile(lambda arg: arg != '--', args))) config_file_paths = cls.get_config_file_paths(env=env, args=args) config_files_products = [filecontent_for(p) for p in config_file_paths] pre_bootstrap_config = Config.load_file_contents(config_files_products) initial_bootstrap_options = cls.parse_bootstrap_options( env, bargs, pre_bootstrap_config) bootstrap_option_values = initial_bootstrap_options.for_global_scope() # Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped # from (typically pants.ini), then config override, then rcfiles. full_configpaths = pre_bootstrap_config.sources() if bootstrap_option_values.pantsrc: rcfiles = [ os.path.expanduser(str(rcfile)) for rcfile in bootstrap_option_values.pantsrc_files ] existing_rcfiles = list(filter(os.path.exists, rcfiles)) full_configpaths.extend(existing_rcfiles) full_config_files_products = [ filecontent_for(p) for p in full_configpaths ] post_bootstrap_config = Config.load_file_contents( full_config_files_products, seed_values=bootstrap_option_values) env_tuples = tuple(sorted(env.items(), key=lambda x: x[0])) return cls(env_tuples=env_tuples, bootstrap_args=bargs, args=args, config=post_bootstrap_config)
def create(cls, env=None, args=None): """Parses the minimum amount of configuration necessary to create an OptionsBootstrapper. :param env: An environment dictionary, or None to use `os.environ`. :param args: An args array, or None to use `sys.argv`. """ env = {k: v for k, v in (os.environ if env is None else env).items() if k.startswith('PANTS_')} args = tuple(sys.argv if args is None else args) flags = set() short_flags = set() def filecontent_for(path): return FileContent(ensure_text(path), read_file(path, binary_mode=True)) def capture_the_flags(*args, **kwargs): for arg in args: flags.add(arg) if len(arg) == 2: short_flags.add(arg) elif kwargs.get('type') == bool: flags.add('--no-{}'.format(arg[2:])) GlobalOptionsRegistrar.register_bootstrap_options(capture_the_flags) def is_bootstrap_option(arg): components = arg.split('=', 1) if components[0] in flags: return True for flag in short_flags: if arg.startswith(flag): return True return False # Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line. # Stop before '--' since args after that are pass-through and may have duplicate names to our # bootstrap options. bargs = tuple(filter(is_bootstrap_option, itertools.takewhile(lambda arg: arg != '--', args))) config_file_paths = cls.get_config_file_paths(env=env, args=args) config_files_products = [filecontent_for(p) for p in config_file_paths] pre_bootstrap_config = Config.load_file_contents(config_files_products) initial_bootstrap_options = cls.parse_bootstrap_options(env, bargs, pre_bootstrap_config) bootstrap_option_values = initial_bootstrap_options.for_global_scope() # Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped # from (typically pants.ini), then config override, then rcfiles. full_configpaths = pre_bootstrap_config.sources() if bootstrap_option_values.pantsrc: rcfiles = [os.path.expanduser(str(rcfile)) for rcfile in bootstrap_option_values.pantsrc_files] existing_rcfiles = list(filter(os.path.exists, rcfiles)) full_configpaths.extend(existing_rcfiles) full_config_files_products = [filecontent_for(p) for p in full_configpaths] post_bootstrap_config = Config.load_file_contents( full_config_files_products, seed_values=bootstrap_option_values ) env_tuples = tuple(sorted(iteritems(env), key=lambda x: x[0])) return cls(env_tuples=env_tuples, bootstrap_args=bargs, args=args, config=post_bootstrap_config)
def context(self, for_task_types=None, options=None, target_roots=None, console_outstream=None, workspace=None): for_task_types = for_task_types or [] options = options or {} option_values = defaultdict(dict) registered_subsystems = set() bootstrap_option_values = None # We fill these in after registering bootstrap options. # We provide our own test-only registration implementation, bypassing argparse. # When testing we set option values directly, so we don't care about cmd-line flags, config, # env vars etc. In fact, for test isolation we explicitly don't want to look at those. # All this does is make the names available in code, with the default values. # Individual tests can then override the option values they care about. def register_func(on_scope): def register(*rargs, **rkwargs): scoped_options = option_values[on_scope] default = rkwargs.get('default') if default is None and rkwargs.get('action') == 'append': default = [] for flag_name in rargs: option_name = flag_name.lstrip('-').replace('-', '_') scoped_options[option_name] = default register.bootstrap = bootstrap_option_values register.scope = on_scope return register # TODO: This sequence is a bit repetitive of the real registration sequence. # Register bootstrap options and grab their default values for use in subsequent registration. GlobalOptionsRegistrar.register_bootstrap_options(register_func(Options.GLOBAL_SCOPE)) bootstrap_option_values = create_option_values(copy.copy(option_values[Options.GLOBAL_SCOPE])) # Now register the full global scope options. GlobalOptionsRegistrar.register_options(register_func(Options.GLOBAL_SCOPE)) # Now register task and subsystem options for relevant tasks. for task_type in for_task_types: scope = task_type.options_scope if scope is None: raise TaskError('You must set a scope on your task type before using it in tests.') task_type.register_options(register_func(scope)) for subsystem in (set(task_type.global_subsystems()) | set(task_type.task_subsystems()) | self._build_configuration.subsystems()): if subsystem not in registered_subsystems: subsystem.register_options(register_func(subsystem.options_scope)) registered_subsystems.add(subsystem) # Now default option values override with any caller-specified values. # TODO(benjy): Get rid of the options arg, and require tests to call set_options. for scope, opts in options.items(): for key, val in opts.items(): option_values[scope][key] = val for scope, opts in self.options.items(): for key, val in opts.items(): option_values[scope][key] = val # Make inner scopes inherit option values from their enclosing scopes. all_scopes = set(option_values.keys()) for task_type in for_task_types: # Make sure we know about pre-task subsystem scopes. all_scopes.update([si.scope for si in task_type.known_scope_infos()]) # Iterating in sorted order guarantees that we see outer scopes before inner scopes, # and therefore only have to inherit from our immediately enclosing scope. for scope in sorted(all_scopes): if scope != Options.GLOBAL_SCOPE: enclosing_scope = scope.rpartition('.')[0] opts = option_values[scope] for key, val in option_values.get(enclosing_scope, {}).items(): if key not in opts: # Inner scope values override the inherited ones. opts[key] = val context = create_context(options=option_values, target_roots=target_roots, build_graph=self.build_graph, build_file_parser=self.build_file_parser, address_mapper=self.address_mapper, console_outstream=console_outstream, workspace=workspace) Subsystem._options = context.options return context
def create_options_for_optionables(optionables, extra_scopes=None, options=None, options_fingerprintable=None, passthru_args=None): """Create a fake Options object for testing with appropriate defaults for the given optionables. Any scoped `options` provided will override defaults, behaving as-if set on the command line. :param iterable optionables: A series of `Optionable` types to register default options for. :param iterable extra_scopes: An optional series of extra known scopes in play. :param dict options: A dict of scope -> (dict of option name -> value) representing option values explicitly set via the command line. :param dict options_fingerprintable: A dict of scope -> (dict of option name -> option type) representing the fingerprintable options and the scopes they are registered for. :param list passthru_args: A list of passthrough args (specified after `--` on the command line). :returns: A fake `Options` object with defaults populated for the given `optionables` and any explicitly set `options` overlayed. """ all_options = defaultdict(dict) fingerprintable_options = defaultdict(dict) bootstrap_option_values = None # NB(cosmicexplorer): we do this again for all_options after calling # register_func below, this is a hack if options: for scope, opts in options.items(): all_options[scope].update(opts) if options_fingerprintable: for scope, opts in options_fingerprintable.items(): fingerprintable_options[scope].update(opts) def complete_scopes(scopes): """Return all enclosing scopes. This is similar to what `complete_scopes` does in `pants.option.options.Options` without creating `ScopeInfo`s. """ completed_scopes = set(scopes) for scope in scopes: while scope != GLOBAL_SCOPE: if scope not in completed_scopes: completed_scopes.add(scope) scope = enclosing_scope(scope) return completed_scopes def register_func(on_scope): scoped_options = all_options[on_scope] scoped_fingerprintables = fingerprintable_options[on_scope] register = _options_registration_function(scoped_options, scoped_fingerprintables) register.bootstrap = bootstrap_option_values register.scope = on_scope return register # TODO: This sequence is a bit repetitive of the real registration sequence. # Register bootstrap options and grab their default values for use in subsequent registration. GlobalOptionsRegistrar.register_bootstrap_options(register_func(GLOBAL_SCOPE)) bootstrap_option_values = _FakeOptionValues(all_options[GLOBAL_SCOPE].copy()) # Now register the full global scope options. GlobalOptionsRegistrar.register_options(register_func(GLOBAL_SCOPE)) for optionable in optionables: optionable.register_options(register_func(optionable.options_scope)) # Make inner scopes inherit option values from their enclosing scopes. all_scopes = set(all_options.keys()) # TODO(John Sirois): Kill extra scopes one this goes in: # https://github.com/pantsbuild/pants/issues/1957 # For now we need a way for users of this utility to provide extra derived scopes out of band. # With #1957 resolved, the extra scopes will be embedded in the Optionable's option_scope # directly. if extra_scopes: all_scopes.update(extra_scopes) all_scopes = complete_scopes(all_scopes) # We need to update options before completing them based on inner/outer relation. if options: for scope, opts in options.items(): all_options[scope].update(opts) # Iterating in sorted order guarantees that we see outer scopes before inner scopes, # and therefore only have to inherit from our immediately enclosing scope. for s in sorted(all_scopes): if s != GLOBAL_SCOPE: scope = enclosing_scope(s) opts = all_options[s] for key, val in all_options.get(scope, {}).items(): if key not in opts: # Inner scope values override the inherited ones. opts[key] = val return create_options(all_options, passthru_args=passthru_args, fingerprintable_options=fingerprintable_options)
def produce_and_set_bootstrap_options(self): """Cooperatively populates the internal bootstrap_options cache with a producer of `FileContent`.""" flags = set() short_flags = set() def capture_the_flags(*args, **kwargs): for arg in args: flags.add(arg) if len(arg) == 2: short_flags.add(arg) elif kwargs.get('type') == bool: flags.add('--no-{}'.format(arg[2:])) GlobalOptionsRegistrar.register_bootstrap_options(capture_the_flags) def is_bootstrap_option(arg): components = arg.split('=', 1) if components[0] in flags: return True for flag in short_flags: if arg.startswith(flag): return True return False # Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line. # Stop before '--' since args after that are pass-through and may have duplicate names to our # bootstrap options. bargs = list(filter(is_bootstrap_option, itertools.takewhile(lambda arg: arg != '--', self._args))) config_file_paths = self.get_config_file_paths(env=self._env, args=self._args) config_files_products = yield config_file_paths pre_bootstrap_config = Config.load_file_contents(config_files_products) def bootstrap_options_from_config(config): bootstrap_options = Options.create( env=self._env, config=config, known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()], args=bargs, option_tracker=self._option_tracker ) def register_global(*args, **kwargs): ## Only use of Options.register? bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs) GlobalOptionsRegistrar.register_bootstrap_options(register_global) return bootstrap_options initial_bootstrap_options = bootstrap_options_from_config(pre_bootstrap_config) bootstrap_option_values = initial_bootstrap_options.for_global_scope() # Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped # from (typically pants.ini), then config override, then rcfiles. full_configpaths = pre_bootstrap_config.sources() if bootstrap_option_values.pantsrc: rcfiles = [os.path.expanduser(rcfile) for rcfile in bootstrap_option_values.pantsrc_files] existing_rcfiles = list(filter(os.path.exists, rcfiles)) full_configpaths.extend(existing_rcfiles) full_config_files_products = yield full_configpaths self._post_bootstrap_config = Config.load_file_contents( full_config_files_products, seed_values=bootstrap_option_values ) # Now recompute the bootstrap options with the full config. This allows us to pick up # bootstrap values (such as backends) from a config override file, for example. self._bootstrap_options = bootstrap_options_from_config(self._post_bootstrap_config)
def context(self, for_task_types=None, options=None, target_roots=None, console_outstream=None, workspace=None): for_task_types = for_task_types or [] options = options or {} option_values = defaultdict(dict) registered_subsystems = set() bootstrap_option_values = None # We fill these in after registering bootstrap options. # We provide our own test-only registration implementation, bypassing argparse. # When testing we set option values directly, so we don't care about cmd-line flags, config, # env vars etc. In fact, for test isolation we explicitly don't want to look at those. # All this does is make the names available in code, with the default values. # Individual tests can then override the option values they care about. def register_func(on_scope): def register(*rargs, **rkwargs): scoped_options = option_values[on_scope] default = rkwargs.get('default') if default is None and rkwargs.get('action') == 'append': default = [] for flag_name in rargs: option_name = flag_name.lstrip('-').replace('-', '_') scoped_options[option_name] = default register.bootstrap = bootstrap_option_values register.scope = on_scope return register # TODO: This sequence is a bit repetitive of the real registration sequence. # Register bootstrap options and grab their default values for use in subsequent registration. GlobalOptionsRegistrar.register_bootstrap_options( register_func(Options.GLOBAL_SCOPE)) bootstrap_option_values = create_option_values( copy.copy(option_values[Options.GLOBAL_SCOPE])) # Now register the full global scope options. GlobalOptionsRegistrar.register_options( register_func(Options.GLOBAL_SCOPE)) # Now register task and subsystem options for relevant tasks. for task_type in for_task_types: scope = task_type.options_scope if scope is None: raise TaskError( 'You must set a scope on your task type before using it in tests.' ) task_type.register_options(register_func(scope)) for subsystem in (set(task_type.global_subsystems()) | set(task_type.task_subsystems()) | self._build_configuration.subsystems()): if subsystem not in registered_subsystems: subsystem.register_options( register_func(subsystem.options_scope)) registered_subsystems.add(subsystem) # Now default option values override with any caller-specified values. # TODO(benjy): Get rid of the options arg, and require tests to call set_options. for scope, opts in options.items(): for key, val in opts.items(): option_values[scope][key] = val for scope, opts in self.options.items(): for key, val in opts.items(): option_values[scope][key] = val # Make inner scopes inherit option values from their enclosing scopes. all_scopes = set(option_values.keys()) for task_type in for_task_types: # Make sure we know about pre-task subsystem scopes. all_scopes.update( [si.scope for si in task_type.known_scope_infos()]) # Iterating in sorted order guarantees that we see outer scopes before inner scopes, # and therefore only have to inherit from our immediately enclosing scope. for scope in sorted(all_scopes): if scope != Options.GLOBAL_SCOPE: enclosing_scope = scope.rpartition('.')[0] opts = option_values[scope] for key, val in option_values.get(enclosing_scope, {}).items(): if key not in opts: # Inner scope values override the inherited ones. opts[key] = val context = create_context(options=option_values, target_roots=target_roots, build_graph=self.build_graph, build_file_parser=self.build_file_parser, address_mapper=self.address_mapper, console_outstream=console_outstream, workspace=workspace) Subsystem._options = context.options return context