def bootstrap_options_from_config(config):
bootstrap_options = Options.create(env=self._env, config=config,
known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()], args=bargs)
def register_global(*args, **kwargs):
bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs)
GlobalOptionsRegistrar.register_bootstrap_options(register_global)
return bootstrap_options
def create_options_for_optionables(optionables, extra_scopes=None, options=None):
"""Create a fake Options object for testing with appropriate defaults for the given optionables.
Any scoped `options` provided will override defaults, behaving as-if set on the command line.
:param iterable optionables: A series of `Optionable` types to register default options for.
:param iterable extra_scopes: An optional series of extra known scopes in play.
:param dict options: A dict of scope -> (dict of option name -> value) representing option values
explicitly set via the command line.
:returns: A fake `Options` object with defaults populated for the given `optionables` and any
explicitly set `options` overlayed.
"""
all_options = defaultdict(dict)
bootstrap_option_values = None
def register_func(on_scope):
scoped_options = all_options[on_scope]
register = _options_registration_function(scoped_options)
register.bootstrap = bootstrap_option_values
register.scope = on_scope
return register
# TODO: This sequence is a bit repetitive of the real registration sequence.
# Register bootstrap options and grab their default values for use in subsequent registration.
GlobalOptionsRegistrar.register_bootstrap_options(register_func(GLOBAL_SCOPE))
bootstrap_option_values = create_option_values(all_options[GLOBAL_SCOPE].copy())
# Now register the full global scope options.
GlobalOptionsRegistrar.register_options(register_func(GLOBAL_SCOPE))
for optionable in optionables:
optionable.register_options(register_func(optionable.options_scope))
# Make inner scopes inherit option values from their enclosing scopes.
all_scopes = set(all_options.keys())
# TODO(John Sirois): Kill extra scopes one this goes in:
# https://github.com/pantsbuild/pants/issues/1957
# For now we need a way for users of this utility to provide extra derived scopes out of band.
# With #1957 resolved, the extra scopes will be embedded in the Optionable's option_scope
# directly.
if extra_scopes:
all_scopes.update(extra_scopes)
# Iterating in sorted order guarantees that we see outer scopes before inner scopes,
# and therefore only have to inherit from our immediately enclosing scope.
for s in sorted(all_scopes):
if s != GLOBAL_SCOPE:
enclosing_scope = s.rpartition(".")[0]
opts = all_options[s]
for key, val in all_options.get(enclosing_scope, {}).items():
if key not in opts: # Inner scope values override the inherited ones.
opts[key] = val
if options:
for scope, opts in options.items():
all_options[scope].update(opts)
return create_options(all_options)
def create_options_for_optionables(optionables,
options=None,
options_fingerprintable=None,
passthru_args=None):
"""Create a fake Options object for testing with appropriate defaults for the given optionables.
Any scoped `options` provided will override defaults, behaving as-if set on the command line.
:param iterable optionables: A series of `Optionable` types to register default options for.
:param dict options: A dict of scope -> (dict of option name -> value) representing option values
explicitly set via the command line.
:param dict options_fingerprintable: A dict of scope -> (dict of option name -> option type)
representing the fingerprintable options
and the scopes they are registered for.
:param list passthru_args: A list of passthrough args (specified after `--` on the command line).
:returns: A fake `Options` object with defaults populated for the given `optionables` and any
explicitly set `options` overlayed.
"""
all_options = defaultdict(dict)
fingerprintable_options = defaultdict(dict)
bootstrap_option_values = None
if options_fingerprintable:
for scope, opts in options_fingerprintable.items():
fingerprintable_options[scope].update(opts)
def register_func(on_scope):
scoped_options = all_options[on_scope]
scoped_fingerprintables = fingerprintable_options[on_scope]
register = _options_registration_function(scoped_options,
scoped_fingerprintables)
register.bootstrap = bootstrap_option_values
register.scope = on_scope
return register
# TODO: This sequence is a bit repetitive of the real registration sequence.
# Register bootstrap options and grab their default values for use in subsequent registration.
GlobalOptionsRegistrar.register_bootstrap_options(
register_func(GLOBAL_SCOPE))
bootstrap_option_values = _FakeOptionValues(
all_options[GLOBAL_SCOPE].copy())
# Now register the full global scope options.
GlobalOptionsRegistrar.register_options(register_func(GLOBAL_SCOPE))
for optionable in optionables:
optionable.register_options(register_func(optionable.options_scope))
if options:
for scope, opts in options.items():
all_options[scope].update(opts)
return create_options(all_options,
passthru_args=passthru_args,
fingerprintable_options=fingerprintable_options)
def create_options_for_optionables(optionables,
options=None,
options_fingerprintable=None,
passthru_args=None):
"""Create a fake Options object for testing with appropriate defaults for the given optionables.
Any scoped `options` provided will override defaults, behaving as-if set on the command line.
:param iterable optionables: A series of `Optionable` types to register default options for.
:param dict options: A dict of scope -> (dict of option name -> value) representing option values
explicitly set via the command line.
:param dict options_fingerprintable: A dict of scope -> (dict of option name -> option type)
representing the fingerprintable options
and the scopes they are registered for.
:param list passthru_args: A list of passthrough args (specified after `--` on the command line).
:returns: A fake `Options` object with defaults populated for the given `optionables` and any
explicitly set `options` overlayed.
"""
all_options = defaultdict(dict)
fingerprintable_options = defaultdict(dict)
bootstrap_option_values = None
if options_fingerprintable:
for scope, opts in options_fingerprintable.items():
fingerprintable_options[scope].update(opts)
def register_func(on_scope):
scoped_options = all_options[on_scope]
scoped_fingerprintables = fingerprintable_options[on_scope]
register = _options_registration_function(scoped_options, scoped_fingerprintables)
register.bootstrap = bootstrap_option_values
register.scope = on_scope
return register
# TODO: This sequence is a bit repetitive of the real registration sequence.
# Register bootstrap options and grab their default values for use in subsequent registration.
GlobalOptionsRegistrar.register_bootstrap_options(register_func(GLOBAL_SCOPE))
bootstrap_option_values = _FakeOptionValues(all_options[GLOBAL_SCOPE].copy())
# Now register the full global scope options.
GlobalOptionsRegistrar.register_options(register_func(GLOBAL_SCOPE))
for optionable in optionables:
optionable.register_options(register_func(optionable.options_scope))
if options:
for scope, opts in options.items():
all_options[scope].update(opts)
return create_options(all_options,
passthru_args=passthru_args,
fingerprintable_options=fingerprintable_options)
def parse_bootstrap_options(env, args, config):
bootstrap_options = Options.create(
env=env,
config=config,
known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()],
args=args,
)
def register_global(*args, **kwargs):
## Only use of Options.register?
bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs)
GlobalOptionsRegistrar.register_bootstrap_options(register_global)
return bootstrap_options
def parse_bootstrap_options(env, args, config):
bootstrap_options = Options.create(
env=env,
config=config,
known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()],
args=args,
)
def register_global(*args, **kwargs):
## Only use of Options.register?
bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs)
GlobalOptionsRegistrar.register_bootstrap_options(register_global)
return bootstrap_options
def produce_and_set_bootstrap_options(self):
"""Cooperatively populates the internal bootstrap_options cache with
a producer of `FileContent`."""
flags = set()
short_flags = set()
def capture_the_flags(*args, **kwargs):
for arg in args:
flags.add(arg)
if len(arg) == 2:
short_flags.add(arg)
elif kwargs.get('type') == bool:
flags.add('--no-{}'.format(arg[2:]))
GlobalOptionsRegistrar.register_bootstrap_options(capture_the_flags)
def is_bootstrap_option(arg):
components = arg.split('=', 1)
if components[0] in flags:
return True
for flag in short_flags:
if arg.startswith(flag):
return True
return False
# Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
# Stop before '--' since args after that are pass-through and may have duplicate names to our
# bootstrap options.
bargs = list(
filter(is_bootstrap_option,
itertools.takewhile(lambda arg: arg != '--', self._args)))
config_file_paths = self.get_config_file_paths(env=self._env,
args=self._args)
config_files_products = yield config_file_paths
pre_bootstrap_config = Config.load_file_contents(config_files_products)
def bootstrap_options_from_config(config):
bootstrap_options = Options.create(
env=self._env,
config=config,
known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()],
args=bargs,
option_tracker=self._option_tracker)
def register_global(*args, **kwargs):
## Only use of Options.register?
bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs)
GlobalOptionsRegistrar.register_bootstrap_options(register_global)
return bootstrap_options
initial_bootstrap_options = bootstrap_options_from_config(
pre_bootstrap_config)
bootstrap_option_values = initial_bootstrap_options.for_global_scope()
# Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped
# from (typically pants.ini), then config override, then rcfiles.
full_configpaths = pre_bootstrap_config.sources()
if bootstrap_option_values.pantsrc:
rcfiles = [
os.path.expanduser(rcfile)
for rcfile in bootstrap_option_values.pantsrc_files
]
existing_rcfiles = list(filter(os.path.exists, rcfiles))
full_configpaths.extend(existing_rcfiles)
full_config_files_products = yield full_configpaths
self._post_bootstrap_config = Config.load_file_contents(
full_config_files_products, seed_values=bootstrap_option_values)
# Now recompute the bootstrap options with the full config. This allows us to pick up
# bootstrap values (such as backends) from a config override file, for example.
self._bootstrap_options = bootstrap_options_from_config(
self._post_bootstrap_config)
def create_options_for_optionables(optionables,
extra_scopes=None,
options=None):
"""Create a fake Options object for testing with appropriate defaults for the given optionables.
Any scoped `options` provided will override defaults, behaving as-if set on the command line.
:param iterable optionables: A series of `Optionable` types to register default options for.
:param iterable extra_scopes: An optional series of extra known scopes in play.
:param dict options: A dict of scope -> (dict of option name -> value) representing option values
explicitly set via the command line.
:returns: A fake `Options` object with defaults populated for the given `optionables` and any
explicitly set `options` overlayed.
"""
all_options = defaultdict(dict)
bootstrap_option_values = None
def complete_scopes(scopes):
"""Return all enclosing scopes.
This is similar to what `complete_scopes` does in `pants.option.options.Options` without
creating `ScopeInfo`s.
"""
completed_scopes = set(scopes)
for scope in scopes:
while scope != '':
if scope not in completed_scopes:
completed_scopes.add(scope)
scope = enclosing_scope(scope)
return completed_scopes
def register_func(on_scope):
scoped_options = all_options[on_scope]
register = _options_registration_function(scoped_options)
register.bootstrap = bootstrap_option_values
register.scope = on_scope
return register
# TODO: This sequence is a bit repetitive of the real registration sequence.
# Register bootstrap options and grab their default values for use in subsequent registration.
GlobalOptionsRegistrar.register_bootstrap_options(
register_func(GLOBAL_SCOPE))
bootstrap_option_values = create_option_values(
all_options[GLOBAL_SCOPE].copy())
# Now register the full global scope options.
GlobalOptionsRegistrar.register_options(register_func(GLOBAL_SCOPE))
for optionable in optionables:
optionable.register_options(register_func(optionable.options_scope))
# Make inner scopes inherit option values from their enclosing scopes.
all_scopes = set(all_options.keys())
# TODO(John Sirois): Kill extra scopes one this goes in:
# https://github.com/pantsbuild/pants/issues/1957
# For now we need a way for users of this utility to provide extra derived scopes out of band.
# With #1957 resolved, the extra scopes will be embedded in the Optionable's option_scope
# directly.
if extra_scopes:
all_scopes.update(extra_scopes)
all_scopes = complete_scopes(all_scopes)
# Iterating in sorted order guarantees that we see outer scopes before inner scopes,
# and therefore only have to inherit from our immediately enclosing scope.
for s in sorted(all_scopes):
if s != GLOBAL_SCOPE:
scope = enclosing_scope(s)
opts = all_options[s]
for key, val in all_options.get(scope, {}).items():
if key not in opts: # Inner scope values override the inherited ones.
opts[key] = val
if options:
for scope, opts in options.items():
all_options[scope].update(opts)
return create_options(all_options)
def get_bootstrap_options(self):
""":returns: an Options instance that only knows about the bootstrap options.
:rtype: :class:`Options`
"""
if not self._bootstrap_options:
flags = set()
short_flags = set()
def capture_the_flags(*args, **kwargs):
for arg in args:
flags.add(arg)
if len(arg) == 2:
short_flags.add(arg)
elif is_boolean_flag(kwargs):
flags.add('--no-{}'.format(arg[2:]))
GlobalOptionsRegistrar.register_bootstrap_options(capture_the_flags)
def is_bootstrap_option(arg):
components = arg.split('=', 1)
if components[0] in flags:
return True
for flag in short_flags:
if arg.startswith(flag):
return True
return False
# Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
# Stop before '--' since args after that are pass-through and may have duplicate names to our
# bootstrap options.
bargs = filter(is_bootstrap_option, itertools.takewhile(lambda arg: arg != '--', self._args))
configpaths = [self._configpath] if self._configpath else None
pre_bootstrap_config = Config.load(configpaths)
def bootstrap_options_from_config(config):
bootstrap_options = Options.create(env=self._env, config=config,
known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()], args=bargs)
def register_global(*args, **kwargs):
bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs)
GlobalOptionsRegistrar.register_bootstrap_options(register_global)
return bootstrap_options
initial_bootstrap_options = bootstrap_options_from_config(pre_bootstrap_config)
bootstrap_option_values = initial_bootstrap_options.for_global_scope()
# Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped
# from (typically pants.ini), then config override, then rcfiles.
full_configpaths = pre_bootstrap_config.sources()
if bootstrap_option_values.config_override:
full_configpaths.append(bootstrap_option_values.config_override)
if bootstrap_option_values.pantsrc:
rcfiles = [os.path.expanduser(rcfile) for rcfile in bootstrap_option_values.pantsrc_files]
existing_rcfiles = filter(os.path.exists, rcfiles)
full_configpaths.extend(existing_rcfiles)
self._post_bootstrap_config = Config.load(full_configpaths,
seed_values=bootstrap_option_values)
# Now recompute the bootstrap options with the full config. This allows us to pick up
# bootstrap values (such as backends) from a config override file, for example.
self._bootstrap_options = bootstrap_options_from_config(self._post_bootstrap_config)
return self._bootstrap_options
def get_bootstrap_options(self):
""":returns: an Options instance that only knows about the bootstrap options.
:rtype: :class:`Options`
"""
if not self._bootstrap_options:
flags = set()
short_flags = set()
def capture_the_flags(*args, **kwargs):
for arg in args:
flags.add(arg)
if len(arg) == 2:
short_flags.add(arg)
elif is_boolean_flag(kwargs):
flags.add('--no-{}'.format(arg[2:]))
GlobalOptionsRegistrar.register_bootstrap_options(
capture_the_flags)
def is_bootstrap_option(arg):
components = arg.split('=', 1)
if components[0] in flags:
return True
for flag in short_flags:
if arg.startswith(flag):
return True
return False
# Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
# Stop before '--' since args after that are pass-through and may have duplicate names to our
# bootstrap options.
bargs = filter(
is_bootstrap_option,
itertools.takewhile(lambda arg: arg != '--', self._args))
configpaths = [self._configpath] if self._configpath else None
pre_bootstrap_config = Config.load(configpaths)
def bootstrap_options_from_config(config):
bootstrap_options = Options.create(
env=self._env,
config=config,
known_scope_infos=[
GlobalOptionsRegistrar.get_scope_info()
],
args=bargs,
option_tracker=self._option_tracker)
def register_global(*args, **kwargs):
bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs)
GlobalOptionsRegistrar.register_bootstrap_options(
register_global)
return bootstrap_options
initial_bootstrap_options = bootstrap_options_from_config(
pre_bootstrap_config)
bootstrap_option_values = initial_bootstrap_options.for_global_scope(
)
# Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped
# from (typically pants.ini), then config override, then rcfiles.
full_configpaths = pre_bootstrap_config.sources()
if bootstrap_option_values.config_override:
full_configpaths.extend(
bootstrap_option_values.config_override)
if bootstrap_option_values.pantsrc:
rcfiles = [
os.path.expanduser(rcfile)
for rcfile in bootstrap_option_values.pantsrc_files
]
existing_rcfiles = filter(os.path.exists, rcfiles)
full_configpaths.extend(existing_rcfiles)
self._post_bootstrap_config = Config.load(
full_configpaths, seed_values=bootstrap_option_values)
# Now recompute the bootstrap options with the full config. This allows us to pick up
# bootstrap values (such as backends) from a config override file, for example.
self._bootstrap_options = bootstrap_options_from_config(
self._post_bootstrap_config)
return self._bootstrap_options
def create(cls, env=None, args=None):
"""Parses the minimum amount of configuration necessary to create an OptionsBootstrapper.
:param env: An environment dictionary, or None to use `os.environ`.
:param args: An args array, or None to use `sys.argv`.
"""
env = {
k: v
for k, v in (os.environ if env is None else env).items()
if k.startswith('PANTS_')
}
args = tuple(sys.argv if args is None else args)
flags = set()
short_flags = set()
# TODO: This codepath probably shouldn't be using FileContent, which is a very v2 engine thing.
def filecontent_for(path):
is_executable = os.stat(
path).st_mode & stat.S_IXUSR == stat.S_IXUSR
return FileContent(
ensure_text(path),
read_file(path, binary_mode=True),
is_executable=is_executable,
)
def capture_the_flags(*args, **kwargs):
for arg in args:
flags.add(arg)
if len(arg) == 2:
short_flags.add(arg)
elif kwargs.get('type') == bool:
flags.add(f'--no-{arg[2:]}')
GlobalOptionsRegistrar.register_bootstrap_options(capture_the_flags)
def is_bootstrap_option(arg):
components = arg.split('=', 1)
if components[0] in flags:
return True
for flag in short_flags:
if arg.startswith(flag):
return True
return False
# Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
# Stop before '--' since args after that are pass-through and may have duplicate names to our
# bootstrap options.
bargs = tuple(
filter(is_bootstrap_option,
itertools.takewhile(lambda arg: arg != '--', args)))
config_file_paths = cls.get_config_file_paths(env=env, args=args)
config_files_products = [filecontent_for(p) for p in config_file_paths]
pre_bootstrap_config = Config.load_file_contents(config_files_products)
initial_bootstrap_options = cls.parse_bootstrap_options(
env, bargs, pre_bootstrap_config)
bootstrap_option_values = initial_bootstrap_options.for_global_scope()
# Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped
# from (typically pants.ini), then config override, then rcfiles.
full_configpaths = pre_bootstrap_config.sources()
if bootstrap_option_values.pantsrc:
rcfiles = [
os.path.expanduser(str(rcfile))
for rcfile in bootstrap_option_values.pantsrc_files
]
existing_rcfiles = list(filter(os.path.exists, rcfiles))
full_configpaths.extend(existing_rcfiles)
full_config_files_products = [
filecontent_for(p) for p in full_configpaths
]
post_bootstrap_config = Config.load_file_contents(
full_config_files_products, seed_values=bootstrap_option_values)
env_tuples = tuple(sorted(env.items(), key=lambda x: x[0]))
return cls(env_tuples=env_tuples,
bootstrap_args=bargs,
args=args,
config=post_bootstrap_config)
def create(cls, env=None, args=None):
"""Parses the minimum amount of configuration necessary to create an OptionsBootstrapper.
:param env: An environment dictionary, or None to use `os.environ`.
:param args: An args array, or None to use `sys.argv`.
"""
env = {k: v for k, v in (os.environ if env is None else env).items()
if k.startswith('PANTS_')}
args = tuple(sys.argv if args is None else args)
flags = set()
short_flags = set()
def filecontent_for(path):
return FileContent(ensure_text(path), read_file(path, binary_mode=True))
def capture_the_flags(*args, **kwargs):
for arg in args:
flags.add(arg)
if len(arg) == 2:
short_flags.add(arg)
elif kwargs.get('type') == bool:
flags.add('--no-{}'.format(arg[2:]))
GlobalOptionsRegistrar.register_bootstrap_options(capture_the_flags)
def is_bootstrap_option(arg):
components = arg.split('=', 1)
if components[0] in flags:
return True
for flag in short_flags:
if arg.startswith(flag):
return True
return False
# Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
# Stop before '--' since args after that are pass-through and may have duplicate names to our
# bootstrap options.
bargs = tuple(filter(is_bootstrap_option, itertools.takewhile(lambda arg: arg != '--', args)))
config_file_paths = cls.get_config_file_paths(env=env, args=args)
config_files_products = [filecontent_for(p) for p in config_file_paths]
pre_bootstrap_config = Config.load_file_contents(config_files_products)
initial_bootstrap_options = cls.parse_bootstrap_options(env, bargs, pre_bootstrap_config)
bootstrap_option_values = initial_bootstrap_options.for_global_scope()
# Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped
# from (typically pants.ini), then config override, then rcfiles.
full_configpaths = pre_bootstrap_config.sources()
if bootstrap_option_values.pantsrc:
rcfiles = [os.path.expanduser(str(rcfile)) for rcfile in bootstrap_option_values.pantsrc_files]
existing_rcfiles = list(filter(os.path.exists, rcfiles))
full_configpaths.extend(existing_rcfiles)
full_config_files_products = [filecontent_for(p) for p in full_configpaths]
post_bootstrap_config = Config.load_file_contents(
full_config_files_products,
seed_values=bootstrap_option_values
)
env_tuples = tuple(sorted(iteritems(env), key=lambda x: x[0]))
return cls(env_tuples=env_tuples, bootstrap_args=bargs, args=args, config=post_bootstrap_config)
def context(self, for_task_types=None, options=None, target_roots=None,
console_outstream=None, workspace=None):
for_task_types = for_task_types or []
options = options or {}
option_values = defaultdict(dict)
registered_subsystems = set()
bootstrap_option_values = None # We fill these in after registering bootstrap options.
# We provide our own test-only registration implementation, bypassing argparse.
# When testing we set option values directly, so we don't care about cmd-line flags, config,
# env vars etc. In fact, for test isolation we explicitly don't want to look at those.
# All this does is make the names available in code, with the default values.
# Individual tests can then override the option values they care about.
def register_func(on_scope):
def register(*rargs, **rkwargs):
scoped_options = option_values[on_scope]
default = rkwargs.get('default')
if default is None and rkwargs.get('action') == 'append':
default = []
for flag_name in rargs:
option_name = flag_name.lstrip('-').replace('-', '_')
scoped_options[option_name] = default
register.bootstrap = bootstrap_option_values
register.scope = on_scope
return register
# TODO: This sequence is a bit repetitive of the real registration sequence.
# Register bootstrap options and grab their default values for use in subsequent registration.
GlobalOptionsRegistrar.register_bootstrap_options(register_func(Options.GLOBAL_SCOPE))
bootstrap_option_values = create_option_values(copy.copy(option_values[Options.GLOBAL_SCOPE]))
# Now register the full global scope options.
GlobalOptionsRegistrar.register_options(register_func(Options.GLOBAL_SCOPE))
# Now register task and subsystem options for relevant tasks.
for task_type in for_task_types:
scope = task_type.options_scope
if scope is None:
raise TaskError('You must set a scope on your task type before using it in tests.')
task_type.register_options(register_func(scope))
for subsystem in (set(task_type.global_subsystems()) |
set(task_type.task_subsystems()) |
self._build_configuration.subsystems()):
if subsystem not in registered_subsystems:
subsystem.register_options(register_func(subsystem.options_scope))
registered_subsystems.add(subsystem)
# Now default option values override with any caller-specified values.
# TODO(benjy): Get rid of the options arg, and require tests to call set_options.
for scope, opts in options.items():
for key, val in opts.items():
option_values[scope][key] = val
for scope, opts in self.options.items():
for key, val in opts.items():
option_values[scope][key] = val
# Make inner scopes inherit option values from their enclosing scopes.
all_scopes = set(option_values.keys())
for task_type in for_task_types: # Make sure we know about pre-task subsystem scopes.
all_scopes.update([si.scope for si in task_type.known_scope_infos()])
# Iterating in sorted order guarantees that we see outer scopes before inner scopes,
# and therefore only have to inherit from our immediately enclosing scope.
for scope in sorted(all_scopes):
if scope != Options.GLOBAL_SCOPE:
enclosing_scope = scope.rpartition('.')[0]
opts = option_values[scope]
for key, val in option_values.get(enclosing_scope, {}).items():
if key not in opts: # Inner scope values override the inherited ones.
opts[key] = val
context = create_context(options=option_values,
target_roots=target_roots,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
address_mapper=self.address_mapper,
console_outstream=console_outstream,
workspace=workspace)
Subsystem._options = context.options
return context
def create_options_for_optionables(optionables,
extra_scopes=None,
options=None,
options_fingerprintable=None,
passthru_args=None):
"""Create a fake Options object for testing with appropriate defaults for the given optionables.
Any scoped `options` provided will override defaults, behaving as-if set on the command line.
:param iterable optionables: A series of `Optionable` types to register default options for.
:param iterable extra_scopes: An optional series of extra known scopes in play.
:param dict options: A dict of scope -> (dict of option name -> value) representing option values
explicitly set via the command line.
:param dict options_fingerprintable: A dict of scope -> (dict of option name -> option type)
representing the fingerprintable options
and the scopes they are registered for.
:param list passthru_args: A list of passthrough args (specified after `--` on the command line).
:returns: A fake `Options` object with defaults populated for the given `optionables` and any
explicitly set `options` overlayed.
"""
all_options = defaultdict(dict)
fingerprintable_options = defaultdict(dict)
bootstrap_option_values = None
# NB(cosmicexplorer): we do this again for all_options after calling
# register_func below, this is a hack
if options:
for scope, opts in options.items():
all_options[scope].update(opts)
if options_fingerprintable:
for scope, opts in options_fingerprintable.items():
fingerprintable_options[scope].update(opts)
def complete_scopes(scopes):
"""Return all enclosing scopes.
This is similar to what `complete_scopes` does in `pants.option.options.Options` without
creating `ScopeInfo`s.
"""
completed_scopes = set(scopes)
for scope in scopes:
while scope != GLOBAL_SCOPE:
if scope not in completed_scopes:
completed_scopes.add(scope)
scope = enclosing_scope(scope)
return completed_scopes
def register_func(on_scope):
scoped_options = all_options[on_scope]
scoped_fingerprintables = fingerprintable_options[on_scope]
register = _options_registration_function(scoped_options, scoped_fingerprintables)
register.bootstrap = bootstrap_option_values
register.scope = on_scope
return register
# TODO: This sequence is a bit repetitive of the real registration sequence.
# Register bootstrap options and grab their default values for use in subsequent registration.
GlobalOptionsRegistrar.register_bootstrap_options(register_func(GLOBAL_SCOPE))
bootstrap_option_values = _FakeOptionValues(all_options[GLOBAL_SCOPE].copy())
# Now register the full global scope options.
GlobalOptionsRegistrar.register_options(register_func(GLOBAL_SCOPE))
for optionable in optionables:
optionable.register_options(register_func(optionable.options_scope))
# Make inner scopes inherit option values from their enclosing scopes.
all_scopes = set(all_options.keys())
# TODO(John Sirois): Kill extra scopes one this goes in:
# https://github.com/pantsbuild/pants/issues/1957
# For now we need a way for users of this utility to provide extra derived scopes out of band.
# With #1957 resolved, the extra scopes will be embedded in the Optionable's option_scope
# directly.
if extra_scopes:
all_scopes.update(extra_scopes)
all_scopes = complete_scopes(all_scopes)
# We need to update options before completing them based on inner/outer relation.
if options:
for scope, opts in options.items():
all_options[scope].update(opts)
# Iterating in sorted order guarantees that we see outer scopes before inner scopes,
# and therefore only have to inherit from our immediately enclosing scope.
for s in sorted(all_scopes):
if s != GLOBAL_SCOPE:
scope = enclosing_scope(s)
opts = all_options[s]
for key, val in all_options.get(scope, {}).items():
if key not in opts: # Inner scope values override the inherited ones.
opts[key] = val
return create_options(all_options,
passthru_args=passthru_args,
fingerprintable_options=fingerprintable_options)
def produce_and_set_bootstrap_options(self):
"""Cooperatively populates the internal bootstrap_options cache with
a producer of `FileContent`."""
flags = set()
short_flags = set()
def capture_the_flags(*args, **kwargs):
for arg in args:
flags.add(arg)
if len(arg) == 2:
short_flags.add(arg)
elif kwargs.get('type') == bool:
flags.add('--no-{}'.format(arg[2:]))
GlobalOptionsRegistrar.register_bootstrap_options(capture_the_flags)
def is_bootstrap_option(arg):
components = arg.split('=', 1)
if components[0] in flags:
return True
for flag in short_flags:
if arg.startswith(flag):
return True
return False
# Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
# Stop before '--' since args after that are pass-through and may have duplicate names to our
# bootstrap options.
bargs = list(filter(is_bootstrap_option, itertools.takewhile(lambda arg: arg != '--', self._args)))
config_file_paths = self.get_config_file_paths(env=self._env, args=self._args)
config_files_products = yield config_file_paths
pre_bootstrap_config = Config.load_file_contents(config_files_products)
def bootstrap_options_from_config(config):
bootstrap_options = Options.create(
env=self._env,
config=config,
known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()],
args=bargs,
option_tracker=self._option_tracker
)
def register_global(*args, **kwargs):
## Only use of Options.register?
bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs)
GlobalOptionsRegistrar.register_bootstrap_options(register_global)
return bootstrap_options
initial_bootstrap_options = bootstrap_options_from_config(pre_bootstrap_config)
bootstrap_option_values = initial_bootstrap_options.for_global_scope()
# Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped
# from (typically pants.ini), then config override, then rcfiles.
full_configpaths = pre_bootstrap_config.sources()
if bootstrap_option_values.pantsrc:
rcfiles = [os.path.expanduser(rcfile) for rcfile in bootstrap_option_values.pantsrc_files]
existing_rcfiles = list(filter(os.path.exists, rcfiles))
full_configpaths.extend(existing_rcfiles)
full_config_files_products = yield full_configpaths
self._post_bootstrap_config = Config.load_file_contents(
full_config_files_products,
seed_values=bootstrap_option_values
)
# Now recompute the bootstrap options with the full config. This allows us to pick up
# bootstrap values (such as backends) from a config override file, for example.
self._bootstrap_options = bootstrap_options_from_config(self._post_bootstrap_config)
def context(self,
for_task_types=None,
options=None,
target_roots=None,
console_outstream=None,
workspace=None):
for_task_types = for_task_types or []
options = options or {}
option_values = defaultdict(dict)
registered_subsystems = set()
bootstrap_option_values = None # We fill these in after registering bootstrap options.
# We provide our own test-only registration implementation, bypassing argparse.
# When testing we set option values directly, so we don't care about cmd-line flags, config,
# env vars etc. In fact, for test isolation we explicitly don't want to look at those.
# All this does is make the names available in code, with the default values.
# Individual tests can then override the option values they care about.
def register_func(on_scope):
def register(*rargs, **rkwargs):
scoped_options = option_values[on_scope]
default = rkwargs.get('default')
if default is None and rkwargs.get('action') == 'append':
default = []
for flag_name in rargs:
option_name = flag_name.lstrip('-').replace('-', '_')
scoped_options[option_name] = default
register.bootstrap = bootstrap_option_values
register.scope = on_scope
return register
# TODO: This sequence is a bit repetitive of the real registration sequence.
# Register bootstrap options and grab their default values for use in subsequent registration.
GlobalOptionsRegistrar.register_bootstrap_options(
register_func(Options.GLOBAL_SCOPE))
bootstrap_option_values = create_option_values(
copy.copy(option_values[Options.GLOBAL_SCOPE]))
# Now register the full global scope options.
GlobalOptionsRegistrar.register_options(
register_func(Options.GLOBAL_SCOPE))
# Now register task and subsystem options for relevant tasks.
for task_type in for_task_types:
scope = task_type.options_scope
if scope is None:
raise TaskError(
'You must set a scope on your task type before using it in tests.'
)
task_type.register_options(register_func(scope))
for subsystem in (set(task_type.global_subsystems())
| set(task_type.task_subsystems())
| self._build_configuration.subsystems()):
if subsystem not in registered_subsystems:
subsystem.register_options(
register_func(subsystem.options_scope))
registered_subsystems.add(subsystem)
# Now default option values override with any caller-specified values.
# TODO(benjy): Get rid of the options arg, and require tests to call set_options.
for scope, opts in options.items():
for key, val in opts.items():
option_values[scope][key] = val
for scope, opts in self.options.items():
for key, val in opts.items():
option_values[scope][key] = val
# Make inner scopes inherit option values from their enclosing scopes.
all_scopes = set(option_values.keys())
for task_type in for_task_types: # Make sure we know about pre-task subsystem scopes.
all_scopes.update(
[si.scope for si in task_type.known_scope_infos()])
# Iterating in sorted order guarantees that we see outer scopes before inner scopes,
# and therefore only have to inherit from our immediately enclosing scope.
for scope in sorted(all_scopes):
if scope != Options.GLOBAL_SCOPE:
enclosing_scope = scope.rpartition('.')[0]
opts = option_values[scope]
for key, val in option_values.get(enclosing_scope, {}).items():
if key not in opts: # Inner scope values override the inherited ones.
opts[key] = val
context = create_context(options=option_values,
target_roots=target_roots,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
address_mapper=self.address_mapper,
console_outstream=console_outstream,
workspace=workspace)
Subsystem._options = context.options
return context
