def create(
cls, env: Mapping[str, str], args: Sequence[str], *, allow_pantsrc: bool
) -> OptionsBootstrapper:
"""Parses the minimum amount of configuration necessary to create an OptionsBootstrapper.
:param env: An environment dictionary, or None to use `os.environ`.
:param args: An args array, or None to use `sys.argv`.
:param allow_pantsrc: True to allow pantsrc files to be used. Unless tests are expecting to
consume pantsrc files, they should pass False in order to avoid reading files from
absolute paths. Production usecases should pass True to allow options values to make the
decision of whether to respect pantsrc files.
"""
with warnings.catch_warnings(record=True):
env = {k: v for k, v in env.items() if k.startswith("PANTS_")}
args = tuple(args)
flags = set()
short_flags = set()
# We can't use pants.engine.fs.FileContent here because it would cause a circular dep.
@dataclass(frozen=True)
class FileContent:
path: str
content: bytes
def filecontent_for(path: str) -> FileContent:
return FileContent(
ensure_text(path),
read_file(path, binary_mode=True),
)
def capture_the_flags(*args: str, **kwargs) -> None:
for arg in args:
flags.add(arg)
if len(arg) == 2:
short_flags.add(arg)
elif kwargs.get("type") == bool:
flags.add(f"--no-{arg[2:]}")
GlobalOptions.register_bootstrap_options(capture_the_flags)
def is_bootstrap_option(arg: str) -> bool:
components = arg.split("=", 1)
if components[0] in flags:
return True
for flag in short_flags:
if arg.startswith(flag):
return True
return False
# Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
# Stop before '--' since args after that are pass-through and may have duplicate names to our
# bootstrap options.
bargs = ("./pants",) + tuple(
filter(is_bootstrap_option, itertools.takewhile(lambda arg: arg != "--", args))
)
config_file_paths = cls.get_config_file_paths(env=env, args=args)
config_files_products = [filecontent_for(p) for p in config_file_paths]
pre_bootstrap_config = Config.load_file_contents(config_files_products)
initial_bootstrap_options = cls.parse_bootstrap_options(
env, bargs, pre_bootstrap_config
)
bootstrap_option_values = initial_bootstrap_options.for_global_scope()
# Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped
# from (typically pants.toml), then config override, then rcfiles.
full_config_paths = pre_bootstrap_config.sources()
if allow_pantsrc and bootstrap_option_values.pantsrc:
rcfiles = [
os.path.expanduser(str(rcfile))
for rcfile in bootstrap_option_values.pantsrc_files
]
existing_rcfiles = list(filter(os.path.exists, rcfiles))
full_config_paths.extend(existing_rcfiles)
full_config_files_products = [filecontent_for(p) for p in full_config_paths]
post_bootstrap_config = Config.load_file_contents(
full_config_files_products,
seed_values=bootstrap_option_values.as_dict(),
)
env_tuples = tuple(sorted(env.items(), key=lambda x: x[0]))
return cls(
env_tuples=env_tuples, bootstrap_args=bargs, args=args, config=post_bootstrap_config
)
def produce_and_set_bootstrap_options(self):
"""Cooperatively populates the internal bootstrap_options cache with
a producer of `FileContent`."""
flags = set()
short_flags = set()
def capture_the_flags(*args, **kwargs):
for arg in args:
flags.add(arg)
if len(arg) == 2:
short_flags.add(arg)
elif kwargs.get('type') == bool:
flags.add('--no-{}'.format(arg[2:]))
GlobalOptionsRegistrar.register_bootstrap_options(capture_the_flags)
def is_bootstrap_option(arg):
components = arg.split('=', 1)
if components[0] in flags:
return True
for flag in short_flags:
if arg.startswith(flag):
return True
return False
# Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
# Stop before '--' since args after that are pass-through and may have duplicate names to our
# bootstrap options.
bargs = list(
filter(is_bootstrap_option,
itertools.takewhile(lambda arg: arg != '--', self._args)))
config_file_paths = self.get_config_file_paths(env=self._env,
args=self._args)
config_files_products = yield config_file_paths
pre_bootstrap_config = Config.load_file_contents(config_files_products)
def bootstrap_options_from_config(config):
bootstrap_options = Options.create(
env=self._env,
config=config,
known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()],
args=bargs,
option_tracker=self._option_tracker)
def register_global(*args, **kwargs):
## Only use of Options.register?
bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs)
GlobalOptionsRegistrar.register_bootstrap_options(register_global)
return bootstrap_options
initial_bootstrap_options = bootstrap_options_from_config(
pre_bootstrap_config)
bootstrap_option_values = initial_bootstrap_options.for_global_scope()
# Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped
# from (typically pants.ini), then config override, then rcfiles.
full_configpaths = pre_bootstrap_config.sources()
if bootstrap_option_values.pantsrc:
rcfiles = [
os.path.expanduser(rcfile)
for rcfile in bootstrap_option_values.pantsrc_files
]
existing_rcfiles = list(filter(os.path.exists, rcfiles))
full_configpaths.extend(existing_rcfiles)
full_config_files_products = yield full_configpaths
self._post_bootstrap_config = Config.load_file_contents(
full_config_files_products, seed_values=bootstrap_option_values)
# Now recompute the bootstrap options with the full config. This allows us to pick up
# bootstrap values (such as backends) from a config override file, for example.
self._bootstrap_options = bootstrap_options_from_config(
self._post_bootstrap_config)
def create(cls, env=None, args=None):
"""Parses the minimum amount of configuration necessary to create an OptionsBootstrapper.
:param env: An environment dictionary, or None to use `os.environ`.
:param args: An args array, or None to use `sys.argv`.
"""
env = {
k: v
for k, v in (os.environ if env is None else env).items()
if k.startswith('PANTS_')
}
args = tuple(sys.argv if args is None else args)
flags = set()
short_flags = set()
# TODO: This codepath probably shouldn't be using FileContent, which is a very v2 engine thing.
def filecontent_for(path):
is_executable = os.stat(
path).st_mode & stat.S_IXUSR == stat.S_IXUSR
return FileContent(
ensure_text(path),
read_file(path, binary_mode=True),
is_executable=is_executable,
)
def capture_the_flags(*args, **kwargs):
for arg in args:
flags.add(arg)
if len(arg) == 2:
short_flags.add(arg)
elif kwargs.get('type') == bool:
flags.add(f'--no-{arg[2:]}')
GlobalOptionsRegistrar.register_bootstrap_options(capture_the_flags)
def is_bootstrap_option(arg):
components = arg.split('=', 1)
if components[0] in flags:
return True
for flag in short_flags:
if arg.startswith(flag):
return True
return False
# Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
# Stop before '--' since args after that are pass-through and may have duplicate names to our
# bootstrap options.
bargs = tuple(
filter(is_bootstrap_option,
itertools.takewhile(lambda arg: arg != '--', args)))
config_file_paths = cls.get_config_file_paths(env=env, args=args)
config_files_products = [filecontent_for(p) for p in config_file_paths]
pre_bootstrap_config = Config.load_file_contents(config_files_products)
initial_bootstrap_options = cls.parse_bootstrap_options(
env, bargs, pre_bootstrap_config)
bootstrap_option_values = initial_bootstrap_options.for_global_scope()
# Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped
# from (typically pants.ini), then config override, then rcfiles.
full_configpaths = pre_bootstrap_config.sources()
if bootstrap_option_values.pantsrc:
rcfiles = [
os.path.expanduser(str(rcfile))
for rcfile in bootstrap_option_values.pantsrc_files
]
existing_rcfiles = list(filter(os.path.exists, rcfiles))
full_configpaths.extend(existing_rcfiles)
full_config_files_products = [
filecontent_for(p) for p in full_configpaths
]
post_bootstrap_config = Config.load_file_contents(
full_config_files_products, seed_values=bootstrap_option_values)
env_tuples = tuple(sorted(env.items(), key=lambda x: x[0]))
return cls(env_tuples=env_tuples,
bootstrap_args=bargs,
args=args,
config=post_bootstrap_config)
def test_get_all_help_info():
class Global(Subsystem):
"""Global options."""
options_scope = GLOBAL_SCOPE
@classmethod
def register_options(cls, register):
register("-o", "--opt1", type=int, default=42, help="Option 1")
class Foo(Subsystem):
"""A foo."""
options_scope = "foo"
@classmethod
def register_options(cls, register):
register("--opt2", type=bool, default=True, help="Option 2")
register("--opt3", advanced=True, choices=["a", "b", "c"])
class Bar(GoalSubsystem):
"""The bar goal."""
name = "bar"
options = Options.create(
env={},
config=Config.load_file_contents(""),
known_scope_infos=[
Global.get_scope_info(),
Foo.get_scope_info(),
Bar.get_scope_info()
],
args=["./pants"],
bootstrap_option_values=None,
)
Global.register_options_on_scope(options)
Foo.register_options_on_scope(options)
Bar.register_options_on_scope(options)
def fake_consumed_scopes_mapper(scope: str) -> Tuple[str, ...]:
return ("somescope", f"used_by_{scope or 'GLOBAL_SCOPE'}")
all_help_info = HelpInfoExtracter.get_all_help_info(
options, UnionMembership({}), fake_consumed_scopes_mapper)
all_help_info_dict = dataclasses.asdict(all_help_info)
expected_all_help_info_dict = {
"scope_to_help_info": {
GLOBAL_SCOPE: {
"scope":
GLOBAL_SCOPE,
"description":
"Global options.",
"is_goal":
False,
"basic": ({
"display_args": ("-o=<int>", "--opt1=<int>"),
"comma_separated_display_args": "-o=<int>, --opt1=<int>",
"scoped_cmd_line_args": ("-o", "--opt1"),
"unscoped_cmd_line_args": ("-o", "--opt1"),
"config_key": "opt1",
"env_var": "PANTS_OPT1",
"value_history": {
"ranked_values": (
{
"rank": Rank.NONE,
"value": None,
"details": None
},
{
"rank": Rank.HARDCODED,
"value": 42,
"details": None
},
),
},
"typ": int,
"default": 42,
"help": "Option 1",
"deprecated_message": None,
"removal_version": None,
"removal_hint": None,
"choices": None,
"comma_separated_choices": None,
}, ),
"advanced":
tuple(),
"deprecated":
tuple(),
},
"foo": {
"scope":
"foo",
"description":
"A foo.",
"is_goal":
False,
"basic": ({
"display_args": ("--[no-]foo-opt2", ),
"comma_separated_display_args":
"--[no-]foo-opt2",
"scoped_cmd_line_args": ("--foo-opt2", "--no-foo-opt2"),
"unscoped_cmd_line_args": ("--opt2", "--no-opt2"),
"config_key":
"opt2",
"env_var":
"PANTS_FOO_OPT2",
"value_history": {
"ranked_values": (
{
"rank": Rank.NONE,
"value": None,
"details": None
},
{
"rank": Rank.HARDCODED,
"value": True,
"details": None
},
),
},
"typ":
bool,
"default":
True,
"help":
"Option 2",
"deprecated_message":
None,
"removal_version":
None,
"removal_hint":
None,
"choices":
None,
"comma_separated_choices":
None,
}, ),
"advanced": ({
"display_args": ("--foo-opt3=<str>", ),
"comma_separated_display_args": "--foo-opt3=<str>",
"scoped_cmd_line_args": ("--foo-opt3", ),
"unscoped_cmd_line_args": ("--opt3", ),
"config_key": "opt3",
"env_var": "PANTS_FOO_OPT3",
"value_history": {
"ranked_values": ({
"rank": Rank.NONE,
"value": None,
"details": None
}, ),
},
"typ": str,
"default": None,
"help": "No help available.",
"deprecated_message": None,
"removal_version": None,
"removal_hint": None,
"choices": ("a", "b", "c"),
"comma_separated_choices": "a, b, c",
}, ),
"deprecated":
tuple(),
},
"bar": {
"scope": "bar",
"description": "The bar goal.",
"is_goal": True,
"basic": tuple(),
"advanced": tuple(),
"deprecated": tuple(),
},
},
"name_to_goal_info": {
"bar": {
"name": "bar",
"description": "The bar goal.",
"consumed_scopes": ("somescope", "used_by_bar"),
"is_implemented": True,
}
},
}
assert expected_all_help_info_dict == all_help_info_dict
def test_get_all_help_info():
class Global(Subsystem):
options_scope = GLOBAL_SCOPE
help = "Global options."
@classmethod
def register_options(cls, register):
register("-o", "--opt1", type=int, default=42, help="Option 1")
class Foo(Subsystem):
options_scope = "foo"
help = "A foo."
@classmethod
def register_options(cls, register):
register("--opt2", type=bool, default=True, help="Option 2")
register("--opt3", advanced=True, choices=["a", "b", "c"])
class Bar(GoalSubsystem):
name = "bar"
help = "The bar goal."
class QuxField(StringField):
alias = "qux"
default = "blahblah"
help = "A qux string."
class QuuxField(IntField):
alias = "quux"
required = True
help = "A quux int.\n\nMust be non-zero. Or zero. Whatever you like really."
class BazLibrary(Target):
alias = "baz_library"
help = "A library of baz-es.\n\nUse it however you like."
core_fields = [QuxField, QuuxField]
options = Options.create(
env={},
config=Config.load_file_contents(""),
known_scope_infos=[
Global.get_scope_info(),
Foo.get_scope_info(),
Bar.get_scope_info()
],
args=["./pants"],
bootstrap_option_values=None,
)
Global.register_options_on_scope(options)
Foo.register_options_on_scope(options)
Bar.register_options_on_scope(options)
def fake_consumed_scopes_mapper(scope: str) -> Tuple[str, ...]:
return ("somescope", f"used_by_{scope or 'GLOBAL_SCOPE'}")
all_help_info = HelpInfoExtracter.get_all_help_info(
options,
UnionMembership({}),
fake_consumed_scopes_mapper,
RegisteredTargetTypes({BazLibrary.alias: BazLibrary}),
)
all_help_info_dict = dataclasses.asdict(all_help_info)
expected_all_help_info_dict = {
"scope_to_help_info": {
GLOBAL_SCOPE: {
"scope":
GLOBAL_SCOPE,
"description":
"Global options.",
"is_goal":
False,
"basic": ({
"display_args": ("-o=<int>", "--opt1=<int>"),
"comma_separated_display_args": "-o=<int>, --opt1=<int>",
"scoped_cmd_line_args": ("-o", "--opt1"),
"unscoped_cmd_line_args": ("-o", "--opt1"),
"config_key": "opt1",
"env_var": "PANTS_OPT1",
"value_history": {
"ranked_values": (
{
"rank": Rank.NONE,
"value": None,
"details": None
},
{
"rank": Rank.HARDCODED,
"value": 42,
"details": None
},
),
},
"typ": int,
"default": 42,
"help": "Option 1",
"deprecation_active": False,
"deprecated_message": None,
"removal_version": None,
"removal_hint": None,
"choices": None,
"comma_separated_choices": None,
}, ),
"advanced":
tuple(),
"deprecated":
tuple(),
},
"foo": {
"scope":
"foo",
"description":
"A foo.",
"is_goal":
False,
"basic": ({
"display_args": ("--[no-]foo-opt2", ),
"comma_separated_display_args":
"--[no-]foo-opt2",
"scoped_cmd_line_args": ("--foo-opt2", "--no-foo-opt2"),
"unscoped_cmd_line_args": ("--opt2", "--no-opt2"),
"config_key":
"opt2",
"env_var":
"PANTS_FOO_OPT2",
"value_history": {
"ranked_values": (
{
"rank": Rank.NONE,
"value": None,
"details": None
},
{
"rank": Rank.HARDCODED,
"value": True,
"details": None
},
),
},
"typ":
bool,
"default":
True,
"help":
"Option 2",
"deprecation_active":
False,
"deprecated_message":
None,
"removal_version":
None,
"removal_hint":
None,
"choices":
None,
"comma_separated_choices":
None,
}, ),
"advanced": ({
"display_args": ("--foo-opt3=<str>", ),
"comma_separated_display_args": "--foo-opt3=<str>",
"scoped_cmd_line_args": ("--foo-opt3", ),
"unscoped_cmd_line_args": ("--opt3", ),
"config_key": "opt3",
"env_var": "PANTS_FOO_OPT3",
"value_history": {
"ranked_values": ({
"rank": Rank.NONE,
"value": None,
"details": None
}, ),
},
"typ": str,
"default": None,
"help": "No help available.",
"deprecation_active": False,
"deprecated_message": None,
"removal_version": None,
"removal_hint": None,
"choices": ("a", "b", "c"),
"comma_separated_choices": "a, b, c",
}, ),
"deprecated":
tuple(),
},
"bar": {
"scope": "bar",
"description": "The bar goal.",
"is_goal": True,
"basic": tuple(),
"advanced": tuple(),
"deprecated": tuple(),
},
},
"name_to_goal_info": {
"bar": {
"name": "bar",
"description": "The bar goal.",
"consumed_scopes": ("somescope", "used_by_bar"),
"is_implemented": True,
}
},
"name_to_target_type_info": {
"baz_library": {
"alias":
"baz_library",
"summary":
"A library of baz-es.",
"description":
"A library of baz-es.\n\nUse it however you like.",
"fields": (
{
"alias": "qux",
"default": "'blahblah'",
"description": "A qux string.",
"required": False,
"type_hint": "str | None",
},
{
"alias":
"quux",
"default":
None,
"description":
"A quux int.\n\nMust be non-zero. Or zero. "
"Whatever you like really.",
"required":
True,
"type_hint":
"int",
},
),
}
},
}
assert expected_all_help_info_dict == all_help_info_dict
def create(cls, env=None, args=None):
"""Parses the minimum amount of configuration necessary to create an OptionsBootstrapper.
:param env: An environment dictionary, or None to use `os.environ`.
:param args: An args array, or None to use `sys.argv`.
"""
env = {k: v for k, v in (os.environ if env is None else env).items()
if k.startswith('PANTS_')}
args = tuple(sys.argv if args is None else args)
flags = set()
short_flags = set()
def filecontent_for(path):
return FileContent(ensure_text(path), read_file(path, binary_mode=True))
def capture_the_flags(*args, **kwargs):
for arg in args:
flags.add(arg)
if len(arg) == 2:
short_flags.add(arg)
elif kwargs.get('type') == bool:
flags.add('--no-{}'.format(arg[2:]))
GlobalOptionsRegistrar.register_bootstrap_options(capture_the_flags)
def is_bootstrap_option(arg):
components = arg.split('=', 1)
if components[0] in flags:
return True
for flag in short_flags:
if arg.startswith(flag):
return True
return False
# Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
# Stop before '--' since args after that are pass-through and may have duplicate names to our
# bootstrap options.
bargs = tuple(filter(is_bootstrap_option, itertools.takewhile(lambda arg: arg != '--', args)))
config_file_paths = cls.get_config_file_paths(env=env, args=args)
config_files_products = [filecontent_for(p) for p in config_file_paths]
pre_bootstrap_config = Config.load_file_contents(config_files_products)
initial_bootstrap_options = cls.parse_bootstrap_options(env, bargs, pre_bootstrap_config)
bootstrap_option_values = initial_bootstrap_options.for_global_scope()
# Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped
# from (typically pants.ini), then config override, then rcfiles.
full_configpaths = pre_bootstrap_config.sources()
if bootstrap_option_values.pantsrc:
rcfiles = [os.path.expanduser(str(rcfile)) for rcfile in bootstrap_option_values.pantsrc_files]
existing_rcfiles = list(filter(os.path.exists, rcfiles))
full_configpaths.extend(existing_rcfiles)
full_config_files_products = [filecontent_for(p) for p in full_configpaths]
post_bootstrap_config = Config.load_file_contents(
full_config_files_products,
seed_values=bootstrap_option_values
)
env_tuples = tuple(sorted(iteritems(env), key=lambda x: x[0]))
return cls(env_tuples=env_tuples, bootstrap_args=bargs, args=args, config=post_bootstrap_config)
def produce_and_set_bootstrap_options(self):
"""Cooperatively populates the internal bootstrap_options cache with
a producer of `FileContent`."""
flags = set()
short_flags = set()
def capture_the_flags(*args, **kwargs):
for arg in args:
flags.add(arg)
if len(arg) == 2:
short_flags.add(arg)
elif kwargs.get('type') == bool:
flags.add('--no-{}'.format(arg[2:]))
GlobalOptionsRegistrar.register_bootstrap_options(capture_the_flags)
def is_bootstrap_option(arg):
components = arg.split('=', 1)
if components[0] in flags:
return True
for flag in short_flags:
if arg.startswith(flag):
return True
return False
# Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
# Stop before '--' since args after that are pass-through and may have duplicate names to our
# bootstrap options.
bargs = list(filter(is_bootstrap_option, itertools.takewhile(lambda arg: arg != '--', self._args)))
config_file_paths = self.get_config_file_paths(env=self._env, args=self._args)
config_files_products = yield config_file_paths
pre_bootstrap_config = Config.load_file_contents(config_files_products)
def bootstrap_options_from_config(config):
bootstrap_options = Options.create(
env=self._env,
config=config,
known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()],
args=bargs,
option_tracker=self._option_tracker
)
def register_global(*args, **kwargs):
## Only use of Options.register?
bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs)
GlobalOptionsRegistrar.register_bootstrap_options(register_global)
return bootstrap_options
initial_bootstrap_options = bootstrap_options_from_config(pre_bootstrap_config)
bootstrap_option_values = initial_bootstrap_options.for_global_scope()
# Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped
# from (typically pants.ini), then config override, then rcfiles.
full_configpaths = pre_bootstrap_config.sources()
if bootstrap_option_values.pantsrc:
rcfiles = [os.path.expanduser(rcfile) for rcfile in bootstrap_option_values.pantsrc_files]
existing_rcfiles = list(filter(os.path.exists, rcfiles))
full_configpaths.extend(existing_rcfiles)
full_config_files_products = yield full_configpaths
self._post_bootstrap_config = Config.load_file_contents(
full_config_files_products,
seed_values=bootstrap_option_values
)
# Now recompute the bootstrap options with the full config. This allows us to pick up
# bootstrap values (such as backends) from a config override file, for example.
self._bootstrap_options = bootstrap_options_from_config(self._post_bootstrap_config)
