def resolve(cls, keystore_config_file):
"""Parse a keystore config file and return a list of Keystore objects."""
keystore_config_file = os.path.expanduser(keystore_config_file)
try:
keystore_config = Config.load([keystore_config_file])
except IOError as e:
raise KeystoreResolver.Error('Problem parsing keystore config file at {}: '
'{}'.format(keystore_config_file, e))
def create_key(key_name):
"""Instantiate Keystore objects."""
def get_config_value(section, option):
val = keystore_config.get(section, option)
if val is None or val == '':
raise KeystoreResolver.Error('Required keystore config value {}.{} is '
'not defined.'.format(section, option))
return val
keystore = Keystore(keystore_name=key_name,
build_type=get_config_value(key_name, 'build_type'),
keystore_location=get_config_value(key_name, 'keystore_location'),
keystore_alias=get_config_value(key_name, 'keystore_alias'),
keystore_password=get_config_value(key_name, 'keystore_password'),
key_password=get_config_value(key_name, 'key_password'))
return keystore
keys = {}
for name in keystore_config.sections():
try:
keys[name] = create_key(name)
except Config.ConfigError as e:
raise KeystoreResolver.Error(e)
return keys
def _create_config(self, config):
with open(os.path.join(safe_mkdtemp(), 'test_config.ini'), 'w') as fp:
for section, options in config.items():
fp.write('[{}]\n'.format(section))
for key, value in options.items():
fp.write('{}: {}\n'.format(key, value))
return Config.load(configpaths=[fp.name])
def _create_config(self, config):
with open(os.path.join(safe_mkdtemp(), 'test_config.ini'), 'w') as fp:
for section, options in config.items():
fp.write('[{}]\n'.format(section))
for key, value in options.items():
fp.write('{}: {}\n'.format(key, value))
return Config.load(configpaths=[fp.name])
def _setup_config() -> Config:
parsed_config = Config.load(
file_contents=[
FileContent("file1.toml", FILE_1.content.encode()),
FileContent("file2.toml", FILE_2.content.encode()),
],
seed_values={"buildroot": "fake_buildroot"},
)
assert ["file1.toml", "file2.toml"] == parsed_config.sources()
return parsed_config
def do_test(args, kwargs, expected_default):
# Defaults are computed in the parser and added into the kwargs, so we
# must jump through this hoop in this test.
parser = Parser(env={}, config=Config.load([]),
scope_info=GlobalOptionsRegistrar.get_scope_info(),
parent_parser=None, option_tracker=OptionTracker())
parser.register(*args, **kwargs)
oshi = HelpInfoExtracter.get_option_scope_help_info_from_parser(parser).basic
self.assertEquals(1, len(oshi))
ohi = oshi[0]
self.assertEqual(expected_default, ohi.default)
def do_test(args, kwargs, expected_default):
# Defaults are computed in the parser and added into the kwargs, so we
# must jump through this hoop in this test.
parser = Parser(env={}, config=Config.load([]),
scope_info=GlobalOptionsRegistrar.get_scope_info(),
parent_parser=None, option_tracker=OptionTracker())
parser.register(*args, **kwargs)
oshi = HelpInfoExtracter.get_option_scope_help_info_from_parser(parser).basic
self.assertEquals(1, len(oshi))
ohi = oshi[0]
self.assertEqual(expected_default, ohi.default)
def _setup_config(self, config1_content: str, config2_content: str, *, suffix: str) -> Config:
with temporary_file(binary_mode=False, suffix=suffix) as config1, \
temporary_file(binary_mode=False, suffix=suffix) as config2:
config1.write(config1_content)
config1.close()
config2.write(config2_content)
config2.close()
parsed_config = Config.load(
config_paths=[config1.name, config2.name], seed_values={"buildroot": self.build_root}
)
assert [config1.name, config2.name] == parsed_config.sources()
return parsed_config
def setUp(self):
self.ini1_content = textwrap.dedent(
"""
[DEFAULT]
name: foo
answer: 42
scale: 1.2
path: /a/b/%(answer)s
embed: %(path)s::foo
disclaimer:
Let it be known
that.
blank_section:
[a]
list: [1, 2, 3, %(answer)s]
listappend: +[7, 8, 9]
[b]
preempt: True
dict: {
'a': 1,
'b': %(answer)s,
'c': ['%(answer)s', %(answer)s]
}
"""
)
self.ini2_content = textwrap.dedent(
"""
[a]
fast: True
[b]
preempt: False
[defined_section]
"""
)
with temporary_file(binary_mode=False) as ini1, \
temporary_file(binary_mode=False) as ini2, \
temporary_file_path() as buildroot:
ini1.write(self.ini1_content)
ini1.close()
ini2.write(self.ini2_content)
ini2.close()
self.config = Config.load(
config_paths=[ini1.name, ini2.name], seed_values={"buildroot": buildroot}
)
self.assertEqual([ini1.name, ini2.name], self.config.sources())
def do_test(kwargs, expected_basic=False, expected_advanced=False):
def exp_to_len(exp):
return int(exp) # True -> 1, False -> 0.
parser = Parser(
env={},
config=Config.load([]),
scope_info=GlobalOptions.get_scope_info(),
)
parser.register("--foo", **kwargs)
oshi = HelpInfoExtracter("").get_option_scope_help_info(
"", parser, False)
assert exp_to_len(expected_basic) == len(oshi.basic)
assert exp_to_len(expected_advanced) == len(oshi.advanced)
def _setup_config() -> Config:
with temporary_file(binary_mode=False,
suffix=".toml") as config1, temporary_file(
binary_mode=False, suffix=".toml") as config2:
config1.write(FILE_1.content)
config1.close()
config2.write(FILE_2.content)
config2.close()
parsed_config = Config.load(
config_paths=[config1.name, config2.name],
seed_values={"buildroot": "fake_buildroot"},
)
assert [config1.name, config2.name] == parsed_config.sources()
return parsed_config
def test_register_options_blessed(caplog) -> None:
class GoodToGo(Subsystem):
options_scope = "good-to-go"
options = Options.create(
env={},
config=Config.load([]),
known_scope_infos=[GoodToGo.get_scope_info()],
args=["./pants"],
bootstrap_option_values=None,
)
GoodToGo.register_options_on_scope(options)
assert not caplog.records, "The current blessed means of registering options should never warn."
def _format_for_global_scope(show_advanced, show_deprecated, args, kwargs):
parser = Parser(
env={},
config=Config.load([]),
scope_info=GlobalOptions.get_scope_info(),
parent_parser=None,
)
parser.register(*args, **kwargs)
# Force a parse to generate the derivation history.
parser.parse_args(Parser.ParseArgsRequest((), OptionValueContainerBuilder(), [], False))
oshi = HelpInfoExtracter("").get_option_scope_help_info("", parser, False)
return HelpFormatter(
show_advanced=show_advanced, show_deprecated=show_deprecated, color=False
).format_options(oshi)
def setUp(self):
self.ini1_content = textwrap.dedent(
"""
[DEFAULT]
name: foo
answer: 42
scale: 1.2
path: /a/b/%(answer)s
embed: %(path)s::foo
disclaimer:
Let it be known
that.
blank_section:
[a]
list: [1, 2, 3, %(answer)s]
listappend: +[7, 8, 9]
[b]
preempt: True
dict: {
'a': 1,
'b': %(answer)s,
'c': ['%(answer)s', %(answer)s]
}
"""
)
self.ini2_content = textwrap.dedent(
"""
[a]
fast: True
[b]
preempt: False
[defined_section]
"""
)
with temporary_file(binary_mode=False) as ini1:
ini1.write(self.ini1_content)
ini1.close()
with temporary_file(binary_mode=False) as ini2:
ini2.write(self.ini2_content)
ini2.close()
self.config = Config.load(config_paths=[ini1.name, ini2.name])
self.assertEqual([ini1.name, ini2.name], self.config.sources())
def do_test(args, kwargs, expected_default_str):
# Defaults are computed in the parser and added into the kwargs, so we
# must jump through this hoop in this test.
parser = Parser(
env={},
config=Config.load([]),
scope_info=GlobalOptions.get_scope_info(),
)
parser.register(*args, **kwargs)
oshi = HelpInfoExtracter(parser.scope).get_option_scope_help_info(
"description", parser, False)
assert oshi.description == "description"
assert len(oshi.basic) == 1
ohi = oshi.basic[0]
assert to_help_str(ohi.default) == expected_default_str
def setUp(self):
with temporary_file() as ini1:
ini1.write(textwrap.dedent(
"""
[DEFAULT]
name: foo
answer: 42
scale: 1.2
path: /a/b/%(answer)s
embed: %(path)s::foo
disclaimer:
Let it be known
that.
blank_section:
[a]
list: [1, 2, 3, %(answer)s]
listappend: +[7, 8, 9]
[b]
preempt: True
dict: {
'a': 1,
'b': %(answer)s,
'c': ['%(answer)s', %(answer)s]
}
"""))
ini1.close()
with temporary_file() as ini2:
ini2.write(textwrap.dedent(
"""
[a]
fast: True
[b]
preempt: False
[defined_section]
"""))
ini2.close()
self.config = Config.load(configpaths=[ini1.name, ini2.name])
self.assertEqual([ini1.name, ini2.name], self.config.sources())
def setUp(self):
with temporary_file() as ini1:
ini1.write(textwrap.dedent(
"""
[DEFAULT]
name: foo
answer: 42
scale: 1.2
path: /a/b/%(answer)s
embed: %(path)s::foo
disclaimer:
Let it be known
that.
blank_section:
[a]
list: [1, 2, 3, %(answer)s]
[b]
preempt: True
dict: {
'a': 1,
'b': %(answer)s,
'c': ['%(answer)s', %(answer)s]
}
"""))
ini1.close()
with temporary_file() as ini2:
ini2.write(textwrap.dedent(
"""
[a]
fast: True
[b]
preempt: False
[defined_section]
"""))
ini2.close()
self.config = Config.load(configpaths=[ini1.name, ini2.name])
def resolve(cls, keystore_config_file):
"""Parse a keystore config file and return a list of Keystore objects."""
keystore_config_file = os.path.expanduser(keystore_config_file)
try:
keystore_config = Config.load([keystore_config_file])
except IOError as e:
raise KeystoreResolver.Error(
'Problem parsing keystore config file at {}: '
'{}'.format(keystore_config_file, e))
def create_key(key_name):
"""Instantiate Keystore objects."""
def get_config_value(section, option):
val = keystore_config.get(section, option)
if val is None or val == '':
raise KeystoreResolver.Error(
'Required keystore config value {}.{} is '
'not defined.'.format(section, option))
return val
keystore = Keystore(
keystore_name=key_name,
build_type=get_config_value(key_name, 'build_type'),
keystore_location=get_config_value(key_name,
'keystore_location'),
keystore_alias=get_config_value(key_name, 'keystore_alias'),
keystore_password=get_config_value(key_name,
'keystore_password'),
key_password=get_config_value(key_name, 'key_password'))
return keystore
keys = {}
for name in keystore_config.sections():
try:
keys[name] = create_key(name)
except Config.ConfigError as e:
raise KeystoreResolver.Error(e)
return keys
def test_get_all_help_info():
class Global(Subsystem):
options_scope = GLOBAL_SCOPE
help = "Global options."
opt1 = IntOption("-o", "--opt1", default=42, help="Option 1")
class Foo(Subsystem):
options_scope = "foo"
help = "A foo."
opt2 = BoolOption("--opt2",
default=True,
advanced=True,
help="Option 2")
class Bar(GoalSubsystem):
name = "bar"
help = "The bar goal."
deprecated_options_scope = "bar-old"
deprecated_options_scope_removal_version = "9.9.999"
class QuxField(StringField):
alias = "qux"
default = "blahblah"
help = "A qux string."
class QuuxField(IntField):
alias = "quux"
required = True
help = "A quux int.\n\nMust be non-zero. Or zero. Whatever you like really."
class BazLibrary(Target):
alias = "baz_library"
help = "A library of baz-es.\n\nUse it however you like."
core_fields = [QuxField, QuuxField]
options = Options.create(
env={},
config=Config.load([]),
known_scope_infos=[
Global.get_scope_info(),
Foo.get_scope_info(),
Bar.get_scope_info()
],
args=["./pants"],
bootstrap_option_values=None,
)
Global.register_options_on_scope(options)
Foo.register_options_on_scope(options)
Bar.register_options_on_scope(options)
@rule
def rule_info_test(foo: Foo) -> Target:
"""This rule is for testing info extraction only."""
def fake_consumed_scopes_mapper(scope: str) -> Tuple[str, ...]:
return ("somescope", f"used_by_{scope or 'GLOBAL_SCOPE'}")
bc_builder = BuildConfiguration.Builder()
bc_builder.register_subsystems("help_info_extracter_test", (Foo, Bar))
bc_builder.register_target_types("help_info_extracter_test",
(BazLibrary, ))
bc_builder.register_rules("help_info_extracter_test",
collect_rules(locals()))
all_help_info = HelpInfoExtracter.get_all_help_info(
options,
UnionMembership({}),
fake_consumed_scopes_mapper,
RegisteredTargetTypes({BazLibrary.alias: BazLibrary}),
bc_builder.create(),
)
all_help_info_dict = all_help_info.asdict()
expected_all_help_info_dict = {
"scope_to_help_info": {
GLOBAL_SCOPE: {
"scope":
GLOBAL_SCOPE,
"description":
"Global options.",
"provider":
"",
"is_goal":
False,
"deprecated_scope":
None,
"basic": ({
"display_args": ("-o=<int>", "--opt1=<int>"),
"comma_separated_display_args": "-o=<int>, --opt1=<int>",
"scoped_cmd_line_args": ("-o", "--opt1"),
"unscoped_cmd_line_args": ("-o", "--opt1"),
"config_key": "opt1",
"env_var": "PANTS_OPT1",
"value_history": {
"ranked_values": (
{
"rank": Rank.NONE,
"value": None,
"details": None
},
{
"rank": Rank.HARDCODED,
"value": 42,
"details": None
},
),
},
"typ": int,
"default": 42,
"help": "Option 1",
"deprecation_active": False,
"deprecated_message": None,
"removal_version": None,
"removal_hint": None,
"choices": None,
"comma_separated_choices": None,
}, ),
"advanced":
tuple(),
"deprecated":
tuple(),
},
"foo": {
"scope":
"foo",
"provider":
"help_info_extracter_test",
"description":
"A foo.",
"is_goal":
False,
"deprecated_scope":
None,
"basic": (),
"advanced": ({
"display_args": ("--[no-]foo-opt2", ),
"comma_separated_display_args":
"--[no-]foo-opt2",
"scoped_cmd_line_args": ("--foo-opt2", "--no-foo-opt2"),
"unscoped_cmd_line_args": ("--opt2", "--no-opt2"),
"config_key":
"opt2",
"env_var":
"PANTS_FOO_OPT2",
"value_history": {
"ranked_values": (
{
"rank": Rank.NONE,
"value": None,
"details": None
},
{
"rank": Rank.HARDCODED,
"value": True,
"details": None
},
),
},
"typ":
bool,
"default":
True,
"help":
"Option 2",
"deprecation_active":
False,
"deprecated_message":
None,
"removal_version":
None,
"removal_hint":
None,
"choices":
None,
"comma_separated_choices":
None,
}, ),
"deprecated":
tuple(),
},
"bar": {
"scope": "bar",
"provider": "help_info_extracter_test",
"description": "The bar goal.",
"is_goal": True,
"deprecated_scope": "bar-old",
"basic": tuple(),
"advanced": tuple(),
"deprecated": tuple(),
},
"bar-old": {
"scope": "bar-old",
"provider": "help_info_extracter_test",
"description": "The bar goal.",
"is_goal": True,
"deprecated_scope": "bar-old",
"basic": tuple(),
"advanced": tuple(),
"deprecated": tuple(),
},
},
"rule_output_type_to_rule_infos": {
"Foo": ({
"description": None,
"help": "A foo.",
"input_gets": ("Get(ScopedOptions, Scope, ..)", ),
"input_types": (),
"name": "construct_scope_foo",
"output_desc": None,
"output_type": "Foo",
"provider": "help_info_extracter_test",
}, ),
"Target": ({
"description":
None,
"help":
"This rule is for testing info extraction only.",
"input_gets": (),
"input_types": ("Foo", ),
"name":
"pants.help.help_info_extracter_test.rule_info_test",
"output_desc":
("A Target represents an addressable set of metadata.\n\n Set the "
"`help` class property with a description, which will be used in "
"`./pants help`. For the\n best rendering, use soft wrapping (e.g. "
"implicit string concatenation) within paragraphs, but\n hard wrapping "
"(`\n`) to separate distinct paragraphs and/or lists.\n "),
"output_type":
"Target",
"provider":
"help_info_extracter_test",
}, ),
},
"name_to_goal_info": {
"bar": {
"name": "bar",
"provider": "help_info_extracter_test",
"description": "The bar goal.",
"consumed_scopes": ("somescope", "used_by_bar"),
"is_implemented": True,
},
"bar-old": {
"name": "bar",
"provider": "help_info_extracter_test",
"description": "The bar goal.",
"consumed_scopes": ("somescope", "used_by_bar-old"),
"is_implemented": True,
},
},
"name_to_target_type_info": {
"baz_library": {
"alias":
"baz_library",
"provider":
"help_info_extracter_test",
"summary":
"A library of baz-es.",
"description":
"A library of baz-es.\n\nUse it however you like.",
"fields": (
{
"alias": "qux",
"provider": "",
"default": "'blahblah'",
"description": "A qux string.",
"required": False,
"type_hint": "str | None",
},
{
"alias":
"quux",
"provider":
"",
"default":
None,
"description":
"A quux int.\n\nMust be non-zero. Or zero. "
"Whatever you like really.",
"required":
True,
"type_hint":
"int",
},
),
}
},
}
assert expected_all_help_info_dict == all_help_info_dict
def parser() -> Parser:
return Parser(
env={},
config=Config.load([]),
scope_info=GlobalOptions.get_scope_info(),
)
def test_empty(self):
config = Config.load([])
self.assertEqual([], config.sections())
def create(cls, env: Mapping[str, str], args: Sequence[str], *,
allow_pantsrc: bool) -> OptionsBootstrapper:
"""Parses the minimum amount of configuration necessary to create an OptionsBootstrapper.
:param env: An environment dictionary.
:param args: An args array.
:param allow_pantsrc: True to allow pantsrc files to be used. Unless tests are expecting to
consume pantsrc files, they should pass False in order to avoid reading files from
absolute paths. Production usecases should pass True to allow options values to make the
decision of whether to respect pantsrc files.
"""
with warnings.catch_warnings(record=True):
# We can't use pants.engine.fs.FileContent here because it would cause a circular dep.
@dataclass(frozen=True)
class FileContent:
path: str
content: bytes
def filecontent_for(path: str) -> FileContent:
return FileContent(
ensure_text(path),
read_file(path, binary_mode=True),
)
env = {k: v for k, v in env.items() if k.startswith("PANTS_")}
bargs = cls._get_bootstrap_args(args)
config_file_paths = cls.get_config_file_paths(env=env, args=args)
config_files_products = [
filecontent_for(p) for p in config_file_paths
]
pre_bootstrap_config = Config.load(config_files_products)
initial_bootstrap_options = cls.parse_bootstrap_options(
env, bargs, pre_bootstrap_config)
bootstrap_option_values = initial_bootstrap_options.for_global_scope(
)
# Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped
# from (typically pants.toml), then config override, then rcfiles.
full_config_paths = pre_bootstrap_config.sources()
if allow_pantsrc and bootstrap_option_values.pantsrc:
rcfiles = [
os.path.expanduser(str(rcfile))
for rcfile in bootstrap_option_values.pantsrc_files
]
existing_rcfiles = list(filter(os.path.exists, rcfiles))
full_config_paths.extend(existing_rcfiles)
full_config_files_products = [
filecontent_for(p) for p in full_config_paths
]
post_bootstrap_config = Config.load(
full_config_files_products,
seed_values=bootstrap_option_values.as_dict(),
)
env_tuples = tuple(sorted(env.items(), key=lambda x: x[0]))
# Finally, we expand any aliases and re-populates the bootstrap args, in case there was
# any from an alias.
# stuhood: This could potentially break the rust client when aliases are used:
# https://github.com/pantsbuild/pants/pull/13228#discussion_r728223889
alias = CliAlias.from_dict(
post_bootstrap_config.get("cli", "alias", dict, {}))
args = alias.expand_args(tuple(args))
bargs = cls._get_bootstrap_args(args)
return cls(
env_tuples=env_tuples,
bootstrap_args=bargs,
args=args,
config=post_bootstrap_config,
alias=alias,
)
def test_empty(self) -> None:
config = Config.load([])
assert config.sections() == []
assert config.sources() == []
assert config.has_section("DEFAULT") is False
assert config.has_option(section="DEFAULT", option="name") is False
def create(
cls, env: Mapping[str, str], args: Sequence[str], *, allow_pantsrc: bool
) -> OptionsBootstrapper:
"""Parses the minimum amount of configuration necessary to create an OptionsBootstrapper.
:param env: An environment dictionary.
:param args: An args array.
:param allow_pantsrc: True to allow pantsrc files to be used. Unless tests are expecting to
consume pantsrc files, they should pass False in order to avoid reading files from
absolute paths. Production usecases should pass True to allow options values to make the
decision of whether to respect pantsrc files.
"""
with warnings.catch_warnings(record=True):
# We can't use pants.engine.fs.FileContent here because it would cause a circular dep.
@dataclass(frozen=True)
class FileContent:
path: str
content: bytes
def filecontent_for(path: str) -> FileContent:
return FileContent(
ensure_text(path),
read_file(path, binary_mode=True),
)
bargs = cls._get_bootstrap_args(args)
config_file_paths = cls.get_config_file_paths(env=env, args=args)
config_files_products = [filecontent_for(p) for p in config_file_paths]
pre_bootstrap_config = Config.load(config_files_products, env=env)
initial_bootstrap_options = cls.parse_bootstrap_options(
env, bargs, pre_bootstrap_config
)
bootstrap_option_values = initial_bootstrap_options.for_global_scope()
# Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped
# from (typically pants.toml), then config override, then rcfiles.
full_config_paths = pre_bootstrap_config.sources()
if allow_pantsrc and bootstrap_option_values.pantsrc:
rcfiles = [
os.path.expanduser(str(rcfile))
for rcfile in bootstrap_option_values.pantsrc_files
]
existing_rcfiles = list(filter(os.path.exists, rcfiles))
full_config_paths.extend(existing_rcfiles)
full_config_files_products = [filecontent_for(p) for p in full_config_paths]
post_bootstrap_config = Config.load(
full_config_files_products,
seed_values=bootstrap_option_values.as_dict(),
env=env,
)
# Finally, we expand any aliases and re-populate the bootstrap args, in case there
# were any from aliases.
# stuhood: This could potentially break the rust client when aliases are used:
# https://github.com/pantsbuild/pants/pull/13228#discussion_r728223889
alias_vals = post_bootstrap_config.get("cli", "alias")
alias_dict = parse_expression(
name="cli.alias",
val=alias_vals[-1] if alias_vals else "{}",
acceptable_types=dict,
)
alias = CliAlias.from_dict(alias_dict)
args = alias.expand_args(tuple(args))
bargs = cls._get_bootstrap_args(args)
# We need to set this env var to allow various static help strings to reference the
# right name (via `pants.util.docutil`), and we need to do it as early as possible to
# avoid needing to lazily import code to avoid chicken-and-egg-problems. This is the
# earliest place it makes sense to do so and is generically used by both the local and
# remote pants runners.
os.environ["PANTS_BIN_NAME"] = bootstrap_option_values.pants_bin_name
env_tuples = tuple(
sorted(
(item for item in env.items() if item[0].startswith("PANTS_")),
)
)
return cls(
env_tuples=env_tuples,
bootstrap_args=bargs,
args=args,
config=post_bootstrap_config,
alias=alias,
)
def test_empty(self) -> None:
config = Config.load([])
assert config.sources() == []
def test_get_all_help_info():
class Global(Subsystem):
options_scope = GLOBAL_SCOPE
help = "Global options."
opt1 = IntOption("--opt1", default=42, help="Option 1")
# This is special in having a short option `-l`. Make sure it works.
level = LogLevelOption()
class Foo(Subsystem):
options_scope = "foo"
help = "A foo."
opt2 = BoolOption("--opt2",
default=True,
advanced=True,
help="Option 2")
class Bar(GoalSubsystem):
name = "bar"
help = "The bar goal."
deprecated_options_scope = "bar-old"
deprecated_options_scope_removal_version = "9.9.999"
class QuxField(StringField):
alias = "qux"
default = "blahblah"
help = "A qux string."
class QuuxField(IntField):
alias = "quux"
required = True
help = "A quux int.\n\nMust be non-zero. Or zero. Whatever you like really."
class BazLibrary(Target):
alias = "baz_library"
help = "A library of baz-es.\n\nUse it however you like."
core_fields = [QuxField, QuuxField]
options = Options.create(
env={},
config=Config.load([]),
known_scope_infos=[
Global.get_scope_info(),
Foo.get_scope_info(),
Bar.get_scope_info()
],
args=["./pants"],
bootstrap_option_values=None,
)
Global.register_options_on_scope(options)
Foo.register_options_on_scope(options)
Bar.register_options_on_scope(options)
@rule
def rule_info_test(foo: Foo) -> Target:
"""This rule is for testing info extraction only."""
def fake_consumed_scopes_mapper(scope: str) -> Tuple[str, ...]:
return ("somescope", f"used_by_{scope or 'GLOBAL_SCOPE'}")
bc_builder = BuildConfiguration.Builder()
bc_builder.register_subsystems("help_info_extracter_test", (Foo, Bar))
bc_builder.register_target_types("help_info_extracter_test",
(BazLibrary, ))
bc_builder.register_rules("help_info_extracter_test",
collect_rules(locals()))
all_help_info = HelpInfoExtracter.get_all_help_info(
options,
UnionMembership({}),
fake_consumed_scopes_mapper,
RegisteredTargetTypes({BazLibrary.alias: BazLibrary}),
bc_builder.create(),
)
all_help_info_dict = all_help_info.asdict()
expected_all_help_info_dict = {
"scope_to_help_info": {
GLOBAL_SCOPE: {
"scope":
GLOBAL_SCOPE,
"description":
"Global options.",
"provider":
"",
"is_goal":
False,
"deprecated_scope":
None,
"basic": (
{
"display_args": ("--opt1=<int>", ),
"comma_separated_display_args": "--opt1=<int>",
"scoped_cmd_line_args": ("--opt1", ),
"unscoped_cmd_line_args": ("--opt1", ),
"config_key": "opt1",
"env_var": "PANTS_OPT1",
"value_history": {
"ranked_values": (
{
"rank": Rank.NONE,
"value": None,
"details": None
},
{
"rank": Rank.HARDCODED,
"value": 42,
"details": None
},
),
},
"typ": int,
"default": 42,
"help": "Option 1",
"deprecation_active": False,
"deprecated_message": None,
"removal_version": None,
"removal_hint": None,
"choices": None,
"comma_separated_choices": None,
},
{
"display_args":
("-l=<LogLevel>", "--level=<LogLevel>"),
"comma_separated_display_args":
"-l=<LogLevel>, --level=<LogLevel>",
"scoped_cmd_line_args": ("-l", "--level"),
"unscoped_cmd_line_args": ("-l", "--level"),
"config_key":
"level",
"env_var":
"PANTS_LEVEL",
"value_history": {
"ranked_values": (
{
"rank": Rank.NONE,
"value": None,
"details": None
},
{
"rank": Rank.HARDCODED,
"value": LogLevel.INFO,
"details": None
},
),
},
"typ":
LogLevel,
"default":
LogLevel.INFO,
"help":
"Set the logging level.",
"deprecation_active":
False,
"deprecated_message":
None,
"removal_version":
None,
"removal_hint":
None,
"choices": ("trace", "debug", "info", "warn", "error"),
"comma_separated_choices":
"trace, debug, info, warn, error",
},
),
"advanced":
tuple(),
"deprecated":
tuple(),
},
"foo": {
"scope":
"foo",
"provider":
"help_info_extracter_test",
"description":
"A foo.",
"is_goal":
False,
"deprecated_scope":
None,
"basic": (),
"advanced": ({
"display_args": ("--[no-]foo-opt2", ),
"comma_separated_display_args":
"--[no-]foo-opt2",
"scoped_cmd_line_args": ("--foo-opt2", "--no-foo-opt2"),
"unscoped_cmd_line_args": ("--opt2", "--no-opt2"),
"config_key":
"opt2",
"env_var":
"PANTS_FOO_OPT2",
"value_history": {
"ranked_values": (
{
"rank": Rank.NONE,
"value": None,
"details": None
},
{
"rank": Rank.HARDCODED,
"value": True,
"details": None
},
),
},
"typ":
bool,
"default":
True,
"help":
"Option 2",
"deprecation_active":
False,
"deprecated_message":
None,
"removal_version":
None,
"removal_hint":
None,
"choices":
None,
"comma_separated_choices":
None,
}, ),
"deprecated":
tuple(),
},
"bar": {
"scope": "bar",
"provider": "help_info_extracter_test",
"description": "The bar goal.",
"is_goal": True,
"deprecated_scope": "bar-old",
"basic": tuple(),
"advanced": tuple(),
"deprecated": tuple(),
},
"bar-old": {
"scope": "bar-old",
"provider": "help_info_extracter_test",
"description": "The bar goal.",
"is_goal": True,
"deprecated_scope": "bar-old",
"basic": tuple(),
"advanced": tuple(),
"deprecated": tuple(),
},
},
"name_to_goal_info": {
"bar": {
"name": "bar",
"provider": "help_info_extracter_test",
"description": "The bar goal.",
"consumed_scopes": ("somescope", "used_by_bar"),
"is_implemented": True,
},
"bar-old": {
"name": "bar",
"provider": "help_info_extracter_test",
"description": "The bar goal.",
"consumed_scopes": ("somescope", "used_by_bar-old"),
"is_implemented": True,
},
},
"name_to_target_type_info": {
"baz_library": {
"alias":
"baz_library",
"provider":
"help_info_extracter_test",
"summary":
"A library of baz-es.",
"description":
"A library of baz-es.\n\nUse it however you like.",
"fields": (
{
"alias": "qux",
"provider": "",
"default": "'blahblah'",
"description": "A qux string.",
"required": False,
"type_hint": "str | None",
},
{
"alias":
"quux",
"provider":
"",
"default":
None,
"description":
"A quux int.\n\nMust be non-zero. Or zero. "
"Whatever you like really.",
"required":
True,
"type_hint":
"int",
},
),
}
},
"name_to_rule_info": {
"construct_scope_foo": {
"description": None,
"documentation": "A foo.",
"input_gets": ("Get(ScopedOptions, Scope, ..)", ),
"input_types": (),
"name": "construct_scope_foo",
"output_type": "Foo",
"provider": "help_info_extracter_test",
},
"pants.help.help_info_extracter_test.test_get_all_help_info.rule_info_test":
{
"description": None,
"documentation":
"This rule is for testing info extraction only.",
"input_gets": (),
"input_types": ("Foo", ),
"name":
"pants.help.help_info_extracter_test.test_get_all_help_info.rule_info_test",
"output_type": "Target",
"provider": "help_info_extracter_test",
},
},
"name_to_api_type_info": {
"pants.help.help_info_extracter_test.Foo": {
"consumed_by_rules":
("pants.help.help_info_extracter_test.test_get_all_help_info.rule_info_test",
),
"dependees": ("help_info_extracter_test", ),
"dependencies": ("pants.option.scope", ),
"documentation":
None,
"is_union":
False,
"module":
"pants.help.help_info_extracter_test",
"name":
"Foo",
"provider":
"help_info_extracter_test",
"returned_by_rules": ("construct_scope_foo", ),
"union_members": (),
"union_type":
None,
"used_in_rules": (),
},
"pants.engine.target.Target": {
"consumed_by_rules": (),
"dependees": (),
"dependencies": (),
"documentation":
("A Target represents an addressable set of metadata.\n\n Set the `help` "
"class property with a description, which will be used in `./pants help`. For "
"the\n best rendering, use soft wrapping (e.g. implicit string concatenation"
") within paragraphs, but\n hard wrapping (`\n`) to separate distinct "
"paragraphs and/or lists.\n "),
"is_union":
False,
"module":
"pants.engine.target",
"name":
"Target",
"provider":
"help_info_extracter_test",
"returned_by_rules":
("pants.help.help_info_extracter_test.test_get_all_help_info.rule_info_test",
),
"union_members": (),
"union_type":
None,
"used_in_rules": (),
},
"pants.option.scope.Scope": {
"consumed_by_rules": (),
"dependees": (),
"dependencies": (),
"documentation": "An options scope.",
"is_union": False,
"module": "pants.option.scope",
"name": "Scope",
"provider": "pants.option.scope",
"returned_by_rules": (),
"union_members": (),
"union_type": None,
"used_in_rules": ("construct_scope_foo", ),
},
},
}
# Break down this colossal structure into pieces so it is easier to spot where the issue is.
# Check keys equality first, then contents
assert set(expected_all_help_info_dict) == set(all_help_info_dict)
for key in all_help_info_dict:
actual = all_help_info_dict[key]
expected = expected_all_help_info_dict[key]
assert expected == actual
def get_bootstrap_options(self):
""":returns: an Options instance that only knows about the bootstrap options.
:rtype: :class:`Options`
"""
if not self._bootstrap_options:
flags = set()
short_flags = set()
def capture_the_flags(*args, **kwargs):
for arg in args:
flags.add(arg)
if len(arg) == 2:
short_flags.add(arg)
elif kwargs.get('type') == bool:
flags.add('--no-{}'.format(arg[2:]))
GlobalOptionsRegistrar.register_bootstrap_options(
capture_the_flags)
def is_bootstrap_option(arg):
components = arg.split('=', 1)
if components[0] in flags:
return True
for flag in short_flags:
if arg.startswith(flag):
return True
return False
# Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
# Stop before '--' since args after that are pass-through and may have duplicate names to our
# bootstrap options.
bargs = filter(
is_bootstrap_option,
itertools.takewhile(lambda arg: arg != '--', self._args))
configpaths = self.get_config_file_paths(env=self._env,
args=self._args)
pre_bootstrap_config = Config.load(configpaths)
def bootstrap_options_from_config(config):
bootstrap_options = Options.create(
env=self._env,
config=config,
known_scope_infos=[
GlobalOptionsRegistrar.get_scope_info()
],
args=bargs,
option_tracker=self._option_tracker)
def register_global(*args, **kwargs):
bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs)
GlobalOptionsRegistrar.register_bootstrap_options(
register_global)
return bootstrap_options
initial_bootstrap_options = bootstrap_options_from_config(
pre_bootstrap_config)
bootstrap_option_values = initial_bootstrap_options.for_global_scope(
)
# Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped
# from (typically pants.ini), then config override, then rcfiles.
full_configpaths = pre_bootstrap_config.sources()
if bootstrap_option_values.pantsrc:
rcfiles = [
os.path.expanduser(rcfile)
for rcfile in bootstrap_option_values.pantsrc_files
]
existing_rcfiles = filter(os.path.exists, rcfiles)
full_configpaths.extend(existing_rcfiles)
self._post_bootstrap_config = Config.load(
full_configpaths, seed_values=bootstrap_option_values)
# Now recompute the bootstrap options with the full config. This allows us to pick up
# bootstrap values (such as backends) from a config override file, for example.
self._bootstrap_options = bootstrap_options_from_config(
self._post_bootstrap_config)
return self._bootstrap_options
def get_bootstrap_options(self):
""":returns: an Options instance that only knows about the bootstrap options.
:rtype: :class:`Options`
"""
if not self._bootstrap_options:
flags = set()
short_flags = set()
def capture_the_flags(*args, **kwargs):
for arg in args:
flags.add(arg)
if len(arg) == 2:
short_flags.add(arg)
elif is_boolean_flag(kwargs):
flags.add('--no-{}'.format(arg[2:]))
GlobalOptionsRegistrar.register_bootstrap_options(capture_the_flags)
def is_bootstrap_option(arg):
components = arg.split('=', 1)
if components[0] in flags:
return True
for flag in short_flags:
if arg.startswith(flag):
return True
return False
# Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
# Stop before '--' since args after that are pass-through and may have duplicate names to our
# bootstrap options.
bargs = filter(is_bootstrap_option, itertools.takewhile(lambda arg: arg != '--', self._args))
configpaths = self.get_config_file_paths(env=self._env, args=self._args)
pre_bootstrap_config = Config.load(configpaths)
def bootstrap_options_from_config(config):
bootstrap_options = Options.create(env=self._env, config=config,
known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()], args=bargs,
option_tracker=self._option_tracker)
def register_global(*args, **kwargs):
bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs)
GlobalOptionsRegistrar.register_bootstrap_options(register_global)
return bootstrap_options
initial_bootstrap_options = bootstrap_options_from_config(pre_bootstrap_config)
bootstrap_option_values = initial_bootstrap_options.for_global_scope()
# Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped
# from (typically pants.ini), then config override, then rcfiles.
full_configpaths = pre_bootstrap_config.sources()
if bootstrap_option_values.config_override:
full_configpaths.extend(bootstrap_option_values.config_override)
if bootstrap_option_values.pantsrc:
rcfiles = [os.path.expanduser(rcfile) for rcfile in bootstrap_option_values.pantsrc_files]
existing_rcfiles = filter(os.path.exists, rcfiles)
full_configpaths.extend(existing_rcfiles)
self._post_bootstrap_config = Config.load(full_configpaths,
seed_values=bootstrap_option_values)
# Now recompute the bootstrap options with the full config. This allows us to pick up
# bootstrap values (such as backends) from a config override file, for example.
self._bootstrap_options = bootstrap_options_from_config(self._post_bootstrap_config)
return self._bootstrap_options
def test_get_all_help_info():
class Global(Subsystem):
options_scope = GLOBAL_SCOPE
help = "Global options."
@classmethod
def register_options(cls, register):
register("-o", "--opt1", type=int, default=42, help="Option 1")
class Foo(Subsystem):
options_scope = "foo"
help = "A foo."
@classmethod
def register_options(cls, register):
register("--opt2", type=bool, default=True, help="Option 2")
register("--opt3", advanced=True, choices=["a", "b", "c"])
class Bar(GoalSubsystem):
name = "bar"
help = "The bar goal."
class QuxField(StringField):
alias = "qux"
default = "blahblah"
help = "A qux string."
class QuuxField(IntField):
alias = "quux"
required = True
help = "A quux int.\n\nMust be non-zero. Or zero. Whatever you like really."
class BazLibrary(Target):
alias = "baz_library"
help = "A library of baz-es.\n\nUse it however you like."
core_fields = [QuxField, QuuxField]
options = Options.create(
env={},
config=Config.load([]),
known_scope_infos=[
Global.get_scope_info(),
Foo.get_scope_info(),
Bar.get_scope_info()
],
args=["./pants"],
bootstrap_option_values=None,
)
Global.register_options_on_scope(options)
Foo.register_options_on_scope(options)
Bar.register_options_on_scope(options)
def fake_consumed_scopes_mapper(scope: str) -> Tuple[str, ...]:
return ("somescope", f"used_by_{scope or 'GLOBAL_SCOPE'}")
all_help_info = HelpInfoExtracter.get_all_help_info(
options,
UnionMembership({}),
fake_consumed_scopes_mapper,
RegisteredTargetTypes({BazLibrary.alias: BazLibrary}),
)
all_help_info_dict = dataclasses.asdict(all_help_info)
expected_all_help_info_dict = {
"scope_to_help_info": {
GLOBAL_SCOPE: {
"scope":
GLOBAL_SCOPE,
"description":
"Global options.",
"is_goal":
False,
"basic": ({
"display_args": ("-o=<int>", "--opt1=<int>"),
"comma_separated_display_args": "-o=<int>, --opt1=<int>",
"scoped_cmd_line_args": ("-o", "--opt1"),
"unscoped_cmd_line_args": ("-o", "--opt1"),
"config_key": "opt1",
"env_var": "PANTS_OPT1",
"value_history": {
"ranked_values": (
{
"rank": Rank.NONE,
"value": None,
"details": None
},
{
"rank": Rank.HARDCODED,
"value": 42,
"details": None
},
),
},
"typ": int,
"default": 42,
"help": "Option 1",
"deprecation_active": False,
"deprecated_message": None,
"removal_version": None,
"removal_hint": None,
"choices": None,
"comma_separated_choices": None,
}, ),
"advanced":
tuple(),
"deprecated":
tuple(),
},
"foo": {
"scope":
"foo",
"description":
"A foo.",
"is_goal":
False,
"basic": ({
"display_args": ("--[no-]foo-opt2", ),
"comma_separated_display_args":
"--[no-]foo-opt2",
"scoped_cmd_line_args": ("--foo-opt2", "--no-foo-opt2"),
"unscoped_cmd_line_args": ("--opt2", "--no-opt2"),
"config_key":
"opt2",
"env_var":
"PANTS_FOO_OPT2",
"value_history": {
"ranked_values": (
{
"rank": Rank.NONE,
"value": None,
"details": None
},
{
"rank": Rank.HARDCODED,
"value": True,
"details": None
},
),
},
"typ":
bool,
"default":
True,
"help":
"Option 2",
"deprecation_active":
False,
"deprecated_message":
None,
"removal_version":
None,
"removal_hint":
None,
"choices":
None,
"comma_separated_choices":
None,
}, ),
"advanced": ({
"display_args": ("--foo-opt3=<str>", ),
"comma_separated_display_args": "--foo-opt3=<str>",
"scoped_cmd_line_args": ("--foo-opt3", ),
"unscoped_cmd_line_args": ("--opt3", ),
"config_key": "opt3",
"env_var": "PANTS_FOO_OPT3",
"value_history": {
"ranked_values": ({
"rank": Rank.NONE,
"value": None,
"details": None
}, ),
},
"typ": str,
"default": None,
"help": "No help available.",
"deprecation_active": False,
"deprecated_message": None,
"removal_version": None,
"removal_hint": None,
"choices": ("a", "b", "c"),
"comma_separated_choices": "a, b, c",
}, ),
"deprecated":
tuple(),
},
"bar": {
"scope": "bar",
"description": "The bar goal.",
"is_goal": True,
"basic": tuple(),
"advanced": tuple(),
"deprecated": tuple(),
},
},
"name_to_goal_info": {
"bar": {
"name": "bar",
"description": "The bar goal.",
"consumed_scopes": ("somescope", "used_by_bar"),
"is_implemented": True,
}
},
"name_to_target_type_info": {
"baz_library": {
"alias":
"baz_library",
"summary":
"A library of baz-es.",
"description":
"A library of baz-es.\n\nUse it however you like.",
"fields": (
{
"alias": "qux",
"default": "'blahblah'",
"description": "A qux string.",
"required": False,
"type_hint": "str | None",
},
{
"alias":
"quux",
"default":
None,
"description":
"A quux int.\n\nMust be non-zero. Or zero. "
"Whatever you like really.",
"required":
True,
"type_hint":
"int",
},
),
}
},
}
assert expected_all_help_info_dict == all_help_info_dict
def test_empty(self): config = Config.load([]) self.assertEquals([], config.sections())
def create(
cls, env: Mapping[str, str], args: Sequence[str], *, allow_pantsrc: bool
) -> OptionsBootstrapper:
"""Parses the minimum amount of configuration necessary to create an OptionsBootstrapper.
:param env: An environment dictionary.
:param args: An args array.
:param allow_pantsrc: True to allow pantsrc files to be used. Unless tests are expecting to
consume pantsrc files, they should pass False in order to avoid reading files from
absolute paths. Production usecases should pass True to allow options values to make the
decision of whether to respect pantsrc files.
"""
with warnings.catch_warnings(record=True):
env = {k: v for k, v in env.items() if k.startswith("PANTS_")}
args = tuple(args)
flags = set()
short_flags = set()
# We can't use pants.engine.fs.FileContent here because it would cause a circular dep.
@dataclass(frozen=True)
class FileContent:
path: str
content: bytes
def filecontent_for(path: str) -> FileContent:
return FileContent(
ensure_text(path),
read_file(path, binary_mode=True),
)
def capture_the_flags(*args: str, **kwargs) -> None:
for arg in args:
flags.add(arg)
if len(arg) == 2:
short_flags.add(arg)
elif kwargs.get("type") == bool:
flags.add(f"--no-{arg[2:]}")
GlobalOptions.register_bootstrap_options(capture_the_flags)
def is_bootstrap_option(arg: str) -> bool:
components = arg.split("=", 1)
if components[0] in flags:
return True
for flag in short_flags:
if arg.startswith(flag):
return True
return False
# Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
# Stop before '--' since args after that are pass-through and may have duplicate names to our
# bootstrap options.
bargs = ("./pants",) + tuple(
filter(is_bootstrap_option, itertools.takewhile(lambda arg: arg != "--", args))
)
config_file_paths = cls.get_config_file_paths(env=env, args=args)
config_files_products = [filecontent_for(p) for p in config_file_paths]
pre_bootstrap_config = Config.load(config_files_products)
initial_bootstrap_options = cls.parse_bootstrap_options(
env, bargs, pre_bootstrap_config
)
bootstrap_option_values = initial_bootstrap_options.for_global_scope()
# Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped
# from (typically pants.toml), then config override, then rcfiles.
full_config_paths = pre_bootstrap_config.sources()
if allow_pantsrc and bootstrap_option_values.pantsrc:
rcfiles = [
os.path.expanduser(str(rcfile))
for rcfile in bootstrap_option_values.pantsrc_files
]
existing_rcfiles = list(filter(os.path.exists, rcfiles))
full_config_paths.extend(existing_rcfiles)
full_config_files_products = [filecontent_for(p) for p in full_config_paths]
post_bootstrap_config = Config.load(
full_config_files_products,
seed_values=bootstrap_option_values.as_dict(),
)
env_tuples = tuple(sorted(env.items(), key=lambda x: x[0]))
return cls(
env_tuples=env_tuples, bootstrap_args=bargs, args=args, config=post_bootstrap_config
)
