def verify_configs(self, global_config: Config) -> None:
"""Verify all loaded configs have correct scopes and options."""
error_log = []
for config in global_config.configs():
for section in config.sections():
scope = GLOBAL_SCOPE if section == GLOBAL_SCOPE_CONFIG_SECTION else section
try:
# TODO(#10834): this is broken for subscopes. Once we fix global options to no
# longer be included in self.for_scope(), we should set
# inherit_from_enclosing_scope=True.
valid_options_under_scope = set(
self.for_scope(scope,
inherit_from_enclosing_scope=False))
# Only catch ConfigValidationError. Other exceptions will be raised directly.
except Config.ConfigValidationError:
error_log.append(
f"Invalid scope [{section}] in {config.config_path}")
else:
# All the options specified under [`section`] in `config` excluding bootstrap defaults.
all_options_under_scope = set(
config.values.options(section)) - set(
config.values.defaults)
for option in sorted(all_options_under_scope):
if option not in valid_options_under_scope:
error_log.append(
f"Invalid option '{option}' under [{section}] in {config.config_path}"
)
if error_log:
for error in error_log:
logger.error(error)
raise Config.ConfigValidationError(
"Invalid config entries detected. See log for details on which entries to update or "
"remove.\n(Specify --no-verify-config to disable this check.)")
def verify_configs(self, global_config: Config) -> None:
"""Verify all loaded configs have correct scopes and options."""
section_to_valid_options = {}
for scope in self.known_scope_to_info:
section = GLOBAL_SCOPE_CONFIG_SECTION if scope == GLOBAL_SCOPE else scope
section_to_valid_options[section] = set(self.for_scope(scope, check_deprecations=False))
global_config.verify(section_to_valid_options)
def verify_configs_against_options(self, options):
"""Verify all loaded configs have correct scopes and options.
:param options: Fully bootstrapped valid options.
:return: None.
"""
error_log = []
for config in self.config.configs():
for section in config.sections():
if section == GLOBAL_SCOPE_CONFIG_SECTION:
scope = GLOBAL_SCOPE
else:
scope = section
try:
valid_options_under_scope = set(options.for_scope(scope))
# Only catch ConfigValidationError. Other exceptions will be raised directly.
except Config.ConfigValidationError:
error_log.append("Invalid scope [{}] in {}".format(section, config.configpath))
else:
# All the options specified under [`section`] in `config` excluding bootstrap defaults.
all_options_under_scope = (set(config.configparser.options(section)) -
set(config.configparser.defaults()))
for option in all_options_under_scope:
if option not in valid_options_under_scope:
error_log.append("Invalid option '{}' under [{}] in {}".format(option, section, config.configpath))
if error_log:
for error in error_log:
logger.error(error)
raise Config.ConfigValidationError("Invalid config entries detected. "
"See log for details on which entries to update or remove.\n"
"(Specify --no-verify-config to disable this check.)")
def verify_configs_against_options(self, options: Options) -> None:
"""Verify all loaded configs have correct scopes and options.
:param options: Fully bootstrapped valid options.
"""
error_log = []
for config in self.config.configs():
for section in config.sections():
scope = GLOBAL_SCOPE if section == GLOBAL_SCOPE_CONFIG_SECTION else section
try:
valid_options_under_scope = set(
options.for_scope(scope, include_passive_options=True))
# Only catch ConfigValidationError. Other exceptions will be raised directly.
except Config.ConfigValidationError:
error_log.append(
f"Invalid scope [{section}] in {config.config_path}")
else:
# All the options specified under [`section`] in `config` excluding bootstrap defaults.
all_options_under_scope = set(
config.values.options(section)) - set(
config.values.defaults)
for option in sorted(all_options_under_scope):
if option not in valid_options_under_scope:
error_log.append(
f"Invalid option '{option}' under [{section}] in {config.config_path}"
)
if error_log:
for error in error_log:
logger.error(error)
raise Config.ConfigValidationError(
"Invalid config entries detected. See log for details on which entries to update or "
"remove.\n(Specify --no-verify-config to disable this check.)")
def _create_config(self, config):
with open(os.path.join(safe_mkdtemp(), 'test_config.ini'), 'w') as fp:
for section, options in config.items():
fp.write('[{}]\n'.format(section))
for key, value in options.items():
fp.write('{}: {}\n'.format(key, value))
return Config.load(configpaths=[fp.name])
def _create_config(self, config):
with open(os.path.join(safe_mkdtemp(), 'test_config.ini'), 'w') as fp:
for section, options in config.items():
fp.write('[{}]\n'.format(section))
for key, value in options.items():
fp.write('{}: {}\n'.format(key, value))
return Config.load(configpaths=[fp.name])
def resolve(cls, keystore_config_file):
"""Parse a keystore config file and return a list of Keystore objects."""
keystore_config_file = os.path.expanduser(keystore_config_file)
try:
keystore_config = Config.load([keystore_config_file])
except IOError as e:
raise KeystoreResolver.Error('Problem parsing keystore config file at {}: '
'{}'.format(keystore_config_file, e))
def create_key(key_name):
"""Instantiate Keystore objects."""
def get_config_value(section, option):
val = keystore_config.get(section, option)
if val is None or val == '':
raise KeystoreResolver.Error('Required keystore config value {}.{} is '
'not defined.'.format(section, option))
return val
keystore = Keystore(keystore_name=key_name,
build_type=get_config_value(key_name, 'build_type'),
keystore_location=get_config_value(key_name, 'keystore_location'),
keystore_alias=get_config_value(key_name, 'keystore_alias'),
keystore_password=get_config_value(key_name, 'keystore_password'),
key_password=get_config_value(key_name, 'key_password'))
return keystore
keys = {}
for name in keystore_config.sections():
try:
keys[name] = create_key(name)
except Config.ConfigError as e:
raise KeystoreResolver.Error(e)
return keys
def get_parser(self, scope: str) -> Parser:
"""Returns the parser for the given scope, so code can register on it directly."""
try:
return self._parser_by_scope[scope]
except KeyError:
raise Config.ConfigValidationError(
f"No such options scope: {scope}")
def _setup_config() -> Config:
parsed_config = Config.load(
file_contents=[
FileContent("file1.toml", FILE_1.content.encode()),
FileContent("file2.toml", FILE_2.content.encode()),
],
seed_values={"buildroot": "fake_buildroot"},
)
assert ["file1.toml", "file2.toml"] == parsed_config.sources()
return parsed_config
def setUp(self) -> None:
self.config = self._setup_config()
self.default_seed_values = Config._determine_seed_values(
seed_values={"buildroot": self.build_root},
)
self.expected_combined_values = {
**FILE_1.expected_options,
**FILE_2.expected_options,
"a": {**FILE_2.expected_options["a"], **FILE_1.expected_options["a"]},
}
def do_test(args, kwargs, expected_default):
# Defaults are computed in the parser and added into the kwargs, so we
# must jump through this hoop in this test.
parser = Parser(env={}, config=Config.load([]),
scope_info=GlobalOptionsRegistrar.get_scope_info(),
parent_parser=None, option_tracker=OptionTracker())
parser.register(*args, **kwargs)
oshi = HelpInfoExtracter.get_option_scope_help_info_from_parser(parser).basic
self.assertEquals(1, len(oshi))
ohi = oshi[0]
self.assertEqual(expected_default, ohi.default)
def do_test(args, kwargs, expected_default):
# Defaults are computed in the parser and added into the kwargs, so we
# must jump through this hoop in this test.
parser = Parser(env={}, config=Config.load([]),
scope_info=GlobalOptionsRegistrar.get_scope_info(),
parent_parser=None, option_tracker=OptionTracker())
parser.register(*args, **kwargs)
oshi = HelpInfoExtracter.get_option_scope_help_info_from_parser(parser).basic
self.assertEquals(1, len(oshi))
ohi = oshi[0]
self.assertEqual(expected_default, ohi.default)
def _setup_config(self, config1_content: str, config2_content: str, *, suffix: str) -> Config:
with temporary_file(binary_mode=False, suffix=suffix) as config1, \
temporary_file(binary_mode=False, suffix=suffix) as config2:
config1.write(config1_content)
config1.close()
config2.write(config2_content)
config2.close()
parsed_config = Config.load(
config_paths=[config1.name, config2.name], seed_values={"buildroot": self.build_root}
)
assert [config1.name, config2.name] == parsed_config.sources()
return parsed_config
def setUp(self):
self.ini1_content = textwrap.dedent(
"""
[DEFAULT]
name: foo
answer: 42
scale: 1.2
path: /a/b/%(answer)s
embed: %(path)s::foo
disclaimer:
Let it be known
that.
blank_section:
[a]
list: [1, 2, 3, %(answer)s]
listappend: +[7, 8, 9]
[b]
preempt: True
dict: {
'a': 1,
'b': %(answer)s,
'c': ['%(answer)s', %(answer)s]
}
"""
)
self.ini2_content = textwrap.dedent(
"""
[a]
fast: True
[b]
preempt: False
[defined_section]
"""
)
with temporary_file(binary_mode=False) as ini1, \
temporary_file(binary_mode=False) as ini2, \
temporary_file_path() as buildroot:
ini1.write(self.ini1_content)
ini1.close()
ini2.write(self.ini2_content)
ini2.close()
self.config = Config.load(
config_paths=[ini1.name, ini2.name], seed_values={"buildroot": buildroot}
)
self.assertEqual([ini1.name, ini2.name], self.config.sources())
def _format_for_global_scope(show_advanced, show_deprecated, args, kwargs):
parser = Parser(
env={},
config=Config.load([]),
scope_info=GlobalOptions.get_scope_info(),
parent_parser=None,
)
parser.register(*args, **kwargs)
# Force a parse to generate the derivation history.
parser.parse_args(Parser.ParseArgsRequest((), OptionValueContainerBuilder(), [], False))
oshi = HelpInfoExtracter("").get_option_scope_help_info("", parser, False)
return HelpFormatter(
show_advanced=show_advanced, show_deprecated=show_deprecated, color=False
).format_options(oshi)
def _setup_config() -> Config:
with temporary_file(binary_mode=False,
suffix=".toml") as config1, temporary_file(
binary_mode=False, suffix=".toml") as config2:
config1.write(FILE_1.content)
config1.close()
config2.write(FILE_2.content)
config2.close()
parsed_config = Config.load(
config_paths=[config1.name, config2.name],
seed_values={"buildroot": "fake_buildroot"},
)
assert [config1.name, config2.name] == parsed_config.sources()
return parsed_config
def do_test(kwargs, expected_basic=False, expected_advanced=False):
def exp_to_len(exp):
return int(exp) # True -> 1, False -> 0.
parser = Parser(
env={},
config=Config.load([]),
scope_info=GlobalOptions.get_scope_info(),
)
parser.register("--foo", **kwargs)
oshi = HelpInfoExtracter("").get_option_scope_help_info(
"", parser, False)
assert exp_to_len(expected_basic) == len(oshi.basic)
assert exp_to_len(expected_advanced) == len(oshi.advanced)
def test_register_options_blessed(caplog) -> None:
class GoodToGo(Subsystem):
options_scope = "good-to-go"
options = Options.create(
env={},
config=Config.load([]),
known_scope_infos=[GoodToGo.get_scope_info()],
args=["./pants"],
bootstrap_option_values=None,
)
GoodToGo.register_options_on_scope(options)
assert not caplog.records, "The current blessed means of registering options should never warn."
def setUp(self):
self.ini1_content = textwrap.dedent(
"""
[DEFAULT]
name: foo
answer: 42
scale: 1.2
path: /a/b/%(answer)s
embed: %(path)s::foo
disclaimer:
Let it be known
that.
blank_section:
[a]
list: [1, 2, 3, %(answer)s]
listappend: +[7, 8, 9]
[b]
preempt: True
dict: {
'a': 1,
'b': %(answer)s,
'c': ['%(answer)s', %(answer)s]
}
"""
)
self.ini2_content = textwrap.dedent(
"""
[a]
fast: True
[b]
preempt: False
[defined_section]
"""
)
with temporary_file(binary_mode=False) as ini1:
ini1.write(self.ini1_content)
ini1.close()
with temporary_file(binary_mode=False) as ini2:
ini2.write(self.ini2_content)
ini2.close()
self.config = Config.load(config_paths=[ini1.name, ini2.name])
self.assertEqual([ini1.name, ini2.name], self.config.sources())
def do_test(args, kwargs, expected_default_str):
# Defaults are computed in the parser and added into the kwargs, so we
# must jump through this hoop in this test.
parser = Parser(
env={},
config=Config.load([]),
scope_info=GlobalOptions.get_scope_info(),
)
parser.register(*args, **kwargs)
oshi = HelpInfoExtracter(parser.scope).get_option_scope_help_info(
"description", parser, False)
assert oshi.description == "description"
assert len(oshi.basic) == 1
ohi = oshi.basic[0]
assert to_help_str(ohi.default) == expected_default_str
def setUp(self):
with temporary_file() as ini1:
ini1.write(textwrap.dedent(
"""
[DEFAULT]
name: foo
answer: 42
scale: 1.2
path: /a/b/%(answer)s
embed: %(path)s::foo
disclaimer:
Let it be known
that.
blank_section:
[a]
list: [1, 2, 3, %(answer)s]
listappend: +[7, 8, 9]
[b]
preempt: True
dict: {
'a': 1,
'b': %(answer)s,
'c': ['%(answer)s', %(answer)s]
}
"""))
ini1.close()
with temporary_file() as ini2:
ini2.write(textwrap.dedent(
"""
[a]
fast: True
[b]
preempt: False
[defined_section]
"""))
ini2.close()
self.config = Config.load(configpaths=[ini1.name, ini2.name])
self.assertEqual([ini1.name, ini2.name], self.config.sources())
def setUp(self):
with temporary_file() as ini1:
ini1.write(textwrap.dedent(
"""
[DEFAULT]
name: foo
answer: 42
scale: 1.2
path: /a/b/%(answer)s
embed: %(path)s::foo
disclaimer:
Let it be known
that.
blank_section:
[a]
list: [1, 2, 3, %(answer)s]
[b]
preempt: True
dict: {
'a': 1,
'b': %(answer)s,
'c': ['%(answer)s', %(answer)s]
}
"""))
ini1.close()
with temporary_file() as ini2:
ini2.write(textwrap.dedent(
"""
[a]
fast: True
[b]
preempt: False
[defined_section]
"""))
ini2.close()
self.config = Config.load(configpaths=[ini1.name, ini2.name])
def setUp(self) -> None:
self.config = _setup_config()
self.default_seed_values = Config._determine_seed_values(
seed_values={"buildroot": "fake_buildroot"},
env={"NAME": "foo"},
)
self.expected_combined_values: dict[str, dict[str, list[str]]] = {
"a": {
"list": ['["1", "2", "3", "42"]'],
"list2": ["+[7, 8, 9]"],
"list3": ['-["x", "y", "z"]'],
"fast": ["True"],
},
"b": {
"preempt": ["True", "False"]
},
"c": {
"name": ["overridden_from_default"],
"interpolated_from_section":
["overridden_from_default is interpolated"],
"recursively_interpolated_from_section":
["overridden_from_default is interpolated (again)"],
},
"d": {
"dict_val":
["{'add': 0, 'remove': 0, 'nested': {'nested_key': 'foo'}}"],
"list": ["+[0, 1],-[8, 9]"],
},
"empty_section": {},
"list_merging": {
"list1": ["[]", "[11, 22]"],
"list2": ["[1, 2]", "+[33]"],
"list3": ["+[3, 4]", "+[8, 9],-[4, 55]"],
"list4": ["-[5]", "[66]"],
"list5": ["[6, 7]"],
"list6": ["+[77, 88]"],
},
}
def resolve(cls, keystore_config_file):
"""Parse a keystore config file and return a list of Keystore objects."""
keystore_config_file = os.path.expanduser(keystore_config_file)
try:
keystore_config = Config.load([keystore_config_file])
except IOError as e:
raise KeystoreResolver.Error(
'Problem parsing keystore config file at {}: '
'{}'.format(keystore_config_file, e))
def create_key(key_name):
"""Instantiate Keystore objects."""
def get_config_value(section, option):
val = keystore_config.get(section, option)
if val is None or val == '':
raise KeystoreResolver.Error(
'Required keystore config value {}.{} is '
'not defined.'.format(section, option))
return val
keystore = Keystore(
keystore_name=key_name,
build_type=get_config_value(key_name, 'build_type'),
keystore_location=get_config_value(key_name,
'keystore_location'),
keystore_alias=get_config_value(key_name, 'keystore_alias'),
keystore_password=get_config_value(key_name,
'keystore_password'),
key_password=get_config_value(key_name, 'key_password'))
return keystore
keys = {}
for name in keystore_config.sections():
try:
keys[name] = create_key(name)
except Config.ConfigError as e:
raise KeystoreResolver.Error(e)
return keys
async def check_default_tools(
console: Console,
real_opts: _Options,
) -> CheckDefaultTools:
# The real options know about all the registered tools.
for scope, si in real_opts.options.known_scope_to_info.items():
if si.subsystem_cls and issubclass(si.subsystem_cls, ExternalTool):
tool_cls = si.subsystem_cls
console.print_stdout(f"Checking {console.cyan(tool_cls.name)}:")
for known_version in tool_cls.default_known_versions:
ver, plat_val, sha256, length = tool_cls.split_known_version_str(
known_version)
# Note that we don't want to use the real option values here - we want to
# verify that the *defaults* aren't broken. However the get_request_for() method
# requires an instance (since it can consult option values, including custom
# options for specific tools, that we don't know about), so we construct a
# default one, but we force the --version to the one we're checking (which will
# typically be the same as the default version, but doesn't have to be, if the
# tool provides default_known_versions for versions other than default_version).
args = ("./pants", f"--{scope}-version={ver}")
blank_opts = await Get(
_Options,
SessionValues({
OptionsBootstrapper:
OptionsBootstrapper(tuple(), ("./pants", ), args,
Config(tuple()), CliAlias())
}),
)
instance = tool_cls(blank_opts.options.for_scope(scope))
req = instance.get_request_for(plat_val, sha256, length)
console.write_stdout(f" version {ver} for {plat_val}... ")
# TODO: We'd like to run all the requests concurrently, but since we can't catch
# engine exceptions, we wouldn't have an easy way to output which one failed.
await Get(DownloadedExternalTool, ExternalToolRequest, req)
console.print_stdout(console.sigil_succeeded())
return CheckDefaultTools(exit_code=0)
def load_config(p):
"""We need direct access to the underlying configparser, so we don't load via Config."""
cp = Config._create_parser()
with open(p, "r") as ini:
cp.readfp(ini)
return cp
def test_empty(self) -> None:
config = Config.load([])
assert config.sections() == []
assert config.sources() == []
assert config.has_section("DEFAULT") is False
assert config.has_option(section="DEFAULT", option="name") is False
def test_empty(self): config = Config.load([]) self.assertEquals([], config.sections())
def load_config(p):
"""We need direct access to the underlying configparser, so we don't load via Config."""
cp = Config._create_parser()
with open(p, 'r') as ini:
cp.readfp(ini)
return cp
def create(
cls, env: Mapping[str, str], args: Sequence[str], *, allow_pantsrc: bool
) -> OptionsBootstrapper:
"""Parses the minimum amount of configuration necessary to create an OptionsBootstrapper.
:param env: An environment dictionary, or None to use `os.environ`.
:param args: An args array, or None to use `sys.argv`.
:param allow_pantsrc: True to allow pantsrc files to be used. Unless tests are expecting to
consume pantsrc files, they should pass False in order to avoid reading files from
absolute paths. Production usecases should pass True to allow options values to make the
decision of whether to respect pantsrc files.
"""
with warnings.catch_warnings(record=True):
env = {k: v for k, v in env.items() if k.startswith("PANTS_")}
args = tuple(args)
flags = set()
short_flags = set()
# We can't use pants.engine.fs.FileContent here because it would cause a circular dep.
@dataclass(frozen=True)
class FileContent:
path: str
content: bytes
def filecontent_for(path: str) -> FileContent:
return FileContent(
ensure_text(path),
read_file(path, binary_mode=True),
)
def capture_the_flags(*args: str, **kwargs) -> None:
for arg in args:
flags.add(arg)
if len(arg) == 2:
short_flags.add(arg)
elif kwargs.get("type") == bool:
flags.add(f"--no-{arg[2:]}")
GlobalOptions.register_bootstrap_options(capture_the_flags)
def is_bootstrap_option(arg: str) -> bool:
components = arg.split("=", 1)
if components[0] in flags:
return True
for flag in short_flags:
if arg.startswith(flag):
return True
return False
# Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
# Stop before '--' since args after that are pass-through and may have duplicate names to our
# bootstrap options.
bargs = ("./pants",) + tuple(
filter(is_bootstrap_option, itertools.takewhile(lambda arg: arg != "--", args))
)
config_file_paths = cls.get_config_file_paths(env=env, args=args)
config_files_products = [filecontent_for(p) for p in config_file_paths]
pre_bootstrap_config = Config.load_file_contents(config_files_products)
initial_bootstrap_options = cls.parse_bootstrap_options(
env, bargs, pre_bootstrap_config
)
bootstrap_option_values = initial_bootstrap_options.for_global_scope()
# Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped
# from (typically pants.toml), then config override, then rcfiles.
full_config_paths = pre_bootstrap_config.sources()
if allow_pantsrc and bootstrap_option_values.pantsrc:
rcfiles = [
os.path.expanduser(str(rcfile))
for rcfile in bootstrap_option_values.pantsrc_files
]
existing_rcfiles = list(filter(os.path.exists, rcfiles))
full_config_paths.extend(existing_rcfiles)
full_config_files_products = [filecontent_for(p) for p in full_config_paths]
post_bootstrap_config = Config.load_file_contents(
full_config_files_products,
seed_values=bootstrap_option_values.as_dict(),
)
env_tuples = tuple(sorted(env.items(), key=lambda x: x[0]))
return cls(
env_tuples=env_tuples, bootstrap_args=bargs, args=args, config=post_bootstrap_config
)
def check_config_file(path):
cp = Config._create_parser()
with open(path, 'r') as ini:
cp.readfp(ini)
print('Checking config file at {} for unmigrated keys.'.format(path), file=sys.stderr)
def section(s):
return cyan('[{}]'.format(s))
for src, dst in migrations.items():
check_option(cp, src, dst)
# Special-case handling of per-task subsystem options, so we can sweep them up in all
# sections easily.
def check_task_subsystem_options(subsystem_sec, options_map, sections=None):
sections = sections or cp.sections()
for src_sec in ['DEFAULT'] + sections:
dst_sec = subsystem_sec if src_sec == 'DEFAULT' else '{}.{}'.format(subsystem_sec, src_sec)
for src_key, dst_key in options_map.items():
check_option(cp, (src_sec, src_key), (dst_sec, dst_key))
artifact_cache_options_map = {
'read_from_artifact_cache': 'read',
'write_to_artifact_cache': 'write',
'overwrite_cache_artifacts': 'overwrite',
'read_artifact_caches': 'read_from',
'write_artifact_caches': 'write_to',
'cache_compression': 'compression_level',
}
check_task_subsystem_options('cache', artifact_cache_options_map)
jvm_options_map = {
'jvm_options': 'options',
'args': 'program_args',
'debug': 'debug',
'debug_port': 'debug_port',
'debug_args': 'debug_args',
}
jvm_options_sections = [
'repl.scala', 'test.junit', 'run.jvm', 'bench', 'doc.javadoc', 'doc.scaladoc'
]
check_task_subsystem_options('jvm', jvm_options_map, sections=jvm_options_sections)
# Check that all values are parseable.
for sec in ['DEFAULT'] + cp.sections():
for key, value in cp.items(sec):
value = value.strip()
if value.startswith('['):
try:
custom_types.list_option(value)
except ParseError:
print('Value of {key} in section {section} is not a valid '
'JSON list.'.format(key=green(key), section=section(sec)))
elif value.startswith('{'):
try:
custom_types.dict_option(value)
except ParseError:
print('Value of {key} in section {section} is not a valid '
'JSON object.'.format(key=green(key), section=section(sec)))
def get_bootstrap_options(self):
""":returns: an Options instance that only knows about the bootstrap options.
:rtype: :class:`Options`
"""
if not self._bootstrap_options:
flags = set()
short_flags = set()
def capture_the_flags(*args, **kwargs):
for arg in args:
flags.add(arg)
if len(arg) == 2:
short_flags.add(arg)
elif is_boolean_flag(kwargs):
flags.add('--no-{}'.format(arg[2:]))
GlobalOptionsRegistrar.register_bootstrap_options(
capture_the_flags)
def is_bootstrap_option(arg):
components = arg.split('=', 1)
if components[0] in flags:
return True
for flag in short_flags:
if arg.startswith(flag):
return True
return False
# Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
# Stop before '--' since args after that are pass-through and may have duplicate names to our
# bootstrap options.
bargs = filter(
is_bootstrap_option,
itertools.takewhile(lambda arg: arg != '--', self._args))
configpaths = [self._configpath] if self._configpath else None
pre_bootstrap_config = Config.load(configpaths)
def bootstrap_options_from_config(config):
bootstrap_options = Options.create(
env=self._env,
config=config,
known_scope_infos=[
GlobalOptionsRegistrar.get_scope_info()
],
args=bargs,
option_tracker=self._option_tracker)
def register_global(*args, **kwargs):
bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs)
GlobalOptionsRegistrar.register_bootstrap_options(
register_global)
return bootstrap_options
initial_bootstrap_options = bootstrap_options_from_config(
pre_bootstrap_config)
bootstrap_option_values = initial_bootstrap_options.for_global_scope(
)
# Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped
# from (typically pants.ini), then config override, then rcfiles.
full_configpaths = pre_bootstrap_config.sources()
if bootstrap_option_values.config_override:
full_configpaths.extend(
bootstrap_option_values.config_override)
if bootstrap_option_values.pantsrc:
rcfiles = [
os.path.expanduser(rcfile)
for rcfile in bootstrap_option_values.pantsrc_files
]
existing_rcfiles = filter(os.path.exists, rcfiles)
full_configpaths.extend(existing_rcfiles)
self._post_bootstrap_config = Config.load(
full_configpaths, seed_values=bootstrap_option_values)
# Now recompute the bootstrap options with the full config. This allows us to pick up
# bootstrap values (such as backends) from a config override file, for example.
self._bootstrap_options = bootstrap_options_from_config(
self._post_bootstrap_config)
return self._bootstrap_options
def test_empty(self):
config = Config.load([])
self.assertEqual([], config.sections())
def test_empty(self) -> None:
config = Config.load([])
assert config.sources() == []
def produce_and_set_bootstrap_options(self):
"""Cooperatively populates the internal bootstrap_options cache with
a producer of `FileContent`."""
flags = set()
short_flags = set()
def capture_the_flags(*args, **kwargs):
for arg in args:
flags.add(arg)
if len(arg) == 2:
short_flags.add(arg)
elif kwargs.get('type') == bool:
flags.add('--no-{}'.format(arg[2:]))
GlobalOptionsRegistrar.register_bootstrap_options(capture_the_flags)
def is_bootstrap_option(arg):
components = arg.split('=', 1)
if components[0] in flags:
return True
for flag in short_flags:
if arg.startswith(flag):
return True
return False
# Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
# Stop before '--' since args after that are pass-through and may have duplicate names to our
# bootstrap options.
bargs = list(
filter(is_bootstrap_option,
itertools.takewhile(lambda arg: arg != '--', self._args)))
config_file_paths = self.get_config_file_paths(env=self._env,
args=self._args)
config_files_products = yield config_file_paths
pre_bootstrap_config = Config.load_file_contents(config_files_products)
def bootstrap_options_from_config(config):
bootstrap_options = Options.create(
env=self._env,
config=config,
known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()],
args=bargs,
option_tracker=self._option_tracker)
def register_global(*args, **kwargs):
## Only use of Options.register?
bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs)
GlobalOptionsRegistrar.register_bootstrap_options(register_global)
return bootstrap_options
initial_bootstrap_options = bootstrap_options_from_config(
pre_bootstrap_config)
bootstrap_option_values = initial_bootstrap_options.for_global_scope()
# Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped
# from (typically pants.ini), then config override, then rcfiles.
full_configpaths = pre_bootstrap_config.sources()
if bootstrap_option_values.pantsrc:
rcfiles = [
os.path.expanduser(rcfile)
for rcfile in bootstrap_option_values.pantsrc_files
]
existing_rcfiles = list(filter(os.path.exists, rcfiles))
full_configpaths.extend(existing_rcfiles)
full_config_files_products = yield full_configpaths
self._post_bootstrap_config = Config.load_file_contents(
full_config_files_products, seed_values=bootstrap_option_values)
# Now recompute the bootstrap options with the full config. This allows us to pick up
# bootstrap values (such as backends) from a config override file, for example.
self._bootstrap_options = bootstrap_options_from_config(
self._post_bootstrap_config)
def create(cls, env=None, args=None):
"""Parses the minimum amount of configuration necessary to create an OptionsBootstrapper.
:param env: An environment dictionary, or None to use `os.environ`.
:param args: An args array, or None to use `sys.argv`.
"""
env = {k: v for k, v in (os.environ if env is None else env).items()
if k.startswith('PANTS_')}
args = tuple(sys.argv if args is None else args)
flags = set()
short_flags = set()
def filecontent_for(path):
return FileContent(ensure_text(path), read_file(path, binary_mode=True))
def capture_the_flags(*args, **kwargs):
for arg in args:
flags.add(arg)
if len(arg) == 2:
short_flags.add(arg)
elif kwargs.get('type') == bool:
flags.add('--no-{}'.format(arg[2:]))
GlobalOptionsRegistrar.register_bootstrap_options(capture_the_flags)
def is_bootstrap_option(arg):
components = arg.split('=', 1)
if components[0] in flags:
return True
for flag in short_flags:
if arg.startswith(flag):
return True
return False
# Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
# Stop before '--' since args after that are pass-through and may have duplicate names to our
# bootstrap options.
bargs = tuple(filter(is_bootstrap_option, itertools.takewhile(lambda arg: arg != '--', args)))
config_file_paths = cls.get_config_file_paths(env=env, args=args)
config_files_products = [filecontent_for(p) for p in config_file_paths]
pre_bootstrap_config = Config.load_file_contents(config_files_products)
initial_bootstrap_options = cls.parse_bootstrap_options(env, bargs, pre_bootstrap_config)
bootstrap_option_values = initial_bootstrap_options.for_global_scope()
# Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped
# from (typically pants.ini), then config override, then rcfiles.
full_configpaths = pre_bootstrap_config.sources()
if bootstrap_option_values.pantsrc:
rcfiles = [os.path.expanduser(str(rcfile)) for rcfile in bootstrap_option_values.pantsrc_files]
existing_rcfiles = list(filter(os.path.exists, rcfiles))
full_configpaths.extend(existing_rcfiles)
full_config_files_products = [filecontent_for(p) for p in full_configpaths]
post_bootstrap_config = Config.load_file_contents(
full_config_files_products,
seed_values=bootstrap_option_values
)
env_tuples = tuple(sorted(iteritems(env), key=lambda x: x[0]))
return cls(env_tuples=env_tuples, bootstrap_args=bargs, args=args, config=post_bootstrap_config)
def get_parser_by_scope(self, scope):
try:
return self._parser_by_scope[scope]
except KeyError:
raise Config.ConfigValidationError('No such options scope: {}'.format(scope))
def check_config_file(path):
cp = Config._create_parser()
with open(path, 'r') as ini:
cp.readfp(ini)
print('Checking config file at {} for unmigrated keys.'.format(path),
file=sys.stderr)
def section(s):
return cyan('[{}]'.format(s))
for src, dst in migrations.items():
check_option(cp, src, dst)
# Special-case handling of per-task subsystem options, so we can sweep them up in all
# sections easily.
def check_task_subsystem_options(subsystem_sec,
options_map,
sections=None):
sections = sections or cp.sections()
for src_sec in ['DEFAULT'] + sections:
dst_sec = subsystem_sec if src_sec == 'DEFAULT' else '{}.{}'.format(
subsystem_sec, src_sec)
for src_key, dst_key in options_map.items():
check_option(cp, (src_sec, src_key), (dst_sec, dst_key))
artifact_cache_options_map = {
'read_from_artifact_cache': 'read',
'write_to_artifact_cache': 'write',
'overwrite_cache_artifacts': 'overwrite',
'read_artifact_caches': 'read_from',
'write_artifact_caches': 'write_to',
'cache_compression': 'compression_level',
}
check_task_subsystem_options('cache', artifact_cache_options_map)
jvm_options_map = {
'jvm_options': 'options',
'args': 'program_args',
'debug': 'debug',
'debug_port': 'debug_port',
'debug_args': 'debug_args',
}
jvm_options_sections = [
'repl.scala', 'test.junit', 'run.jvm', 'bench', 'doc.javadoc',
'doc.scaladoc'
]
check_task_subsystem_options('jvm',
jvm_options_map,
sections=jvm_options_sections)
# Check that all values are parseable.
for sec in ['DEFAULT'] + cp.sections():
for key, value in cp.items(sec):
value = value.strip()
if value.startswith('['):
try:
custom_types.list_option(value)
except ParseError:
print('Value of {key} in section {section} is not a valid '
'JSON list.'.format(key=green(key),
section=section(sec)))
elif value.startswith('{'):
try:
custom_types.dict_option(value)
except ParseError:
print('Value of {key} in section {section} is not a valid '
'JSON object.'.format(key=green(key),
section=section(sec)))
def get_bootstrap_options(self):
""":returns: an Options instance that only knows about the bootstrap options.
:rtype: :class:`Options`
"""
if not self._bootstrap_options:
flags = set()
short_flags = set()
def capture_the_flags(*args, **kwargs):
for arg in args:
flags.add(arg)
if len(arg) == 2:
short_flags.add(arg)
elif is_boolean_flag(kwargs):
flags.add('--no-{}'.format(arg[2:]))
GlobalOptionsRegistrar.register_bootstrap_options(capture_the_flags)
def is_bootstrap_option(arg):
components = arg.split('=', 1)
if components[0] in flags:
return True
for flag in short_flags:
if arg.startswith(flag):
return True
return False
# Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
# Stop before '--' since args after that are pass-through and may have duplicate names to our
# bootstrap options.
bargs = filter(is_bootstrap_option, itertools.takewhile(lambda arg: arg != '--', self._args))
configpaths = self.get_config_file_paths(env=self._env, args=self._args)
pre_bootstrap_config = Config.load(configpaths)
def bootstrap_options_from_config(config):
bootstrap_options = Options.create(env=self._env, config=config,
known_scope_infos=[GlobalOptionsRegistrar.get_scope_info()], args=bargs,
option_tracker=self._option_tracker)
def register_global(*args, **kwargs):
bootstrap_options.register(GLOBAL_SCOPE, *args, **kwargs)
GlobalOptionsRegistrar.register_bootstrap_options(register_global)
return bootstrap_options
initial_bootstrap_options = bootstrap_options_from_config(pre_bootstrap_config)
bootstrap_option_values = initial_bootstrap_options.for_global_scope()
# Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped
# from (typically pants.ini), then config override, then rcfiles.
full_configpaths = pre_bootstrap_config.sources()
if bootstrap_option_values.config_override:
full_configpaths.extend(bootstrap_option_values.config_override)
if bootstrap_option_values.pantsrc:
rcfiles = [os.path.expanduser(rcfile) for rcfile in bootstrap_option_values.pantsrc_files]
existing_rcfiles = filter(os.path.exists, rcfiles)
full_configpaths.extend(existing_rcfiles)
self._post_bootstrap_config = Config.load(full_configpaths,
seed_values=bootstrap_option_values)
# Now recompute the bootstrap options with the full config. This allows us to pick up
# bootstrap values (such as backends) from a config override file, for example.
self._bootstrap_options = bootstrap_options_from_config(self._post_bootstrap_config)
return self._bootstrap_options
def create(cls, env=None, args=None):
"""Parses the minimum amount of configuration necessary to create an OptionsBootstrapper.
:param env: An environment dictionary, or None to use `os.environ`.
:param args: An args array, or None to use `sys.argv`.
"""
env = {
k: v
for k, v in (os.environ if env is None else env).items()
if k.startswith('PANTS_')
}
args = tuple(sys.argv if args is None else args)
flags = set()
short_flags = set()
# TODO: This codepath probably shouldn't be using FileContent, which is a very v2 engine thing.
def filecontent_for(path):
is_executable = os.stat(
path).st_mode & stat.S_IXUSR == stat.S_IXUSR
return FileContent(
ensure_text(path),
read_file(path, binary_mode=True),
is_executable=is_executable,
)
def capture_the_flags(*args, **kwargs):
for arg in args:
flags.add(arg)
if len(arg) == 2:
short_flags.add(arg)
elif kwargs.get('type') == bool:
flags.add(f'--no-{arg[2:]}')
GlobalOptionsRegistrar.register_bootstrap_options(capture_the_flags)
def is_bootstrap_option(arg):
components = arg.split('=', 1)
if components[0] in flags:
return True
for flag in short_flags:
if arg.startswith(flag):
return True
return False
# Take just the bootstrap args, so we don't choke on other global-scope args on the cmd line.
# Stop before '--' since args after that are pass-through and may have duplicate names to our
# bootstrap options.
bargs = tuple(
filter(is_bootstrap_option,
itertools.takewhile(lambda arg: arg != '--', args)))
config_file_paths = cls.get_config_file_paths(env=env, args=args)
config_files_products = [filecontent_for(p) for p in config_file_paths]
pre_bootstrap_config = Config.load_file_contents(config_files_products)
initial_bootstrap_options = cls.parse_bootstrap_options(
env, bargs, pre_bootstrap_config)
bootstrap_option_values = initial_bootstrap_options.for_global_scope()
# Now re-read the config, post-bootstrapping. Note the order: First whatever we bootstrapped
# from (typically pants.ini), then config override, then rcfiles.
full_configpaths = pre_bootstrap_config.sources()
if bootstrap_option_values.pantsrc:
rcfiles = [
os.path.expanduser(str(rcfile))
for rcfile in bootstrap_option_values.pantsrc_files
]
existing_rcfiles = list(filter(os.path.exists, rcfiles))
full_configpaths.extend(existing_rcfiles)
full_config_files_products = [
filecontent_for(p) for p in full_configpaths
]
post_bootstrap_config = Config.load_file_contents(
full_config_files_products, seed_values=bootstrap_option_values)
env_tuples = tuple(sorted(env.items(), key=lambda x: x[0]))
return cls(env_tuples=env_tuples,
bootstrap_args=bargs,
args=args,
config=post_bootstrap_config)
