def group(value_component_type, process_val_func) -> list[RankedValue]:
# We group any values that are merged together, so that the history can reflect
# merges vs. replacements in a useful way. E.g., if we merge [a, b] and [c],
# and then replace it with [d, e], the history will contain:
# - [d, e] (from command-line flag)
# - [a, b, c] (from env var, from config)
# And similarly for dicts.
grouped: list[list[RankedValue]] = [[]]
for ranked_val in ranked_vals:
if ranked_val.value and ranked_val.value.action == value_component_type.REPLACE:
grouped.append([])
grouped[-1].append(ranked_val)
return [
RankedValue(
grp[-1].rank,
process_val_func(
value_component_type.merge(
rv.value for rv in grp if rv.value is not None
).val
),
", ".join(rv.details for rv in grp if rv.details),
)
for grp in grouped
if grp
]
def execute(self):
requested_compiler = JvmPlatform.global_instance().get_options(
).compiler
if requested_compiler != self.compiler_name:
return
if requested_compiler == self.Compiler.ZINC and self.compiler_name == self.Compiler.RSC:
# Issue a deprecation warning (above) and rewrite zinc to rsc, as zinc is being deprecated.
JvmPlatform.global_instance().get_options().compiler = RankedValue(
0, self.compiler_name)
elif requested_compiler != self.compiler_name:
# If the requested compiler is not the one supported by this task, log and abort
self.context.log.debug(
'Requested an unsupported compiler [{}], aborting'.format(
requested_compiler))
return
# In case we have no relevant targets and return early, create the requested product maps.
self.create_empty_extra_products()
relevant_targets = list(self.context.targets(predicate=self.select))
if not relevant_targets:
return
# Clone the compile_classpath to the runtime_classpath.
classpath_product = self.create_runtime_classpath()
fingerprint_strategy = DependencyContext.global_instance(
).create_fingerprint_strategy(classpath_product)
# Note, JVM targets are validated (`vts.update()`) as they succeed. As a result,
# we begin writing artifacts out to the cache immediately instead of waiting for
# all targets to finish.
with self.invalidated(relevant_targets,
invalidate_dependents=True,
fingerprint_strategy=fingerprint_strategy,
topological_order=True) as invalidation_check:
compile_contexts = {
vt.target: self.create_compile_context(vt.target,
vt.results_dir)
for vt in invalidation_check.all_vts
}
self.do_compile(
invalidation_check,
compile_contexts,
classpath_product,
)
if not self.get_options().use_classpath_jars:
# Once compilation has completed, replace the classpath entry for each target with
# its jar'd representation.
for ccs in compile_contexts.values():
cc = self.select_runtime_context(ccs)
for conf in self._confs:
classpath_product.remove_for_target(
cc.target, [(conf, cc.classes_dir)])
classpath_product.add_for_target(
cc.target, [(conf, cc.jar_file)])
def test_rsc_dep_for_scala_java_and_test_targets(self):
self.set_options(workflow=RankedValue(
value=RscCompile.JvmCompileWorkflowType.rsc_and_zinc,
rank=RankedValue.CONFIG,
))
self.init_dependencies_for_scala_libraries()
scala_dep = self.make_target('scala/classpath:scala_dep',
target_type=ScalaLibrary,
sources=['com/example/Bar.scala'])
java_target = self.make_target('java/classpath:java_lib',
target_type=JavaLibrary,
sources=['com/example/Foo.java'],
dependencies=[scala_dep],
tags={'use-compiler:zinc-only'})
scala_target = self.make_target('scala/classpath:scala_lib',
target_type=ScalaLibrary,
sources=['com/example/Foo.scala'],
dependencies=[scala_dep])
test_target = self.make_target('scala/classpath:scala_test',
target_type=JUnitTests,
sources=['com/example/Test.scala'],
dependencies=[scala_target],
tags={'use-compiler:zinc-only'})
with temporary_dir() as tmp_dir:
invalid_targets = [
java_target, scala_target, scala_dep, test_target
]
task = self.create_task_with_target_roots(
target_roots=[java_target, scala_target, test_target])
jobs = task._create_compile_jobs(
compile_contexts=self.create_compile_contexts(
invalid_targets, task, tmp_dir),
invalid_targets=invalid_targets,
invalid_vts=self.wrap_in_vts(invalid_targets),
classpath_product=None)
dependee_graph = self.construct_dependee_graph_str(jobs, task)
self.assertEqual(
dedent("""
zinc[zinc-java](java/classpath:java_lib) <- {}
rsc(scala/classpath:scala_lib) <- {
zinc[zinc-only](scala/classpath:scala_test)
}
zinc[rsc-and-zinc](scala/classpath:scala_lib) <- {}
rsc(scala/classpath:scala_dep) <- {
rsc(scala/classpath:scala_lib),
zinc[rsc-and-zinc](scala/classpath:scala_lib),
zinc[zinc-only](scala/classpath:scala_test)
}
zinc[rsc-and-zinc](scala/classpath:scala_dep) <- {
zinc[zinc-java](java/classpath:java_lib)
}
zinc[zinc-only](scala/classpath:scala_test) <- {}""").
strip(), dependee_graph)
def test_unknown_values(self) -> None:
ob = OptionValueContainerBuilder()
ob.foo = RankedValue(Rank.HARDCODED, 1)
o = ob.build()
self.assertEqual(1, o.foo)
with self.assertRaises(AttributeError):
o.bar
def test_deepcopy(self) -> None:
# copy semantics can get hairy when overriding __setattr__/__getattr__, so we test them.
o = OptionValueContainer()
o.foo = RankedValue(RankedValue.FLAG, 1)
o.bar = RankedValue(RankedValue.FLAG, {"a": 111})
p = copy.deepcopy(o)
# Verify that the result is in fact a copy.
self.assertEqual(1, p.foo) # Has original attribute.
o.baz = RankedValue(RankedValue.FLAG, 42)
self.assertFalse(hasattr(p, "baz")) # Does not have attribute added after the copy.
# Verify that it's a deep copy by modifying a referent in o and reading it in p.
# TODO: add type hints to ranked_value.py and option_value_container.py so that this works.
o.bar["b"] = 222 # type: ignore[index]
self.assertEqual({"a": 111}, p.bar)
def test_deepcopy(self):
# copy semantics can get hairy when overriding __setattr__/__getattr__, so we test them.
o = OptionValueContainer()
o.foo = RankedValue(RankedValue.FLAG, 1)
o.bar = RankedValue(RankedValue.FLAG, {'a': 111})
p = copy.deepcopy(o)
# Verify that the result is in fact a copy.
self.assertEqual(1, p.foo) # Has original attribute.
o.baz = RankedValue(RankedValue.FLAG, 42)
self.assertFalse(hasattr(
p, 'baz')) # Does not have attribute added after the copy.
# Verify that it's a deep copy by modifying a referent in o and reading it in p.
o.bar['b'] = 222
self.assertEqual({'a': 111}, p.bar)
def test_unknown_values(self) -> None:
ob = OptionValueContainerBuilder()
ob.foo = RankedValue(Rank.HARDCODED, 1)
o = ob.build()
assert 1 == o.foo
with pytest.raises(AttributeError):
o.bar
def test_copy(self) -> None:
# copy semantics can get hairy when overriding __setattr__/__getattr__, so we test them.
ob = OptionValueContainerBuilder()
ob.foo = RankedValue(Rank.FLAG, 1)
ob.bar = RankedValue(Rank.FLAG, {"a": 111})
p = ob.build()
z = ob.build()
# Verify that the result is in fact a copy.
assert 1 == p.foo # Has original attribute.
ob.baz = RankedValue(Rank.FLAG, 42)
assert not hasattr(
p, "baz") # Does not have attribute added after the copy.
# Verify that it's a shallow copy by modifying a referent in o and reading it in p.
p.bar["b"] = 222
assert {"a": 111, "b": 222} == z.bar
def test_value_ranking(self) -> None:
ob = OptionValueContainerBuilder()
ob.foo = RankedValue(Rank.CONFIG, 11)
o = ob.build()
assert 11 == o.foo
assert Rank.CONFIG == o.get_rank("foo")
ob.foo = RankedValue(Rank.HARDCODED, 22)
o = ob.build()
assert 11 == o.foo
assert Rank.CONFIG == o.get_rank("foo")
ob.foo = RankedValue(Rank.ENVIRONMENT, 33)
o = ob.build()
assert 33 == o.foo
assert Rank.ENVIRONMENT == o.get_rank("foo")
ob.foo = RankedValue(Rank.FLAG, 44)
o = ob.build()
assert 44 == o.foo
assert Rank.FLAG == o.get_rank("foo")
def setUp(self):
super().setUp()
# We're tied tightly to pex implementation details here faking out a python binary that outputs
# only one value no matter what arguments, environment or input stream it has attached. That
# value is the interpreter identity which is a JSON dict with exactly the following keys:
# binary, python_tag, abi_tag, platform_tag, version, supported_tags, env_markers.
def fake_interpreter(python_tag: str, abi_tag: str, version: Tuple[int, int, int]):
interpreter_dir = safe_mkdtemp()
binary = os.path.join(interpreter_dir, "python")
values = dict(
binary=binary,
python_tag=python_tag,
abi_tag=abi_tag,
platform_tag="",
version=version,
supported_tags=[],
env_markers={},
)
id_str = json.dumps(values)
with open(binary, "w") as fp:
fp.write(
dedent(
f"""
#!{PythonInterpreter.get().binary}
from __future__ import print_function
print({id_str!r})
"""
).strip()
)
chmod_plus_x(binary)
return PythonInterpreter.from_binary(binary)
# impl, abi, impl_version, major, minor, patch
self.fake_interpreters = [
fake_interpreter(python_tag="ip", abi_tag="ip2", version=(2, 77, 777)),
fake_interpreter(python_tag="ip", abi_tag="ip2", version=(2, 88, 888)),
fake_interpreter(python_tag="ip", abi_tag="ip2", version=(2, 99, 999)),
]
self.set_options_for_scope(
PythonSetup.options_scope,
interpreter_constraints=RankedValue(RankedValue.CONFIG, ["IronPython>=2.55"]),
interpreter_search_paths=[interpreter.binary for interpreter in self.fake_interpreters],
)
self.reqtgt = self.make_target(
spec="req", target_type=PythonRequirementLibrary, requirements=[],
)
self.tgt1 = self._fake_target("tgt1")
self.tgt2 = self._fake_target("tgt2", compatibility=["IronPython>2.77.777"])
self.tgt3 = self._fake_target("tgt3", compatibility=["IronPython>2.88.888"])
self.tgt4 = self._fake_target("tgt4", compatibility=["IronPython<2.99.999"])
self.tgt20 = self._fake_target("tgt20", dependencies=[self.tgt2])
self.tgt30 = self._fake_target("tgt30", dependencies=[self.tgt3])
self.tgt40 = self._fake_target("tgt40", dependencies=[self.tgt4])
def test_value_ranking(self) -> None:
ob = OptionValueContainerBuilder()
ob.foo = RankedValue(Rank.CONFIG, 11)
o = ob.build()
self.assertEqual(11, o.foo)
self.assertEqual(Rank.CONFIG, o.get_rank("foo"))
ob.foo = RankedValue(Rank.HARDCODED, 22)
o = ob.build()
self.assertEqual(11, o.foo)
self.assertEqual(Rank.CONFIG, o.get_rank("foo"))
ob.foo = RankedValue(Rank.ENVIRONMENT, 33)
o = ob.build()
self.assertEqual(33, o.foo)
self.assertEqual(Rank.ENVIRONMENT, o.get_rank("foo"))
ob.foo = RankedValue(Rank.FLAG, 44)
o = ob.build()
self.assertEqual(44, o.foo)
self.assertEqual(Rank.FLAG, o.get_rank("foo"))
def create_option_value_container(
default_rank: Rank = Rank.NONE, **options: RankedValue | Value
) -> OptionValueContainer:
scoped_options = OptionValueContainerBuilder()
for key, value in options.items():
if not isinstance(value, RankedValue):
value = RankedValue(default_rank, value)
setattr(scoped_options, key, value)
return scoped_options.build()
def _create_scoped_options(
default_rank: Rank, **options: Union[RankedValue,
Value]) -> OptionValueContainer:
scoped_options = OptionValueContainer()
for key, value in options.items():
if not isinstance(value, RankedValue):
value = RankedValue(default_rank, value)
setattr(scoped_options, key, value)
return scoped_options
def _format_record(self, record):
value_color = green if self.get_options().colors else lambda x: x
rank_color = self._rank_color(record.rank)
return '{value} {rank}'.format(
value=value_color(str(record.value)),
rank=rank_color('(from {rank}{details})'.format(
rank=RankedValue.get_rank_name(record.rank),
details=' {}'.format(record.details) if record.details else '',
)),
)
def _format_record(self, record):
value_color = green if self.get_options().colors else lambda x: x
rank_color = self._rank_color(record.rank)
return '{value} {rank}'.format(
value=value_color(str(record.value)),
rank=rank_color('(from {rank}{details})'.format(
rank=RankedValue.get_rank_name(record.rank),
details=' {}'.format(record.details) if record.details else '',
)),
)
def register_options(cls, register):
super(ExplainOptionsTask, cls).register_options(register)
register('--scope', help='Only show options in this scope.')
register('--name', help='Only show options with this name.')
register('--rank', choices=RankedValue.get_names(),
help='Only show options with at least this importance.')
register('--show-history', action='store_true', default=False,
help='Show the previous values options had before being overridden.')
register('--only-overridden', action='store_true', default=False,
help='Only show values that overrode defaults.')
def register(*args, **kwargs):
default = kwargs.get('default')
if default is None:
if kwargs.get('type') == bool:
default = False
if kwargs.get('type') == list:
default = []
for flag_name in args:
normalized_flag_name = flag_name.lstrip('-').replace('-', '_')
defaults[normalized_flag_name] = RankedValue(RankedValue.HARDCODED, default)
def test_resolve_conflicting_options(self) -> None:
resolve_options = partial(
resolve_conflicting_options,
old_option="my_opt",
new_option="my_opt",
old_scope="old-scope",
new_scope="new-scope",
)
old_val = "ancient"
new_val = "modern"
old_default_rv = RankedValue(RankedValue.HARDCODED, old_val)
new_default_rv = RankedValue(RankedValue.HARDCODED, new_val)
old_configured_rv = RankedValue(RankedValue.FLAG, old_val)
new_configured_rv = RankedValue(RankedValue.FLAG, new_val)
def assert_option_resolved(
*,
old_configured: bool = False,
new_configured: bool = False,
expected: str,
) -> None:
old_container, new_container = OptionValueContainer(
), OptionValueContainer()
old_container.my_opt = old_configured_rv if old_configured else old_default_rv
new_container.my_opt = new_configured_rv if new_configured else new_default_rv
assert resolve_options(old_container=old_container,
new_container=new_container) == expected
assert_option_resolved(expected=new_val)
assert_option_resolved(old_configured=True, expected=old_val)
assert_option_resolved(new_configured=True, expected=new_val)
# both configured -> raise an error
old_container, new_container = OptionValueContainer(
), OptionValueContainer()
old_container.my_opt = old_configured_rv
new_container.my_opt = new_configured_rv
with pytest.raises(ValueError) as e:
resolve_options(old_container=old_container,
new_container=new_container)
assert "--old-scope-my-opt" in str(e.value)
assert "--new-scope-my-opt" in str(e.value)
def register(*args, **kwargs):
default = kwargs.get('default')
if default is None:
action = kwargs.get('action')
if action == 'store_true':
default = False
if action == 'append':
default = []
for flag_name in args:
normalized_flag_name = flag_name.lstrip('-').replace('-', '_')
defaults[normalized_flag_name] = RankedValue(RankedValue.HARDCODED, default)
def test_find_python_interpreter_constraints_from_lockfile() -> None:
def assert_ics(
lockfile: str,
expected: list[str],
*,
ics: RankedValue = RankedValue(Rank.HARDCODED,
Black.default_interpreter_constraints),
metadata: PythonLockfileMetadata
| None = PythonLockfileMetadata.new(
InterpreterConstraints(["==2.7.*"]), set()),
) -> None:
black = create_subsystem(
Black,
lockfile=lockfile,
interpreter_constraints=ics,
version="v",
extra_requirements=[],
)
loaded_lock = LoadedLockfile(
EMPTY_DIGEST,
"black.lock",
metadata=metadata,
requirement_estimate=1,
is_pex_native=True,
constraints_strings=None,
original_lockfile=Lockfile("black.lock",
file_path_description_of_origin="foo",
resolve_name="black"),
)
result = run_rule_with_mocks(
_find_python_interpreter_constraints_from_lockfile,
rule_args=[black],
mock_gets=[
MockGet(
output_type=LoadedLockfile,
input_type=LoadedLockfileRequest,
mock=lambda _: loaded_lock,
)
],
)
assert result == InterpreterConstraints(expected)
# If ICs are set by user, always use those.
for lockfile in (NO_TOOL_LOCKFILE, DEFAULT_TOOL_LOCKFILE, "black.lock"):
assert_ics(lockfile, ["==3.8.*"],
ics=RankedValue(Rank.CONFIG, ["==3.8.*"]))
assert_ics(NO_TOOL_LOCKFILE, Black.default_interpreter_constraints)
assert_ics(DEFAULT_TOOL_LOCKFILE, Black.default_interpreter_constraints)
assert_ics("black.lock", ["==2.7.*"])
assert_ics("black.lock",
Black.default_interpreter_constraints,
metadata=None)
def _support_current_pants_plugin_options_usage(self):
export_options = self.get_options()
try:
ivy_options = self.context.options.for_scope('resolve.ivy')
except OptionsError:
# No resolve.ivy task installed, so continue silently.
ivy_options = []
for name in set.intersection(set(export_options), set(ivy_options)):
if not ivy_options.is_default(name):
setattr(export_options, name,
RankedValue(RankedValue.FLAG, ivy_options[name]))
def register_options(cls, register):
super(ExplainOptionsTask, cls).register_options(register)
register('--scope', help='Only show options in this scope.')
register('--name', help='Only show options with this name.')
register('--rank', choices=RankedValue.get_names(),
help='Only show options with at least this importance.')
register('--show-history', type=bool,
help='Show the previous values options had before being overridden.')
register('--only-overridden', type=bool,
help='Only show values that overrode defaults.')
register('--skip-inherited', type=bool, default=True,
help='Do not show inherited options, unless their values differ from their parents.')
def test_indexing(self) -> None:
o = OptionValueContainer()
o.foo = RankedValue(RankedValue.CONFIG, 1)
self.assertEqual(1, o["foo"])
self.assertEqual(1, o.get("foo"))
self.assertEqual(1, o.get("foo", 2))
self.assertIsNone(o.get("unknown"))
self.assertEqual(2, o.get("unknown", 2))
with self.assertRaises(AttributeError):
o["bar"]
def test_indexing(self):
o = OptionValueContainer()
o.foo = RankedValue(RankedValue.CONFIG, 1)
self.assertEqual(1, o['foo'])
self.assertEqual(1, o.get('foo'))
self.assertEqual(1, o.get('foo', 2))
self.assertIsNone(o.get('unknown'))
self.assertEqual(2, o.get('unknown', 2))
with self.assertRaises(AttributeError):
o['bar']
def test_invalidation_for_global_constraints(self):
# Because the system is setup with interpreter constraints, the task should
# invalidate on the first run.
self._select_interpreter_and_get_version([self.tgt1], should_invalidate=True)
self.set_options_for_scope(
PythonSetup.options_scope,
interpreter_constraints=RankedValue(RankedValue.CONFIG, ["IronPython>2.77.777"]),
)
# After changing the global interpreter constraints, the task should invalidate.
self._select_interpreter_and_get_version([self.tgt1], should_invalidate=True)
# If the global constraints don't change, the task should not invalidate.
self._select_interpreter_and_get_version([self.tgt1], should_invalidate=False)
def test_is_python2(constraints, compatibilities):
Subsystem.reset()
init_subsystem(
PythonSetup,
{
PythonSetup.options_scope: {
"interpreter_constraints": RankedValue(Rank.CONFIG,
constraints)
}
},
)
assert is_python2(compatibilities, PythonSetup.global_instance())
def setUp(self):
super(SelectInterpreterTest, self).setUp()
# We're tied tightly to pex implementation details here faking out a python binary that outputs
# only one value no matter what arguments, environment or input stream it has attached. That
# value is the interpreter identity which is - minimally, one line containing:
# <impl> <abi> <impl_version> <major> <minor> <patch>
def fake_interpreter(id_str):
interpreter_dir = safe_mkdtemp()
binary = os.path.join(interpreter_dir, 'python')
with open(binary, 'w') as fp:
fp.write(
dedent("""
#!{}
from __future__ import print_function
print({!r})
""".format(PythonInterpreter.get().binary, id_str)).strip())
chmod_plus_x(binary)
return PythonInterpreter.from_binary(binary)
# impl, abi, impl_version, major, minor, patch
self.fake_interpreters = [
fake_interpreter('ip ip2 2 2 77 777'),
fake_interpreter('ip ip2 2 2 88 888'),
fake_interpreter('ip ip2 2 2 99 999')
]
self.set_options_for_scope(
PythonSetup.options_scope,
interpreter_constraints=RankedValue(RankedValue.CONFIG,
['IronPython>=2.55']),
interpreter_search_paths=[
interpreter.binary for interpreter in self.fake_interpreters
])
self.reqtgt = self.make_target(
spec='req',
target_type=PythonRequirementLibrary,
requirements=[],
)
self.tgt1 = self._fake_target('tgt1')
self.tgt2 = self._fake_target('tgt2',
compatibility=['IronPython>2.77.777'])
self.tgt3 = self._fake_target('tgt3',
compatibility=['IronPython>2.88.888'])
self.tgt4 = self._fake_target('tgt4',
compatibility=['IronPython<2.99.999'])
self.tgt20 = self._fake_target('tgt20', dependencies=[self.tgt2])
self.tgt30 = self._fake_target('tgt30', dependencies=[self.tgt3])
self.tgt40 = self._fake_target('tgt40', dependencies=[self.tgt4])
def test_indexing(self) -> None:
ob = OptionValueContainerBuilder()
ob.foo = RankedValue(Rank.CONFIG, 1)
o = ob.build()
assert 1 == o["foo"]
assert 1 == o.get("foo")
assert 1 == o.get("foo", 2)
assert o.get("unknown") is None
assert 2 == o.get("unknown", 2)
with pytest.raises(AttributeError):
o["bar"]
def _compute_default(self, dest, kwargs):
"""Compute the default value to use for an option's registration.
The source of the default value is chosen according to the ranking in RankedValue.
"""
config_section = 'DEFAULT' if self._scope == GLOBAL_SCOPE else self._scope
env_var = 'PANTS_{0}_{1}'.format(config_section.upper().replace('.', '_'), dest.upper())
value_type = kwargs.get('type', str)
env_val_str = self._env.get(env_var) if self._env else None
env_val = None if env_val_str is None else value_type(env_val_str)
config_val = self._config.get(config_section, dest, default=None) if self._config else None
hardcoded_val = kwargs.get('default')
return RankedValue.choose(None, env_val, config_val, hardcoded_val)
def _format_record(self, record):
simple_rank = RankedValue.get_rank_name(record.rank)
if self.is_json():
return record.value, simple_rank
elif self.is_text():
simple_value = str(record.value)
value_color = green if self.get_options().colors else lambda x: x
formatted_value = value_color(simple_value)
rank_color = self._rank_color(record.rank)
formatted_rank = "(from {rank}{details})".format(
rank=simple_rank,
details=rank_color(" {}".format(record.details)) if record.details else "",
)
return "{value} {rank}".format(value=formatted_value, rank=formatted_rank,)
def normalize_kwargs(orig_args, orig_kwargs):
nkwargs = copy.copy(orig_kwargs)
dest = self.parse_dest(*orig_args, **nkwargs)
nkwargs["dest"] = dest
if not ("default" in nkwargs and isinstance(nkwargs["default"], RankedValue)):
type_arg = nkwargs.get("type", str)
member_type = nkwargs.get("member_type", str)
default_val = self.to_value_type(
nkwargs.get("default"), type_arg, member_type, dest
)
if isinstance(default_val, (ListValueComponent, DictValueComponent)):
default_val = default_val.val
nkwargs["default"] = RankedValue(Rank.HARDCODED, default_val)
return nkwargs
def register_options(cls, register):
super(ExplainOptionsTask, cls).register_options(register)
register('--scope', help='Only show options in this scope. Use GLOBAL for global scope.')
register('--name', help='Only show options with this name.')
register('--rank', choices=RankedValue.get_names(),
help='Only show options with at least this importance.')
register('--show-history', type=bool,
help='Show the previous values options had before being overridden.')
register('--only-overridden', type=bool,
help='Only show values that overrode defaults.')
register('--skip-inherited', type=bool, default=True,
help='Do not show inherited options, unless their values differ from their parents.')
register('--output-format', choices=['text', 'json'], default='text',
help='Specify the format options will be printed.')
def test_resolve_conflicting_options() -> None:
old_val = "ancient"
new_val = "modern"
old_default_rv = RankedValue(Rank.HARDCODED, old_val)
new_default_rv = RankedValue(Rank.HARDCODED, new_val)
old_configured_rv = RankedValue(Rank.FLAG, old_val)
new_configured_rv = RankedValue(Rank.FLAG, new_val)
def option_resolved(*,
old_configured: bool = False,
new_configured: bool = False):
old_container_builder, new_container_builder = (
OptionValueContainerBuilder(),
OptionValueContainerBuilder(),
)
old_container_builder.my_opt = old_configured_rv if old_configured else old_default_rv
new_container_builder.my_opt = new_configured_rv if new_configured else new_default_rv
old_container = old_container_builder.build()
new_container = new_container_builder.build()
return resolve_conflicting_options(
old_option="my_opt",
new_option="my_opt",
old_scope="old-scope",
new_scope="new-scope",
old_container=old_container,
new_container=new_container,
)
assert option_resolved() == new_val
assert option_resolved(old_configured=True) == old_val
assert option_resolved(new_configured=True) == new_val
# both configured -> raise an error
with pytest.raises(ValueError) as e:
option_resolved(old_configured=True, new_configured=True)
assert "--old-scope-my-opt" in str(e.value)
assert "--new-scope-my-opt" in str(e.value)
def _format_record(self, record):
simple_rank = RankedValue.get_rank_name(record.rank)
if self.is_json():
return record.value, simple_rank
elif self.is_text():
simple_value = str(record.value)
value_color = green if self.get_options().colors else lambda x: x
formatted_value = value_color(simple_value)
rank_color = self._rank_color(record.rank)
formatted_rank = '(from {rank}{details})'.format(
rank=simple_rank,
details=rank_color(' {}'.format(record.details)) if record.details else '',
)
return '{value} {rank}'.format(
value=formatted_value,
rank=formatted_rank,
)
def _compute_default(self, dest, kwargs):
"""Compute the default value to use for an option's registration.
The source of the default value is chosen according to the ranking in RankedValue.
"""
config_section = 'DEFAULT' if self._scope == GLOBAL_SCOPE else self._scope
udest = dest.upper()
if self._scope == GLOBAL_SCOPE:
# For convenience, we allow three forms of env var for global scope options.
# The fully-specified env var is PANTS_DEFAULT_FOO, which is uniform with PANTS_<SCOPE>_FOO
# for all the other scopes. However we also allow simply PANTS_FOO. And if the option name
# itself starts with 'pants-' then we also allow simply FOO. E.g., PANTS_WORKDIR instead of
# PANTS_PANTS_WORKDIR or PANTS_DEFAULT_PANTS_WORKDIR. We take the first specified value we
# find, in this order: PANTS_DEFAULT_FOO, PANTS_FOO, FOO.
env_vars = ['PANTS_DEFAULT_{0}'.format(udest), 'PANTS_{0}'.format(udest)]
if udest.startswith('PANTS_'):
env_vars.append(udest)
else:
sanitized_env_var_scope = self._ENV_SANITIZER_RE.sub('_', config_section.upper())
env_vars = ['PANTS_{0}_{1}'.format(sanitized_env_var_scope, udest)]
value_type = self.str_to_bool if is_boolean_flag(kwargs) else kwargs.get('type', str)
env_val_str = None
if self._env:
for env_var in env_vars:
if env_var in self._env:
env_val_str = self._env.get(env_var)
break
env_val = None if env_val_str is None else value_type(env_val_str)
if kwargs.get('action') == 'append':
config_val_strs = self._config.getlist(config_section, dest) if self._config else None
config_val = (None if config_val_strs is None else
[value_type(config_val_str) for config_val_str in config_val_strs])
default = []
else:
config_val_str = (self._config.get(config_section, dest, default=None)
if self._config else None)
config_val = None if config_val_str is None else value_type(config_val_str)
default = None
hardcoded_val = kwargs.get('default')
return RankedValue.choose(None, env_val, config_val, hardcoded_val, default)
def _rank_filter(self, rank):
pattern = self.get_options().rank
if not pattern:
return True
return rank >= RankedValue.get_rank_value(pattern)
def _compute_value(self, dest, kwargs, flag_vals):
"""Compute the value to use for an option.
The source of the default value is chosen according to the ranking in RankedValue.
"""
is_fromfile = kwargs.get('fromfile', False)
action = kwargs.get('action')
if is_fromfile and action and action != 'append':
raise ParseError('Cannot fromfile {} with an action ({}) in scope {}'
.format(dest, action, self._scope))
config_section = GLOBAL_SCOPE_CONFIG_SECTION if self._scope == GLOBAL_SCOPE else self._scope
udest = dest.upper()
if self._scope == GLOBAL_SCOPE:
# For convenience, we allow three forms of env var for global scope options.
# The fully-specified env var is PANTS_DEFAULT_FOO, which is uniform with PANTS_<SCOPE>_FOO
# for all the other scopes. However we also allow simply PANTS_FOO. And if the option name
# itself starts with 'pants-' then we also allow simply FOO. E.g., PANTS_WORKDIR instead of
# PANTS_PANTS_WORKDIR or PANTS_DEFAULT_PANTS_WORKDIR. We take the first specified value we
# find, in this order: PANTS_DEFAULT_FOO, PANTS_FOO, FOO.
env_vars = ['PANTS_DEFAULT_{0}'.format(udest), 'PANTS_{0}'.format(udest)]
if udest.startswith('PANTS_'):
env_vars.append(udest)
else:
sanitized_env_var_scope = self._ENV_SANITIZER_RE.sub('_', config_section.upper())
env_vars = ['PANTS_{0}_{1}'.format(sanitized_env_var_scope, udest)]
value_type = self.str_to_bool if is_boolean_flag(kwargs) else kwargs.get('type', str)
env_val_str = None
if self._env:
for env_var in env_vars:
if env_var in self._env:
env_val_str = self._env.get(env_var)
break
config_val_str = self._config.get(config_section, dest, default=None)
config_source_file = self._config.get_source_for_option(config_section, dest)
if config_source_file is not None:
config_source_file = os.path.relpath(config_source_file)
def expand(val_str):
if is_fromfile and val_str and val_str.startswith('@') and not val_str.startswith('@@'):
fromfile = val_str[1:]
try:
with open(fromfile) as fp:
return fp.read().strip()
except IOError as e:
raise self.FromfileError('Failed to read {} in {} from file {}: {}'.format(
dest, self._scope_str(), fromfile, e))
else:
# Support a literal @ for fromfile values via @@.
return val_str[1:] if is_fromfile and val_str.startswith('@@') else val_str
def parse_typed_list(val_str):
return None if val_str is None else [value_type(x) for x in list_option(expand(val_str))]
def parse_typed_item(val_str):
return None if val_str is None else value_type(expand(val_str))
flag_val = None
if flag_vals:
if action == 'append':
flag_val = [parse_typed_item(v) for v in flag_vals]
elif len(flag_vals) > 1:
raise ParseError('Multiple cmd line flags specified for option {} in {}'.format(
dest, self._scope_str()))
else:
flag_val = parse_typed_item(flag_vals[0])
default, parse = ([], parse_typed_list) if action == 'append' else (None, parse_typed_item)
config_val = parse(config_val_str)
env_val = parse(env_val_str)
hardcoded_val = kwargs.get('default')
config_details = 'in {}'.format(config_source_file) if config_source_file else None
# Note: ranked_vals is guaranteed to have at least one element, and none of the values
# of any of its elements will be None.
ranked_vals = list(reversed(list(RankedValue.prioritized_iter(
flag_val, env_val, config_val, hardcoded_val, default))))
choices = kwargs.get('choices')
for ranked_val in ranked_vals:
details = config_details if ranked_val.rank == RankedValue.CONFIG else None
self._option_tracker.record_option(scope=self._scope,
option=dest,
value=ranked_val.value,
rank=ranked_val.rank,
deprecation_version=kwargs.get('deprecated_version'),
details=details)
def check(val):
if choices is not None and val is not None and val not in choices:
raise ParseError('{} is not an allowed value for option {} in {}. '
'Must be one of: {}'.format(
val, dest, self._scope_str(), choices
))
return val
if action == 'append':
merged_rank = ranked_vals[-1].rank
merged_val = [check(val) for vals in ranked_vals for val in vals.value]
return RankedValue(merged_rank, merged_val)
else:
map(lambda rv: check(rv.value), ranked_vals)
return ranked_vals[-1]
def _compute_value(self, dest, kwargs, flag_val_strs):
"""Compute the value to use for an option.
The source of the default value is chosen according to the ranking in RankedValue.
"""
# Helper function to convert a string to a value of the option's type.
def to_value_type(val_str):
if val_str is None:
return None
elif kwargs.get('type') == bool:
return self._ensure_bool(val_str)
else:
return self._wrap_type(kwargs.get('type', str))(val_str)
# Helper function to expand a fromfile=True value string, if needed.
def expand(val_str):
if kwargs.get('fromfile', False) and val_str and val_str.startswith('@'):
if val_str.startswith('@@'): # Support a literal @ for fromfile values via @@.
return val_str[1:]
else:
fromfile = val_str[1:]
try:
with open(fromfile) as fp:
return fp.read().strip()
except IOError as e:
raise self.FromfileError('Failed to read {} in {} from file {}: {}'.format(
dest, self._scope_str(), fromfile, e))
else:
return val_str
# Get value from config files, and capture details about its derivation.
config_details = None
config_section = GLOBAL_SCOPE_CONFIG_SECTION if self._scope == GLOBAL_SCOPE else self._scope
config_val_str = expand(self._config.get(config_section, dest, default=None))
config_source_file = self._config.get_source_for_option(config_section, dest)
if config_source_file is not None:
config_source_file = os.path.relpath(config_source_file)
config_details = 'in {}'.format(config_source_file)
# Get value from environment, and capture details about its derivation.
udest = dest.upper()
if self._scope == GLOBAL_SCOPE:
# For convenience, we allow three forms of env var for global scope options.
# The fully-specified env var is PANTS_GLOBAL_FOO, which is uniform with PANTS_<SCOPE>_FOO
# for all the other scopes. However we also allow simply PANTS_FOO. And if the option name
# itself starts with 'pants-' then we also allow simply FOO. E.g., PANTS_WORKDIR instead of
# PANTS_PANTS_WORKDIR or PANTS_GLOBAL_PANTS_WORKDIR. We take the first specified value we
# find, in this order: PANTS_GLOBAL_FOO, PANTS_FOO, FOO.
env_vars = ['PANTS_GLOBAL_{0}'.format(udest), 'PANTS_{0}'.format(udest)]
if udest.startswith('PANTS_'):
env_vars.append(udest)
else:
sanitized_env_var_scope = self._ENV_SANITIZER_RE.sub('_', self._scope.upper())
env_vars = ['PANTS_{0}_{1}'.format(sanitized_env_var_scope, udest)]
env_val_str = None
env_details = None
if self._env:
for env_var in env_vars:
if env_var in self._env:
env_val_str = expand(self._env.get(env_var))
env_details = 'from env var {}'.format(env_var)
break
# Get value from cmd-line flags.
flag_vals = [to_value_type(expand(x)) for x in flag_val_strs]
if is_list_option(kwargs):
# Note: It's important to set flag_val to None if no flags were specified, so we can
# distinguish between no flags set vs. explicit setting of the value to [].
flag_val = ListValueComponent.merge(flag_vals) if flag_vals else None
elif len(flag_vals) > 1:
raise ParseError('Multiple cmd line flags specified for option {} in {}'.format(
dest, self._scope_str()))
elif len(flag_vals) == 1:
flag_val = flag_vals[0]
else:
flag_val = None
# Rank all available values.
# Note that some of these values may already be of the value type, but type conversion
# is idempotent, so this is OK.
values_to_rank = [to_value_type(x) for x in
[flag_val, env_val_str, config_val_str, kwargs.get('default'), None]]
# Note that ranked_vals will always have at least one element, and all elements will be
# instances of RankedValue (so none will be None, although they may wrap a None value).
ranked_vals = list(reversed(list(RankedValue.prioritized_iter(*values_to_rank))))
# Record info about the derivation of each of the values.
for ranked_val in ranked_vals:
if ranked_val.rank == RankedValue.CONFIG:
details = config_details
elif ranked_val.rank == RankedValue.ENVIRONMENT:
details = env_details
else:
details = None
self._option_tracker.record_option(scope=self._scope,
option=dest,
value=ranked_val.value,
rank=ranked_val.rank,
deprecation_version=kwargs.get('removal_version'),
details=details)
# Helper function to check various validity constraints on final option values.
def check(val):
if val is not None:
choices = kwargs.get('choices')
if choices is not None and val not in choices:
raise ParseError('{} is not an allowed value for option {} in {}. '
'Must be one of: {}'.format(val, dest, self._scope_str(), choices))
elif kwargs.get('type') == file_option and not os.path.isfile(val):
raise ParseError('File value {} for option {} in {} does not exist.'.format(
val, dest, self._scope_str()))
# Generate the final value from all available values, and check that it (or its members,
# if a list) are in the set of allowed choices.
if is_list_option(kwargs):
merged_rank = ranked_vals[-1].rank
merged_val = ListValueComponent.merge(
[rv.value for rv in ranked_vals if rv.value is not None]).val
merged_val = [self._wrap_type(kwargs.get('member_type', str))(x) for x in merged_val]
map(check, merged_val)
ret = RankedValue(merged_rank, merged_val)
else:
ret = ranked_vals[-1]
check(ret.value)
# All done!
return ret
def _compute_default(self, dest, kwargs):
"""Compute the default value to use for an option's registration.
The source of the default value is chosen according to the ranking in RankedValue.
"""
is_fromfile = kwargs.get('fromfile', False)
action = kwargs.get('action')
if is_fromfile and action:
raise ParseError('Cannot fromfile {} with an action ({}) in scope {}'
.format(dest, action, self._scope))
config_section = 'DEFAULT' if self._scope == GLOBAL_SCOPE else self._scope
udest = dest.upper()
if self._scope == GLOBAL_SCOPE:
# For convenience, we allow three forms of env var for global scope options.
# The fully-specified env var is PANTS_DEFAULT_FOO, which is uniform with PANTS_<SCOPE>_FOO
# for all the other scopes. However we also allow simply PANTS_FOO. And if the option name
# itself starts with 'pants-' then we also allow simply FOO. E.g., PANTS_WORKDIR instead of
# PANTS_PANTS_WORKDIR or PANTS_DEFAULT_PANTS_WORKDIR. We take the first specified value we
# find, in this order: PANTS_DEFAULT_FOO, PANTS_FOO, FOO.
env_vars = ['PANTS_DEFAULT_{0}'.format(udest), 'PANTS_{0}'.format(udest)]
if udest.startswith('PANTS_'):
env_vars.append(udest)
else:
sanitized_env_var_scope = self._ENV_SANITIZER_RE.sub('_', config_section.upper())
env_vars = ['PANTS_{0}_{1}'.format(sanitized_env_var_scope, udest)]
value_type = self.str_to_bool if is_boolean_flag(kwargs) else kwargs.get('type', str)
env_val_str = None
if self._env:
for env_var in env_vars:
if env_var in self._env:
env_val_str = self._env.get(env_var)
break
def expand(val_str):
if is_fromfile and val_str and val_str.startswith('@') and not val_str.startswith('@@'):
fromfile = val_str[1:]
try:
with open(fromfile) as fp:
return fp.read().strip()
except IOError as e:
raise self.FromfileError('Failed to read {} from file {}: {}'.format(dest, fromfile, e))
else:
# Support a literal @ for fromfile values via @@.
return val_str[1:] if is_fromfile and val_str.startswith('@@') else val_str
if is_fromfile:
kwargs['type'] = lambda flag_val_str: value_type(expand(flag_val_str)) # Expand flag values.
env_val = None if env_val_str is None else value_type(expand(env_val_str)) # Expand env values.
config_val = None
if action == 'append':
config_val_strs = self._config.getlist(config_section, dest) if self._config else None
if config_val_strs is not None:
config_val = [value_type(config_val_str) for config_val_str in config_val_strs]
default = []
else:
config_val_str = (self._config.get(config_section, dest, default=None)
if self._config else None)
if config_val_str is not None:
config_val = value_type(expand(config_val_str)) # Expand config values.
default = None
hardcoded_val = kwargs.get('default') # We don't expand hard coded defaults.
return RankedValue.choose(None, env_val, config_val, hardcoded_val, default)
def _compute_value(self, dest, kwargs, flag_val_strs):
"""Compute the value to use for an option.
The source of the default value is chosen according to the ranking in RankedValue.
"""
# Helper function to convert a string to a value of the option's type.
def to_value_type(val_str):
if val_str is None:
return None
elif kwargs.get('type') == bool:
return self._ensure_bool(val_str)
else:
type_arg = kwargs.get('type', str)
try:
return self._wrap_type(type_arg)(val_str)
except TypeError as e:
raise ParseError(
"Error applying type '{}' to option value '{}', for option '--{}' in {}: {}"
.format(type_arg.__name__, val_str, dest, self._scope_str(), e))
# Helper function to expand a fromfile=True value string, if needed.
# May return a string or a dict/list decoded from a json/yaml file.
def expand(val_or_str):
if (kwargs.get('fromfile', True) and val_or_str and
isinstance(val_or_str, str) and val_or_str.startswith('@')):
if val_or_str.startswith('@@'): # Support a literal @ for fromfile values via @@.
return val_or_str[1:]
else:
fromfile = val_or_str[1:]
try:
with open(fromfile, 'r') as fp:
s = fp.read().strip()
if fromfile.endswith('.json'):
return json.loads(s)
elif fromfile.endswith('.yml') or fromfile.endswith('.yaml'):
return yaml.safe_load(s)
else:
return s
except (IOError, ValueError, yaml.YAMLError) as e:
raise self.FromfileError('Failed to read {} in {} from file {}: {}'.format(
dest, self._scope_str(), fromfile, e))
else:
return val_or_str
# Get value from config files, and capture details about its derivation.
config_details = None
config_section = GLOBAL_SCOPE_CONFIG_SECTION if self._scope == GLOBAL_SCOPE else self._scope
config_default_val_or_str = expand(self._config.get(Config.DEFAULT_SECTION, dest, default=None))
config_val_or_str = expand(self._config.get(config_section, dest, default=None))
config_source_file = (self._config.get_source_for_option(config_section, dest) or
self._config.get_source_for_option(Config.DEFAULT_SECTION, dest))
if config_source_file is not None:
config_source_file = os.path.relpath(config_source_file)
config_details = 'in {}'.format(config_source_file)
# Get value from environment, and capture details about its derivation.
udest = dest.upper()
if self._scope == GLOBAL_SCOPE:
# For convenience, we allow three forms of env var for global scope options.
# The fully-specified env var is PANTS_GLOBAL_FOO, which is uniform with PANTS_<SCOPE>_FOO
# for all the other scopes. However we also allow simply PANTS_FOO. And if the option name
# itself starts with 'pants-' then we also allow simply FOO. E.g., PANTS_WORKDIR instead of
# PANTS_PANTS_WORKDIR or PANTS_GLOBAL_PANTS_WORKDIR. We take the first specified value we
# find, in this order: PANTS_GLOBAL_FOO, PANTS_FOO, FOO.
env_vars = ['PANTS_GLOBAL_{0}'.format(udest), 'PANTS_{0}'.format(udest)]
if udest.startswith('PANTS_'):
env_vars.append(udest)
else:
sanitized_env_var_scope = self._ENV_SANITIZER_RE.sub('_', self._scope.upper())
env_vars = ['PANTS_{0}_{1}'.format(sanitized_env_var_scope, udest)]
env_val_or_str = None
env_details = None
if self._env:
for env_var in env_vars:
if env_var in self._env:
env_val_or_str = expand(self._env.get(env_var))
env_details = 'from env var {}'.format(env_var)
break
# Get value from cmd-line flags.
flag_vals = [to_value_type(expand(x)) for x in flag_val_strs]
if is_list_option(kwargs):
# Note: It's important to set flag_val to None if no flags were specified, so we can
# distinguish between no flags set vs. explicit setting of the value to [].
flag_val = ListValueComponent.merge(flag_vals) if flag_vals else None
elif is_dict_option(kwargs):
# Note: It's important to set flag_val to None if no flags were specified, so we can
# distinguish between no flags set vs. explicit setting of the value to {}.
flag_val = DictValueComponent.merge(flag_vals) if flag_vals else None
elif len(flag_vals) > 1:
raise ParseError('Multiple cmd line flags specified for option {} in {}'.format(
dest, self._scope_str()))
elif len(flag_vals) == 1:
flag_val = flag_vals[0]
else:
flag_val = None
# Rank all available values.
# Note that some of these values may already be of the value type, but type conversion
# is idempotent, so this is OK.
values_to_rank = [to_value_type(x) for x in
[flag_val, env_val_or_str, config_val_or_str,
config_default_val_or_str, kwargs.get('default'), None]]
# Note that ranked_vals will always have at least one element, and all elements will be
# instances of RankedValue (so none will be None, although they may wrap a None value).
ranked_vals = list(reversed(list(RankedValue.prioritized_iter(*values_to_rank))))
def record_option(value, rank, option_details=None):
deprecation_version = kwargs.get('removal_version')
self._option_tracker.record_option(scope=self._scope,
option=dest,
value=value,
rank=rank,
deprecation_version=deprecation_version,
details=option_details)
# Record info about the derivation of each of the contributing values.
detail_history = []
for ranked_val in ranked_vals:
if ranked_val.rank in (RankedValue.CONFIG, RankedValue.CONFIG_DEFAULT):
details = config_details
elif ranked_val.rank == RankedValue.ENVIRONMENT:
details = env_details
else:
details = None
if details:
detail_history.append(details)
record_option(value=ranked_val.value, rank=ranked_val.rank, option_details=details)
# Helper function to check various validity constraints on final option values.
def check(val):
if val is not None:
choices = kwargs.get('choices')
# If the `type` argument has an `all_variants` attribute, use that as `choices` if not
# already set. Using an attribute instead of checking a subclass allows `type` arguments
# which are functions to have an implicit fallback `choices` set as well.
if choices is None and 'type' in kwargs:
type_arg = kwargs.get('type')
if hasattr(type_arg, 'all_variants'):
choices = list(type_arg.all_variants)
# TODO: convert this into an enum() pattern match!
if choices is not None and val not in choices:
raise ParseError('`{}` is not an allowed value for option {} in {}. '
'Must be one of: {}'.format(val, dest, self._scope_str(), choices))
elif kwargs.get('type') == dir_option and not os.path.isdir(val):
raise ParseError('Directory value `{}` for option {} in {} does not exist.'.format(
val, dest, self._scope_str()))
elif kwargs.get('type') == file_option and not os.path.isfile(val):
raise ParseError('File value `{}` for option {} in {} does not exist.'.format(
val, dest, self._scope_str()))
# Generate the final value from all available values, and check that it (or its members,
# if a list) are in the set of allowed choices.
if is_list_option(kwargs):
merged_rank = ranked_vals[-1].rank
merged_val = ListValueComponent.merge(
[rv.value for rv in ranked_vals if rv.value is not None]).val
# TODO: run `check()` for all elements of a list option too!!!
merged_val = [self._convert_member_type(kwargs.get('member_type', str), x)
for x in merged_val]
for val in merged_val:
check(val)
ret = RankedValue(merged_rank, merged_val)
elif is_dict_option(kwargs):
# TODO: convert `member_type` for dict values too!
merged_rank = ranked_vals[-1].rank
merged_val = DictValueComponent.merge(
[rv.value for rv in ranked_vals if rv.value is not None]).val
for val in merged_val:
check(val)
ret = RankedValue(merged_rank, merged_val)
else:
ret = ranked_vals[-1]
check(ret.value)
# Record info about the derivation of the final value.
merged_details = ', '.join(detail_history) if detail_history else None
record_option(value=ret.value, rank=ret.rank, option_details=merged_details)
# All done!
return ret
def _compute_default(self, kwargs):
"""Compute the default value to use for an option's registration.
The source of the default value is chosen according to the ranking in RankedValue.
Note: Only call if kwargs has a 'dest' key set.
"""
dest = kwargs['dest']
is_fromfile = kwargs.get('fromfile', False)
action = kwargs.get('action')
if is_fromfile and action and action != 'append':
raise ParseError('Cannot fromfile {} with an action ({}) in scope {}'
.format(dest, action, self._scope))
config_section = 'DEFAULT' if self._scope == GLOBAL_SCOPE else self._scope
udest = dest.upper()
if self._scope == GLOBAL_SCOPE:
# For convenience, we allow three forms of env var for global scope options.
# The fully-specified env var is PANTS_DEFAULT_FOO, which is uniform with PANTS_<SCOPE>_FOO
# for all the other scopes. However we also allow simply PANTS_FOO. And if the option name
# itself starts with 'pants-' then we also allow simply FOO. E.g., PANTS_WORKDIR instead of
# PANTS_PANTS_WORKDIR or PANTS_DEFAULT_PANTS_WORKDIR. We take the first specified value we
# find, in this order: PANTS_DEFAULT_FOO, PANTS_FOO, FOO.
env_vars = ['PANTS_DEFAULT_{0}'.format(udest), 'PANTS_{0}'.format(udest)]
if udest.startswith('PANTS_'):
env_vars.append(udest)
else:
sanitized_env_var_scope = self._ENV_SANITIZER_RE.sub('_', config_section.upper())
env_vars = ['PANTS_{0}_{1}'.format(sanitized_env_var_scope, udest)]
value_type = self.str_to_bool if is_boolean_flag(kwargs) else kwargs.get('type', str)
env_val_str = None
if self._env:
for env_var in env_vars:
if env_var in self._env:
env_val_str = self._env.get(env_var)
break
config_val_str = self._config.get(config_section, dest, default=None)
config_source_file = self._config.get_source_for_option(config_section, dest)
if config_source_file is not None:
config_source_file = os.path.relpath(config_source_file)
def expand(val_str):
if is_fromfile and val_str and val_str.startswith('@') and not val_str.startswith('@@'):
fromfile = val_str[1:]
try:
with open(fromfile) as fp:
return fp.read().strip()
except IOError as e:
raise self.FromfileError('Failed to read {} from file {}: {}'.format(dest, fromfile, e))
else:
# Support a literal @ for fromfile values via @@.
return val_str[1:] if is_fromfile and val_str.startswith('@@') else val_str
def parse_typed_list(val_str):
return None if val_str is None else [value_type(x) for x in list_option(expand(val_str))]
def parse_typed_item(val_str):
return None if val_str is None else value_type(expand(val_str))
# Handle the forthcoming conversions argparse will need to do by placing our parse hook - we
# handle the conversions for env and config ourselves below. Unlike the env and config
# handling, `action='append'` does not need to be handled specially since appended flag values
# come as single items' thus only `parse_typed_item` is ever needed for the flag value type
# conversions.
if is_fromfile:
kwargs['type'] = parse_typed_item
default, parse = ([], parse_typed_list) if action == 'append' else (None, parse_typed_item)
config_val = parse(config_val_str)
env_val = parse(env_val_str)
hardcoded_val = kwargs.get('default')
config_details = 'in {}'.format(config_source_file) if config_source_file else None
choices = list(RankedValue.prioritized_iter(None, env_val, config_val, hardcoded_val, default))
for choice in reversed(choices):
details = config_details if choice.rank == RankedValue.CONFIG else None
self._option_tracker.record_option(scope=self._scope, option=dest, value=choice.value,
rank=choice.rank, details=details)
return choices[0]
