def check(val):
if val is not None:
choices = kwargs.get('choices')
if choices is None and 'type' in kwargs:
type_arg = kwargs.get('type')
if inspect.isclass(type_arg) and issubclass(
type_arg, Enum):
choices = list(type_arg)
# TODO: convert this into an enum() pattern match!
if choices is not None and val not in choices:
raise ParseError(
'`{}` is not an allowed value for option {} in {}. '
'Must be one of: {}'.format(val, dest,
self._scope_str(),
choices))
elif kwargs.get(
'type') == dir_option and not os.path.isdir(val):
raise ParseError(
'Directory value `{}` for option {} in {} does not exist.'
.format(val, dest, self._scope_str()))
elif kwargs.get(
'type') == file_option and not os.path.isfile(val):
raise ParseError(
'File value `{}` for option {} in {} does not exist.'.
format(val, dest, self._scope_str()))
def check(val):
if val is not None:
choices = kwargs.get('choices')
# If the `type` argument has an `all_variants` attribute, use that as `choices` if not
# already set. Using an attribute instead of checking a subclass allows `type` arguments
# which are functions to have an implicit fallback `choices` set as well.
if choices is None and 'type' in kwargs:
type_arg = kwargs.get('type')
if hasattr(type_arg, 'all_variants'):
choices = list(type_arg.all_variants)
# TODO: convert this into an enum() pattern match!
if choices is not None and val not in choices:
raise ParseError(
'`{}` is not an allowed value for option {} in {}. '
'Must be one of: {}'.format(val, dest,
self._scope_str(),
choices))
elif kwargs.get(
'type') == dir_option and not os.path.isdir(val):
raise ParseError(
'Directory value `{}` for option {} in {} does not exist.'
.format(val, dest, self._scope_str()))
elif kwargs.get(
'type') == file_option and not os.path.isfile(val):
raise ParseError(
'File value `{}` for option {} in {} does not exist.'.
format(val, dest, self._scope_str()))
def check_dir_exists(val) -> None:
error_prefix = f"Directory value `{val}` for option `{dest}` in `{self._scope_str()}`"
try:
path = Path(val)
path_with_buildroot = Path(get_buildroot(), val)
except TypeError:
raise ParseError(f"{error_prefix} cannot be parsed as a directory path.")
if not path.is_dir() and not path_with_buildroot.is_dir():
raise ParseError(f"{error_prefix} does not exist.")
def check(val):
if val is not None:
choices = kwargs.get('choices')
if choices is not None and val not in choices:
raise ParseError('`{}` is not an allowed value for option {} in {}. '
'Must be one of: {}'.format(val, dest, self._scope_str(), choices))
elif kwargs.get('type') == dir_option and not os.path.isdir(val):
raise ParseError('Directory value `{}` for option {} in {} does not exist.'.format(
val, dest, self._scope_str()))
elif kwargs.get('type') == file_option and not os.path.isfile(val):
raise ParseError('File value `{}` for option {} in {} does not exist.'.format(
val, dest, self._scope_str()))
def _convert_member_type(member_type, value):
if member_type == dict:
return DictValueComponent.create(value).val
try:
return member_type(value)
except ValueError as error:
raise ParseError(str(error))
def merge(cls, components):
"""Merges components into a single component, applying their actions appropriately.
This operation is associative: M(M(a, b), c) == M(a, M(b, c)) == M(a, b, c).
:param list components: an iterable of instances of ListValueComponent.
:return: An instance representing the result of merging the components.
:rtype: `ListValueComponent`
"""
# Note that action of the merged component is MODIFY until the first REPLACE is encountered.
# This guarantees associativity.
action = cls.MODIFY
appends = []
filters = []
for component in components:
if component._action is cls.REPLACE:
appends = component._appends
filters = component._filters
action = cls.REPLACE
elif component._action is cls.MODIFY:
appends.extend(component._appends)
filters.extend(component._filters)
else:
raise ParseError('Unknown action for list value: {}'.format(
component.action))
return cls(action, appends, filters)
def register(self, *args, **kwargs):
"""Register an option, using argparse params.
Custom extensions to argparse params:
:param advanced: if True, the option will be suppressed when displaying help.
:param deprecated_version: Mark an option as deprecated. The value is a semver that indicates
the release at which the option should be removed from the code.
:param deprecated_hint: A message to display to the user when displaying help for or invoking
a deprecated option.
"""
if self._frozen:
raise RegistrationError(
'Cannot register option {0} in scope {1} after registering options '
'in any of its inner scopes.'.format(args[0], self._scope))
# Prevent further registration in enclosing scopes.
ancestor = self._parent_parser
while ancestor:
ancestor._freeze()
ancestor = ancestor._parent_parser
self._validate(args, kwargs)
dest = self._set_dest(args, kwargs)
if 'recursive' in kwargs:
if self._scope_info.category == ScopeInfo.SUBSYSTEM:
raise ParseError(
'Option {} in scope {} registered as recursive, but subsystem options '
'may not set recursive=True.'.format(args[0], self.scope))
kwargs[
'recursive_root'] = True # So we can distinguish the original registrar.
if self._scope_info.category == ScopeInfo.SUBSYSTEM:
kwargs['subsystem'] = True
self._register(
dest, args,
kwargs) # Note: May modify kwargs (to remove recursive_root).
def create(cls, value):
"""Interpret value as either a dict or something to extend another dict with.
:param value: The value to convert. Can be an instance of DictValueComponent, a dict,
or a string representation (possibly prefixed by +) of a dict.
:rtype: `DictValueComponent`
"""
if isinstance(value, bytes):
value = value.decode('utf-8')
if isinstance(value, cls): # Ensure idempotency.
action = value.action
val = value.val
elif isinstance(
value,
dict): # Ensure we can handle dict-typed default values.
action = cls.REPLACE
val = value
elif value.startswith('{'):
action = cls.REPLACE
val = _convert(value, dict)
elif value.startswith('+{'):
action = cls.EXTEND
val = _convert(value[1:], dict)
else:
raise ParseError('Invalid dict value: {}'.format(value))
return cls(action, dict(val))
def _parse_error(s, msg):
"""Return a ParseError with a usefully formatted message, for the caller to throw.
:param s: The option value we're parsing.
:param msg: An extra message to add to the ParseError.
"""
return ParseError('Error while parsing option value {0}: {1}'.format(s, msg))
def add_flag_val(v: Optional[Union[int, float, bool, str]]) -> None:
if v is None:
if implicit_value is None:
raise ParseError(f'Missing value for command line flag {arg} in {self._scope_str()}')
flag_vals.append(implicit_value)
else:
flag_vals.append(v)
def check(val):
if choices is not None and val is not None and val not in choices:
raise ParseError('{} is not an allowed value for option {} in {}. '
'Must be one of: {}'.format(
val, dest, self._scope_str(), choices
))
return val
def _raise_error_for_invalid_flag_names(self, flag_value_map, all_scoped_flag_names,
levenshtein_max_distance):
"""Identify similar option names to unconsumed flags and raise a ParseError with those names."""
matching_flags = {}
for flag_name in flag_value_map.keys():
# We will be matching option names without their leading hyphens, in order to capture both
# short and long-form options.
flag_normalized_unscoped_name = re.sub(r'^-+', '', flag_name)
flag_normalized_scoped_name = (
f"{self.scope.replace('.', '-')}-{flag_normalized_unscoped_name}"
if self.scope != GLOBAL_SCOPE
else flag_normalized_unscoped_name)
substring_matching_option_names = []
levenshtein_matching_option_names = defaultdict(list)
for other_scoped_flag in all_scoped_flag_names:
other_complete_flag_name = other_scoped_flag.scoped_arg
other_normalized_scoped_name = other_scoped_flag.normalized_scoped_arg
other_normalized_unscoped_name = other_scoped_flag.normalized_arg
if flag_normalized_unscoped_name == other_normalized_unscoped_name:
# If the unscoped option name itself matches, but the scope doesn't, display it.
substring_matching_option_names.append(other_complete_flag_name)
elif other_normalized_scoped_name.startswith(flag_normalized_scoped_name):
# If the invalid scoped option name is the beginning of another scoped option name,
# display it. This will also suggest long-form options such as --verbose for an attempted
# -v (if -v isn't defined as an option).
substring_matching_option_names.append(other_complete_flag_name)
else:
# If an unscoped option name is similar to the unscoped option from the command line
# according to --option-name-check-distance, display the matching scoped option name. This
# covers misspellings!
unscoped_option_levenshtein_distance = Levenshtein.distance(flag_normalized_unscoped_name, other_normalized_unscoped_name)
if unscoped_option_levenshtein_distance <= levenshtein_max_distance:
# NB: We order the matched flags by Levenshtein distance compared to the entire option string!
fully_scoped_levenshtein_distance = Levenshtein.distance(flag_normalized_scoped_name, other_normalized_scoped_name)
levenshtein_matching_option_names[fully_scoped_levenshtein_distance].append(other_complete_flag_name)
# If any option name matched or started with the invalid flag in any scope, put that
# first. Then, display the option names matching in order of overall edit distance, in a deterministic way.
all_matching_scoped_option_names = substring_matching_option_names + [
flag
for distance in sorted(levenshtein_matching_option_names.keys())
for flag in sorted(levenshtein_matching_option_names[distance])
]
if all_matching_scoped_option_names:
matching_flags[flag_name] = all_matching_scoped_option_names
if matching_flags:
suggestions_message = ' Suggestions:\n{}'.format('\n'.join(
'{}: [{}]'.format(flag_name, ', '.join(matches))
for flag_name, matches in matching_flags.items()
))
else:
suggestions_message = ''
raise ParseError(
'Unrecognized command line flags on {scope}: {flags}.{suggestions_message}'
.format(scope=self._scope_str(),
flags=', '.join(flag_value_map.keys()),
suggestions_message=suggestions_message))
def file_option(s):
"""Same type as 'str', but indicates string represents a filepath.
:API: public
"""
if not os.path.isfile(s):
raise ParseError('Options file "{filepath}" does not exist.'.format(filepath=s))
return s
def add_flag_val(v: int | float | bool | str | None) -> None:
if v is None:
if implicit_value is None:
raise ParseError(
f"Missing value for command line flag {arg} in {self._scope_str()}"
)
flag_vals.append(implicit_value)
else:
flag_vals.append(v)
def add_flag_val(v):
if v is None:
if implicit_value is None:
raise ParseError('Missing value for command line flag {} in {}'.format(
arg, self._scope_str()))
else:
flag_vals.append(implicit_value)
else:
flag_vals.append(v)
def to_value_type(self, val_str, type_arg, member_type, dest):
"""Convert a string to a value of the option's type."""
if val_str is None:
return None
if type_arg == bool:
return self.ensure_bool(val_str)
try:
if type_arg == list:
return ListValueComponent.create(val_str, member_type=member_type)
if type_arg == dict:
return DictValueComponent.create(val_str)
return type_arg(val_str)
except (TypeError, ValueError) as e:
if issubclass(type_arg, Enum):
choices = ", ".join(f"{choice.value}" for choice in type_arg)
raise ParseError(f"Invalid choice '{val_str}'. Choose from: {choices}")
raise ParseError(
f"Error applying type '{type_arg.__name__}' to option value '{val_str}': {e}"
)
def workspace_path(s: str) -> str:
"""Same type as 'str', but indicates string represents a directory path that is relative to
either the build root, or a BUILD file if prefix with `./`.
:API: public
"""
if s.startswith("/"):
raise ParseError(
f"Invalid value: `{s}`. Expected a relative path, optionally in the form "
"`./relative/path` to make it relative to the BUILD files rather than the build root."
)
return s
def to_value_type(val_str):
if val_str is None:
return None
if kwargs.get("type") == bool:
return self._ensure_bool(val_str)
type_arg = kwargs.get("type", str)
try:
return self._wrap_type(type_arg)(val_str)
except (TypeError, ValueError) as e:
raise ParseError(
f"Error applying type '{type_arg.__name__}' to option value '{val_str}', for option "
f"'--{dest}' in {self._scope_str()}: {e}")
def memory_size(s: str | int | float) -> int:
"""A string that normalizes the suffixes {GiB, MiB, KiB, B} into the number of bytes.
:API: public
"""
if isinstance(s, (int, float)):
return int(s)
if not s:
raise ParseError("Missing value.")
original = s
s = s.lower().strip()
try:
return int(float(s))
except ValueError:
pass
invalid = ParseError(
f"Invalid value: `{original}`. Expected either a bare number or a number with one of "
f"`GiB`, `MiB`, `KiB`, or `B`.")
def convert_to_bytes(power_of_2) -> int:
try:
return int(float(s[:-3]) * (2**power_of_2)) # type: ignore[index]
except TypeError:
raise invalid
if s.endswith("gib"):
return convert_to_bytes(30)
elif s.endswith("mib"):
return convert_to_bytes(20)
elif s.endswith("kib"):
return convert_to_bytes(10)
elif s.endswith("b"):
try:
return int(float(s[:-1]))
except TypeError:
raise invalid
raise invalid
def to_value_type(val_str):
if val_str is None:
return None
elif kwargs.get('type') == bool:
return self._ensure_bool(val_str)
else:
type_arg = kwargs.get('type', str)
try:
return self._wrap_type(type_arg)(val_str)
except TypeError as e:
raise ParseError(
"Error applying type '{}' to option value '{}', for option '--{}' in {}: {}"
.format(type_arg.__name__, val_str, dest, self._scope_str(), e))
def _convert(val, acceptable_types):
"""Ensure that val is one of the acceptable types, converting it if needed.
:param val: The value we're parsing (either a string or one of the acceptable types).
:param acceptable_types: A tuple of expected types for val.
:returns: The parsed value.
:raises :class:`pants.options.errors.ParseError`: if there was a problem parsing the val as an
acceptable type.
"""
if isinstance(val, acceptable_types):
return val
try:
return parse_expression(val, acceptable_types)
except ValueError as e:
raise ParseError(str(e)) from e
def check_scalar_value(val):
if val is None:
return
choices = kwargs.get("choices")
if choices is None and "type" in kwargs:
if inspect.isclass(type_arg) and issubclass(type_arg, Enum):
choices = list(type_arg)
if choices is not None and val not in choices:
raise ParseError(
f"`{val}` is not an allowed value for option {dest} in {self._scope_str()}. "
f"Must be one of: {choices}")
elif type_arg == file_option:
check_file_exists(val)
elif type_arg == dir_option:
check_dir_exists(val)
def merge(cls, components: Iterable["DictValueComponent"]) -> "DictValueComponent":
"""Merges components into a single component, applying their actions appropriately.
This operation is associative: M(M(a, b), c) == M(a, M(b, c)) == M(a, b, c)."""
# Note that action of the merged component is EXTEND until the first REPLACE is encountered.
# This guarantees associativity.
action = cls.EXTEND
val = {}
for component in components:
if component.action is cls.REPLACE:
val = component.val
action = cls.REPLACE
elif component.action is cls.EXTEND:
val.update(component.val)
else:
raise ParseError(f'Unknown action for dict value: {component.action}')
return cls(action, val)
def to_value_type(self, val_str, type_arg, member_type, dest):
"""Convert a string to a value of the option's type."""
if val_str is None:
return None
if type_arg == bool:
return self._ensure_bool(val_str)
try:
if type_arg == list:
return ListValueComponent.create(val_str, member_type=member_type)
if type_arg == dict:
return DictValueComponent.create(val_str)
return type_arg(val_str)
except (TypeError, ValueError) as e:
raise ParseError(
f"Error applying type '{type_arg.__name__}' to option value '{val_str}', "
f"for option '--{dest}' in {self._scope_str()}: {e}"
)
def check(val):
if val is None:
return
choices = kwargs.get("choices")
type_arg = kwargs.get("type")
if choices is None and "type" in kwargs:
if inspect.isclass(type_arg) and issubclass(type_arg, Enum):
choices = list(type_arg)
# TODO: convert this into an enum() pattern match!
if choices is not None and val not in choices:
raise ParseError(
f"`{val}` is not an allowed value for option {dest} in {self._scope_str()}. Must be one of: {choices}"
)
if type_arg == file_option:
check_file_exists(val)
elif type_arg == dir_option:
check_dir_exists(val)
def merge(cls, components: Iterable["ListValueComponent"]) -> "ListValueComponent":
"""Merges components into a single component, applying their actions appropriately.
This operation is associative: M(M(a, b), c) == M(a, M(b, c)) == M(a, b, c)."""
# Note that action of the merged component is MODIFY until the first REPLACE is encountered.
# This guarantees associativity.
action = cls.MODIFY
appends = []
filters = []
for component in components:
if component._action is cls.REPLACE:
appends = component._appends
filters = component._filters
action = cls.REPLACE
elif component._action is cls.MODIFY:
appends.extend(component._appends)
filters.extend(component._filters)
else:
raise ParseError(f'Unknown action for list value: {component._action}')
return cls(action, appends, filters)
def merge(cls, components):
"""Merges components into a single component, applying their actions appropriately.
This operation is associative: M(M(a, b), c) == M(a, M(b, c)) == M(a, b, c).
:param list components: an iterable of instances of DictValueComponent.
:return: An instance representing the result of merging the components.
:rtype: `DictValueComponent`
"""
# Note that action of the merged component is EXTEND until the first REPLACE is encountered.
# This guarantees associativity.
action = cls.EXTEND
val = {}
for component in components:
if component.action is cls.REPLACE:
val = component.val
action = cls.REPLACE
elif component.action is cls.EXTEND:
val.update(component.val)
else:
raise ParseError('Unknown action for dict value: {}'.format(component.action))
return cls(action, val)
def _compute_value(self, dest, kwargs, flag_vals):
"""Compute the value to use for an option.
The source of the default value is chosen according to the ranking in RankedValue.
"""
is_fromfile = kwargs.get('fromfile', False)
action = kwargs.get('action')
if is_fromfile and action and action != 'append':
raise ParseError('Cannot fromfile {} with an action ({}) in scope {}'
.format(dest, action, self._scope))
config_section = GLOBAL_SCOPE_CONFIG_SECTION if self._scope == GLOBAL_SCOPE else self._scope
udest = dest.upper()
if self._scope == GLOBAL_SCOPE:
# For convenience, we allow three forms of env var for global scope options.
# The fully-specified env var is PANTS_DEFAULT_FOO, which is uniform with PANTS_<SCOPE>_FOO
# for all the other scopes. However we also allow simply PANTS_FOO. And if the option name
# itself starts with 'pants-' then we also allow simply FOO. E.g., PANTS_WORKDIR instead of
# PANTS_PANTS_WORKDIR or PANTS_DEFAULT_PANTS_WORKDIR. We take the first specified value we
# find, in this order: PANTS_DEFAULT_FOO, PANTS_FOO, FOO.
env_vars = ['PANTS_DEFAULT_{0}'.format(udest), 'PANTS_{0}'.format(udest)]
if udest.startswith('PANTS_'):
env_vars.append(udest)
else:
sanitized_env_var_scope = self._ENV_SANITIZER_RE.sub('_', config_section.upper())
env_vars = ['PANTS_{0}_{1}'.format(sanitized_env_var_scope, udest)]
value_type = self.str_to_bool if is_boolean_flag(kwargs) else kwargs.get('type', str)
env_val_str = None
if self._env:
for env_var in env_vars:
if env_var in self._env:
env_val_str = self._env.get(env_var)
break
config_val_str = self._config.get(config_section, dest, default=None)
config_source_file = self._config.get_source_for_option(config_section, dest)
if config_source_file is not None:
config_source_file = os.path.relpath(config_source_file)
def expand(val_str):
if is_fromfile and val_str and val_str.startswith('@') and not val_str.startswith('@@'):
fromfile = val_str[1:]
try:
with open(fromfile) as fp:
return fp.read().strip()
except IOError as e:
raise self.FromfileError('Failed to read {} in {} from file {}: {}'.format(
dest, self._scope_str(), fromfile, e))
else:
# Support a literal @ for fromfile values via @@.
return val_str[1:] if is_fromfile and val_str.startswith('@@') else val_str
def parse_typed_list(val_str):
return None if val_str is None else [value_type(x) for x in list_option(expand(val_str))]
def parse_typed_item(val_str):
return None if val_str is None else value_type(expand(val_str))
flag_val = None
if flag_vals:
if action == 'append':
flag_val = [parse_typed_item(v) for v in flag_vals]
elif len(flag_vals) > 1:
raise ParseError('Multiple cmd line flags specified for option {} in {}'.format(
dest, self._scope_str()))
else:
flag_val = parse_typed_item(flag_vals[0])
default, parse = ([], parse_typed_list) if action == 'append' else (None, parse_typed_item)
config_val = parse(config_val_str)
env_val = parse(env_val_str)
hardcoded_val = kwargs.get('default')
config_details = 'in {}'.format(config_source_file) if config_source_file else None
# Note: ranked_vals is guaranteed to have at least one element, and none of the values
# of any of its elements will be None.
ranked_vals = list(reversed(list(RankedValue.prioritized_iter(
flag_val, env_val, config_val, hardcoded_val, default))))
choices = kwargs.get('choices')
for ranked_val in ranked_vals:
details = config_details if ranked_val.rank == RankedValue.CONFIG else None
self._option_tracker.record_option(scope=self._scope,
option=dest,
value=ranked_val.value,
rank=ranked_val.rank,
deprecation_version=kwargs.get('deprecated_version'),
details=details)
def check(val):
if choices is not None and val is not None and val not in choices:
raise ParseError('{} is not an allowed value for option {} in {}. '
'Must be one of: {}'.format(
val, dest, self._scope_str(), choices
))
return val
if action == 'append':
merged_rank = ranked_vals[-1].rank
merged_val = [check(val) for vals in ranked_vals for val in vals.value]
return RankedValue(merged_rank, merged_val)
else:
map(lambda rv: check(rv.value), ranked_vals)
return ranked_vals[-1]
def parse_args(self, flags, namespace):
"""Set values for this parser's options on the namespace object."""
flag_value_map = self._create_flag_value_map(flags)
for args, kwargs in self._unnormalized_option_registrations_iter():
self._validate(args, kwargs)
dest = kwargs.get('dest') or self._select_dest(args)
is_bool = is_boolean_flag(kwargs)
def consume_flag(flag):
self._check_deprecated(dest, kwargs)
del flag_value_map[flag]
# Compute the values provided on the command line for this option. Note tha there may be
# multiple values, for any combination of the following reasons:
# - The user used the same flag multiple times.
# - The user specified a boolean flag (--foo) and its inverse (--no-foo).
# - The option has multiple names, and the user used more than one of them.
#
# We also check if the option is deprecated, but we only do so if the option is explicitly
# specified as a command-line flag, so we don't spam users with deprecated option values
# specified in config, which isn't something they control.
implicit_value = kwargs.get('implicit_value')
flag_vals = []
def add_flag_val(v):
if v is None:
if implicit_value is None:
raise ParseError('Missing value for command line flag {} in {}'.format(
arg, self._scope_str()))
else:
flag_vals.append(implicit_value)
else:
flag_vals.append(v)
for arg in args:
if is_bool:
if arg in flag_value_map:
flag_vals.append('true' if kwargs['action'] == 'store_true' else 'false')
consume_flag(arg)
elif self._inverse_arg(arg) in flag_value_map:
flag_vals.append('false' if kwargs['action'] == 'store_true' else 'true')
consume_flag(self._inverse_arg(arg))
else:
if arg in flag_value_map:
for v in flag_value_map[arg]:
add_flag_val(v)
consume_flag(arg)
# Get the value for this option, falling back to defaults as needed.
try:
val = self._compute_value(dest, kwargs, flag_vals)
except ParseError as e:
# Reraise a new exception with context on the option being processed at the time of error.
# Note that other exception types can be raised here that are caught by ParseError (e.g.
# BooleanConversionError), hence we reference the original exception type by e.__class__.
raise type(e)(
'Error computing value for {} in {} (may also be from PANTS_* environment variables):\n'
'{}'.format(', '.join(args), self._scope_str(), e)
)
setattr(namespace, dest, val)
# See if there are any unconsumed flags remaining.
if flag_value_map:
raise ParseError('Unrecognized command line flags on {}: {}'.format(
self._scope_str(), ', '.join(flag_value_map.keys())))
return namespace
def _compute_value(self, dest, kwargs, flag_val_strs):
"""Compute the value to use for an option.
The source of the default value is chosen according to the ranking in RankedValue.
"""
# Helper function to convert a string to a value of the option's type.
def to_value_type(val_str):
if val_str is None:
return None
elif kwargs.get('type') == bool:
return self._ensure_bool(val_str)
else:
return self._wrap_type(kwargs.get('type', str))(val_str)
# Helper function to expand a fromfile=True value string, if needed.
def expand(val_str):
if kwargs.get('fromfile',
False) and val_str and val_str.startswith('@'):
if val_str.startswith(
'@@'
): # Support a literal @ for fromfile values via @@.
return val_str[1:]
else:
fromfile = val_str[1:]
try:
with open(fromfile) as fp:
return fp.read().strip()
except IOError as e:
raise self.FromfileError(
'Failed to read {} in {} from file {}: {}'.format(
dest, self._scope_str(), fromfile, e))
else:
return val_str
# Get value from config files, and capture details about its derivation.
config_details = None
config_section = GLOBAL_SCOPE_CONFIG_SECTION if self._scope == GLOBAL_SCOPE else self._scope
config_val_str = expand(
self._config.get(config_section, dest, default=None))
config_source_file = self._config.get_source_for_option(
config_section, dest)
if config_source_file is not None:
config_source_file = os.path.relpath(config_source_file)
config_details = 'in {}'.format(config_source_file)
# Get value from environment, and capture details about its derivation.
udest = dest.upper()
if self._scope == GLOBAL_SCOPE:
# For convenience, we allow three forms of env var for global scope options.
# The fully-specified env var is PANTS_GLOBAL_FOO, which is uniform with PANTS_<SCOPE>_FOO
# for all the other scopes. However we also allow simply PANTS_FOO. And if the option name
# itself starts with 'pants-' then we also allow simply FOO. E.g., PANTS_WORKDIR instead of
# PANTS_PANTS_WORKDIR or PANTS_GLOBAL_PANTS_WORKDIR. We take the first specified value we
# find, in this order: PANTS_GLOBAL_FOO, PANTS_FOO, FOO.
env_vars = [
'PANTS_GLOBAL_{0}'.format(udest), 'PANTS_{0}'.format(udest)
]
if udest.startswith('PANTS_'):
env_vars.append(udest)
else:
sanitized_env_var_scope = self._ENV_SANITIZER_RE.sub(
'_', self._scope.upper())
env_vars = ['PANTS_{0}_{1}'.format(sanitized_env_var_scope, udest)]
env_val_str = None
env_details = None
if self._env:
for env_var in env_vars:
if env_var in self._env:
env_val_str = expand(self._env.get(env_var))
env_details = 'from env var {}'.format(env_var)
break
# Get value from cmd-line flags.
flag_vals = [to_value_type(expand(x)) for x in flag_val_strs]
if is_list_option(kwargs):
# Note: It's important to set flag_val to None if no flags were specified, so we can
# distinguish between no flags set vs. explicit setting of the value to [].
flag_val = ListValueComponent.merge(
flag_vals) if flag_vals else None
elif is_dict_option(kwargs):
# Note: It's important to set flag_val to None if no flags were specified, so we can
# distinguish between no flags set vs. explicit setting of the value to {}.
flag_val = DictValueComponent.merge(
flag_vals) if flag_vals else None
elif len(flag_vals) > 1:
raise ParseError(
'Multiple cmd line flags specified for option {} in {}'.format(
dest, self._scope_str()))
elif len(flag_vals) == 1:
flag_val = flag_vals[0]
else:
flag_val = None
# Rank all available values.
# Note that some of these values may already be of the value type, but type conversion
# is idempotent, so this is OK.
values_to_rank = [
to_value_type(x) for x in [
flag_val, env_val_str, config_val_str,
kwargs.get('default'), None
]
]
# Note that ranked_vals will always have at least one element, and all elements will be
# instances of RankedValue (so none will be None, although they may wrap a None value).
ranked_vals = list(
reversed(list(RankedValue.prioritized_iter(*values_to_rank))))
# Record info about the derivation of each of the values.
for ranked_val in ranked_vals:
if ranked_val.rank == RankedValue.CONFIG:
details = config_details
elif ranked_val.rank == RankedValue.ENVIRONMENT:
details = env_details
else:
details = None
self._option_tracker.record_option(
scope=self._scope,
option=dest,
value=ranked_val.value,
rank=ranked_val.rank,
deprecation_version=kwargs.get('removal_version'),
details=details)
# Helper function to check various validity constraints on final option values.
def check(val):
if val is not None:
choices = kwargs.get('choices')
if choices is not None and val not in choices:
raise ParseError(
'`{}` is not an allowed value for option {} in {}. '
'Must be one of: {}'.format(val, dest,
self._scope_str(),
choices))
elif kwargs.get(
'type') == dir_option and not os.path.isdir(val):
raise ParseError(
'Directory value `{}` for option {} in {} does not exist.'
.format(val, dest, self._scope_str()))
elif kwargs.get(
'type') == file_option and not os.path.isfile(val):
raise ParseError(
'File value `{}` for option {} in {} does not exist.'.
format(val, dest, self._scope_str()))
# Generate the final value from all available values, and check that it (or its members,
# if a list) are in the set of allowed choices.
if is_list_option(kwargs):
merged_rank = ranked_vals[-1].rank
merged_val = ListValueComponent.merge(
[rv.value for rv in ranked_vals if rv.value is not None]).val
merged_val = [
self._convert_member_type(kwargs.get('member_type', str), x)
for x in merged_val
]
map(check, merged_val)
ret = RankedValue(merged_rank, merged_val)
elif is_dict_option(kwargs):
merged_rank = ranked_vals[-1].rank
merged_val = DictValueComponent.merge(
[rv.value for rv in ranked_vals if rv.value is not None]).val
map(check, merged_val)
ret = RankedValue(merged_rank, merged_val)
else:
ret = ranked_vals[-1]
check(ret.value)
# All done!
return ret