def _compute_value(self, dest, kwargs, flag_val_strs):
"""Compute the value to use for an option.
The source of the default value is chosen according to the ranking in RankedValue.
"""
# Helper function to convert a string to a value of the option's type.
def to_value_type(val_str):
if val_str is None:
return None
elif kwargs.get('type') == bool:
return self._ensure_bool(val_str)
else:
return self._wrap_type(kwargs.get('type', str))(val_str)
# Helper function to expand a fromfile=True value string, if needed.
def expand(val_str):
if kwargs.get('fromfile',
False) and val_str and val_str.startswith('@'):
if val_str.startswith(
'@@'
): # Support a literal @ for fromfile values via @@.
return val_str[1:]
else:
fromfile = val_str[1:]
try:
with open(fromfile) as fp:
return fp.read().strip()
except IOError as e:
raise self.FromfileError(
'Failed to read {} in {} from file {}: {}'.format(
dest, self._scope_str(), fromfile, e))
else:
return val_str
# Get value from config files, and capture details about its derivation.
config_details = None
config_section = GLOBAL_SCOPE_CONFIG_SECTION if self._scope == GLOBAL_SCOPE else self._scope
config_val_str = expand(
self._config.get(config_section, dest, default=None))
config_source_file = self._config.get_source_for_option(
config_section, dest)
if config_source_file is not None:
config_source_file = os.path.relpath(config_source_file)
config_details = 'in {}'.format(config_source_file)
# Get value from environment, and capture details about its derivation.
udest = dest.upper()
if self._scope == GLOBAL_SCOPE:
# For convenience, we allow three forms of env var for global scope options.
# The fully-specified env var is PANTS_GLOBAL_FOO, which is uniform with PANTS_<SCOPE>_FOO
# for all the other scopes. However we also allow simply PANTS_FOO. And if the option name
# itself starts with 'pants-' then we also allow simply FOO. E.g., PANTS_WORKDIR instead of
# PANTS_PANTS_WORKDIR or PANTS_GLOBAL_PANTS_WORKDIR. We take the first specified value we
# find, in this order: PANTS_GLOBAL_FOO, PANTS_FOO, FOO.
env_vars = [
'PANTS_GLOBAL_{0}'.format(udest), 'PANTS_{0}'.format(udest)
]
if udest.startswith('PANTS_'):
env_vars.append(udest)
else:
sanitized_env_var_scope = self._ENV_SANITIZER_RE.sub(
'_', self._scope.upper())
env_vars = ['PANTS_{0}_{1}'.format(sanitized_env_var_scope, udest)]
env_val_str = None
env_details = None
if self._env:
for env_var in env_vars:
if env_var in self._env:
env_val_str = expand(self._env.get(env_var))
env_details = 'from env var {}'.format(env_var)
break
# Get value from cmd-line flags.
flag_vals = [to_value_type(expand(x)) for x in flag_val_strs]
if is_list_option(kwargs):
# Note: It's important to set flag_val to None if no flags were specified, so we can
# distinguish between no flags set vs. explicit setting of the value to [].
flag_val = ListValueComponent.merge(
flag_vals) if flag_vals else None
elif is_dict_option(kwargs):
# Note: It's important to set flag_val to None if no flags were specified, so we can
# distinguish between no flags set vs. explicit setting of the value to {}.
flag_val = DictValueComponent.merge(
flag_vals) if flag_vals else None
elif len(flag_vals) > 1:
raise ParseError(
'Multiple cmd line flags specified for option {} in {}'.format(
dest, self._scope_str()))
elif len(flag_vals) == 1:
flag_val = flag_vals[0]
else:
flag_val = None
# Rank all available values.
# Note that some of these values may already be of the value type, but type conversion
# is idempotent, so this is OK.
values_to_rank = [
to_value_type(x) for x in [
flag_val, env_val_str, config_val_str,
kwargs.get('default'), None
]
]
# Note that ranked_vals will always have at least one element, and all elements will be
# instances of RankedValue (so none will be None, although they may wrap a None value).
ranked_vals = list(
reversed(list(RankedValue.prioritized_iter(*values_to_rank))))
# Record info about the derivation of each of the values.
for ranked_val in ranked_vals:
if ranked_val.rank == RankedValue.CONFIG:
details = config_details
elif ranked_val.rank == RankedValue.ENVIRONMENT:
details = env_details
else:
details = None
self._option_tracker.record_option(
scope=self._scope,
option=dest,
value=ranked_val.value,
rank=ranked_val.rank,
deprecation_version=kwargs.get('removal_version'),
details=details)
# Helper function to check various validity constraints on final option values.
def check(val):
if val is not None:
choices = kwargs.get('choices')
if choices is not None and val not in choices:
raise ParseError(
'`{}` is not an allowed value for option {} in {}. '
'Must be one of: {}'.format(val, dest,
self._scope_str(),
choices))
elif kwargs.get(
'type') == dir_option and not os.path.isdir(val):
raise ParseError(
'Directory value `{}` for option {} in {} does not exist.'
.format(val, dest, self._scope_str()))
elif kwargs.get(
'type') == file_option and not os.path.isfile(val):
raise ParseError(
'File value `{}` for option {} in {} does not exist.'.
format(val, dest, self._scope_str()))
# Generate the final value from all available values, and check that it (or its members,
# if a list) are in the set of allowed choices.
if is_list_option(kwargs):
merged_rank = ranked_vals[-1].rank
merged_val = ListValueComponent.merge(
[rv.value for rv in ranked_vals if rv.value is not None]).val
merged_val = [
self._convert_member_type(kwargs.get('member_type', str), x)
for x in merged_val
]
map(check, merged_val)
ret = RankedValue(merged_rank, merged_val)
elif is_dict_option(kwargs):
merged_rank = ranked_vals[-1].rank
merged_val = DictValueComponent.merge(
[rv.value for rv in ranked_vals if rv.value is not None]).val
map(check, merged_val)
ret = RankedValue(merged_rank, merged_val)
else:
ret = ranked_vals[-1]
check(ret.value)
# All done!
return ret
def _compute_value(self, dest, kwargs, flag_vals):
"""Compute the value to use for an option.
The source of the default value is chosen according to the ranking in RankedValue.
"""
is_fromfile = kwargs.get('fromfile', False)
action = kwargs.get('action')
if is_fromfile and action and action != 'append':
raise ParseError('Cannot fromfile {} with an action ({}) in scope {}'
.format(dest, action, self._scope))
config_section = GLOBAL_SCOPE_CONFIG_SECTION if self._scope == GLOBAL_SCOPE else self._scope
udest = dest.upper()
if self._scope == GLOBAL_SCOPE:
# For convenience, we allow three forms of env var for global scope options.
# The fully-specified env var is PANTS_DEFAULT_FOO, which is uniform with PANTS_<SCOPE>_FOO
# for all the other scopes. However we also allow simply PANTS_FOO. And if the option name
# itself starts with 'pants-' then we also allow simply FOO. E.g., PANTS_WORKDIR instead of
# PANTS_PANTS_WORKDIR or PANTS_DEFAULT_PANTS_WORKDIR. We take the first specified value we
# find, in this order: PANTS_DEFAULT_FOO, PANTS_FOO, FOO.
env_vars = ['PANTS_DEFAULT_{0}'.format(udest), 'PANTS_{0}'.format(udest)]
if udest.startswith('PANTS_'):
env_vars.append(udest)
else:
sanitized_env_var_scope = self._ENV_SANITIZER_RE.sub('_', config_section.upper())
env_vars = ['PANTS_{0}_{1}'.format(sanitized_env_var_scope, udest)]
value_type = self.str_to_bool if is_boolean_flag(kwargs) else kwargs.get('type', str)
env_val_str = None
if self._env:
for env_var in env_vars:
if env_var in self._env:
env_val_str = self._env.get(env_var)
break
config_val_str = self._config.get(config_section, dest, default=None)
config_source_file = self._config.get_source_for_option(config_section, dest)
if config_source_file is not None:
config_source_file = os.path.relpath(config_source_file)
def expand(val_str):
if is_fromfile and val_str and val_str.startswith('@') and not val_str.startswith('@@'):
fromfile = val_str[1:]
try:
with open(fromfile) as fp:
return fp.read().strip()
except IOError as e:
raise self.FromfileError('Failed to read {} in {} from file {}: {}'.format(
dest, self._scope_str(), fromfile, e))
else:
# Support a literal @ for fromfile values via @@.
return val_str[1:] if is_fromfile and val_str.startswith('@@') else val_str
def parse_typed_list(val_str):
return None if val_str is None else [value_type(x) for x in list_option(expand(val_str))]
def parse_typed_item(val_str):
return None if val_str is None else value_type(expand(val_str))
flag_val = None
if flag_vals:
if action == 'append':
flag_val = [parse_typed_item(v) for v in flag_vals]
elif len(flag_vals) > 1:
raise ParseError('Multiple cmd line flags specified for option {} in {}'.format(
dest, self._scope_str()))
else:
flag_val = parse_typed_item(flag_vals[0])
default, parse = ([], parse_typed_list) if action == 'append' else (None, parse_typed_item)
config_val = parse(config_val_str)
env_val = parse(env_val_str)
hardcoded_val = kwargs.get('default')
config_details = 'in {}'.format(config_source_file) if config_source_file else None
# Note: ranked_vals is guaranteed to have at least one element, and none of the values
# of any of its elements will be None.
ranked_vals = list(reversed(list(RankedValue.prioritized_iter(
flag_val, env_val, config_val, hardcoded_val, default))))
choices = kwargs.get('choices')
for ranked_val in ranked_vals:
details = config_details if ranked_val.rank == RankedValue.CONFIG else None
self._option_tracker.record_option(scope=self._scope,
option=dest,
value=ranked_val.value,
rank=ranked_val.rank,
deprecation_version=kwargs.get('deprecated_version'),
details=details)
def check(val):
if choices is not None and val is not None and val not in choices:
raise ParseError('{} is not an allowed value for option {} in {}. '
'Must be one of: {}'.format(
val, dest, self._scope_str(), choices
))
return val
if action == 'append':
merged_rank = ranked_vals[-1].rank
merged_val = [check(val) for vals in ranked_vals for val in vals.value]
return RankedValue(merged_rank, merged_val)
else:
map(lambda rv: check(rv.value), ranked_vals)
return ranked_vals[-1]
def _compute_value(self, dest, kwargs, flag_val_strs):
"""Compute the value to use for an option.
The source of the default value is chosen according to the ranking in RankedValue.
"""
# Helper function to convert a string to a value of the option's type.
def to_value_type(val_str):
if val_str is None:
return None
if kwargs.get("type") == bool:
return self._ensure_bool(val_str)
type_arg = kwargs.get("type", str)
try:
return self._wrap_type(type_arg)(val_str)
except (TypeError, ValueError) as e:
raise ParseError(
f"Error applying type '{type_arg.__name__}' to option value '{val_str}', for option "
f"'--{dest}' in {self._scope_str()}: {e}")
# Helper function to expand a fromfile=True value string, if needed.
# May return a string or a dict/list decoded from a json/yaml file.
def expand(val_or_str):
if (kwargs.get("fromfile", True) and isinstance(val_or_str, str)
and val_or_str.startswith("@")):
if val_or_str.startswith(
"@@"
): # Support a literal @ for fromfile values via @@.
return val_or_str[1:]
else:
fromfile = val_or_str[1:]
try:
with open(fromfile, "r") as fp:
s = fp.read().strip()
if fromfile.endswith(".json"):
return json.loads(s)
elif fromfile.endswith(
".yml") or fromfile.endswith(".yaml"):
return yaml.safe_load(s)
else:
return s
except (IOError, ValueError, yaml.YAMLError) as e:
raise FromfileError(
f"Failed to read {dest} in {self._scope_str()} from file {fromfile}: {e!r}"
)
else:
return val_or_str
# Get value from config files, and capture details about its derivation.
config_details = None
config_section = GLOBAL_SCOPE_CONFIG_SECTION if self._scope == GLOBAL_SCOPE else self._scope
config_default_val_or_str = expand(
self._config.get(Config.DEFAULT_SECTION, dest, default=None))
config_val_or_str = expand(
self._config.get(config_section, dest, default=None))
config_source_file = self._config.get_source_for_option(
config_section, dest) or self._config.get_source_for_option(
Config.DEFAULT_SECTION, dest)
if config_source_file is not None:
config_source_file = os.path.relpath(config_source_file)
config_details = f"in {config_source_file}"
# Get value from environment, and capture details about its derivation.
udest = dest.upper()
if self._scope == GLOBAL_SCOPE:
# For convenience, we allow three forms of env var for global scope options.
# The fully-specified env var is PANTS_GLOBAL_FOO, which is uniform with PANTS_<SCOPE>_FOO
# for all the other scopes. However we also allow simply PANTS_FOO. And if the option name
# itself starts with 'pants-' then we also allow simply FOO. E.g., PANTS_WORKDIR instead of
# PANTS_PANTS_WORKDIR or PANTS_GLOBAL_PANTS_WORKDIR. We take the first specified value we
# find, in this order: PANTS_GLOBAL_FOO, PANTS_FOO, FOO.
env_vars = [f"PANTS_GLOBAL_{udest}", f"PANTS_{udest}"]
if udest.startswith("PANTS_"):
env_vars.append(udest)
else:
sanitized_env_var_scope = self._ENV_SANITIZER_RE.sub(
"_", self._scope.upper())
env_vars = [f"PANTS_{sanitized_env_var_scope}_{udest}"]
env_val_or_str = None
env_details = None
if self._env:
for env_var in env_vars:
if env_var in self._env:
env_val_or_str = expand(self._env.get(env_var))
env_details = f"from env var {env_var}"
break
# Get value from cmd-line flags.
flag_vals = [to_value_type(expand(x)) for x in flag_val_strs]
if is_list_option(kwargs):
# Note: It's important to set flag_val to None if no flags were specified, so we can
# distinguish between no flags set vs. explicit setting of the value to [].
flag_val = ListValueComponent.merge(
flag_vals) if flag_vals else None
elif is_dict_option(kwargs):
# Note: It's important to set flag_val to None if no flags were specified, so we can
# distinguish between no flags set vs. explicit setting of the value to {}.
flag_val = DictValueComponent.merge(
flag_vals) if flag_vals else None
elif len(flag_vals) > 1:
raise ParseError(
"Multiple cmd line flags specified for option {} in {}".format(
dest, self._scope_str()))
elif len(flag_vals) == 1:
flag_val = flag_vals[0]
else:
flag_val = None
# Rank all available values.
# Note that some of these values may already be of the value type, but type conversion
# is idempotent, so this is OK.
values_to_rank = [
to_value_type(x) for x in [
flag_val,
env_val_or_str,
config_val_or_str,
config_default_val_or_str,
kwargs.get("default"),
None,
]
]
# Note that ranked_vals will always have at least one element, and all elements will be
# instances of RankedValue (so none will be None, although they may wrap a None value).
ranked_vals = list(
reversed(list(RankedValue.prioritized_iter(*values_to_rank))))
def record_option(value, rank, option_details=None):
deprecation_version = kwargs.get("removal_version")
self._option_tracker.record_option(
scope=self._scope,
option=dest,
value=value,
rank=rank,
deprecation_version=deprecation_version,
details=option_details,
)
# Record info about the derivation of each of the contributing values.
detail_history = []
for ranked_val in ranked_vals:
if ranked_val.rank in (RankedValue.CONFIG,
RankedValue.CONFIG_DEFAULT):
details = config_details
elif ranked_val.rank == RankedValue.ENVIRONMENT:
details = env_details
else:
details = None
if details:
detail_history.append(details)
record_option(value=ranked_val.value,
rank=ranked_val.rank,
option_details=details)
# Helper function to check various validity constraints on final option values.
def check(val):
if val is None:
return
choices = kwargs.get("choices")
type_arg = kwargs.get("type")
if choices is None and "type" in kwargs:
if inspect.isclass(type_arg) and issubclass(type_arg, Enum):
choices = list(type_arg)
# TODO: convert this into an enum() pattern match!
if choices is not None and val not in choices:
raise ParseError(
"`{}` is not an allowed value for option {} in {}. "
"Must be one of: {}".format(val, dest, self._scope_str(),
choices))
if type_arg == file_option:
check_file_exists(val)
if type_arg == dir_option:
check_dir_exists(val)
def check_file_exists(val) -> None:
error_prefix = f"File value `{val}` for option `{dest}` in `{self._scope_str()}`"
try:
path = Path(val)
path_with_buildroot = Path(get_buildroot(), val)
except TypeError:
raise ParseError(
f"{error_prefix} cannot be parsed as a file path.")
if not path.is_file() and not path_with_buildroot.is_file():
raise ParseError(f"{error_prefix} does not exist.")
def check_dir_exists(val) -> None:
error_prefix = f"Directory value `{val}` for option `{dest}` in `{self._scope_str()}`"
try:
path = Path(val)
path_with_buildroot = Path(get_buildroot(), val)
except TypeError:
raise ParseError(
f"{error_prefix} cannot be parsed as a directory path.")
if not path.is_dir() and not path_with_buildroot.is_dir():
raise ParseError(f"{error_prefix} does not exist.")
# Generate the final value from all available values, and check that it (or its members,
# if a list) are in the set of allowed choices.
if is_list_option(kwargs):
merged_rank = ranked_vals[-1].rank
merged_val = ListValueComponent.merge(
[rv.value for rv in ranked_vals if rv.value is not None]).val
# TODO: run `check()` for all elements of a list option too!!!
merged_val = [
self._convert_member_type(kwargs.get("member_type", str), x)
for x in merged_val
]
if kwargs.get("member_type") == shell_str:
merged_val = flatten_shlexed_list(merged_val)
for val in merged_val:
check(val)
ret = RankedValue(merged_rank, merged_val)
elif is_dict_option(kwargs):
# TODO: convert `member_type` for dict values too!
merged_rank = ranked_vals[-1].rank
merged_val = DictValueComponent.merge(
[rv.value for rv in ranked_vals if rv.value is not None]).val
for val in merged_val:
check(val)
ret = RankedValue(merged_rank, merged_val)
else:
ret = ranked_vals[-1]
check(ret.value)
# Record info about the derivation of the final value.
merged_details = ", ".join(detail_history) if detail_history else None
record_option(value=ret.value,
rank=ret.rank,
option_details=merged_details)
# All done!
return ret
def _compute_value(self, dest, kwargs, flag_val_strs):
"""Compute the value to use for an option.
The source of the default value is chosen according to the ranking in RankedValue.
"""
# Helper function to convert a string to a value of the option's type.
def to_value_type(val_str):
if val_str is None:
return None
elif kwargs.get('type') == bool:
return self._ensure_bool(val_str)
else:
return self._wrap_type(kwargs.get('type', str))(val_str)
# Helper function to expand a fromfile=True value string, if needed.
def expand(val_str):
if kwargs.get('fromfile', False) and val_str and val_str.startswith('@'):
if val_str.startswith('@@'): # Support a literal @ for fromfile values via @@.
return val_str[1:]
else:
fromfile = val_str[1:]
try:
with open(fromfile) as fp:
return fp.read().strip()
except IOError as e:
raise self.FromfileError('Failed to read {} in {} from file {}: {}'.format(
dest, self._scope_str(), fromfile, e))
else:
return val_str
# Get value from config files, and capture details about its derivation.
config_details = None
config_section = GLOBAL_SCOPE_CONFIG_SECTION if self._scope == GLOBAL_SCOPE else self._scope
config_val_str = expand(self._config.get(config_section, dest, default=None))
config_source_file = self._config.get_source_for_option(config_section, dest)
if config_source_file is not None:
config_source_file = os.path.relpath(config_source_file)
config_details = 'in {}'.format(config_source_file)
# Get value from environment, and capture details about its derivation.
udest = dest.upper()
if self._scope == GLOBAL_SCOPE:
# For convenience, we allow three forms of env var for global scope options.
# The fully-specified env var is PANTS_GLOBAL_FOO, which is uniform with PANTS_<SCOPE>_FOO
# for all the other scopes. However we also allow simply PANTS_FOO. And if the option name
# itself starts with 'pants-' then we also allow simply FOO. E.g., PANTS_WORKDIR instead of
# PANTS_PANTS_WORKDIR or PANTS_GLOBAL_PANTS_WORKDIR. We take the first specified value we
# find, in this order: PANTS_GLOBAL_FOO, PANTS_FOO, FOO.
env_vars = ['PANTS_GLOBAL_{0}'.format(udest), 'PANTS_{0}'.format(udest)]
if udest.startswith('PANTS_'):
env_vars.append(udest)
else:
sanitized_env_var_scope = self._ENV_SANITIZER_RE.sub('_', self._scope.upper())
env_vars = ['PANTS_{0}_{1}'.format(sanitized_env_var_scope, udest)]
env_val_str = None
env_details = None
if self._env:
for env_var in env_vars:
if env_var in self._env:
env_val_str = expand(self._env.get(env_var))
env_details = 'from env var {}'.format(env_var)
break
# Get value from cmd-line flags.
flag_vals = [to_value_type(expand(x)) for x in flag_val_strs]
if is_list_option(kwargs):
# Note: It's important to set flag_val to None if no flags were specified, so we can
# distinguish between no flags set vs. explicit setting of the value to [].
flag_val = ListValueComponent.merge(flag_vals) if flag_vals else None
elif len(flag_vals) > 1:
raise ParseError('Multiple cmd line flags specified for option {} in {}'.format(
dest, self._scope_str()))
elif len(flag_vals) == 1:
flag_val = flag_vals[0]
else:
flag_val = None
# Rank all available values.
# Note that some of these values may already be of the value type, but type conversion
# is idempotent, so this is OK.
values_to_rank = [to_value_type(x) for x in
[flag_val, env_val_str, config_val_str, kwargs.get('default'), None]]
# Note that ranked_vals will always have at least one element, and all elements will be
# instances of RankedValue (so none will be None, although they may wrap a None value).
ranked_vals = list(reversed(list(RankedValue.prioritized_iter(*values_to_rank))))
# Record info about the derivation of each of the values.
for ranked_val in ranked_vals:
if ranked_val.rank == RankedValue.CONFIG:
details = config_details
elif ranked_val.rank == RankedValue.ENVIRONMENT:
details = env_details
else:
details = None
self._option_tracker.record_option(scope=self._scope,
option=dest,
value=ranked_val.value,
rank=ranked_val.rank,
deprecation_version=kwargs.get('removal_version'),
details=details)
# Helper function to check various validity constraints on final option values.
def check(val):
if val is not None:
choices = kwargs.get('choices')
if choices is not None and val not in choices:
raise ParseError('{} is not an allowed value for option {} in {}. '
'Must be one of: {}'.format(val, dest, self._scope_str(), choices))
elif kwargs.get('type') == file_option and not os.path.isfile(val):
raise ParseError('File value {} for option {} in {} does not exist.'.format(
val, dest, self._scope_str()))
# Generate the final value from all available values, and check that it (or its members,
# if a list) are in the set of allowed choices.
if is_list_option(kwargs):
merged_rank = ranked_vals[-1].rank
merged_val = ListValueComponent.merge(
[rv.value for rv in ranked_vals if rv.value is not None]).val
merged_val = [self._wrap_type(kwargs.get('member_type', str))(x) for x in merged_val]
map(check, merged_val)
ret = RankedValue(merged_rank, merged_val)
else:
ret = ranked_vals[-1]
check(ret.value)
# All done!
return ret
def _compute_value(self, dest, kwargs, flag_val_strs, passthru_arg_strs):
"""Compute the value to use for an option.
The source of the value is chosen according to the ranking in Rank.
"""
type_arg = kwargs.get("type", str)
member_type = kwargs.get("member_type", str)
def to_value_type(val_str):
return self.to_value_type(val_str, type_arg, member_type, dest)
# Helper function to expand a fromfile=True value string, if needed.
# May return a string or a dict/list decoded from a json/yaml file.
def expand(val_or_str):
if (
kwargs.get("fromfile", True)
and isinstance(val_or_str, str)
and val_or_str.startswith("@")
):
if val_or_str.startswith("@@"): # Support a literal @ for fromfile values via @@.
return val_or_str[1:]
else:
fromfile = val_or_str[1:]
try:
with open(fromfile, "r") as fp:
s = fp.read().strip()
if fromfile.endswith(".json"):
return json.loads(s)
elif fromfile.endswith(".yml") or fromfile.endswith(".yaml"):
return yaml.safe_load(s)
else:
return s
except (IOError, ValueError, yaml.YAMLError) as e:
raise FromfileError(
f"Failed to read {dest} in {self._scope_str()} from file {fromfile}: {e!r}"
)
else:
return val_or_str
# Get value from config files, and capture details about its derivation.
config_details = None
config_section = GLOBAL_SCOPE_CONFIG_SECTION if self._scope == GLOBAL_SCOPE else self._scope
config_default_val_or_str = expand(
self._config.get(Config.DEFAULT_SECTION, dest, default=None)
)
config_val_or_str = expand(self._config.get(config_section, dest, default=None))
config_source_file = self._config.get_source_for_option(
config_section, dest
) or self._config.get_source_for_option(Config.DEFAULT_SECTION, dest)
if config_source_file is not None:
config_source_file = os.path.relpath(config_source_file)
config_details = f"from {config_source_file}"
# Get value from environment, and capture details about its derivation.
env_vars = self.get_env_var_names(self._scope, dest)
env_val_or_str = None
env_details = None
if self._env:
for env_var in env_vars:
if env_var in self._env:
env_val_or_str = expand(self._env.get(env_var))
env_details = f"from env var {env_var}"
break
# Get value from cmd-line flags.
flag_vals = [to_value_type(expand(x)) for x in flag_val_strs]
if kwargs.get("passthrough"):
# NB: Passthrough arguments are either of type `str` or `shell_str`
# (see self._validate): the former never need interpretation, and the latter do not
# need interpretation when they have been provided directly via `sys.argv` as the
# passthrough args have been.
flag_vals.append(
ListValueComponent(ListValueComponent.MODIFY, [*passthru_arg_strs], [])
)
if is_list_option(kwargs):
# Note: It's important to set flag_val to None if no flags were specified, so we can
# distinguish between no flags set vs. explicit setting of the value to [].
flag_val = ListValueComponent.merge(flag_vals) if flag_vals else None
elif is_dict_option(kwargs):
# Note: It's important to set flag_val to None if no flags were specified, so we can
# distinguish between no flags set vs. explicit setting of the value to {}.
flag_val = DictValueComponent.merge(flag_vals) if flag_vals else None
elif len(flag_vals) > 1:
raise ParseError(
f"Multiple cmd line flags specified for option {dest} in {self._scope_str()}"
)
elif len(flag_vals) == 1:
flag_val = flag_vals[0]
else:
flag_val = None
flag_details = None if flag_val is None else "from command-line flag"
# Rank all available values.
# Note that some of these values may already be of the value type, but type conversion
# is idempotent, so this is OK.
values_to_rank = [
(to_value_type(x), detail)
for (x, detail) in [
(flag_val, flag_details),
(env_val_or_str, env_details),
(config_val_or_str, config_details),
(config_default_val_or_str, config_details),
(kwargs.get("default"), None),
(None, None),
]
]
# Note that ranked_vals will always have at least one element, and all elements will be
# instances of RankedValue (so none will be None, although they may wrap a None value).
ranked_vals = list(reversed(list(RankedValue.prioritized_iter(*values_to_rank))))
def group(value_component_type, process_val_func) -> List[RankedValue]:
# We group any values that are merged together, so that the history can reflect
# merges vs. replacements in a useful way. E.g., if we merge [a, b] and [c],
# and then replace it with [d, e], the history will contain:
# - [d, e] (from command-line flag)
# - [a, b, c] (from env var, from config)
# And similarly for dicts.
grouped: List[List[RankedValue]] = [[]]
for ranked_val in ranked_vals:
if ranked_val.value and ranked_val.value.action == value_component_type.REPLACE:
grouped.append([])
grouped[-1].append(ranked_val)
return [
RankedValue(
grp[-1].rank,
process_val_func(
value_component_type.merge(
rv.value for rv in grp if rv.value is not None
).val
),
", ".join(rv.details for rv in grp if rv.details),
)
for grp in grouped
if grp
]
if is_list_option(kwargs):
def process_list(lst):
lst = [self._convert_member_type(member_type, val) for val in lst]
if member_type == shell_str:
lst = flatten_shlexed_list(lst)
return lst
historic_ranked_vals = group(ListValueComponent, process_list)
elif is_dict_option(kwargs):
historic_ranked_vals = group(DictValueComponent, lambda x: x)
else:
historic_ranked_vals = ranked_vals
value_history = OptionValueHistory(tuple(historic_ranked_vals))
# Helper function to check various validity constraints on final option values.
def check_scalar_value(val):
if val is None:
return
choices = kwargs.get("choices")
if choices is None and "type" in kwargs:
if inspect.isclass(type_arg) and issubclass(type_arg, Enum):
choices = list(type_arg)
if choices is not None and val not in choices:
raise ParseError(
f"`{val}` is not an allowed value for option {dest} in {self._scope_str()}. "
f"Must be one of: {choices}"
)
elif type_arg == file_option:
check_file_exists(val)
elif type_arg == dir_option:
check_dir_exists(val)
def check_file_exists(val) -> None:
error_prefix = f"File value `{val}` for option `{dest}` in `{self._scope_str()}`"
try:
path = Path(val)
path_with_buildroot = Path(get_buildroot(), val)
except TypeError:
raise ParseError(f"{error_prefix} cannot be parsed as a file path.")
if not path.is_file() and not path_with_buildroot.is_file():
raise ParseError(f"{error_prefix} does not exist.")
def check_dir_exists(val) -> None:
error_prefix = f"Directory value `{val}` for option `{dest}` in `{self._scope_str()}`"
try:
path = Path(val)
path_with_buildroot = Path(get_buildroot(), val)
except TypeError:
raise ParseError(f"{error_prefix} cannot be parsed as a directory path.")
if not path.is_dir() and not path_with_buildroot.is_dir():
raise ParseError(f"{error_prefix} does not exist.")
# Validate the final value.
final_val = value_history.final_value
if isinstance(final_val.value, list):
for component in final_val.value:
check_scalar_value(component)
if inspect.isclass(member_type) and issubclass(member_type, Enum):
if len(final_val.value) != len(set(final_val.value)):
raise ParseError(f"Duplicate enum values specified in list: {final_val.value}")
elif isinstance(final_val.value, dict):
for component in final_val.value.values():
check_scalar_value(component)
else:
check_scalar_value(final_val.value)
return value_history
def _compute_value(self, dest, kwargs, flag_val_strs):
"""Compute the value to use for an option.
The source of the default value is chosen according to the ranking in RankedValue.
"""
# Helper function to convert a string to a value of the option's type.
def to_value_type(val_str):
if val_str is None:
return None
elif kwargs.get('type') == bool:
return self._ensure_bool(val_str)
else:
type_arg = kwargs.get('type', str)
try:
return self._wrap_type(type_arg)(val_str)
except TypeError as e:
raise ParseError(
"Error applying type '{}' to option value '{}', for option '--{}' in {}: {}"
.format(type_arg.__name__, val_str, dest,
self._scope_str(), e))
# Helper function to expand a fromfile=True value string, if needed.
# May return a string or a dict/list decoded from a json/yaml file.
def expand(val_or_str):
if (kwargs.get('fromfile', True) and val_or_str
and isinstance(val_or_str, str)
and val_or_str.startswith('@')):
if val_or_str.startswith(
'@@'
): # Support a literal @ for fromfile values via @@.
return val_or_str[1:]
else:
fromfile = val_or_str[1:]
try:
with open(fromfile, 'r') as fp:
s = fp.read().strip()
if fromfile.endswith('.json'):
return json.loads(s)
elif fromfile.endswith(
'.yml') or fromfile.endswith('.yaml'):
return yaml.safe_load(s)
else:
return s
except (IOError, ValueError, yaml.YAMLError) as e:
raise self.FromfileError(
'Failed to read {} in {} from file {}: {}'.format(
dest, self._scope_str(), fromfile, e))
else:
return val_or_str
# Get value from config files, and capture details about its derivation.
config_details = None
config_section = GLOBAL_SCOPE_CONFIG_SECTION if self._scope == GLOBAL_SCOPE else self._scope
config_default_val_or_str = expand(
self._config.get(Config.DEFAULT_SECTION, dest, default=None))
config_val_or_str = expand(
self._config.get(config_section, dest, default=None))
config_source_file = (self._config.get_source_for_option(
config_section, dest) or self._config.get_source_for_option(
Config.DEFAULT_SECTION, dest))
if config_source_file is not None:
config_source_file = os.path.relpath(config_source_file)
config_details = 'in {}'.format(config_source_file)
# Get value from environment, and capture details about its derivation.
udest = dest.upper()
if self._scope == GLOBAL_SCOPE:
# For convenience, we allow three forms of env var for global scope options.
# The fully-specified env var is PANTS_GLOBAL_FOO, which is uniform with PANTS_<SCOPE>_FOO
# for all the other scopes. However we also allow simply PANTS_FOO. And if the option name
# itself starts with 'pants-' then we also allow simply FOO. E.g., PANTS_WORKDIR instead of
# PANTS_PANTS_WORKDIR or PANTS_GLOBAL_PANTS_WORKDIR. We take the first specified value we
# find, in this order: PANTS_GLOBAL_FOO, PANTS_FOO, FOO.
env_vars = [
'PANTS_GLOBAL_{0}'.format(udest), 'PANTS_{0}'.format(udest)
]
if udest.startswith('PANTS_'):
env_vars.append(udest)
else:
sanitized_env_var_scope = self._ENV_SANITIZER_RE.sub(
'_', self._scope.upper())
env_vars = ['PANTS_{0}_{1}'.format(sanitized_env_var_scope, udest)]
env_val_or_str = None
env_details = None
if self._env:
for env_var in env_vars:
if env_var in self._env:
env_val_or_str = expand(self._env.get(env_var))
env_details = 'from env var {}'.format(env_var)
break
# Get value from cmd-line flags.
flag_vals = [to_value_type(expand(x)) for x in flag_val_strs]
if is_list_option(kwargs):
# Note: It's important to set flag_val to None if no flags were specified, so we can
# distinguish between no flags set vs. explicit setting of the value to [].
flag_val = ListValueComponent.merge(
flag_vals) if flag_vals else None
elif is_dict_option(kwargs):
# Note: It's important to set flag_val to None if no flags were specified, so we can
# distinguish between no flags set vs. explicit setting of the value to {}.
flag_val = DictValueComponent.merge(
flag_vals) if flag_vals else None
elif len(flag_vals) > 1:
raise ParseError(
'Multiple cmd line flags specified for option {} in {}'.format(
dest, self._scope_str()))
elif len(flag_vals) == 1:
flag_val = flag_vals[0]
else:
flag_val = None
# Rank all available values.
# Note that some of these values may already be of the value type, but type conversion
# is idempotent, so this is OK.
values_to_rank = [
to_value_type(x) for x in [
flag_val, env_val_or_str, config_val_or_str,
config_default_val_or_str,
kwargs.get('default'), None
]
]
# Note that ranked_vals will always have at least one element, and all elements will be
# instances of RankedValue (so none will be None, although they may wrap a None value).
ranked_vals = list(
reversed(list(RankedValue.prioritized_iter(*values_to_rank))))
def record_option(value, rank, option_details=None):
deprecation_version = kwargs.get('removal_version')
self._option_tracker.record_option(
scope=self._scope,
option=dest,
value=value,
rank=rank,
deprecation_version=deprecation_version,
details=option_details)
# Record info about the derivation of each of the contributing values.
detail_history = []
for ranked_val in ranked_vals:
if ranked_val.rank in (RankedValue.CONFIG,
RankedValue.CONFIG_DEFAULT):
details = config_details
elif ranked_val.rank == RankedValue.ENVIRONMENT:
details = env_details
else:
details = None
if details:
detail_history.append(details)
record_option(value=ranked_val.value,
rank=ranked_val.rank,
option_details=details)
# Helper function to check various validity constraints on final option values.
def check(val):
if val is not None:
choices = kwargs.get('choices')
# If the `type` argument has an `all_variants` attribute, use that as `choices` if not
# already set. Using an attribute instead of checking a subclass allows `type` arguments
# which are functions to have an implicit fallback `choices` set as well.
if choices is None and 'type' in kwargs:
type_arg = kwargs.get('type')
if hasattr(type_arg, 'all_variants'):
choices = list(type_arg.all_variants)
# TODO: convert this into an enum() pattern match!
if choices is not None and val not in choices:
raise ParseError(
'`{}` is not an allowed value for option {} in {}. '
'Must be one of: {}'.format(val, dest,
self._scope_str(),
choices))
elif kwargs.get(
'type') == dir_option and not os.path.isdir(val):
raise ParseError(
'Directory value `{}` for option {} in {} does not exist.'
.format(val, dest, self._scope_str()))
elif kwargs.get(
'type') == file_option and not os.path.isfile(val):
raise ParseError(
'File value `{}` for option {} in {} does not exist.'.
format(val, dest, self._scope_str()))
# Generate the final value from all available values, and check that it (or its members,
# if a list) are in the set of allowed choices.
if is_list_option(kwargs):
merged_rank = ranked_vals[-1].rank
merged_val = ListValueComponent.merge(
[rv.value for rv in ranked_vals if rv.value is not None]).val
# TODO: run `check()` for all elements of a list option too!!!
merged_val = [
self._convert_member_type(kwargs.get('member_type', str), x)
for x in merged_val
]
for val in merged_val:
check(val)
ret = RankedValue(merged_rank, merged_val)
elif is_dict_option(kwargs):
# TODO: convert `member_type` for dict values too!
merged_rank = ranked_vals[-1].rank
merged_val = DictValueComponent.merge(
[rv.value for rv in ranked_vals if rv.value is not None]).val
for val in merged_val:
check(val)
ret = RankedValue(merged_rank, merged_val)
else:
ret = ranked_vals[-1]
check(ret.value)
# Record info about the derivation of the final value.
merged_details = ', '.join(detail_history) if detail_history else None
record_option(value=ret.value,
rank=ret.rank,
option_details=merged_details)
# All done!
return ret
def _compute_default(self, kwargs):
"""Compute the default value to use for an option's registration.
The source of the default value is chosen according to the ranking in RankedValue.
Note: Only call if kwargs has a 'dest' key set.
"""
dest = kwargs['dest']
is_fromfile = kwargs.get('fromfile', False)
action = kwargs.get('action')
if is_fromfile and action and action != 'append':
raise ParseError('Cannot fromfile {} with an action ({}) in scope {}'
.format(dest, action, self._scope))
config_section = 'DEFAULT' if self._scope == GLOBAL_SCOPE else self._scope
udest = dest.upper()
if self._scope == GLOBAL_SCOPE:
# For convenience, we allow three forms of env var for global scope options.
# The fully-specified env var is PANTS_DEFAULT_FOO, which is uniform with PANTS_<SCOPE>_FOO
# for all the other scopes. However we also allow simply PANTS_FOO. And if the option name
# itself starts with 'pants-' then we also allow simply FOO. E.g., PANTS_WORKDIR instead of
# PANTS_PANTS_WORKDIR or PANTS_DEFAULT_PANTS_WORKDIR. We take the first specified value we
# find, in this order: PANTS_DEFAULT_FOO, PANTS_FOO, FOO.
env_vars = ['PANTS_DEFAULT_{0}'.format(udest), 'PANTS_{0}'.format(udest)]
if udest.startswith('PANTS_'):
env_vars.append(udest)
else:
sanitized_env_var_scope = self._ENV_SANITIZER_RE.sub('_', config_section.upper())
env_vars = ['PANTS_{0}_{1}'.format(sanitized_env_var_scope, udest)]
value_type = self.str_to_bool if is_boolean_flag(kwargs) else kwargs.get('type', str)
env_val_str = None
if self._env:
for env_var in env_vars:
if env_var in self._env:
env_val_str = self._env.get(env_var)
break
config_val_str = self._config.get(config_section, dest, default=None)
config_source_file = self._config.get_source_for_option(config_section, dest)
if config_source_file is not None:
config_source_file = os.path.relpath(config_source_file)
def expand(val_str):
if is_fromfile and val_str and val_str.startswith('@') and not val_str.startswith('@@'):
fromfile = val_str[1:]
try:
with open(fromfile) as fp:
return fp.read().strip()
except IOError as e:
raise self.FromfileError('Failed to read {} from file {}: {}'.format(dest, fromfile, e))
else:
# Support a literal @ for fromfile values via @@.
return val_str[1:] if is_fromfile and val_str.startswith('@@') else val_str
def parse_typed_list(val_str):
return None if val_str is None else [value_type(x) for x in list_option(expand(val_str))]
def parse_typed_item(val_str):
return None if val_str is None else value_type(expand(val_str))
# Handle the forthcoming conversions argparse will need to do by placing our parse hook - we
# handle the conversions for env and config ourselves below. Unlike the env and config
# handling, `action='append'` does not need to be handled specially since appended flag values
# come as single items' thus only `parse_typed_item` is ever needed for the flag value type
# conversions.
if is_fromfile:
kwargs['type'] = parse_typed_item
default, parse = ([], parse_typed_list) if action == 'append' else (None, parse_typed_item)
config_val = parse(config_val_str)
env_val = parse(env_val_str)
hardcoded_val = kwargs.get('default')
config_details = 'in {}'.format(config_source_file) if config_source_file else None
choices = list(RankedValue.prioritized_iter(None, env_val, config_val, hardcoded_val, default))
for choice in reversed(choices):
details = config_details if choice.rank == RankedValue.CONFIG else None
self._option_tracker.record_option(scope=self._scope, option=dest, value=choice.value,
rank=choice.rank, details=details)
return choices[0]
def _compute_value(self, dest, kwargs, flag_val_strs):
"""Compute the value to use for an option.
The source of the default value is chosen according to the ranking in RankedValue.
"""
# Helper function to convert a string to a value of the option's type.
def to_value_type(val_str):
if val_str is None:
return None
elif kwargs.get('type') == bool:
return self._ensure_bool(val_str)
else:
type_arg = kwargs.get('type', str)
try:
return self._wrap_type(type_arg)(val_str)
except TypeError as e:
raise ParseError(
"Error applying type '{}' to option value '{}', for option '--{}' in {}: {}"
.format(type_arg.__name__, val_str, dest, self._scope_str(), e))
# Helper function to expand a fromfile=True value string, if needed.
# May return a string or a dict/list decoded from a json/yaml file.
def expand(val_or_str):
if (kwargs.get('fromfile', True) and val_or_str and
isinstance(val_or_str, str) and val_or_str.startswith('@')):
if val_or_str.startswith('@@'): # Support a literal @ for fromfile values via @@.
return val_or_str[1:]
else:
fromfile = val_or_str[1:]
try:
with open(fromfile, 'r') as fp:
s = fp.read().strip()
if fromfile.endswith('.json'):
return json.loads(s)
elif fromfile.endswith('.yml') or fromfile.endswith('.yaml'):
return yaml.safe_load(s)
else:
return s
except (IOError, ValueError, yaml.YAMLError) as e:
raise self.FromfileError('Failed to read {} in {} from file {}: {}'.format(
dest, self._scope_str(), fromfile, e))
else:
return val_or_str
# Get value from config files, and capture details about its derivation.
config_details = None
config_section = GLOBAL_SCOPE_CONFIG_SECTION if self._scope == GLOBAL_SCOPE else self._scope
config_default_val_or_str = expand(self._config.get(Config.DEFAULT_SECTION, dest, default=None))
config_val_or_str = expand(self._config.get(config_section, dest, default=None))
config_source_file = (self._config.get_source_for_option(config_section, dest) or
self._config.get_source_for_option(Config.DEFAULT_SECTION, dest))
if config_source_file is not None:
config_source_file = os.path.relpath(config_source_file)
config_details = 'in {}'.format(config_source_file)
# Get value from environment, and capture details about its derivation.
udest = dest.upper()
if self._scope == GLOBAL_SCOPE:
# For convenience, we allow three forms of env var for global scope options.
# The fully-specified env var is PANTS_GLOBAL_FOO, which is uniform with PANTS_<SCOPE>_FOO
# for all the other scopes. However we also allow simply PANTS_FOO. And if the option name
# itself starts with 'pants-' then we also allow simply FOO. E.g., PANTS_WORKDIR instead of
# PANTS_PANTS_WORKDIR or PANTS_GLOBAL_PANTS_WORKDIR. We take the first specified value we
# find, in this order: PANTS_GLOBAL_FOO, PANTS_FOO, FOO.
env_vars = ['PANTS_GLOBAL_{0}'.format(udest), 'PANTS_{0}'.format(udest)]
if udest.startswith('PANTS_'):
env_vars.append(udest)
else:
sanitized_env_var_scope = self._ENV_SANITIZER_RE.sub('_', self._scope.upper())
env_vars = ['PANTS_{0}_{1}'.format(sanitized_env_var_scope, udest)]
env_val_or_str = None
env_details = None
if self._env:
for env_var in env_vars:
if env_var in self._env:
env_val_or_str = expand(self._env.get(env_var))
env_details = 'from env var {}'.format(env_var)
break
# Get value from cmd-line flags.
flag_vals = [to_value_type(expand(x)) for x in flag_val_strs]
if is_list_option(kwargs):
# Note: It's important to set flag_val to None if no flags were specified, so we can
# distinguish between no flags set vs. explicit setting of the value to [].
flag_val = ListValueComponent.merge(flag_vals) if flag_vals else None
elif is_dict_option(kwargs):
# Note: It's important to set flag_val to None if no flags were specified, so we can
# distinguish between no flags set vs. explicit setting of the value to {}.
flag_val = DictValueComponent.merge(flag_vals) if flag_vals else None
elif len(flag_vals) > 1:
raise ParseError('Multiple cmd line flags specified for option {} in {}'.format(
dest, self._scope_str()))
elif len(flag_vals) == 1:
flag_val = flag_vals[0]
else:
flag_val = None
# Rank all available values.
# Note that some of these values may already be of the value type, but type conversion
# is idempotent, so this is OK.
values_to_rank = [to_value_type(x) for x in
[flag_val, env_val_or_str, config_val_or_str,
config_default_val_or_str, kwargs.get('default'), None]]
# Note that ranked_vals will always have at least one element, and all elements will be
# instances of RankedValue (so none will be None, although they may wrap a None value).
ranked_vals = list(reversed(list(RankedValue.prioritized_iter(*values_to_rank))))
def record_option(value, rank, option_details=None):
deprecation_version = kwargs.get('removal_version')
self._option_tracker.record_option(scope=self._scope,
option=dest,
value=value,
rank=rank,
deprecation_version=deprecation_version,
details=option_details)
# Record info about the derivation of each of the contributing values.
detail_history = []
for ranked_val in ranked_vals:
if ranked_val.rank in (RankedValue.CONFIG, RankedValue.CONFIG_DEFAULT):
details = config_details
elif ranked_val.rank == RankedValue.ENVIRONMENT:
details = env_details
else:
details = None
if details:
detail_history.append(details)
record_option(value=ranked_val.value, rank=ranked_val.rank, option_details=details)
# Helper function to check various validity constraints on final option values.
def check(val):
if val is not None:
choices = kwargs.get('choices')
# If the `type` argument has an `all_variants` attribute, use that as `choices` if not
# already set. Using an attribute instead of checking a subclass allows `type` arguments
# which are functions to have an implicit fallback `choices` set as well.
if choices is None and 'type' in kwargs:
type_arg = kwargs.get('type')
if hasattr(type_arg, 'all_variants'):
choices = list(type_arg.all_variants)
# TODO: convert this into an enum() pattern match!
if choices is not None and val not in choices:
raise ParseError('`{}` is not an allowed value for option {} in {}. '
'Must be one of: {}'.format(val, dest, self._scope_str(), choices))
elif kwargs.get('type') == dir_option and not os.path.isdir(val):
raise ParseError('Directory value `{}` for option {} in {} does not exist.'.format(
val, dest, self._scope_str()))
elif kwargs.get('type') == file_option and not os.path.isfile(val):
raise ParseError('File value `{}` for option {} in {} does not exist.'.format(
val, dest, self._scope_str()))
# Generate the final value from all available values, and check that it (or its members,
# if a list) are in the set of allowed choices.
if is_list_option(kwargs):
merged_rank = ranked_vals[-1].rank
merged_val = ListValueComponent.merge(
[rv.value for rv in ranked_vals if rv.value is not None]).val
# TODO: run `check()` for all elements of a list option too!!!
merged_val = [self._convert_member_type(kwargs.get('member_type', str), x)
for x in merged_val]
for val in merged_val:
check(val)
ret = RankedValue(merged_rank, merged_val)
elif is_dict_option(kwargs):
# TODO: convert `member_type` for dict values too!
merged_rank = ranked_vals[-1].rank
merged_val = DictValueComponent.merge(
[rv.value for rv in ranked_vals if rv.value is not None]).val
for val in merged_val:
check(val)
ret = RankedValue(merged_rank, merged_val)
else:
ret = ranked_vals[-1]
check(ret.value)
# Record info about the derivation of the final value.
merged_details = ', '.join(detail_history) if detail_history else None
record_option(value=ret.value, rank=ret.rank, option_details=merged_details)
# All done!
return ret
def _compute_default(self, dest, kwargs):
"""Compute the default value to use for an option's registration.
The source of the default value is chosen according to the ranking in RankedValue.
"""
is_fromfile = kwargs.get('fromfile', False)
action = kwargs.get('action')
if is_fromfile and action and action != 'append':
raise ParseError('Cannot fromfile {} with an action ({}) in scope {}'
.format(dest, action, self._scope))
config_section = 'DEFAULT' if self._scope == GLOBAL_SCOPE else self._scope
udest = dest.upper()
if self._scope == GLOBAL_SCOPE:
# For convenience, we allow three forms of env var for global scope options.
# The fully-specified env var is PANTS_DEFAULT_FOO, which is uniform with PANTS_<SCOPE>_FOO
# for all the other scopes. However we also allow simply PANTS_FOO. And if the option name
# itself starts with 'pants-' then we also allow simply FOO. E.g., PANTS_WORKDIR instead of
# PANTS_PANTS_WORKDIR or PANTS_DEFAULT_PANTS_WORKDIR. We take the first specified value we
# find, in this order: PANTS_DEFAULT_FOO, PANTS_FOO, FOO.
env_vars = ['PANTS_DEFAULT_{0}'.format(udest), 'PANTS_{0}'.format(udest)]
if udest.startswith('PANTS_'):
env_vars.append(udest)
else:
sanitized_env_var_scope = self._ENV_SANITIZER_RE.sub('_', config_section.upper())
env_vars = ['PANTS_{0}_{1}'.format(sanitized_env_var_scope, udest)]
value_type = self.str_to_bool if is_boolean_flag(kwargs) else kwargs.get('type', str)
env_val_str = None
if self._env:
for env_var in env_vars:
if env_var in self._env:
env_val_str = self._env.get(env_var)
break
config_val_str = self._config.get(config_section, dest, default=None)
config_source_file = self._config.get_source_for_option(config_section, dest)
if config_source_file is not None:
config_source_file = os.path.relpath(config_source_file)
def expand(val_str):
if is_fromfile and val_str and val_str.startswith('@') and not val_str.startswith('@@'):
fromfile = val_str[1:]
try:
with open(fromfile) as fp:
return fp.read().strip()
except IOError as e:
raise self.FromfileError('Failed to read {} from file {}: {}'.format(dest, fromfile, e))
else:
# Support a literal @ for fromfile values via @@.
return val_str[1:] if is_fromfile and val_str.startswith('@@') else val_str
def parse_typed_list(val_str):
return None if val_str is None else [value_type(x) for x in list_option(expand(val_str))]
def parse_typed_item(val_str):
return None if val_str is None else value_type(expand(val_str))
# Handle the forthcoming conversions argparse will need to do by placing our parse hook - we
# handle the conversions for env and config ourselves below. Unlike the env and config
# handling, `action='append'` does not need to be handled specially since appended flag values
# come as single items' thus only `parse_typed_item` is ever needed for the flag value type
# conversions.
if is_fromfile:
kwargs['type'] = parse_typed_item
default, parse = ([], parse_typed_list) if action == 'append' else (None, parse_typed_item)
config_val = parse(config_val_str)
env_val = parse(env_val_str)
hardcoded_val = kwargs.get('default')
config_details = 'in {}'.format(config_source_file) if config_source_file else None
choices = list(RankedValue.prioritized_iter(None, env_val, config_val, hardcoded_val, default))
for choice in reversed(choices):
details = config_details if choice.rank == RankedValue.CONFIG else None
self._option_tracker.record_option(scope=self._scope, option=dest, value=choice.value,
rank=choice.rank, details=details)
return choices[0]
