def format_cli(self, value: Optional[ValueType]) -> Optional[List[str]]:
"""
Build a parameter argument (or multiple, if this is a multi-valued parameter).
:return: list of CLI strings -- not escaped. If the parameter should not be expressed, returns None.
:rtype: list[str]|None
"""
pass_as_template = self._get_pass_as_template(value)
if not pass_as_template:
return None
pass_as_bits = pass_as_template.split()
def _format_atom(value: ValueAtomType) -> List[str]:
env = dict(name=self.name, value=value, v=value)
return [bit.format(**env) for bit in pass_as_bits]
if self.multiple == MultipleMode.REPEAT:
out = []
for atom in listify(value):
out.extend(_format_atom(atom))
return out
elif self.multiple == MultipleMode.SEPARATE:
value_list = listify(value)
# Guard against generating a `--foo=` when there are no values.
if value_list:
return _format_atom(
self.multiple_separator.join(
str(atom) for atom in value_list))
return None
elif not self.multiple:
return _format_atom(value)
else:
raise NotImplementedError('unknown multiple type %r' %
self.multiple)
def expand_globs(
sources: Union[str, List[str]],
preprocessor: Callable[[str],
str] = lambda s: s) -> Set[Tuple[str, str]]:
"""Returns a set of paths as a result of expanding all the source wildcards
First item of the resulting tuple item is the expanded path
Second item of the resulting tuple is the logical root path
Example
source: /tmp/foo/**/*.png
output: set(("/tmp/foo/bar/hello.png", "/tmp/foo"), ("/tmp/foo/bar/sub/jello.png", "/tmp/foo"))
source: /tmp/yeah/*.png
output: set(("/tmp/yeah/hello.png", "/tmp/yeah"), ("/tmp/yeah/jello.png", "/tmp/yeah"))
:param sources: Path or list of paths. May contain * or ** wildcards.
:param preprocessor: Preprocessor to be used for each of the paths
:return:
"""
sources = listify(sources)
files_to_compress = set()
for source_path in sources:
source_path = preprocessor(source_path)
glob_pattern = get_glob_pattern(source_path)
root_path = os.path.dirname(glob_pattern).split(
"*", 1)[0] # Handles ** also
for file_path in glob.glob(glob_pattern, recursive=True):
if os.path.isfile(file_path):
files_to_compress.add((file_path, root_path))
if not files_to_compress:
raise ValueError(f"No files to compress at {sources}")
return files_to_compress
def from_urls_and_paths(
cls, urls_and_paths: Union[str, List[str]]) -> "InputInfo":
files = []
for value in listify(urls_and_paths):
if "://" not in value: # The string is a local path
for path in glob.glob(value):
files.append(
FileInfo(
name=os.path.basename(path),
uri=None,
path=path,
size=None,
checksums=None,
))
else: # The string is an URL
files.append(
FileInfo(
name=uri_to_filename(value),
uri=value,
path=None,
size=None,
checksums=None,
))
return cls(files=files)
def __init__(self,
*,
name,
type='string',
optional=False,
min=None,
max=None,
description=None,
default=None,
pass_as=None,
pass_true_as=None,
pass_false_as=None,
choices=None,
multiple=None,
multiple_separator=',') -> None:
self.name = name
self.type = type
self.optional = bool(optional)
self.min = min
self.max = max
self.description = description
self.pass_as = pass_as
self.pass_true_as = pass_true_as
self.pass_false_as = pass_false_as
self.choices = (list(choices) if choices else None)
self.multiple = MultipleMode.cast(multiple)
self.multiple_separator = str(multiple_separator or ',')
self.default = (listify(default) if self.multiple else default)
if self.type == 'flag':
self.optional = True
self.choices = (True, False)
if self.multiple:
raise ValueError('Flag parameters can\'t be multiple')
else:
self.pass_true_as = self.pass_false_as = None
def validate(self, value: ValueType) -> ValueType:
"""
Validate (and possibly typecast) the given parameter value.
:param value: Parameter value
:return: Typecast parameter value
:raises ValidationErrors: if there were validation errors
"""
errors = []
validated_values = []
if not self.multiple and isinstance(value, (list, tuple)):
errors.append('Only a single value is allowed')
for atom in listify(value):
atom = self._validate_type(atom, errors)
atom = self._validate_value(atom, errors)
validated_values.append(atom)
if errors:
raise ValidationErrors(errors)
if self.multiple:
return validated_values
return validated_values[0]
def prepare_inputs(input_dict, verbose=False):
for input_name, input_specs in input_dict.items():
input_specs = listify(input_specs)
multiple_specs = (len(input_specs) > 1)
for filename in input_specs:
obj = _prepare_single_input(input_name, filename, multiple_specs)
if verbose:
echo('Input {name}: {source} -> {target}'.format(
name=style(input_name, bold=True, fg='blue'),
source=style(filename, bold=True),
target=style(obj['destination'], bold=True),
))
yield obj
def convert_input_to_option(self, input: Input) -> Option:
"""
Convert an Input into a click Option.
"""
assert isinstance(input, Input)
option = click.Option(
param_decls=list(generate_sanitized_options(input.name)),
required=(input.default is None and not input.optional),
default=listify(input.default),
metavar='URL',
multiple=True,
help=f'Input "{humanize_identifier(input.name)}"',
)
option.name = f'^{input.name}' # Caretize so we can pick these out of kwargs easily
option.help_group = 'Input Options' # type: ignore[attr-defined]
return option
def sift_cli_inputs(args: argparse.Namespace,
expected_keys: Set[str]) -> Dict[str, List[str]]:
"""Sift inputs from all the command-line args
:param expected_keys: List of expected input names
"""
result = {}
for name, values in vars(args).items():
# Filter out any inputs that we weren't expecting
if name not in expected_keys:
continue
values = [v for v in listify(values) if v is not None]
if len(values) > 0:
result[name] = values
return result
def lint(self, lint_result, context: dict) -> None:
has_pass_as = bool(self._original_data.get('pass-as'))
has_pass_true_as = bool(self._original_data.get('pass-true-as'))
has_pass_false_as = bool(self._original_data.get('pass-false-as'))
context_prefix = 'Step {step}, parameter {param}'.format(
step=context['step'].name,
param=self.name,
)
if self.type == 'flag':
if self._original_data.get('optional'):
lint_result.add_warning(
'{prefix}: `optional` has no effect on flag-type parameters'
.format(prefix=context_prefix, ))
if (has_pass_true_as or has_pass_false_as) and has_pass_as:
lint_result.add_warning(
'{prefix}: `pass-as` has no effect with `pass-true-as`/`pass-false-as`'
.format(prefix=context_prefix, ))
else:
if has_pass_true_as:
lint_result.add_warning(
'{prefix}: `pass-true-as` has no effect on non-flag parameters'
.format(prefix=context_prefix, ))
if has_pass_false_as:
lint_result.add_warning(
'{prefix}: `pass-false-as` has no effect on non-flag parameters'
.format(prefix=context_prefix, ))
if self.default:
errors = []
for default_value in listify(self.default):
if self.type == 'string' and not isinstance(
default_value, str):
errors.append(
'`default` value {value!r} is not a string (got a {type})'
.format(
value=default_value,
type=type(default_value),
))
else:
self._validate_type(default_value, errors)
self._validate_value(default_value, errors)
for message in errors:
lint_result.add_warning('{prefix}: default {message}'.format(
prefix=context_prefix, message=message))
def build_command(
command: Union[str, List[str]],
parameter_map: Union[ParameterMap, LegacyParameterMap, list],
) -> List[str]:
"""
Build command line(s) using the given parameter map.
Even if the passed a single `command`, this function will return a list
of shell commands. It is the caller's responsibility to concatenate them,
likely using the semicolon or double ampersands.
:param command: The command to interpolate params into.
:type command: str|list[str]
:param parameter_map: A ParameterMap object containing parameter knowledge.
:type parameter_map: valohai_yaml.objs.parameter_map.ParameterMap
:return: list of commands
:rtype: list[str]
"""
if isinstance(
parameter_map,
list): # Partially emulate old (pre-0.7) API for this function.
parameter_map = LegacyParameterMap(parameter_map)
out_commands = []
for command in listify(command):
# Only attempt formatting if the string smells like it should be formatted.
# This allows the user to include shell syntax in the commands, if required.
# (There's still naturally the chance for false-positives, so guard against
# those value errors and warn about them.)
if interpolable_re.search(command):
try:
command = interpolable_re.sub(
lambda match: _replace_interpolation(parameter_map, match),
command,
)
except ValueError as exc: # pragma: no cover
warnings.warn(
'failed to interpolate into %r: %s' % (command, exc),
CommandInterpolationWarning)
out_commands.append(command.strip())
return out_commands
def build_command(command, parameters):
"""
Build command line(s) using the given parameter values.
Even if the passed a single `command`, this function will return a list
of shell commands. It is the caller's responsibility to concatenate them,
likely using the semicolon or double ampersands.
:param command: The command to interpolate params into.
:type command: str|list[str]
:param parameter_values: Command line parameters for the {parameters} placeholder.
These are quoted within `build_command`.
:type parameter_values: list[str]
:return: list of commands
:rtype: list[str]
"""
parameters_str = ' '.join(quote(parameter) for parameter in parameters)
# format each command
env = dict(parameters=parameters_str, params=parameters_str)
out_commands = []
for command in listify(command):
# Only attempt formatting if the string smells like it should be formatted.
# This allows the user to include shell syntax in the commands, if required.
# (There's still naturally the chance for false-positives, so guard against
# those value errors and warn about them.)
if any('{%s}' % key in command for key in env):
try:
command = command.format(**env)
except ValueError as exc: # pragma: no cover
warnings.warn(
'failed to interpolate parameters into %r: %s' %
(command, exc), CommandInterpolationWarning)
out_commands.append(command.strip())
return out_commands
def test_listify():
assert listify(None) == []
assert listify([]) == []
assert listify(()) == []
assert listify(6) == [6]
assert listify('foo') == ['foo']