def format_match(match):
name = match.group("name")
try:
value = wildcards[name]
if fail_dynamic and value == dynamic_fill:
raise WildcardError(name)
return str(value) # convert anything into a str
except KeyError as ex:
if fill_missing:
return dynamic_fill
else:
raise WildcardError(str(ex))
def expand(*args, **wildcards):
"""
Expand wildcards in given filepatterns.
Arguments
*args -- first arg: filepatterns as list or one single filepattern,
second arg (optional): a function to combine wildcard values
(itertools.product per default)
**wildcards -- the wildcards as keyword arguments
with their values as lists
"""
filepatterns = args[0]
if len(args) == 1:
combinator = product
elif len(args) == 2:
combinator = args[1]
if isinstance(filepatterns, str):
filepatterns = [filepatterns]
def flatten(wildcards):
for wildcard, values in wildcards.items():
if isinstance(values, str) or not isinstance(values, Iterable):
values = [values]
yield [(wildcard, value) for value in values]
try:
return [
filepattern.format(**comb)
for comb in map(dict, combinator(*flatten(wildcards)))
for filepattern in filepatterns
]
except KeyError as e:
raise WildcardError("No values given for wildcard {}.".format(e))
def expand_log(self, wildcards):
def concretize_logfile(f, wildcards, is_from_callable):
if is_from_callable:
return IOFile(f, rule=self)
else:
return f.apply_wildcards(wildcards,
fill_missing=False,
fail_dynamic=self.dynamic_output)
log = Log()
try:
self._apply_wildcards(log,
self.log,
wildcards,
concretize=concretize_logfile,
property="log")
except WildcardError as e:
raise WildcardError(
"Wildcards in log files cannot be "
"determined from output files:",
str(e),
rule=self,
)
for f in log:
f.check()
return log
def expand_params(self,
wildcards,
input,
output,
resources,
omit_callable=False):
def concretize_param(p, wildcards, is_from_callable):
if not is_from_callable:
if isinstance(p, str):
return apply_wildcards(p, wildcards)
if isinstance(p, list):
return [(apply_wildcards(v, wildcards) if isinstance(
v, str) else v) for v in p]
return p
def handle_incomplete_checkpoint(exception):
"""If checkpoint is incomplete, target it such that it is completed
before this rule gets executed."""
if exception.targetfile in input:
return TBDString()
else:
raise WorkflowError(
"Rule parameter depends on checkpoint but checkpoint output is not defined as input file for the rule. "
"Please add the output of the respective checkpoint to the rule inputs."
)
params = Params()
try:
# When applying wildcards to params, the return type need not be
# a string, so the check is disabled.
self._apply_wildcards(
params,
self.params,
wildcards,
concretize=concretize_param,
check_return_type=False,
omit_callable=omit_callable,
allow_unpack=False,
no_flattening=True,
property="params",
aux_params={
"input": input._plainstrings(),
"resources": resources,
"output": output._plainstrings(),
"threads": resources._cores,
},
incomplete_checkpoint_func=handle_incomplete_checkpoint,
)
except WildcardError as e:
raise WildcardError(
"Wildcards in params cannot be "
"determined from output files. Note that you have "
"to use a function to deactivate automatic wildcard expansion "
"in params strings, e.g., `lambda wildcards: '{test}'`. Also "
"see https://snakemake.readthedocs.io/en/stable/snakefiles/"
"rules.html#non-file-parameters-for-rules:",
str(e),
rule=self,
)
return params
def expand_params(self, wildcards, input, output, resources, omit_callable=False):
def concretize_param(p, wildcards, is_from_callable):
if not is_from_callable and isinstance(p, str):
return apply_wildcards(p, wildcards)
return p
params = Params()
try:
#When applying wildcards to params, the return type need not be
#a string, so the check is disabled.
self._apply_wildcards(params, self.params, wildcards,
concretize=concretize_param,
check_return_type=False,
omit_callable=omit_callable,
allow_unpack=False,
no_flattening=True,
apply_default_remote=False,
aux_params={"input": input.plainstrings(),
"resources": resources,
"output": output.plainstrings(),
"threads": resources._cores},
incomplete_checkpoint_func=lambda e: "<incomplete checkpoint>")
except WildcardError as e:
raise WildcardError(
"Wildcards in params cannot be "
"determined from output files. Note that you have "
"to use a function to deactivate automatic wildcard expansion "
"in params strings, e.g., `lambda wildcards: '{test}'`. Also "
"see https://snakemake.readthedocs.io/en/stable/snakefiles/"
"rules.html#non-file-parameters-for-rules:",
str(e), rule=self)
return params
def expand(*args, **wildcards):
"""
Expand wildcards in given filepatterns.
Arguments
*args -- first arg: filepatterns as list or one single filepattern,
second arg (optional): a function to combine wildcard values
(itertools.product per default)
**wildcards -- the wildcards as keyword arguments
with their values as lists
"""
filepatterns = args[0]
if len(args) == 1:
combinator = product
elif len(args) == 2:
combinator = args[1]
if isinstance(filepatterns, str):
filepatterns = [filepatterns]
def path_to_str(f):
if isinstance(f, Path):
return str(f)
return f
filepatterns = list(map(path_to_str, filepatterns))
if any(map(lambda f: getattr(f, "flags", {}), filepatterns)):
raise WorkflowError(
"Flags in file patterns given to expand() are invalid. "
"Flags (e.g. temp(), directory()) have to be applied outside "
"of expand (e.g. 'temp(expand(\"plots/{sample}.pdf\", sample=SAMPLES))')."
)
# remove unused wildcards to avoid duplicate filepatterns
wildcards = {
filepattern: {
k: v
for k, v in wildcards.items()
if k in re.findall("{([^}\.[!:]+)", filepattern)
}
for filepattern in filepatterns
}
def flatten(wildcards):
for wildcard, values in wildcards.items():
if isinstance(values, str) or not isinstance(
values, collections.abc.Iterable
):
values = [values]
yield [(wildcard, value) for value in values]
try:
return [
filepattern.format(**comb)
for filepattern in filepatterns
for comb in map(dict, combinator(*flatten(wildcards[filepattern])))
]
except KeyError as e:
raise WildcardError("No values given for wildcard {}.".format(e))
def expand_input(self, wildcards, groupid=None):
def concretize_iofile(f, wildcards, is_from_callable):
if is_from_callable:
if isinstance(f, Path):
f = str(f)
return IOFile(f, rule=self).apply_wildcards(
wildcards,
fill_missing=f in self.dynamic_input,
fail_dynamic=self.dynamic_output,
)
else:
return f.apply_wildcards(
wildcards,
fill_missing=f in self.dynamic_input,
fail_dynamic=self.dynamic_output,
)
def handle_incomplete_checkpoint(exception):
"""If checkpoint is incomplete, target it such that it is completed
before this rule gets executed."""
return exception.targetfile
input = InputFiles()
mapping = dict()
try:
incomplete = self._apply_wildcards(
input,
self.input,
wildcards,
concretize=concretize_iofile,
mapping=mapping,
incomplete_checkpoint_func=handle_incomplete_checkpoint,
path_modifier=self.input_modifier,
property="input",
groupid=groupid,
)
except WildcardError as e:
raise WildcardError(
"Wildcards in input files cannot be "
"determined from output files:",
str(e),
rule=self,
)
if self.dependencies:
dependencies = {
f: self.dependencies[f_]
for f, f_ in mapping.items() if f_ in self.dependencies
}
if None in self.dependencies:
dependencies[None] = self.dependencies[None]
else:
dependencies = self.dependencies
for f in input:
f.check()
return input, mapping, dependencies, incomplete
def expand_conda_env(self, wildcards):
try:
conda_env = self.conda_env.apply_wildcards(
wildcards) if self.conda_env else None
except WildcardError as e:
raise WildcardError(
"Wildcards in conda environment file cannot be "
"determined from output files:",
str(e), rule=self)
if conda_env is not None:
conda_env.check()
return conda_env
def expand_benchmark(self, wildcards):
try:
benchmark = self.benchmark.apply_wildcards(
wildcards) if self.benchmark else None
except WildcardError as e:
raise WildcardError(
"Wildcards in benchmark file cannot be "
"determined from output files:",
str(e), rule=self)
if benchmark is not None:
benchmark.check()
return benchmark
def expand(*args, **wildcards):
"""
Expand wildcards in given filepatterns.
Arguments
*args -- first arg: filepatterns as list or one single filepattern,
second arg (optional): a function to combine wildcard values
(itertools.product per default)
**wildcards -- the wildcards as keyword arguments
with their values as lists
"""
filepatterns = args[0]
if len(args) == 1:
combinator = product
elif len(args) == 2:
combinator = args[1]
if isinstance(filepatterns, str):
filepatterns = [filepatterns]
# remove unused wildcards to avoid duplicate filepatterns
wildcards = {
filepattern: {
k: v
for k, v in wildcards.items()
if k in re.findall('{([^}\.[!:]+)', filepattern)
}
for filepattern in filepatterns
}
def flatten(wildcards):
for wildcard, values in wildcards.items():
if isinstance(values,
str) or not isinstance(values, collections.Iterable):
values = [values]
yield [(wildcard, value) for value in values]
try:
return [
filepattern.format(**comb) for filepattern in filepatterns
for comb in map(dict, combinator(*flatten(wildcards[filepattern])))
]
except KeyError as e:
raise WildcardError("No values given for wildcard {}.".format(e))
def expand_input(self, wildcards):
def concretize_iofile(f, wildcards):
if not isinstance(f, _IOFile):
return IOFile(f, rule=self)
else:
return f.apply_wildcards(wildcards,
fill_missing=f in self.dynamic_input,
fail_dynamic=self.dynamic_output)
input = InputFiles()
mapping = dict()
try:
self._apply_wildcards(input,
self.input,
wildcards,
concretize=concretize_iofile,
mapping=mapping)
except WildcardError as e:
raise WildcardError(
"Wildcards in input files cannot be "
"determined from output files:",
str(e),
rule=self)
if self.dependencies:
dependencies = {
f: self.dependencies[f_]
for f, f_ in mapping.items() if f_ in self.dependencies
}
if None in self.dependencies:
dependencies[None] = self.dependencies[None]
else:
dependencies = self.dependencies
for f in input:
f.check()
return input, mapping, dependencies
def expand_params(self,
wildcards,
input,
output,
resources,
omit_callable=False):
def concretize_param(p, wildcards):
if isinstance(p, str):
return apply_wildcards(p, wildcards)
return p
params = Params()
try:
#When applying wildcards to params, the return type need not be
#a string, so the check is disabled.
self._apply_wildcards(params,
self.params,
wildcards,
concretize=concretize_param,
check_return_type=False,
omit_callable=omit_callable,
no_flattening=True,
apply_default_remote=False,
aux_params={
"input": input,
"resources": resources,
"output": output,
"threads": resources._cores
})
except WildcardError as e:
raise WildcardError(
"Wildcards in params cannot be "
"determined from output files:",
str(e),
rule=self)
return params
def expand(*args, **wildcards):
"""
Expand wildcards in given filepatterns.
Arguments
*args -- first arg: filepatterns as list or one single filepattern,
second arg (optional): a function to combine wildcard values
(itertools.product per default)
**wildcards -- the wildcards as keyword arguments
with their values as lists. If allow_missing=True is included
wildcards in filepattern without values will stay unformatted.
"""
filepatterns = args[0]
if len(args) == 1:
combinator = product
elif len(args) == 2:
combinator = args[1]
if isinstance(filepatterns, str):
filepatterns = [filepatterns]
def path_to_str(f):
if isinstance(f, Path):
return str(f)
return f
filepatterns = list(map(path_to_str, filepatterns))
if any(map(lambda f: getattr(f, "flags", {}), filepatterns)):
raise WorkflowError(
"Flags in file patterns given to expand() are invalid. "
"Flags (e.g. temp(), directory()) have to be applied outside "
"of expand (e.g. 'temp(expand(\"plots/{sample}.pdf\", sample=SAMPLES))')."
)
# check if remove missing is provided
format_dict = dict
if "allow_missing" in wildcards and wildcards["allow_missing"] is True:
class FormatDict(dict):
def __missing__(self, key):
return "{" + key + "}"
format_dict = FormatDict
# check that remove missing is not a wildcard in the filepatterns
for filepattern in filepatterns:
if "allow_missing" in re.findall(r"{([^}\.[!:]+)", filepattern):
format_dict = dict
break
# remove unused wildcards to avoid duplicate filepatterns
wildcards = {
filepattern: {
k: v
for k, v in wildcards.items()
if k in re.findall(r"{([^}\.[!:]+)", filepattern)
}
for filepattern in filepatterns
}
def flatten(wildcards):
for wildcard, values in wildcards.items():
if isinstance(values, str) or not isinstance(
values, collections.abc.Iterable
):
values = [values]
yield [(wildcard, value) for value in values]
formatter = string.Formatter()
try:
return [
formatter.vformat(filepattern, (), comb)
for filepattern in filepatterns
for comb in map(format_dict, combinator(*flatten(wildcards[filepattern])))
]
except KeyError as e:
raise WildcardError("No values given for wildcard {}.".format(e))
