def expand_input(self, wildcards):
def concretize_iofile(f, wildcards):
if not isinstance(f, _IOFile):
return IOFile(f, rule=self)
else:
return f.apply_wildcards(wildcards,
fill_missing=f in self.dynamic_input,
fail_dynamic=self.dynamic_output)
input = InputFiles()
mapping = dict()
try:
self._apply_wildcards(input, self.input, wildcards,
concretize=concretize_iofile,
mapping=mapping)
except WildcardError as e:
raise WorkflowError(
"Wildcards in input files cannot be "
"determined from output files:",
str(e), rule=self)
if self.dependencies:
dependencies = {
f: self.dependencies[f_]
for f, f_ in mapping.items() if f_ in self.dependencies
}
if None in self.dependencies:
dependencies[None] = self.dependencies[None]
else:
dependencies = self.dependencies
for f in input:
f.check()
return input, mapping, dependencies
def expand_input(self, wildcards, groupid=None):
def concretize_iofile(f, wildcards, is_from_callable):
if is_from_callable:
if isinstance(f, Path):
f = str(f)
return IOFile(f, rule=self).apply_wildcards(
wildcards,
fill_missing=f in self.dynamic_input,
fail_dynamic=self.dynamic_output,
)
else:
return f.apply_wildcards(
wildcards,
fill_missing=f in self.dynamic_input,
fail_dynamic=self.dynamic_output,
)
def handle_incomplete_checkpoint(exception):
"""If checkpoint is incomplete, target it such that it is completed
before this rule gets executed."""
return exception.targetfile
input = InputFiles()
mapping = dict()
try:
incomplete = self._apply_wildcards(
input,
self.input,
wildcards,
concretize=concretize_iofile,
mapping=mapping,
incomplete_checkpoint_func=handle_incomplete_checkpoint,
path_modifier=self.input_modifier,
property="input",
groupid=groupid,
)
except WildcardError as e:
raise WildcardError(
"Wildcards in input files cannot be "
"determined from output files:",
str(e),
rule=self,
)
if self.dependencies:
dependencies = {
f: self.dependencies[f_]
for f, f_ in mapping.items() if f_ in self.dependencies
}
if None in self.dependencies:
dependencies[None] = self.dependencies[None]
else:
dependencies = self.dependencies
for f in input:
f.check()
return input, mapping, dependencies, incomplete
def __init__(self, *args, lineno=None, snakefile=None):
"""
Create a rule
Arguments
name -- the name of the rule
"""
if len(args) == 2:
name, workflow = args
self.name = name
self.workflow = workflow
self.docstring = None
self.message = None
self._input = InputFiles()
self._output = OutputFiles()
self._params = Params()
self.dynamic_output = set()
self.dynamic_input = set()
self.temp_output = set()
self.protected_output = set()
self.resources = dict(_cores=1)
self.priority = 1
self.version = None
self._log = None
self.wildcard_names = set()
self.lineno = lineno
self.snakefile = snakefile
self.run_func = None
self.shellcmd = None
elif len(args) == 1:
other = args[0]
self.name = other.name
self.workflow = other.workflow
self.docstring = other.docstring
self.message = other.message
self._input = other._input
self._output = other._output
self._params = other._params
self.dynamic_output = other.dynamic_output
self.dynamic_input = other.dynamic_input
self.temp_output = other.temp_output
self.protected_output = other.protected_output
self.resources = other.resources
self.priority = other.priority
self.version = other.version
self._log = other._log
self.wildcard_names = other.wildcard_names
self.lineno = other.lineno
self.snakefile = other.snakefile
self.run_func = other.run_func
self.shellcmd = other.shellcmd
def __init__(self,
input=None,
output=None,
params=None,
wildcards=None,
log="",
config=None):
self.input = InputFiles(self.make_namedlist(input))
self.output = OutputFiles(self.make_namedlist(output))
self.params = Params(self.make_namedlist(params))
self.wildcards = Wildcards(self.make_namedlist(wildcards))
self.log = Log(log)
self.config = config or {}
self.rulename = "mock"
def input(self):
def modify_callable(item):
if is_callable(item):
# For callables ensure that the rule's original path modifier is applied as well.
def inner(wildcards):
return self.rule.apply_path_modifier(
item(wildcards),
self.rule.input_modifier,
property="input")
return inner
else:
# For strings, the path modifier has been already applied.
return item
return InputFiles(toclone=self.rule.input,
strip_constraints=True,
custom_map=modify_callable)
def __init__(self, *args, lineno=None, snakefile=None, restart_times=0):
"""
Create a rule
Arguments
name -- the name of the rule
"""
if len(args) == 2:
name, workflow = args
self.name = name
self.workflow = workflow
self.docstring = None
self.message = None
self._input = InputFiles()
self._output = OutputFiles()
self._params = Params()
self._wildcard_constraints = dict()
self.dependencies = dict()
self.dynamic_output = set()
self.dynamic_input = set()
self.temp_output = set()
self.protected_output = set()
self.touch_output = set()
self.subworkflow_input = dict()
self.shadow_depth = None
self.resources = None
self.priority = 0
self._version = None
self._log = Log()
self._benchmark = None
self._conda_env = None
self._container_img = None
self.is_containerized = False
self.env_modules = None
self.group = None
self._wildcard_names = None
self.lineno = lineno
self.snakefile = snakefile
self.run_func = None
self.shellcmd = None
self.script = None
self.notebook = None
self.wrapper = None
self.cwl = None
self.norun = False
self.is_handover = False
self.is_branched = False
self.is_checkpoint = False
self.restart_times = 0
self.basedir = None
self.path_modifer = None
self.ruleinfo = None
elif len(args) == 1:
other = args[0]
self.name = other.name
self.workflow = other.workflow
self.docstring = other.docstring
self.message = other.message
self._input = InputFiles(other._input)
self._output = OutputFiles(other._output)
self._params = Params(other._params)
self._wildcard_constraints = dict(other._wildcard_constraints)
self.dependencies = dict(other.dependencies)
self.dynamic_output = set(other.dynamic_output)
self.dynamic_input = set(other.dynamic_input)
self.temp_output = set(other.temp_output)
self.protected_output = set(other.protected_output)
self.touch_output = set(other.touch_output)
self.subworkflow_input = dict(other.subworkflow_input)
self.shadow_depth = other.shadow_depth
self.resources = other.resources
self.priority = other.priority
self.version = other.version
self._log = other._log
self._benchmark = other._benchmark
self._conda_env = other._conda_env
self._container_img = other._container_img
self.is_containerized = other.is_containerized
self.env_modules = other.env_modules
self.group = other.group
self._wildcard_names = (set(other._wildcard_names)
if other._wildcard_names is not None else
None)
self.lineno = other.lineno
self.snakefile = other.snakefile
self.run_func = other.run_func
self.shellcmd = other.shellcmd
self.script = other.script
self.notebook = other.notebook
self.wrapper = other.wrapper
self.cwl = other.cwl
self.norun = other.norun
self.is_handover = other.is_handover
self.is_branched = True
self.is_checkpoint = other.is_checkpoint
self.restart_times = other.restart_times
self.basedir = other.basedir
self.path_modifier = other.path_modifier
self.ruleinfo = other.ruleinfo
def __init__(self, *args, lineno=None, snakefile=None,
restart_times=0):
"""
Create a rule
Arguments
name -- the name of the rule
"""
if len(args) == 2:
name, workflow = args
self.name = name
self.workflow = workflow
self.docstring = None
self.message = None
self._input = InputFiles()
self._output = OutputFiles()
self._params = Params()
self._wildcard_constraints = dict()
self.dependencies = dict()
self.dynamic_output = set()
self.dynamic_input = set()
self.temp_output = set()
self.protected_output = set()
self.touch_output = set()
self.subworkflow_input = dict()
self.shadow_depth = None
self.resources = dict(_cores=1, _nodes=1)
self.priority = 0
self._version = None
self._log = Log()
self._benchmark = None
self._conda_env = None
self.wildcard_names = set()
self.lineno = lineno
self.snakefile = snakefile
self.run_func = None
self.shellcmd = None
self.script = None
self.wrapper = None
self.norun = False
self.is_branched = False
self.restart_times = 0
elif len(args) == 1:
other = args[0]
self.name = other.name
self.workflow = other.workflow
self.docstring = other.docstring
self.message = other.message
self._input = InputFiles(other._input)
self._output = OutputFiles(other._output)
self._params = Params(other._params)
self._wildcard_constraints = dict(other._wildcard_constraints)
self.dependencies = dict(other.dependencies)
self.dynamic_output = set(other.dynamic_output)
self.dynamic_input = set(other.dynamic_input)
self.temp_output = set(other.temp_output)
self.protected_output = set(other.protected_output)
self.touch_output = set(other.touch_output)
self.subworkflow_input = dict(other.subworkflow_input)
self.shadow_depth = other.shadow_depth
self.resources = other.resources
self.priority = other.priority
self.version = other.version
self._log = other._log
self._benchmark = other._benchmark
self._conda_env = other._conda_env
self.wildcard_names = set(other.wildcard_names)
self.lineno = other.lineno
self.snakefile = other.snakefile
self.run_func = other.run_func
self.shellcmd = other.shellcmd
self.script = other.script
self.wrapper = other.wrapper
self.norun = other.norun
self.is_branched = True
self.restart_times = other.restart_times
def expand_wildcards(self, wildcards=None):
"""
Expand wildcards depending on the requested output
or given wildcards dict.
"""
def concretize_iofile(f, wildcards):
if not isinstance(f, _IOFile):
return IOFile(f, rule=self)
else:
return f.apply_wildcards(wildcards,
fill_missing=f in self.dynamic_input,
fail_dynamic=self.dynamic_output)
def concretize_param(p, wildcards):
if isinstance(p, str):
return apply_wildcards(p, wildcards)
return p
def check_string_type(f):
if not isinstance(f, str):
raise RuleException(
"Input function did not return str or list of str.",
rule=self)
def _apply_wildcards(newitems,
olditems,
wildcards,
wildcards_obj,
concretize=apply_wildcards,
check_return_type=check_string_type,
ruleio=None,
no_flattening=False):
for name, item in olditems.allitems():
start = len(newitems)
is_iterable = True
if callable(item):
try:
item = item(wildcards_obj)
except (Exception, BaseException) as e:
raise InputFunctionException(e,
rule=self,
wildcards=wildcards)
if not_iterable(item) or no_flattening:
item = [item]
is_iterable = False
for item_ in item:
check_return_type(item_)
concrete = concretize(item_, wildcards)
newitems.append(concrete)
if ruleio is not None:
ruleio[concrete] = item_
if name:
newitems.set_name(
name,
start,
end=len(newitems) if is_iterable else None)
if wildcards is None:
wildcards = dict()
missing_wildcards = self.wildcard_names - set(wildcards.keys())
if missing_wildcards:
raise RuleException(
"Could not resolve wildcards in rule {}:\n{}".format(
self.name, "\n".join(self.wildcard_names)),
lineno=self.lineno,
snakefile=self.snakefile)
ruleio = dict()
try:
input = InputFiles()
wildcards_obj = Wildcards(fromdict=wildcards)
_apply_wildcards(input,
self.input,
wildcards,
wildcards_obj,
concretize=concretize_iofile,
ruleio=ruleio)
params = Params()
#When applying wildcards to params, the return type need not be
#a string, so the check is disabled.
_apply_wildcards(params,
self.params,
wildcards,
wildcards_obj,
concretize=concretize_param,
check_return_type=lambda x: None,
no_flattening=True)
output = OutputFiles(
o.apply_wildcards(wildcards) for o in self.output)
output.take_names(self.output.get_names())
dependencies = {
None if f is None else f.apply_wildcards(wildcards): rule
for f, rule in self.dependencies.items()
}
ruleio.update(dict((f, f_) for f, f_ in zip(output, self.output)))
log = Log()
_apply_wildcards(log,
self.log,
wildcards,
wildcards_obj,
concretize=concretize_iofile)
benchmark = self.benchmark.apply_wildcards(
wildcards) if self.benchmark else None
return input, output, params, log, benchmark, ruleio, dependencies
except WildcardError as ex:
# this can only happen if an input contains an unresolved wildcard.
raise RuleException(
"Wildcards in input, params, log or benchmark file of rule {} cannot be "
"determined from output files:\n{}".format(self, str(ex)),
lineno=self.lineno,
snakefile=self.snakefile)
def expand_wildcards(self, wildcards=None):
"""
Expand wildcards depending on the requested output
or given wildcards dict.
"""
def concretize_iofile(f, wildcards):
if not isinstance(f, _IOFile):
return IOFile(f, rule=self)
else:
return f.apply_wildcards(wildcards,
fill_missing=f in self.dynamic_input,
fail_dynamic=self.dynamic_output)
def _apply_wildcards(newitems,
olditems,
wildcards,
wildcards_obj,
concretize=apply_wildcards,
ruleio=None):
for name, item in olditems.allitems():
start = len(newitems)
if callable(item):
item = item(wildcards_obj)
if not_iterable(item):
item = [item]
for item_ in item:
if not isinstance(item_, str):
raise RuleException(
"Input function did not return str or list of str.",
rule=self)
concrete = concretize(item_, wildcards)
newitems.append(concrete)
if ruleio is not None:
ruleio[concrete] = item_
else:
if not_iterable(item):
item = [item]
for item_ in item:
concrete = concretize(item_, wildcards)
newitems.append(concrete)
if ruleio is not None:
ruleio[concrete] = item_
if name:
newitems.set_name(name, start, end=len(newitems))
if wildcards is None:
wildcards = dict()
# TODO validate
missing_wildcards = self.wildcard_names - set(wildcards.keys())
if missing_wildcards:
raise RuleException(
"Could not resolve wildcards in rule {}:\n{}".format(
self.name, "\n".join(self.wildcard_names)),
lineno=self.lineno,
snakefile=self.snakefile)
ruleio = dict()
try:
input = InputFiles()
wildcards_obj = Wildcards(fromdict=wildcards)
_apply_wildcards(input,
self.input,
wildcards,
wildcards_obj,
concretize=concretize_iofile,
ruleio=ruleio)
params = Params()
_apply_wildcards(params, self.params, wildcards, wildcards_obj)
output = OutputFiles(
o.apply_wildcards(wildcards) for o in self.output)
output.take_names(self.output.get_names())
ruleio.update(dict((f, f_) for f, f_ in zip(output, self.output)))
log = self.log.apply_wildcards(wildcards) if self.log else None
return input, output, params, log, ruleio
except WildcardError as ex:
# this can only happen if an input contains an unresolved wildcard.
raise RuleException(
"Wildcards in input or log file of rule {} cannot be "
"determined from output files:\n{}".format(self, str(ex)),
lineno=self.lineno,
snakefile=self.snakefile)