def expand_output(self, wildcards): output = OutputFiles(o.apply_wildcards(wildcards) for o in self.output) output.take_names(self.output.get_names()) mapping = {f: f_ for f, f_ in zip(output, self.output)} for f in output: f.check() return output, mapping
def expand_output(self, wildcards): output = OutputFiles(o.apply_wildcards(wildcards) for o in self.output) output._take_names(self.output._get_names()) mapping = {f: f_ for f, f_ in zip(output, self.output)} for f in output: f.check() # Note that we do not need to check for duplicate file names after # expansion as all output patterns have contain all wildcards anyway. return output, mapping
def __init__(self, *args, lineno=None, snakefile=None): """ Create a rule Arguments name -- the name of the rule """ if len(args) == 2: name, workflow = args self.name = name self.workflow = workflow self.docstring = None self.message = None self._input = InputFiles() self._output = OutputFiles() self._params = Params() self.dynamic_output = set() self.dynamic_input = set() self.temp_output = set() self.protected_output = set() self.resources = dict(_cores=1) self.priority = 1 self.version = None self._log = None self.wildcard_names = set() self.lineno = lineno self.snakefile = snakefile self.run_func = None self.shellcmd = None elif len(args) == 1: other = args[0] self.name = other.name self.workflow = other.workflow self.docstring = other.docstring self.message = other.message self._input = other._input self._output = other._output self._params = other._params self.dynamic_output = other.dynamic_output self.dynamic_input = other.dynamic_input self.temp_output = other.temp_output self.protected_output = other.protected_output self.resources = other.resources self.priority = other.priority self.version = other.version self._log = other._log self.wildcard_names = other.wildcard_names self.lineno = other.lineno self.snakefile = other.snakefile self.run_func = other.run_func self.shellcmd = other.shellcmd
def __init__(self, input=None, output=None, params=None, wildcards=None, log="", config=None): self.input = InputFiles(self.make_namedlist(input)) self.output = OutputFiles(self.make_namedlist(output)) self.params = Params(self.make_namedlist(params)) self.wildcards = Wildcards(self.make_namedlist(wildcards)) self.log = Log(log) self.config = config or {} self.rulename = "mock"
def __init__(self, *args, lineno=None, snakefile=None, restart_times=0): """ Create a rule Arguments name -- the name of the rule """ if len(args) == 2: name, workflow = args self.name = name self.workflow = workflow self.docstring = None self.message = None self._input = InputFiles() self._output = OutputFiles() self._params = Params() self._wildcard_constraints = dict() self.dependencies = dict() self.dynamic_output = set() self.dynamic_input = set() self.temp_output = set() self.protected_output = set() self.touch_output = set() self.subworkflow_input = dict() self.shadow_depth = None self.resources = None self.priority = 0 self._version = None self._log = Log() self._benchmark = None self._conda_env = None self._container_img = None self.is_containerized = False self.env_modules = None self.group = None self._wildcard_names = None self.lineno = lineno self.snakefile = snakefile self.run_func = None self.shellcmd = None self.script = None self.notebook = None self.wrapper = None self.cwl = None self.norun = False self.is_handover = False self.is_branched = False self.is_checkpoint = False self.restart_times = 0 self.basedir = None self.path_modifer = None self.ruleinfo = None elif len(args) == 1: other = args[0] self.name = other.name self.workflow = other.workflow self.docstring = other.docstring self.message = other.message self._input = InputFiles(other._input) self._output = OutputFiles(other._output) self._params = Params(other._params) self._wildcard_constraints = dict(other._wildcard_constraints) self.dependencies = dict(other.dependencies) self.dynamic_output = set(other.dynamic_output) self.dynamic_input = set(other.dynamic_input) self.temp_output = set(other.temp_output) self.protected_output = set(other.protected_output) self.touch_output = set(other.touch_output) self.subworkflow_input = dict(other.subworkflow_input) self.shadow_depth = other.shadow_depth self.resources = other.resources self.priority = other.priority self.version = other.version self._log = other._log self._benchmark = other._benchmark self._conda_env = other._conda_env self._container_img = other._container_img self.is_containerized = other.is_containerized self.env_modules = other.env_modules self.group = other.group self._wildcard_names = (set(other._wildcard_names) if other._wildcard_names is not None else None) self.lineno = other.lineno self.snakefile = other.snakefile self.run_func = other.run_func self.shellcmd = other.shellcmd self.script = other.script self.notebook = other.notebook self.wrapper = other.wrapper self.cwl = other.cwl self.norun = other.norun self.is_handover = other.is_handover self.is_branched = True self.is_checkpoint = other.is_checkpoint self.restart_times = other.restart_times self.basedir = other.basedir self.path_modifier = other.path_modifier self.ruleinfo = other.ruleinfo
def __init__(self, *args, lineno=None, snakefile=None, restart_times=0): """ Create a rule Arguments name -- the name of the rule """ if len(args) == 2: name, workflow = args self.name = name self.workflow = workflow self.docstring = None self.message = None self._input = InputFiles() self._output = OutputFiles() self._params = Params() self._wildcard_constraints = dict() self.dependencies = dict() self.dynamic_output = set() self.dynamic_input = set() self.temp_output = set() self.protected_output = set() self.touch_output = set() self.subworkflow_input = dict() self.shadow_depth = None self.resources = dict(_cores=1, _nodes=1) self.priority = 0 self._version = None self._log = Log() self._benchmark = None self._conda_env = None self.wildcard_names = set() self.lineno = lineno self.snakefile = snakefile self.run_func = None self.shellcmd = None self.script = None self.wrapper = None self.norun = False self.is_branched = False self.restart_times = 0 elif len(args) == 1: other = args[0] self.name = other.name self.workflow = other.workflow self.docstring = other.docstring self.message = other.message self._input = InputFiles(other._input) self._output = OutputFiles(other._output) self._params = Params(other._params) self._wildcard_constraints = dict(other._wildcard_constraints) self.dependencies = dict(other.dependencies) self.dynamic_output = set(other.dynamic_output) self.dynamic_input = set(other.dynamic_input) self.temp_output = set(other.temp_output) self.protected_output = set(other.protected_output) self.touch_output = set(other.touch_output) self.subworkflow_input = dict(other.subworkflow_input) self.shadow_depth = other.shadow_depth self.resources = other.resources self.priority = other.priority self.version = other.version self._log = other._log self._benchmark = other._benchmark self._conda_env = other._conda_env self.wildcard_names = set(other.wildcard_names) self.lineno = other.lineno self.snakefile = other.snakefile self.run_func = other.run_func self.shellcmd = other.shellcmd self.script = other.script self.wrapper = other.wrapper self.norun = other.norun self.is_branched = True self.restart_times = other.restart_times
def expand_wildcards(self, wildcards=None): """ Expand wildcards depending on the requested output or given wildcards dict. """ def concretize_iofile(f, wildcards): if not isinstance(f, _IOFile): return IOFile(f, rule=self) else: return f.apply_wildcards( wildcards, fill_missing=f in self.dynamic_input, fail_dynamic=self.dynamic_output) def _apply_wildcards( newitems, olditems, wildcards, wildcards_obj, concretize = apply_wildcards, ruleio = None): for name, item in olditems.allitems(): start = len(newitems) if callable(item): item = item(wildcards_obj) if not_iterable(item): item = [item] for item_ in item: if not isinstance(item_, str): raise RuleException("Input function did not return str or list of str.", rule=self) concrete = concretize(item_, wildcards) newitems.append(concrete) if ruleio is not None: ruleio[concrete] = item_ else: if not_iterable(item): item = [item] for item_ in item: concrete = concretize(item_, wildcards) newitems.append(concrete) if ruleio is not None: ruleio[concrete] = item_ if name: newitems.set_name(name, start, end=len(newitems)) if wildcards is None: wildcards = dict() # TODO validate missing_wildcards = self.wildcard_names - set(wildcards.keys()) if missing_wildcards: raise RuleException( "Could not resolve wildcards in rule {}:\n{}".format( self.name, "\n".join(self.wildcard_names)), lineno=self.lineno, snakefile=self.snakefile) ruleio = dict() try: input = InputFiles() wildcards_obj = Wildcards(fromdict=wildcards) _apply_wildcards(input, self.input, wildcards, wildcards_obj, concretize=concretize_iofile, ruleio=ruleio) params = Params() _apply_wildcards(params, self.params, wildcards, wildcards_obj) output = OutputFiles( o.apply_wildcards(wildcards) for o in self.output) output.take_names(self.output.get_names()) ruleio.update(dict((f, f_) for f, f_ in zip(output, self.output))) log = self.log.apply_wildcards(wildcards) if self.log else None return input, output, params, log, ruleio except WildcardError as ex: # this can only happen if an input contains an unresolved wildcard. raise RuleException( "Wildcards in input or log file of rule {} cannot be " "determined from output files:\n{}".format(self, str(ex)), lineno=self.lineno, snakefile=self.snakefile)
def expand_wildcards(self, wildcards=None): """ Expand wildcards depending on the requested output or given wildcards dict. """ def concretize_iofile(f, wildcards): if not isinstance(f, _IOFile): return IOFile(f, rule=self) else: return f.apply_wildcards(wildcards, fill_missing=f in self.dynamic_input, fail_dynamic=self.dynamic_output) def concretize_param(p, wildcards): if isinstance(p, str): return apply_wildcards(p, wildcards) return p def check_string_type(f): if not isinstance(f, str): raise RuleException( "Input function did not return str or list of str.", rule=self) def _apply_wildcards(newitems, olditems, wildcards, wildcards_obj, concretize=apply_wildcards, check_return_type=check_string_type, ruleio=None, no_flattening=False): for name, item in olditems.allitems(): start = len(newitems) is_iterable = True if callable(item): try: item = item(wildcards_obj) except (Exception, BaseException) as e: raise InputFunctionException(e, rule=self, wildcards=wildcards) if not_iterable(item) or no_flattening: item = [item] is_iterable = False for item_ in item: check_return_type(item_) concrete = concretize(item_, wildcards) newitems.append(concrete) if ruleio is not None: ruleio[concrete] = item_ if name: newitems.set_name( name, start, end=len(newitems) if is_iterable else None) if wildcards is None: wildcards = dict() missing_wildcards = self.wildcard_names - set(wildcards.keys()) if missing_wildcards: raise RuleException( "Could not resolve wildcards in rule {}:\n{}".format( self.name, "\n".join(self.wildcard_names)), lineno=self.lineno, snakefile=self.snakefile) ruleio = dict() try: input = InputFiles() wildcards_obj = Wildcards(fromdict=wildcards) _apply_wildcards(input, self.input, wildcards, wildcards_obj, concretize=concretize_iofile, ruleio=ruleio) params = Params() #When applying wildcards to params, the return type need not be #a string, so the check is disabled. _apply_wildcards(params, self.params, wildcards, wildcards_obj, concretize=concretize_param, check_return_type=lambda x: None, no_flattening=True) output = OutputFiles( o.apply_wildcards(wildcards) for o in self.output) output.take_names(self.output.get_names()) dependencies = { None if f is None else f.apply_wildcards(wildcards): rule for f, rule in self.dependencies.items() } ruleio.update(dict((f, f_) for f, f_ in zip(output, self.output))) log = Log() _apply_wildcards(log, self.log, wildcards, wildcards_obj, concretize=concretize_iofile) benchmark = self.benchmark.apply_wildcards( wildcards) if self.benchmark else None return input, output, params, log, benchmark, ruleio, dependencies except WildcardError as ex: # this can only happen if an input contains an unresolved wildcard. raise RuleException( "Wildcards in input, params, log or benchmark file of rule {} cannot be " "determined from output files:\n{}".format(self, str(ex)), lineno=self.lineno, snakefile=self.snakefile)
def expand_wildcards(self, wildcards=None): """ Expand wildcards depending on the requested output or given wildcards dict. """ def concretize_iofile(f, wildcards): if not isinstance(f, _IOFile): return IOFile(f, rule=self) else: return f.apply_wildcards(wildcards, fill_missing=f in self.dynamic_input, fail_dynamic=self.dynamic_output) def concretize_param(p, wildcards): if isinstance(p, str): return apply_wildcards(p, wildcards) return p def check_string_type(f): if not isinstance(f, str): raise RuleException( "Input function did not return str or list of str.", rule=self) def _apply_wildcards(newitems, olditems, wildcards, wildcards_obj, concretize=apply_wildcards, check_return_type=check_string_type, ruleio=None, no_flattening=False): for name, item in olditems.allitems(): start = len(newitems) is_iterable = True if callable(item): try: item = item(wildcards_obj) except (Exception, BaseException) as e: raise InputFunctionException(e, rule=self, wildcards=wildcards) if not_iterable(item) or no_flattening: item = [item] is_iterable = False for item_ in item: check_return_type(item_) concrete = concretize(item_, wildcards) newitems.append(concrete) if ruleio is not None: ruleio[concrete] = item_ if name: newitems.set_name( name, start, end=len(newitems) if is_iterable else None) if wildcards is None: wildcards = dict() missing_wildcards = self.wildcard_names - set(wildcards.keys()) if missing_wildcards: raise RuleException( "Could not resolve wildcards in rule {}:\n{}".format( self.name, "\n".join(self.wildcard_names)), lineno=self.lineno, snakefile=self.snakefile) ruleio = dict() try: input = InputFiles() wildcards_obj = Wildcards(fromdict=wildcards) _apply_wildcards(input, self.input, wildcards, wildcards_obj, concretize=concretize_iofile, ruleio=ruleio) params = Params() #When applying wildcards to params, the return type need not be #a string, so the check is disabled. _apply_wildcards(params, self.params, wildcards, wildcards_obj, concretize=concretize_param, check_return_type=lambda x: None, no_flattening=True) output = OutputFiles(o.apply_wildcards(wildcards) for o in self.output) output.take_names(self.output.get_names()) dependencies = { None if f is None else f.apply_wildcards(wildcards): rule for f, rule in self.dependencies.items() } ruleio.update(dict((f, f_) for f, f_ in zip(output, self.output))) log = Log() _apply_wildcards(log, self.log, wildcards, wildcards_obj, concretize=concretize_iofile) benchmark = self.benchmark.apply_wildcards( wildcards) if self.benchmark else None return input, output, params, log, benchmark, ruleio, dependencies except WildcardError as ex: # this can only happen if an input contains an unresolved wildcard. raise RuleException( "Wildcards in input, params, log or benchmark file of rule {} cannot be " "determined from output files:\n{}".format(self, str(ex)), lineno=self.lineno, snakefile=self.snakefile)
def expand_wildcards(self, wildcards=None): """ Expand wildcards depending on the requested output or given wildcards dict. """ def concretize_iofile(f, wildcards): if not isinstance(f, _IOFile): return IOFile(f, rule=self) else: return f.apply_wildcards(wildcards, fill_missing=f in self.dynamic_input, fail_dynamic=self.dynamic_output) def _apply_wildcards(newitems, olditems, wildcards, wildcards_obj, concretize=apply_wildcards, ruleio=None): for name, item in olditems.allitems(): start = len(newitems) if callable(item): item = item(wildcards_obj) if not_iterable(item): item = [item] for item_ in item: if not isinstance(item_, str): raise RuleException( "Input function did not return str or list of str.", rule=self) concrete = concretize(item_, wildcards) newitems.append(concrete) if ruleio is not None: ruleio[concrete] = item_ else: if not_iterable(item): item = [item] for item_ in item: concrete = concretize(item_, wildcards) newitems.append(concrete) if ruleio is not None: ruleio[concrete] = item_ if name: newitems.set_name(name, start, end=len(newitems)) if wildcards is None: wildcards = dict() # TODO validate missing_wildcards = self.wildcard_names - set(wildcards.keys()) if missing_wildcards: raise RuleException( "Could not resolve wildcards in rule {}:\n{}".format( self.name, "\n".join(self.wildcard_names)), lineno=self.lineno, snakefile=self.snakefile) ruleio = dict() try: input = InputFiles() wildcards_obj = Wildcards(fromdict=wildcards) _apply_wildcards(input, self.input, wildcards, wildcards_obj, concretize=concretize_iofile, ruleio=ruleio) params = Params() _apply_wildcards(params, self.params, wildcards, wildcards_obj) output = OutputFiles( o.apply_wildcards(wildcards) for o in self.output) output.take_names(self.output.get_names()) ruleio.update(dict((f, f_) for f, f_ in zip(output, self.output))) log = self.log.apply_wildcards(wildcards) if self.log else None return input, output, params, log, ruleio except WildcardError as ex: # this can only happen if an input contains an unresolved wildcard. raise RuleException( "Wildcards in input or log file of rule {} cannot be " "determined from output files:\n{}".format(self, str(ex)), lineno=self.lineno, snakefile=self.snakefile)