def format_match(match): name = match.group("name") try: value = wildcards[name] if fail_dynamic and value == dynamic_fill: raise WildcardError(name) return str(value) # convert anything into a str except KeyError as ex: if fill_missing: return dynamic_fill else: raise WildcardError(str(ex))
def expand(*args, **wildcards): """ Expand wildcards in given filepatterns. Arguments *args -- first arg: filepatterns as list or one single filepattern, second arg (optional): a function to combine wildcard values (itertools.product per default) **wildcards -- the wildcards as keyword arguments with their values as lists """ filepatterns = args[0] if len(args) == 1: combinator = product elif len(args) == 2: combinator = args[1] if isinstance(filepatterns, str): filepatterns = [filepatterns] def flatten(wildcards): for wildcard, values in wildcards.items(): if isinstance(values, str) or not isinstance(values, Iterable): values = [values] yield [(wildcard, value) for value in values] try: return [ filepattern.format(**comb) for comb in map(dict, combinator(*flatten(wildcards))) for filepattern in filepatterns ] except KeyError as e: raise WildcardError("No values given for wildcard {}.".format(e))
def expand_log(self, wildcards): def concretize_logfile(f, wildcards, is_from_callable): if is_from_callable: return IOFile(f, rule=self) else: return f.apply_wildcards(wildcards, fill_missing=False, fail_dynamic=self.dynamic_output) log = Log() try: self._apply_wildcards(log, self.log, wildcards, concretize=concretize_logfile, property="log") except WildcardError as e: raise WildcardError( "Wildcards in log files cannot be " "determined from output files:", str(e), rule=self, ) for f in log: f.check() return log
def expand_params(self, wildcards, input, output, resources, omit_callable=False): def concretize_param(p, wildcards, is_from_callable): if not is_from_callable: if isinstance(p, str): return apply_wildcards(p, wildcards) if isinstance(p, list): return [(apply_wildcards(v, wildcards) if isinstance( v, str) else v) for v in p] return p def handle_incomplete_checkpoint(exception): """If checkpoint is incomplete, target it such that it is completed before this rule gets executed.""" if exception.targetfile in input: return TBDString() else: raise WorkflowError( "Rule parameter depends on checkpoint but checkpoint output is not defined as input file for the rule. " "Please add the output of the respective checkpoint to the rule inputs." ) params = Params() try: # When applying wildcards to params, the return type need not be # a string, so the check is disabled. self._apply_wildcards( params, self.params, wildcards, concretize=concretize_param, check_return_type=False, omit_callable=omit_callable, allow_unpack=False, no_flattening=True, property="params", aux_params={ "input": input._plainstrings(), "resources": resources, "output": output._plainstrings(), "threads": resources._cores, }, incomplete_checkpoint_func=handle_incomplete_checkpoint, ) except WildcardError as e: raise WildcardError( "Wildcards in params cannot be " "determined from output files. Note that you have " "to use a function to deactivate automatic wildcard expansion " "in params strings, e.g., `lambda wildcards: '{test}'`. Also " "see https://snakemake.readthedocs.io/en/stable/snakefiles/" "rules.html#non-file-parameters-for-rules:", str(e), rule=self, ) return params
def expand_params(self, wildcards, input, output, resources, omit_callable=False): def concretize_param(p, wildcards, is_from_callable): if not is_from_callable and isinstance(p, str): return apply_wildcards(p, wildcards) return p params = Params() try: #When applying wildcards to params, the return type need not be #a string, so the check is disabled. self._apply_wildcards(params, self.params, wildcards, concretize=concretize_param, check_return_type=False, omit_callable=omit_callable, allow_unpack=False, no_flattening=True, apply_default_remote=False, aux_params={"input": input.plainstrings(), "resources": resources, "output": output.plainstrings(), "threads": resources._cores}, incomplete_checkpoint_func=lambda e: "<incomplete checkpoint>") except WildcardError as e: raise WildcardError( "Wildcards in params cannot be " "determined from output files. Note that you have " "to use a function to deactivate automatic wildcard expansion " "in params strings, e.g., `lambda wildcards: '{test}'`. Also " "see https://snakemake.readthedocs.io/en/stable/snakefiles/" "rules.html#non-file-parameters-for-rules:", str(e), rule=self) return params
def expand(*args, **wildcards): """ Expand wildcards in given filepatterns. Arguments *args -- first arg: filepatterns as list or one single filepattern, second arg (optional): a function to combine wildcard values (itertools.product per default) **wildcards -- the wildcards as keyword arguments with their values as lists """ filepatterns = args[0] if len(args) == 1: combinator = product elif len(args) == 2: combinator = args[1] if isinstance(filepatterns, str): filepatterns = [filepatterns] def path_to_str(f): if isinstance(f, Path): return str(f) return f filepatterns = list(map(path_to_str, filepatterns)) if any(map(lambda f: getattr(f, "flags", {}), filepatterns)): raise WorkflowError( "Flags in file patterns given to expand() are invalid. " "Flags (e.g. temp(), directory()) have to be applied outside " "of expand (e.g. 'temp(expand(\"plots/{sample}.pdf\", sample=SAMPLES))')." ) # remove unused wildcards to avoid duplicate filepatterns wildcards = { filepattern: { k: v for k, v in wildcards.items() if k in re.findall("{([^}\.[!:]+)", filepattern) } for filepattern in filepatterns } def flatten(wildcards): for wildcard, values in wildcards.items(): if isinstance(values, str) or not isinstance( values, collections.abc.Iterable ): values = [values] yield [(wildcard, value) for value in values] try: return [ filepattern.format(**comb) for filepattern in filepatterns for comb in map(dict, combinator(*flatten(wildcards[filepattern]))) ] except KeyError as e: raise WildcardError("No values given for wildcard {}.".format(e))
def expand_input(self, wildcards, groupid=None): def concretize_iofile(f, wildcards, is_from_callable): if is_from_callable: if isinstance(f, Path): f = str(f) return IOFile(f, rule=self).apply_wildcards( wildcards, fill_missing=f in self.dynamic_input, fail_dynamic=self.dynamic_output, ) else: return f.apply_wildcards( wildcards, fill_missing=f in self.dynamic_input, fail_dynamic=self.dynamic_output, ) def handle_incomplete_checkpoint(exception): """If checkpoint is incomplete, target it such that it is completed before this rule gets executed.""" return exception.targetfile input = InputFiles() mapping = dict() try: incomplete = self._apply_wildcards( input, self.input, wildcards, concretize=concretize_iofile, mapping=mapping, incomplete_checkpoint_func=handle_incomplete_checkpoint, path_modifier=self.input_modifier, property="input", groupid=groupid, ) except WildcardError as e: raise WildcardError( "Wildcards in input files cannot be " "determined from output files:", str(e), rule=self, ) if self.dependencies: dependencies = { f: self.dependencies[f_] for f, f_ in mapping.items() if f_ in self.dependencies } if None in self.dependencies: dependencies[None] = self.dependencies[None] else: dependencies = self.dependencies for f in input: f.check() return input, mapping, dependencies, incomplete
def expand_conda_env(self, wildcards): try: conda_env = self.conda_env.apply_wildcards( wildcards) if self.conda_env else None except WildcardError as e: raise WildcardError( "Wildcards in conda environment file cannot be " "determined from output files:", str(e), rule=self) if conda_env is not None: conda_env.check() return conda_env
def expand_benchmark(self, wildcards): try: benchmark = self.benchmark.apply_wildcards( wildcards) if self.benchmark else None except WildcardError as e: raise WildcardError( "Wildcards in benchmark file cannot be " "determined from output files:", str(e), rule=self) if benchmark is not None: benchmark.check() return benchmark
def expand(*args, **wildcards): """ Expand wildcards in given filepatterns. Arguments *args -- first arg: filepatterns as list or one single filepattern, second arg (optional): a function to combine wildcard values (itertools.product per default) **wildcards -- the wildcards as keyword arguments with their values as lists """ filepatterns = args[0] if len(args) == 1: combinator = product elif len(args) == 2: combinator = args[1] if isinstance(filepatterns, str): filepatterns = [filepatterns] # remove unused wildcards to avoid duplicate filepatterns wildcards = { filepattern: { k: v for k, v in wildcards.items() if k in re.findall('{([^}\.[!:]+)', filepattern) } for filepattern in filepatterns } def flatten(wildcards): for wildcard, values in wildcards.items(): if isinstance(values, str) or not isinstance(values, collections.Iterable): values = [values] yield [(wildcard, value) for value in values] try: return [ filepattern.format(**comb) for filepattern in filepatterns for comb in map(dict, combinator(*flatten(wildcards[filepattern]))) ] except KeyError as e: raise WildcardError("No values given for wildcard {}.".format(e))
def expand_input(self, wildcards): def concretize_iofile(f, wildcards): if not isinstance(f, _IOFile): return IOFile(f, rule=self) else: return f.apply_wildcards(wildcards, fill_missing=f in self.dynamic_input, fail_dynamic=self.dynamic_output) input = InputFiles() mapping = dict() try: self._apply_wildcards(input, self.input, wildcards, concretize=concretize_iofile, mapping=mapping) except WildcardError as e: raise WildcardError( "Wildcards in input files cannot be " "determined from output files:", str(e), rule=self) if self.dependencies: dependencies = { f: self.dependencies[f_] for f, f_ in mapping.items() if f_ in self.dependencies } if None in self.dependencies: dependencies[None] = self.dependencies[None] else: dependencies = self.dependencies for f in input: f.check() return input, mapping, dependencies
def expand_params(self, wildcards, input, output, resources, omit_callable=False): def concretize_param(p, wildcards): if isinstance(p, str): return apply_wildcards(p, wildcards) return p params = Params() try: #When applying wildcards to params, the return type need not be #a string, so the check is disabled. self._apply_wildcards(params, self.params, wildcards, concretize=concretize_param, check_return_type=False, omit_callable=omit_callable, no_flattening=True, apply_default_remote=False, aux_params={ "input": input, "resources": resources, "output": output, "threads": resources._cores }) except WildcardError as e: raise WildcardError( "Wildcards in params cannot be " "determined from output files:", str(e), rule=self) return params
def expand(*args, **wildcards): """ Expand wildcards in given filepatterns. Arguments *args -- first arg: filepatterns as list or one single filepattern, second arg (optional): a function to combine wildcard values (itertools.product per default) **wildcards -- the wildcards as keyword arguments with their values as lists. If allow_missing=True is included wildcards in filepattern without values will stay unformatted. """ filepatterns = args[0] if len(args) == 1: combinator = product elif len(args) == 2: combinator = args[1] if isinstance(filepatterns, str): filepatterns = [filepatterns] def path_to_str(f): if isinstance(f, Path): return str(f) return f filepatterns = list(map(path_to_str, filepatterns)) if any(map(lambda f: getattr(f, "flags", {}), filepatterns)): raise WorkflowError( "Flags in file patterns given to expand() are invalid. " "Flags (e.g. temp(), directory()) have to be applied outside " "of expand (e.g. 'temp(expand(\"plots/{sample}.pdf\", sample=SAMPLES))')." ) # check if remove missing is provided format_dict = dict if "allow_missing" in wildcards and wildcards["allow_missing"] is True: class FormatDict(dict): def __missing__(self, key): return "{" + key + "}" format_dict = FormatDict # check that remove missing is not a wildcard in the filepatterns for filepattern in filepatterns: if "allow_missing" in re.findall(r"{([^}\.[!:]+)", filepattern): format_dict = dict break # remove unused wildcards to avoid duplicate filepatterns wildcards = { filepattern: { k: v for k, v in wildcards.items() if k in re.findall(r"{([^}\.[!:]+)", filepattern) } for filepattern in filepatterns } def flatten(wildcards): for wildcard, values in wildcards.items(): if isinstance(values, str) or not isinstance( values, collections.abc.Iterable ): values = [values] yield [(wildcard, value) for value in values] formatter = string.Formatter() try: return [ formatter.vformat(filepattern, (), comb) for filepattern in filepatterns for comb in map(format_dict, combinator(*flatten(wildcards[filepattern]))) ] except KeyError as e: raise WildcardError("No values given for wildcard {}.".format(e))