def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
if self.default is None:
raise YmpRuleError(self.stage,
f"Stage Int Parameter must have 'default' set")
self.regex = f"({self.key}\d+|)"
def filter_input_func(wildcards, input):
outfiles = []
files = ensure_list(getattr(input, name))
if also is None:
extra_files = [[None] for _ in files]
else:
extra_files = [
ensure_list(getattr(input, extra))
for extra in ensure_list(also)
]
all_files = [
fname for fnamell in ([files], extra_files) for fnamel in fnamell
for fname in fnamel if fname is not None
]
files_exist = [os.path.exists(fname) for fname in all_files]
if all(files_exist):
for fname, *extra_fnames in zip(files, *extra_files):
if isinstance(extra_fnames, str):
extra_fnames = [extra_fnames]
if minsize is not None:
if not file_not_empty(fname, minsize=minsize):
continue
if not all(file_not_empty(fn) for fn in extra_fnames):
continue
outfiles.append(fname)
elif any(files_exist):
raise YmpRuleError(None, "Missing files to check for length")
else:
outfiles = files
if join is None:
return outfiles
return join.join(outfiles)
def add_param(self, key, typ, name, value=None, default=None):
"""Add parameter to stage
Example:
>>> with Stage("test") as S
>>> S.add_param("N", "int", "nval", default=50)
>>> rule:
>>> shell: "echo {param.nval}"
This would add a stage "test", optionally callable as "testN123",
printing "50" or in the case of "testN123" printing "123".
Args:
char: The character to use in the Stage name
typ: The type of the parameter (int, flag)
param: Name of parameter in params
value: value ``{param.xyz}`` should be set to if param given
default: default value for ``{{param.xyz}}`` if no param given
"""
if typ == 'flag':
self.params.append(ParamFlag(self, key, name, value, default))
elif typ == 'int':
self.params.append(ParamInt(self, key, name, value, default))
elif typ == 'choice':
self.params.append(ParamChoice(self, key, name, value, default))
else:
raise YmpRuleError(self, f"Unknown Stage Parameter type '{typ}'")
def check_input_func(wildcards, input):
files = [
fname for name in ensure_list(names)
for fname in ensure_list(getattr(input, name))
]
files_exist = [os.path.exists(fname) for fname in files]
if all(files_exist):
nbytes = 0
nlines = 0
for fname in files:
if fname.endswith(".gz"):
openfunc = gzip.open
else:
openfunc = open
with openfunc(fname, "rb") as fd:
btes = fd.read(8192)
while btes:
nlines += btes.count(b"\n")
nbytes += len(btes)
if nbytes >= minbytes and nlines >= minlines:
break
btes = fd.read(8192)
if nbytes < minbytes or nlines < minlines:
return False
elif any(files_exist):
raise YmpRuleError(
None, f"Missing files to check for length: "
f"{files}")
return True
def __enter__(self):
if Stage.active is not None:
raise YmpRuleError(
self, f"Failed to enter stage '{self.name}', "
f"already in stage {self.active.name}'.")
Stage.active = self
return self
def get_value_(self, key, args, kwargs):
stage = Stage.active
if "(" in key:
args = list(args)
if key[-1] != ")":
raise YmpRuleError(
stage, f"Malformed YMP expansion string:'{key}'")
key, _, args_str = key[:-1].partition("(")
for arg_str in args_str.split(","):
try:
arg = int(arg_str)
except ValueError:
try:
arg = float(arg_str)
except ValueError:
arg = arg_str
if isinstance(arg, str) and arg[0] not in ('"', '"'):
if "wc" not in kwargs:
raise ExpandLateException()
arg = getattr(kwargs['wc'], arg)
args += [arg]
# Check Stage variables first. We can do that always:
if hasattr(stage, key):
val = getattr(stage, key)
if hasattr(val, "__call__"):
val = val(args, kwargs)
if val is not None:
return val
# Check StageStack next. This requires the wildcards:
if "wc" not in kwargs:
raise ExpandLateException()
wc = kwargs['wc']
stack = StageStack.get(stage.wc2path(wc), stage)
if hasattr(stack, key):
val = getattr(stack, key)
if hasattr(val, "__call__"):
val = val(args, kwargs)
if val is not None:
return val
# Lastly, check the project:
if hasattr(stack.project, key):
val = getattr(stack.project, key)
if hasattr(val, "__call__"):
val = val(args, kwargs)
if val is not None:
return val
return super().get_value(key, args, kwargs)
def add_param(self, key, typ, name, value=None, default=None) -> bool:
"""Add parameter to stage
Example:
>>> with Stage("test") as S
>>> S.add_param("N", "int", "nval", default=50)
>>> rule:
>>> shell: "echo {param.nval}"
This would add a stage "test", optionally callable as "testN123",
printing "50" or in the case of "testN123" printing "123".
Args:
char: The character to use in the Stage name
typ: The type of the parameter (int, flag)
param: Name of parameter in params
value: value ``{param.xyz}`` should be set to if param given
default: default value for ``{{param.xyz}}`` if no param given
"""
if self.__regex_:
raise RuntimeError("?")
new_param = Param.make(self, typ, key, name, value, default)
for param in self.__params:
if param == new_param:
return False
if key and param.key == key:
raise YmpRuleError(
self,
f"Keys must be uninque. Key '{key}' already used by {param}.\n"
f" while trying to add {new_param}")
if param.name == name:
raise YmpRuleError(
self,
f"Names must be uninque. Name '{name}' already used by {param}.\n"
f" while trying to add {new_param}")
self.__params.append(new_param)
return True
def register(self):
"""Add self to registry"""
cache = self.get_registry()
names = []
for attr in 'name', 'altname':
if hasattr(self, attr):
names += ensure_list(getattr(self, attr))
for name in names:
if (name in cache and self != cache[name]
and (self.filename != cache[name].filename
or self.lineno != cache[name].lineno)):
other = cache[name]
raise YmpRuleError(
self,
f"Failed to create {self.__class__.__name__} '{names[0]}':"
f" already defined in {other.filename}:{other.lineno}")
for name in names:
cache[name] = self
def get_value_(self, key, args, kwargs):
stage = Stage.get_active()
# Fixme: Guard against stage==None?
if "(" in key:
args = list(args)
if key[-1] != ")":
raise YmpRuleError(
stage, f"Malformed YMP expansion string:'{key}'")
key, _, args_str = key[:-1].partition("(")
for arg_str in args_str.split(","):
argname = None
if '=' in arg_str:
argname, arg_str = arg_str.split("=")
try:
arg = int(arg_str)
except ValueError:
try:
arg = float(arg_str)
except ValueError:
arg = arg_str
if isinstance(arg, str):
if arg in ("True", "False"):
arg = bool(arg)
if arg[0] not in ('"', '"'):
if "wc" not in kwargs:
raise ExpandLateException()
arg = getattr(kwargs['wc'], arg)
if not argname:
args += [arg]
else:
kwargs[argname] = arg
# Check Stage variables first. We can do that always:
if hasattr(stage, key):
val = getattr(stage, key)
if hasattr(val, "__call__"):
val = val(args, kwargs)
if val is not None:
return val
# Check StageStack next. This requires the wildcards:
if "wc" not in kwargs:
raise ExpandLateException()
wc = kwargs['wc']
stack = StageStack.instance(stage.wc2path(wc))
if hasattr(stack, key):
val = getattr(stack, key)
if hasattr(val, "__call__"):
val = val(args, kwargs)
if val is not None:
return val
# Check the project:
if hasattr(stack.project, key):
val = getattr(stack.project, key)
if hasattr(val, "__call__"):
val = val(args, kwargs)
if val is not None:
return val
# Expand via super
return super().get_value(key, args, kwargs)
def __init__(self,
env_file: Optional[str] = None,
dag: Optional[object] = None,
singularity_img=None,
name: Optional[str] = None,
packages: Optional[Union[list, str]] = None,
base: str = "none",
channels: Optional[Union[list, str]] = None,
rule: Optional[Rule] = None) -> None:
"""Creates an inline defined conda environment
Args:
name: Name of conda environment (and basename of file)
packages: package(s) to be installed into environment. Version
constraints can be specified in each package string separated from
the package name by whitespace. E.g. ``"blast =2.6*"``
channels: channel(s) to be selected for the environment
base: Select a set of default channels and packages to be added to
the newly created environment. Sets are defined in conda.defaults
in ``yml.yml``
"""
cfg = ymp.get_config()
pseudo_dag = AttrDict({
'workflow': {
'persistence': {
'conda_env_path': cfg.absdir.conda_prefix,
'conda_env_archive_path': cfg.absdir.conda_archive_prefix
}
}
})
# must have either name or env_file:
if (name and env_file) or not (name or env_file):
raise YmpRuleError(
self, "Env must have exactly one of `name` and `file`")
if name:
self.name = name
else:
self.name, _ = op.splitext(op.basename(env_file))
if env_file:
self.dynamic = False
self.filename = env_file
self.lineno = 1
else:
self.dynamic = True
env_file = op.join(cfg.ensuredir.dynamic_envs, f"{name}.yml")
defaults = {
'name':
self.name,
'dependencies':
list(
ensure_list(packages) +
cfg.conda.defaults[base].dependencies),
'channels':
list(
ensure_list(channels) + cfg.conda.defaults[base].channels)
}
yaml = YAML(typ='rt')
yaml.default_flow_style = False
buf = io.StringIO()
yaml.dump(defaults, buf)
contents = buf.getvalue()
disk_contents = ""
if op.exists(env_file):
with open(env_file, "r") as inf:
disk_contents = inf.read()
if contents != disk_contents:
with open(env_file, "w") as out:
out.write(contents)
super().__init__(env_file, pseudo_dag, singularity_img)
self.register()
def __init__(
self,
# Snakemake Params:
env_file: Optional[str] = None,
workflow=None,
env_dir=None,
container_img=None,
cleanup=None,
# YMP Params:
name: Optional[str] = None,
packages: Optional[Union[list, str]] = None,
base: str = "none",
channels: Optional[Union[list, str]] = None) -> None:
"""Creates an inline defined conda environment
Args:
name: Name of conda environment (and basename of file)
packages: package(s) to be installed into environment. Version
constraints can be specified in each package string separated from
the package name by whitespace. E.g. ``"blast =2.6*"``
channels: channel(s) to be selected for the environment
base: Select a set of default channels and packages to be added to
the newly created environment. Sets are defined in conda.defaults
in ``yml.yml``
"""
if 'name' in self.__dict__:
# already initialized
return
cfg = ymp.get_config()
if env_file:
if name:
import pdb
pdb.set_trace()
raise YmpRuleError(
self,
"Env must not have both 'name' and 'env_file' parameters'")
self.dynamic = False
self.name, _ = op.splitext(op.basename(env_file))
self.packages = None
self.base = None
self.channels = None
# Override location for exceptions:
self.filename = env_file
self.lineno = 1
elif name:
self.dynamic = True
self.name = name
self.packages = ensure_list(
packages) + cfg.conda.defaults[base].dependencies
self.channels = ensure_list(
channels) + cfg.conda.defaults[base].channels
env_file = op.join(cfg.ensuredir.dynamic_envs, f"{name}.yml")
contents = self._get_dynamic_contents()
self._update_file(env_file, contents)
else:
raise YmpRuleError(
self, "Env must have either 'name' or 'env_file' parameter")
# Unlike within snakemake, we create these objects before the workflow is fully
# initialized, which means we need to create a fake one:
if not workflow:
workflow = AttrDict({
'persistence': {
'conda_env_path': cfg.ensuredir.conda_prefix,
'conda_env_archive_path':
cfg.ensuredir.conda_archive_prefix,
},
'conda_frontend': cfg.conda.frontend,
'singularity_args': '',
})
super().__init__(env_file, workflow,
env_dir if env_dir else cfg.ensuredir.conda_prefix,
container_img, cleanup)
self.register()
def make(cls, stage: BaseStage, typ: str, key: str, name: str, value,
default) -> "Param":
if typ not in cls.types:
raise YmpRuleError(stage, f"Unknown stage Parameter type '{typ}'")
return cls.types[typ](stage, key, name, value, default)