def log_info(self, skip_dynamic=False, indent=False, printshellcmd=True):
# skip dynamic jobs that will be "executed" only in dryrun mode
if skip_dynamic and self.dag.dynamic(self):
return
priority = self.priority
logger.job_info(
jobid=self.dag.jobid(self),
msg=self.message,
name=self.rule.name,
local=self.dag.workflow.is_local(self.rule),
input=list(format_files(self, self.input, self.dynamic_input)),
output=list(format_files(self, self.output, self.dynamic_output)),
log=list(self.log),
benchmark=self.benchmark,
wildcards=self.wildcards_dict,
reason=str(self.dag.reason(self)),
resources=self.resources,
priority="highest"
if priority == Job.HIGHEST_PRIORITY else priority,
threads=self.threads,
indent=indent,
is_checkpoint=self.rule.is_checkpoint,
printshellcmd=printshellcmd,
)
logger.shellcmd(self.shellcmd, indent=indent)
if self.dynamic_output:
logger.info(
"Subsequent jobs will be added dynamically "
"depending on the output of this job",
indent=True,
)
class shell:
_process_args = {}
_process_prefix = ""
_process_suffix = ""
@classmethod
def executable(cls, cmd):
if os.path.split(cmd)[-1] == "bash":
cls._process_prefix = "set -euo pipefail; "
cls._process_args["executable"] = cmd
@classmethod
def prefix(cls, prefix):
cls._process_prefix = format(prefix, stepout=2)
@classmethod
def suffix(cls, suffix):
cls._process_suffix = format(suffix, stepout=2)
def __new__(cls,
cmd,
*args,
async=False,
iterable=False,
read=False,
**kwargs):
if "stepout" in kwargs:
raise KeyError("Argument stepout is not allowed in shell command.")
cmd = format(cmd, *args, stepout=2, **kwargs)
logger.shellcmd(cmd)
stdout = sp.PIPE if iterable or async or read else STDOUT
close_fds = sys.platform != 'win32'
proc = sp.Popen("{} {} {}".format(cls._process_prefix, cmd.rstrip(),
cls._process_suffix),
bufsize=-1,
shell=True,
stdout=stdout,
close_fds=close_fds,
**cls._process_args)
ret = None
if iterable:
return cls.iter_stdout(proc, cmd)
if read:
ret = proc.stdout.read()
elif async:
return proc
retcode = proc.wait()
if retcode:
raise sp.CalledProcessError(retcode, cmd)
return ret
def __new__(cls,
cmd,
*args,
iterable=False,
read=False,
bench_record=None,
**kwargs):
if "stepout" in kwargs:
raise KeyError("Argument stepout is not allowed in shell command.")
cmd = format(cmd, *args, stepout=2, **kwargs)
context = inspect.currentframe().f_back.f_locals
# add kwargs to context (overwriting the locals of the caller)
context.update(kwargs)
stdout = sp.PIPE if iterable or read else STDOUT
close_fds = sys.platform != "win32"
jobid = context.get("jobid")
if not context.get("is_shell"):
logger.shellcmd(cmd)
env_prefix = ""
conda_env = context.get("conda_env", None)
container_img = context.get("container_img", None)
env_modules = context.get("env_modules", None)
shadow_dir = context.get("shadow_dir", None)
cmd = "{} {} {}".format(cls._process_prefix, cmd.strip(),
cls._process_suffix).strip()
if env_modules:
cmd = env_modules.shellcmd(cmd)
logger.info(
"Activating environment modules: {}".format(env_modules))
if conda_env:
cmd = Conda(container_img).shellcmd(conda_env, cmd)
if container_img:
args = context.get("singularity_args", "")
cmd = singularity.shellcmd(
container_img,
cmd,
args,
shell_executable=cls._process_args["executable"],
container_workdir=shadow_dir,
)
logger.info(
"Activating singularity image {}".format(container_img))
if conda_env:
logger.info("Activating conda environment: {}".format(conda_env))
threads = str(context.get("threads", 1))
# environment variable lists for linear algebra libraries taken from:
# https://stackoverflow.com/a/53224849/2352071
# https://github.com/xianyi/OpenBLAS/tree/59243d49ab8e958bb3872f16a7c0ef8c04067c0a#setting-the-number-of-threads-using-environment-variables
envvars = dict(os.environ)
envvars["OMP_NUM_THREADS"] = threads
envvars["GOTO_NUM_THREADS"] = threads
envvars["OPENBLAS_NUM_THREADS"] = threads
envvars["MKL_NUM_THREADS"] = threads
envvars["VECLIB_MAXIMUM_THREADS"] = threads
envvars["NUMEXPR_NUM_THREADS"] = threads
use_shell = True
if ON_WINDOWS and cls.get_executable():
# If executable is set on Windows shell mode can not be used
# and the executable should be prepended the command together
# with a command prefix (e.g. -c for bash).
use_shell = False
cmd = '"{}" {} {}'.format(cls.get_executable(),
cls._win_command_prefix, argvquote(cmd))
proc = sp.Popen(
cmd,
bufsize=-1,
shell=use_shell,
stdout=stdout,
universal_newlines=iterable or read or None,
close_fds=close_fds,
**cls._process_args,
env=envvars,
)
if jobid is not None:
with cls._lock:
cls._processes[jobid] = proc
ret = None
if iterable:
return cls.iter_stdout(proc, cmd)
if read:
ret = proc.stdout.read()
if bench_record is not None:
from snakemake.benchmark import benchmarked
with benchmarked(proc.pid, bench_record):
retcode = proc.wait()
else:
retcode = proc.wait()
if jobid is not None:
with cls._lock:
del cls._processes[jobid]
if retcode:
raise sp.CalledProcessError(retcode, cmd)
return ret
def __new__(cls,
cmd,
*args,
iterable=False,
read=False,
bench_record=None,
**kwargs):
if "stepout" in kwargs:
raise KeyError("Argument stepout is not allowed in shell command.")
cmd = format(cmd, *args, stepout=2, **kwargs)
context = inspect.currentframe().f_back.f_locals
# add kwargs to context (overwriting the locals of the caller)
context.update(kwargs)
stdout = sp.PIPE if iterable or read else STDOUT
close_fds = sys.platform != "win32"
jobid = context.get("jobid")
if not context.get("is_shell"):
logger.shellcmd(cmd)
env_prefix = ""
conda_env = context.get("conda_env", None)
singularity_img = context.get("singularity_img", None)
env_modules = context.get("env_modules", None)
shadow_dir = context.get("shadow_dir", None)
cmd = "{} {} {}".format(cls._process_prefix, cmd.strip(),
cls._process_suffix).strip()
if env_modules:
cmd = env_modules.shellcmd(cmd)
logger.info(
"Activating environment modules: {}".format(env_modules))
if conda_env:
cmd = Conda(singularity_img).shellcmd(conda_env, cmd)
if singularity_img:
args = context.get("singularity_args", "")
cmd = singularity.shellcmd(
singularity_img,
cmd,
args,
shell_executable=cls._process_args["executable"],
container_workdir=shadow_dir,
)
logger.info(
"Activating singularity image {}".format(singularity_img))
if conda_env:
logger.info("Activating conda environment: {}".format(conda_env))
proc = sp.Popen(cmd,
bufsize=-1,
shell=True,
stdout=stdout,
universal_newlines=iterable or None,
close_fds=close_fds,
**cls._process_args)
if jobid is not None:
with cls._lock:
cls._processes[jobid] = proc
ret = None
if iterable:
return cls.iter_stdout(proc, cmd)
if read:
ret = proc.stdout.read()
if bench_record is not None:
from snakemake.benchmark import benchmarked
with benchmarked(proc.pid, bench_record):
retcode = proc.wait()
else:
retcode = proc.wait()
if jobid is not None:
with cls._lock:
del cls._processes[jobid]
if retcode:
raise sp.CalledProcessError(retcode, cmd)
return ret
def __new__(cls,
cmd,
*args,
iterable=False,
read=False,
bench_record=None,
**kwargs):
if "stepout" in kwargs:
raise KeyError("Argument stepout is not allowed in shell command.")
if ON_WINDOWS and not cls.get_executable():
# If bash is not used on Windows quoting must be handled in a special way
kwargs["quote_func"] = cmd_exe_quote
cmd = format(cmd, *args, stepout=2, **kwargs)
stdout = sp.PIPE if iterable or read else STDOUT
close_fds = sys.platform != "win32"
func_context = inspect.currentframe().f_back.f_locals
if func_context.get(RULEFUNC_CONTEXT_MARKER):
# If this comes from a rule, we expect certain information to be passed
# implicitly via the rule func context, which is added here.
context = func_context
else:
# Otherwise, context is just filled via kwargs.
context = dict()
# add kwargs to context (overwriting the locals of the caller)
context.update(kwargs)
jobid = context.get("jobid")
if not context.get("is_shell"):
logger.shellcmd(cmd)
conda_env = context.get("conda_env", None)
conda_base_path = context.get("conda_base_path", None)
container_img = context.get("container_img", None)
env_modules = context.get("env_modules", None)
shadow_dir = context.get("shadow_dir", None)
resources = context.get("resources", {})
singularity_args = context.get("singularity_args", "")
threads = context.get("threads", 1)
cmd = " ".join((cls._process_prefix, cmd, cls._process_suffix)).strip()
if env_modules:
cmd = env_modules.shellcmd(cmd)
logger.info(
"Activating environment modules: {}".format(env_modules))
if conda_env:
if ON_WINDOWS and not cls.get_executable():
# If we use cmd.exe directly on winodws we need to prepend batch activation script.
cmd = Conda(container_img,
prefix_path=conda_base_path).shellcmd_win(
conda_env, cmd)
else:
cmd = Conda(container_img,
prefix_path=conda_base_path).shellcmd(
conda_env, cmd)
tmpdir = None
if len(cmd.replace("'", r"'\''")) + 2 > MAX_ARG_LEN:
tmpdir = tempfile.mkdtemp(dir=".snakemake", prefix="shell_tmp.")
script = os.path.join(os.path.abspath(tmpdir), "script.sh")
with open(script, "w") as script_fd:
print(cmd, file=script_fd)
os.chmod(script,
os.stat(script).st_mode | stat.S_IXUSR | stat.S_IRUSR)
cmd = '"{}" "{}"'.format(cls.get_executable() or "/bin/sh", script)
if container_img:
cmd = singularity.shellcmd(
container_img,
cmd,
singularity_args,
envvars=None,
shell_executable=cls._process_args["executable"],
container_workdir=shadow_dir,
is_python_script=context.get("is_python_script", False),
)
logger.info(
"Activating singularity image {}".format(container_img))
if conda_env:
logger.info("Activating conda environment: {}".format(
os.path.relpath(conda_env)))
tmpdir_resource = resources.get("tmpdir", None)
# environment variable lists for linear algebra libraries taken from:
# https://stackoverflow.com/a/53224849/2352071
# https://github.com/xianyi/OpenBLAS/tree/59243d49ab8e958bb3872f16a7c0ef8c04067c0a#setting-the-number-of-threads-using-environment-variables
envvars = dict(os.environ)
threads = str(threads)
envvars["OMP_NUM_THREADS"] = threads
envvars["GOTO_NUM_THREADS"] = threads
envvars["OPENBLAS_NUM_THREADS"] = threads
envvars["MKL_NUM_THREADS"] = threads
envvars["VECLIB_MAXIMUM_THREADS"] = threads
envvars["NUMEXPR_NUM_THREADS"] = threads
if tmpdir_resource:
envvars["TMPDIR"] = tmpdir_resource
envvars["TMP"] = tmpdir_resource
envvars["TEMPDIR"] = tmpdir_resource
envvars["TEMP"] = tmpdir_resource
if "additional_envvars" in kwargs:
env = kwargs["additional_envvars"]
if not isinstance(env, dict) or not all(
isinstance(v, str) for v in env.values()):
raise WorkflowError(
"Given environment variables for shell command have to be a dict of strings, "
"but the following was provided instead:\n{}".format(env))
envvars.update(env)
if conda_env and cls.conda_block_conflicting_envvars:
# remove envvars that conflict with conda
for var in ["R_LIBS", "PYTHONPATH", "PERLLIB", "PERL5LIB"]:
try:
del envvars[var]
except KeyError:
pass
use_shell = True
if ON_WINDOWS and cls.get_executable():
# If executable is set on Windows shell mode can not be used
# and the executable should be prepended the command together
# with a command prefix (e.g. -c for bash).
use_shell = False
cmd = '"{}" {} {}'.format(cls.get_executable(),
cls._win_command_prefix, argvquote(cmd))
proc = sp.Popen(
cmd,
bufsize=-1,
shell=use_shell,
stdout=stdout,
universal_newlines=iterable or read or None,
close_fds=close_fds,
**cls._process_args,
env=envvars,
)
if jobid is not None:
with cls._lock:
cls._processes[jobid] = proc
ret = None
if iterable:
return cls.iter_stdout(proc, cmd, tmpdir)
if read:
ret = proc.stdout.read()
if bench_record is not None:
from snakemake.benchmark import benchmarked
with benchmarked(proc.pid, bench_record):
retcode = proc.wait()
else:
retcode = proc.wait()
if tmpdir:
shutil.rmtree(tmpdir)
if jobid is not None:
with cls._lock:
del cls._processes[jobid]
if retcode:
raise sp.CalledProcessError(retcode, cmd)
return ret
def _run(self, job, callback=None, error_callback=None):
if self.max_jobs_per_second:
self._limit_rate()
job.remove_existing_output()
super()._run(job, callback=callback, error_callback=error_callback)
logger.shellcmd(job.shellcmd)
def _run(self, job):
super()._run(job)
logger.shellcmd(job.shellcmd)
def __new__(
cls, cmd, *args, iterable=False, read=False, bench_record=None, **kwargs
):
if "stepout" in kwargs:
raise KeyError("Argument stepout is not allowed in shell command.")
cmd = format(cmd, *args, stepout=2, **kwargs)
context = inspect.currentframe().f_back.f_locals
print(context)
stdout = sp.PIPE if iterable or read else STDOUT
close_fds = sys.platform != "win32"
jobid = context.get("jobid")
if not context.get("is_shell"):
logger.shellcmd(cmd)
env_prefix = ""
conda_env = context.get("conda_env", None)
singularity_img = context.get("singularity_img", None)
shadow_dir = context.get("shadow_dir", None)
cmd = "{} {} {}".format(
format(cls._process_prefix, *args, stepout=2, **kwargs),
cmd.strip(),
format(cls._process_suffix, *args, stepout=2, **kwargs),
).strip()
print(cmd)
conda = None
if conda_env:
cmd = Conda(singularity_img).shellcmd(conda_env, cmd)
if singularity_img:
args = context.get("singularity_args", "")
cmd = singularity.shellcmd(
singularity_img,
cmd,
args,
shell_executable=cls._process_args["executable"],
container_workdir=shadow_dir,
)
logger.info("Activating singularity image {}".format(singularity_img))
if conda_env:
logger.info("Activating conda environment: {}".format(conda_env))
proc = sp.Popen(
cmd,
bufsize=-1,
shell=True,
stdout=stdout,
universal_newlines=iterable or None,
close_fds=close_fds,
**cls._process_args
)
if jobid is not None:
with cls._lock:
cls._processes[jobid] = proc
ret = None
if iterable:
return cls.iter_stdout(proc, cmd)
if read:
ret = proc.stdout.read()
if bench_record is not None:
from benchmark import benchmarked
gpu = None
try:
gpu = [int(x) for x in context.get("params").devices.split(",")]
print("[snakeshell] Attempting to benchmark GPU process with GPUtil on devices: %s" % gpu)
except Exception as e:
pass
rt_bench_path = None
try:
rt_bench_path = context.get("output").rtbench
print("[snakeshell] Attempting to benchmark in real-time to: %s" % rt_bench_path)
except Exception as e:
pass
with benchmarked(proc.pid, bench_record, gpus=gpu, rt_path=rt_bench_path, interval=15):
retcode = proc.wait()
else:
retcode = proc.wait()
if jobid is not None:
with cls._lock:
del cls._processes[jobid]
if retcode:
raise sp.CalledProcessError(retcode, cmd)
return ret
def _run(self, job, callback=None, error_callback=None):
super()._run(job, callback=callback, error_callback=error_callback)
logger.shellcmd(job.shellcmd)
def _run(self, job, callback=None, error_callback=None):
if self.max_jobs_per_second:
self._limit_rate()
job.remove_existing_output()
super()._run(job, callback=callback, error_callback=error_callback)
logger.shellcmd(job.shellcmd)
def _run(self, job):
super()._run(job)
logger.shellcmd(job.shellcmd)
def __new__(cls,
cmd,
*args,
iterable=False,
read=False,
bench_record=None,
**kwargs):
if "stepout" in kwargs:
raise KeyError("Argument stepout is not allowed in shell command.")
cmd = format(cmd, *args, stepout=2, **kwargs)
context = inspect.currentframe().f_back.f_locals
# add kwargs to context (overwriting the locals of the caller)
context.update(kwargs)
stdout = sp.PIPE if iterable or read else STDOUT
close_fds = sys.platform != "win32"
jobid = context.get("jobid")
if not context.get("is_shell"):
logger.shellcmd(cmd)
conda_env = context.get("conda_env", None)
container_img = context.get("container_img", None)
env_modules = context.get("env_modules", None)
shadow_dir = context.get("shadow_dir", None)
cmd = " ".join((cls._process_prefix, cmd, cls._process_suffix)).strip()
if env_modules:
cmd = env_modules.shellcmd(cmd)
logger.info(
"Activating environment modules: {}".format(env_modules))
if conda_env:
cmd = Conda(container_img).shellcmd(conda_env, cmd)
tmpdir = None
if len(cmd.replace("'", r"'\''")) + 2 > MAX_ARG_LEN:
tmpdir = tempfile.mkdtemp(dir=".snakemake", prefix="shell_tmp.")
script = os.path.join(os.path.abspath(tmpdir), "script.sh")
with open(script, "w") as script_fd:
print(cmd, file=script_fd)
os.chmod(script,
os.stat(script).st_mode | stat.S_IXUSR | stat.S_IRUSR)
cmd = '"{}" "{}"'.format(cls.get_executable() or "/bin/sh", script)
if container_img:
args = context.get("singularity_args", "")
cmd = singularity.shellcmd(
container_img,
cmd,
args,
envvars=None,
shell_executable=cls._process_args["executable"],
container_workdir=shadow_dir,
)
logger.info(
"Activating singularity image {}".format(container_img))
if conda_env:
logger.info("Activating conda environment: {}".format(conda_env))
threads = str(context.get("threads", 1))
# environment variable lists for linear algebra libraries taken from:
# https://stackoverflow.com/a/53224849/2352071
# https://github.com/xianyi/OpenBLAS/tree/59243d49ab8e958bb3872f16a7c0ef8c04067c0a#setting-the-number-of-threads-using-environment-variables
envvars = dict(os.environ)
envvars["OMP_NUM_THREADS"] = threads
envvars["GOTO_NUM_THREADS"] = threads
envvars["OPENBLAS_NUM_THREADS"] = threads
envvars["MKL_NUM_THREADS"] = threads
envvars["VECLIB_MAXIMUM_THREADS"] = threads
envvars["NUMEXPR_NUM_THREADS"] = threads
if conda_env and cls.conda_block_conflicting_envvars:
# remove envvars that conflict with conda
for var in ["R_LIBS", "PYTHONPATH", "PERLLIB", "PERL5LIB"]:
try:
del envvars[var]
except KeyError:
pass
use_shell = True
if ON_WINDOWS and cls.get_executable():
# If executable is set on Windows shell mode can not be used
# and the executable should be prepended the command together
# with a command prefix (e.g. -c for bash).
use_shell = False
cmd = '"{}" {} {}'.format(cls.get_executable(),
cls._win_command_prefix, argvquote(cmd))
proc = sp.Popen(
cmd,
bufsize=-1,
shell=use_shell,
stdout=stdout,
universal_newlines=iterable or read or None,
close_fds=close_fds,
**cls._process_args,
env=envvars,
)
if jobid is not None:
with cls._lock:
cls._processes[jobid] = proc
ret = None
if iterable:
return cls.iter_stdout(proc, cmd, tmpdir)
if read:
ret = proc.stdout.read()
if bench_record is not None:
from snakemake.benchmark import benchmarked
with benchmarked(proc.pid, bench_record):
retcode = proc.wait()
else:
retcode = proc.wait()
if tmpdir:
shutil.rmtree(tmpdir)
if jobid is not None:
with cls._lock:
del cls._processes[jobid]
if retcode:
raise sp.CalledProcessError(retcode, cmd)
return ret
def _run(self, job, callback=None, error_callback=None):
super()._run(job, callback=callback, error_callback=error_callback)
logger.shellcmd(job.shellcmd)
