def create_env(env_file, filetype="yaml"):
# Copy env file to env_path (because they can be on
# different volumes and singularity should only mount one).
# In addition, this allows to immediately see what an
# environment in .snakemake/conda contains.
target_env_file = env_path + f".{filetype}"
shutil.copy(env_file, target_env_file)
logger.info(
"Downloading and installing remote packages.")
strict_priority = ([
"conda config --set channel_priority strict &&"
] if self._container_img else [])
subcommand = [self.frontend]
yes_flag = ["--yes"]
if filetype == "yaml":
subcommand.append("env")
yes_flag = []
cmd = (strict_priority + subcommand + [
"create",
"--quiet",
'--file "{}"'.format(target_env_file),
'--prefix "{}"'.format(env_path),
] + yes_flag)
cmd = " ".join(cmd)
if self._container_img:
cmd = singularity.shellcmd(
self._container_img.path,
cmd,
args=self._singularity_args,
envvars=self.get_singularity_envvars(),
)
out = shell.check_output(cmd,
stderr=subprocess.STDOUT,
universal_newlines=True)
# cleanup if requested
if self._cleanup is CondaCleanupMode.tarballs:
logger.info("Cleaning up conda package tarballs.")
shell.check_output("conda clean -y --tarballs")
elif self._cleanup is CondaCleanupMode.cache:
logger.info(
"Cleaning up conda package tarballs and package cache."
)
shell.check_output(
"conda clean -y --tarballs --packages")
return out
def __new__(cls,
cmd,
*args,
iterable=False,
read=False,
bench_record=None,
**kwargs):
if "stepout" in kwargs:
raise KeyError("Argument stepout is not allowed in shell command.")
cmd = format(cmd, *args, stepout=2, **kwargs)
context = inspect.currentframe().f_back.f_locals
# add kwargs to context (overwriting the locals of the caller)
context.update(kwargs)
stdout = sp.PIPE if iterable or read else STDOUT
close_fds = sys.platform != "win32"
jobid = context.get("jobid")
if not context.get("is_shell"):
logger.shellcmd(cmd)
env_prefix = ""
conda_env = context.get("conda_env", None)
container_img = context.get("container_img", None)
env_modules = context.get("env_modules", None)
shadow_dir = context.get("shadow_dir", None)
cmd = "{} {} {}".format(cls._process_prefix, cmd.strip(),
cls._process_suffix).strip()
if env_modules:
cmd = env_modules.shellcmd(cmd)
logger.info(
"Activating environment modules: {}".format(env_modules))
if conda_env:
cmd = Conda(container_img).shellcmd(conda_env, cmd)
if container_img:
args = context.get("singularity_args", "")
cmd = singularity.shellcmd(
container_img,
cmd,
args,
shell_executable=cls._process_args["executable"],
container_workdir=shadow_dir,
)
logger.info(
"Activating singularity image {}".format(container_img))
if conda_env:
logger.info("Activating conda environment: {}".format(conda_env))
threads = str(context.get("threads", 1))
# environment variable lists for linear algebra libraries taken from:
# https://stackoverflow.com/a/53224849/2352071
# https://github.com/xianyi/OpenBLAS/tree/59243d49ab8e958bb3872f16a7c0ef8c04067c0a#setting-the-number-of-threads-using-environment-variables
envvars = dict(os.environ)
envvars["OMP_NUM_THREADS"] = threads
envvars["GOTO_NUM_THREADS"] = threads
envvars["OPENBLAS_NUM_THREADS"] = threads
envvars["MKL_NUM_THREADS"] = threads
envvars["VECLIB_MAXIMUM_THREADS"] = threads
envvars["NUMEXPR_NUM_THREADS"] = threads
use_shell = True
if ON_WINDOWS and cls.get_executable():
# If executable is set on Windows shell mode can not be used
# and the executable should be prepended the command together
# with a command prefix (e.g. -c for bash).
use_shell = False
cmd = '"{}" {} {}'.format(cls.get_executable(),
cls._win_command_prefix, argvquote(cmd))
proc = sp.Popen(
cmd,
bufsize=-1,
shell=use_shell,
stdout=stdout,
universal_newlines=iterable or read or None,
close_fds=close_fds,
**cls._process_args,
env=envvars,
)
if jobid is not None:
with cls._lock:
cls._processes[jobid] = proc
ret = None
if iterable:
return cls.iter_stdout(proc, cmd)
if read:
ret = proc.stdout.read()
if bench_record is not None:
from snakemake.benchmark import benchmarked
with benchmarked(proc.pid, bench_record):
retcode = proc.wait()
else:
retcode = proc.wait()
if jobid is not None:
with cls._lock:
del cls._processes[jobid]
if retcode:
raise sp.CalledProcessError(retcode, cmd)
return ret
def __new__(cls,
cmd,
*args,
iterable=False,
read=False,
bench_record=None,
**kwargs):
if "stepout" in kwargs:
raise KeyError("Argument stepout is not allowed in shell command.")
cmd = format(cmd, *args, stepout=2, **kwargs)
context = inspect.currentframe().f_back.f_locals
# add kwargs to context (overwriting the locals of the caller)
context.update(kwargs)
stdout = sp.PIPE if iterable or read else STDOUT
close_fds = sys.platform != "win32"
jobid = context.get("jobid")
if not context.get("is_shell"):
logger.shellcmd(cmd)
env_prefix = ""
conda_env = context.get("conda_env", None)
singularity_img = context.get("singularity_img", None)
env_modules = context.get("env_modules", None)
shadow_dir = context.get("shadow_dir", None)
cmd = "{} {} {}".format(cls._process_prefix, cmd.strip(),
cls._process_suffix).strip()
if env_modules:
cmd = env_modules.shellcmd(cmd)
logger.info(
"Activating environment modules: {}".format(env_modules))
if conda_env:
cmd = Conda(singularity_img).shellcmd(conda_env, cmd)
if singularity_img:
args = context.get("singularity_args", "")
cmd = singularity.shellcmd(
singularity_img,
cmd,
args,
shell_executable=cls._process_args["executable"],
container_workdir=shadow_dir,
)
logger.info(
"Activating singularity image {}".format(singularity_img))
if conda_env:
logger.info("Activating conda environment: {}".format(conda_env))
proc = sp.Popen(cmd,
bufsize=-1,
shell=True,
stdout=stdout,
universal_newlines=iterable or None,
close_fds=close_fds,
**cls._process_args)
if jobid is not None:
with cls._lock:
cls._processes[jobid] = proc
ret = None
if iterable:
return cls.iter_stdout(proc, cmd)
if read:
ret = proc.stdout.read()
if bench_record is not None:
from snakemake.benchmark import benchmarked
with benchmarked(proc.pid, bench_record):
retcode = proc.wait()
else:
retcode = proc.wait()
if jobid is not None:
with cls._lock:
del cls._processes[jobid]
if retcode:
raise sp.CalledProcessError(retcode, cmd)
return ret
def _get_cmd(self, cmd):
if self.container_img:
return singularity.shellcmd(self.container_img, cmd)
return cmd
def create(self, dryrun=False):
""" Create the conda enviroment."""
from snakemake.shell import shell
# Read env file and create hash.
env_file = self.file
tmp_file = None
url_scheme, *_ = urlparse(env_file)
if (url_scheme and not url_scheme == "file") or (
not url_scheme and env_file.startswith("git+file:/")):
with tempfile.NamedTemporaryFile(delete=False,
suffix=".yaml") as tmp:
tmp.write(self.content)
env_file = tmp.name
tmp_file = tmp.name
env_hash = self.hash
env_path = self.path
# Check for broken environment
if os.path.exists(os.path.join(
env_path, "env_setup_start")) and not os.path.exists(
os.path.join(env_path, "env_setup_done")):
if dryrun:
logger.info(
"Incomplete Conda environment {} will be recreated.".
format(utils.simplify_path(self.file)))
else:
logger.info(
"Removing incomplete Conda environment {}...".format(
utils.simplify_path(self.file)))
shutil.rmtree(env_path, ignore_errors=True)
# Create environment if not already present.
if not os.path.exists(env_path):
if dryrun:
logger.info("Conda environment {} will be created.".format(
utils.simplify_path(self.file)))
return env_path
conda = Conda(self._container_img)
logger.info("Creating conda environment {}...".format(
utils.simplify_path(self.file)))
# Check if env archive exists. Use that if present.
env_archive = self.archive_file
try:
# Touch "start" flag file
os.makedirs(env_path, exist_ok=True)
with open(os.path.join(env_path, "env_setup_start"), "a") as f:
pass
if os.path.exists(env_archive):
logger.info("Installing archived conda packages.")
pkg_list = os.path.join(env_archive, "packages.txt")
if os.path.exists(pkg_list):
# read pacakges in correct order
# this is for newer env archives where the package list
# was stored
packages = [
os.path.join(env_archive, pkg.rstrip())
for pkg in open(pkg_list)
]
else:
# guess order
packages = glob(os.path.join(env_archive, "*.tar.bz2"))
# install packages manually from env archive
cmd = " ".join([
"conda", "create", "--copy", "--prefix '{}'".format(
env_path)
] + packages)
if self._container_img:
cmd = singularity.shellcmd(
self._container_img.path,
cmd,
envvars=self.get_singularity_envvars(),
)
out = shell.check_output(cmd, stderr=subprocess.STDOUT)
else:
# Copy env file to env_path (because they can be on
# different volumes and singularity should only mount one).
# In addition, this allows to immediately see what an
# environment in .snakemake/conda contains.
target_env_file = env_path + ".yaml"
shutil.copy(env_file, target_env_file)
logger.info("Downloading and installing remote packages.")
cmd = " ".join([
"conda",
"env",
"create",
"--file '{}'".format(target_env_file),
"--prefix '{}'".format(env_path),
])
if self._container_img:
cmd = singularity.shellcmd(
self._container_img.path,
cmd,
envvars=self.get_singularity_envvars(),
)
out = shell.check_output(cmd, stderr=subprocess.STDOUT)
# Touch "done" flag file
with open(os.path.join(env_path, "env_setup_done"), "a") as f:
pass
logger.debug(out.decode())
logger.info("Environment for {} created (location: {})".format(
os.path.relpath(env_file), os.path.relpath(env_path)))
except subprocess.CalledProcessError as e:
# remove potential partially installed environment
shutil.rmtree(env_path, ignore_errors=True)
raise CreateCondaEnvironmentException(
"Could not create conda environment from {}:\n".format(
env_file) + e.output.decode())
if tmp_file:
# temporary file was created
os.remove(tmp_file)
return env_path
def create(self, dryrun=False):
"""Create the conda enviroment."""
from snakemake.shell import shell
# Read env file and create hash.
env_file = self.file
tmp_file = None
if not isinstance(env_file, LocalSourceFile) or isinstance(
env_file, LocalGitFile):
with tempfile.NamedTemporaryFile(delete=False,
suffix=".yaml") as tmp:
# write to temp file such that conda can open it
tmp.write(self.content)
env_file = tmp.name
tmp_file = tmp.name
else:
env_file = env_file.get_path_or_uri()
env_hash = self.hash
env_path = self.path
if self.is_containerized:
if not dryrun:
try:
shell.check_output(
singularity.shellcmd(
self._container_img.path,
"[ -d '{}' ]".format(env_path),
args=self._singularity_args,
envvars=self.get_singularity_envvars(),
quiet=True,
),
stderr=subprocess.PIPE,
)
except subprocess.CalledProcessError as e:
raise WorkflowError(
"Unable to find environment in container image. "
"Maybe a conda environment was modified without containerizing again "
"(see snakemake --containerize)?\nDetails:\n{}\n{}".
format(e, e.stderr.decode()))
return env_path
else:
# env should be present in the container
return env_path
# Check for broken environment
if os.path.exists(os.path.join(
env_path, "env_setup_start")) and not os.path.exists(
os.path.join(env_path, "env_setup_done")):
if dryrun:
logger.info(
"Incomplete Conda environment {} will be recreated.".
format(self.file.simplify_path()))
else:
logger.info(
"Removing incomplete Conda environment {}...".format(
self.file.simplify_path()))
shutil.rmtree(env_path, ignore_errors=True)
# Create environment if not already present.
if not os.path.exists(env_path):
if dryrun:
logger.info("Conda environment {} will be created.".format(
self.file.simplify_path()))
return env_path
conda = Conda(self._container_img)
logger.info("Creating conda environment {}...".format(
self.file.simplify_path()))
# Check if env archive exists. Use that if present.
env_archive = self.archive_file
try:
# Touch "start" flag file
os.makedirs(env_path, exist_ok=True)
with open(os.path.join(env_path, "env_setup_start"), "a") as f:
pass
if os.path.exists(env_archive):
logger.info("Installing archived conda packages.")
pkg_list = os.path.join(env_archive, "packages.txt")
if os.path.exists(pkg_list):
# read pacakges in correct order
# this is for newer env archives where the package list
# was stored
packages = [
os.path.join(env_archive, pkg.rstrip())
for pkg in open(pkg_list)
]
else:
# guess order
packages = glob(os.path.join(env_archive, "*.tar.bz2"))
# install packages manually from env archive
cmd = " ".join([
"conda",
"create",
"--quiet",
"--yes",
"--prefix '{}'".format(env_path),
] + packages)
if self._container_img:
cmd = singularity.shellcmd(
self._container_img.path,
cmd,
args=self._singularity_args,
envvars=self.get_singularity_envvars(),
)
out = shell.check_output(cmd,
stderr=subprocess.STDOUT,
universal_newlines=True)
else:
# Copy env file to env_path (because they can be on
# different volumes and singularity should only mount one).
# In addition, this allows to immediately see what an
# environment in .snakemake/conda contains.
target_env_file = env_path + ".yaml"
shutil.copy(env_file, target_env_file)
logger.info("Downloading and installing remote packages.")
cmd = " ".join([
self.frontend,
"env",
"create",
"--quiet",
'--file "{}"'.format(target_env_file),
'--prefix "{}"'.format(env_path),
])
if self._container_img:
cmd = singularity.shellcmd(
self._container_img.path,
cmd,
args=self._singularity_args,
envvars=self.get_singularity_envvars(),
)
out = shell.check_output(cmd,
stderr=subprocess.STDOUT,
universal_newlines=True)
# cleanup if requested
if self._cleanup is CondaCleanupMode.tarballs:
logger.info("Cleaning up conda package tarballs.")
shell.check_output("conda clean -y --tarballs")
elif self._cleanup is CondaCleanupMode.cache:
logger.info(
"Cleaning up conda package tarballs and package cache."
)
shell.check_output(
"conda clean -y --tarballs --packages")
# Touch "done" flag file
with open(os.path.join(env_path, "env_setup_done"), "a") as f:
pass
logger.debug(out)
logger.info("Environment for {} created (location: {})".format(
os.path.relpath(env_file), os.path.relpath(env_path)))
except subprocess.CalledProcessError as e:
# remove potential partially installed environment
shutil.rmtree(env_path, ignore_errors=True)
raise CreateCondaEnvironmentException(
"Could not create conda environment from {}:\n".format(
env_file) + e.output)
if tmp_file:
# temporary file was created
os.remove(tmp_file)
return env_path
def create(self, dryrun=False):
"""Create the conda enviroment."""
from snakemake.shell import shell
self.check_is_file_based()
# Read env file and create hash.
env_file = self.file
deploy_file = None
pin_file = None
tmp_env_file = None
tmp_deploy_file = None
tmp_pin_file = None
if not isinstance(env_file, LocalSourceFile) or isinstance(
env_file, LocalGitFile):
with tempfile.NamedTemporaryFile(delete=False,
suffix=".yaml") as tmp:
# write to temp file such that conda can open it
tmp.write(self.content)
env_file = tmp.name
tmp_env_file = tmp.name
if self.post_deploy_file:
with tempfile.NamedTemporaryFile(
delete=False, suffix=".post-deploy.sh") as tmp:
# write to temp file such that conda can open it
tmp.write(self.content_deploy)
deploy_file = tmp.name
tmp_deploy_file = tmp.name
if self.pin_file:
with tempfile.NamedTemporaryFile(delete=False,
suffix="pin.txt") as tmp:
tmp.write(self.content_pin)
pin_file = tmp.name
tmp_pin_file = tmp.name
else:
env_file = env_file.get_path_or_uri()
deploy_file = self.post_deploy_file
pin_file = self.pin_file
env_path = self.address
if self.is_containerized:
if not dryrun:
try:
shell.check_output(
singularity.shellcmd(
self._container_img.path,
"[ -d '{}' ]".format(env_path),
args=self._singularity_args,
envvars=self.get_singularity_envvars(),
quiet=True,
),
stderr=subprocess.PIPE,
)
except subprocess.CalledProcessError as e:
raise WorkflowError(
"Unable to find environment in container image. "
"Maybe a conda environment was modified without containerizing again "
"(see snakemake --containerize)?\nDetails:\n{}\n{}".
format(e, e.stderr.decode()))
return env_path
else:
# env should be present in the container
return env_path
# Check for broken environment
if os.path.exists(os.path.join(
env_path, "env_setup_start")) and not os.path.exists(
os.path.join(env_path, "env_setup_done")):
if dryrun:
logger.info(
"Incomplete Conda environment {} will be recreated.".
format(self.file.simplify_path()))
else:
logger.info(
"Removing incomplete Conda environment {}...".format(
self.file.simplify_path()))
shutil.rmtree(env_path, ignore_errors=True)
# Create environment if not already present.
if not os.path.exists(env_path):
if dryrun:
logger.info("Conda environment {} will be created.".format(
self.file.simplify_path()))
return env_path
logger.info("Creating conda environment {}...".format(
self.file.simplify_path()))
env_archive = self.archive_file
try:
# Touch "start" flag file
os.makedirs(env_path, exist_ok=True)
with open(os.path.join(env_path, "env_setup_start"), "a") as f:
pass
# Check if env archive exists. Use that if present.
if os.path.exists(env_archive):
logger.info("Installing archived conda packages.")
pkg_list = os.path.join(env_archive, "packages.txt")
if os.path.exists(pkg_list):
# read pacakges in correct order
# this is for newer env archives where the package list
# was stored
packages = [
os.path.join(env_archive, pkg.rstrip())
for pkg in open(pkg_list)
]
else:
# guess order
packages = glob(os.path.join(env_archive, "*.tar.bz2"))
# install packages manually from env archive
cmd = " ".join([
"conda",
"create",
"--quiet",
"--yes",
"--prefix '{}'".format(env_path),
] + packages)
if self._container_img:
cmd = singularity.shellcmd(
self._container_img.path,
cmd,
args=self._singularity_args,
envvars=self.get_singularity_envvars(),
)
out = shell.check_output(cmd,
stderr=subprocess.STDOUT,
universal_newlines=True)
else:
def create_env(env_file, filetype="yaml"):
# Copy env file to env_path (because they can be on
# different volumes and singularity should only mount one).
# In addition, this allows to immediately see what an
# environment in .snakemake/conda contains.
target_env_file = env_path + f".{filetype}"
shutil.copy(env_file, target_env_file)
logger.info(
"Downloading and installing remote packages.")
strict_priority = ([
"conda config --set channel_priority strict &&"
] if self._container_img else [])
subcommand = [self.frontend]
yes_flag = ["--yes"]
if filetype == "yaml":
subcommand.append("env")
yes_flag = []
cmd = (strict_priority + subcommand + [
"create",
"--quiet",
'--file "{}"'.format(target_env_file),
'--prefix "{}"'.format(env_path),
] + yes_flag)
cmd = " ".join(cmd)
if self._container_img:
cmd = singularity.shellcmd(
self._container_img.path,
cmd,
args=self._singularity_args,
envvars=self.get_singularity_envvars(),
)
out = shell.check_output(cmd,
stderr=subprocess.STDOUT,
universal_newlines=True)
# cleanup if requested
if self._cleanup is CondaCleanupMode.tarballs:
logger.info("Cleaning up conda package tarballs.")
shell.check_output("conda clean -y --tarballs")
elif self._cleanup is CondaCleanupMode.cache:
logger.info(
"Cleaning up conda package tarballs and package cache."
)
shell.check_output(
"conda clean -y --tarballs --packages")
return out
if pin_file is not None:
try:
logger.info(
f"Using pinnings from {self.pin_file.get_path_or_uri()}."
)
out = create_env(pin_file, filetype="pin.txt")
except subprocess.CalledProcessError as e:
# remove potential partially installed environment
shutil.rmtree(env_path, ignore_errors=True)
advice = ""
if isinstance(self.file, LocalSourceFile):
advice = (
" If that works, make sure to update the pin file with "
f"'snakedeploy pin-conda-env {self.file.get_path_or_uri()}'."
)
logger.warning(
f"Failed to install conda environment from pin file ({self.pin_file.get_path_or_uri()}). "
f"Trying regular environment definition file.{advice}"
)
out = create_env(env_file, filetype="yaml")
else:
out = create_env(env_file, filetype="yaml")
# Execute post-deplay script if present
if deploy_file:
target_deploy_file = env_path + ".post-deploy.sh"
shutil.copy(deploy_file, target_deploy_file)
self.execute_deployment_script(env_file,
target_deploy_file)
# Touch "done" flag file
with open(os.path.join(env_path, "env_setup_done"), "a") as f:
pass
logger.debug(out)
logger.info(
f"Environment for {self.file.get_path_or_uri()} created (location: {os.path.relpath(env_path)})"
)
except subprocess.CalledProcessError as e:
# remove potential partially installed environment
shutil.rmtree(env_path, ignore_errors=True)
raise CreateCondaEnvironmentException(
f"Could not create conda environment from {env_file}:\nCommand:\n{e.cmd}\nOutput:\n{e.output}"
)
if tmp_env_file:
# temporary file was created
os.remove(tmp_env_file)
if tmp_deploy_file:
os.remove(tmp_deploy_file)
return env_path
def __new__(cls,
cmd,
*args,
iterable=False,
read=False,
bench_record=None,
**kwargs):
if "stepout" in kwargs:
raise KeyError("Argument stepout is not allowed in shell command.")
if ON_WINDOWS and not cls.get_executable():
# If bash is not used on Windows quoting must be handled in a special way
kwargs["quote_func"] = cmd_exe_quote
cmd = format(cmd, *args, stepout=2, **kwargs)
stdout = sp.PIPE if iterable or read else STDOUT
close_fds = sys.platform != "win32"
func_context = inspect.currentframe().f_back.f_locals
if func_context.get(RULEFUNC_CONTEXT_MARKER):
# If this comes from a rule, we expect certain information to be passed
# implicitly via the rule func context, which is added here.
context = func_context
else:
# Otherwise, context is just filled via kwargs.
context = dict()
# add kwargs to context (overwriting the locals of the caller)
context.update(kwargs)
jobid = context.get("jobid")
if not context.get("is_shell"):
logger.shellcmd(cmd)
conda_env = context.get("conda_env", None)
conda_base_path = context.get("conda_base_path", None)
container_img = context.get("container_img", None)
env_modules = context.get("env_modules", None)
shadow_dir = context.get("shadow_dir", None)
resources = context.get("resources", {})
singularity_args = context.get("singularity_args", "")
threads = context.get("threads", 1)
cmd = " ".join((cls._process_prefix, cmd, cls._process_suffix)).strip()
if env_modules:
cmd = env_modules.shellcmd(cmd)
logger.info(
"Activating environment modules: {}".format(env_modules))
if conda_env:
if ON_WINDOWS and not cls.get_executable():
# If we use cmd.exe directly on winodws we need to prepend batch activation script.
cmd = Conda(container_img,
prefix_path=conda_base_path).shellcmd_win(
conda_env, cmd)
else:
cmd = Conda(container_img,
prefix_path=conda_base_path).shellcmd(
conda_env, cmd)
tmpdir = None
if len(cmd.replace("'", r"'\''")) + 2 > MAX_ARG_LEN:
tmpdir = tempfile.mkdtemp(dir=".snakemake", prefix="shell_tmp.")
script = os.path.join(os.path.abspath(tmpdir), "script.sh")
with open(script, "w") as script_fd:
print(cmd, file=script_fd)
os.chmod(script,
os.stat(script).st_mode | stat.S_IXUSR | stat.S_IRUSR)
cmd = '"{}" "{}"'.format(cls.get_executable() or "/bin/sh", script)
if container_img:
cmd = singularity.shellcmd(
container_img,
cmd,
singularity_args,
envvars=None,
shell_executable=cls._process_args["executable"],
container_workdir=shadow_dir,
is_python_script=context.get("is_python_script", False),
)
logger.info(
"Activating singularity image {}".format(container_img))
if conda_env:
logger.info("Activating conda environment: {}".format(
os.path.relpath(conda_env)))
tmpdir_resource = resources.get("tmpdir", None)
# environment variable lists for linear algebra libraries taken from:
# https://stackoverflow.com/a/53224849/2352071
# https://github.com/xianyi/OpenBLAS/tree/59243d49ab8e958bb3872f16a7c0ef8c04067c0a#setting-the-number-of-threads-using-environment-variables
envvars = dict(os.environ)
threads = str(threads)
envvars["OMP_NUM_THREADS"] = threads
envvars["GOTO_NUM_THREADS"] = threads
envvars["OPENBLAS_NUM_THREADS"] = threads
envvars["MKL_NUM_THREADS"] = threads
envvars["VECLIB_MAXIMUM_THREADS"] = threads
envvars["NUMEXPR_NUM_THREADS"] = threads
if tmpdir_resource:
envvars["TMPDIR"] = tmpdir_resource
envvars["TMP"] = tmpdir_resource
envvars["TEMPDIR"] = tmpdir_resource
envvars["TEMP"] = tmpdir_resource
if "additional_envvars" in kwargs:
env = kwargs["additional_envvars"]
if not isinstance(env, dict) or not all(
isinstance(v, str) for v in env.values()):
raise WorkflowError(
"Given environment variables for shell command have to be a dict of strings, "
"but the following was provided instead:\n{}".format(env))
envvars.update(env)
if conda_env and cls.conda_block_conflicting_envvars:
# remove envvars that conflict with conda
for var in ["R_LIBS", "PYTHONPATH", "PERLLIB", "PERL5LIB"]:
try:
del envvars[var]
except KeyError:
pass
use_shell = True
if ON_WINDOWS and cls.get_executable():
# If executable is set on Windows shell mode can not be used
# and the executable should be prepended the command together
# with a command prefix (e.g. -c for bash).
use_shell = False
cmd = '"{}" {} {}'.format(cls.get_executable(),
cls._win_command_prefix, argvquote(cmd))
proc = sp.Popen(
cmd,
bufsize=-1,
shell=use_shell,
stdout=stdout,
universal_newlines=iterable or read or None,
close_fds=close_fds,
**cls._process_args,
env=envvars,
)
if jobid is not None:
with cls._lock:
cls._processes[jobid] = proc
ret = None
if iterable:
return cls.iter_stdout(proc, cmd, tmpdir)
if read:
ret = proc.stdout.read()
if bench_record is not None:
from snakemake.benchmark import benchmarked
with benchmarked(proc.pid, bench_record):
retcode = proc.wait()
else:
retcode = proc.wait()
if tmpdir:
shutil.rmtree(tmpdir)
if jobid is not None:
with cls._lock:
del cls._processes[jobid]
if retcode:
raise sp.CalledProcessError(retcode, cmd)
return ret
def __new__(cls,
cmd,
*args,
iterable=False,
read=False,
bench_record=None,
**kwargs):
if "stepout" in kwargs:
raise KeyError("Argument stepout is not allowed in shell command.")
cmd = format(cmd, *args, stepout=2, **kwargs)
context = inspect.currentframe().f_back.f_locals
# add kwargs to context (overwriting the locals of the caller)
context.update(kwargs)
stdout = sp.PIPE if iterable or read else STDOUT
close_fds = sys.platform != "win32"
jobid = context.get("jobid")
if not context.get("is_shell"):
logger.shellcmd(cmd)
conda_env = context.get("conda_env", None)
container_img = context.get("container_img", None)
env_modules = context.get("env_modules", None)
shadow_dir = context.get("shadow_dir", None)
cmd = " ".join((cls._process_prefix, cmd, cls._process_suffix)).strip()
if env_modules:
cmd = env_modules.shellcmd(cmd)
logger.info(
"Activating environment modules: {}".format(env_modules))
if conda_env:
cmd = Conda(container_img).shellcmd(conda_env, cmd)
tmpdir = None
if len(cmd.replace("'", r"'\''")) + 2 > MAX_ARG_LEN:
tmpdir = tempfile.mkdtemp(dir=".snakemake", prefix="shell_tmp.")
script = os.path.join(os.path.abspath(tmpdir), "script.sh")
with open(script, "w") as script_fd:
print(cmd, file=script_fd)
os.chmod(script,
os.stat(script).st_mode | stat.S_IXUSR | stat.S_IRUSR)
cmd = '"{}" "{}"'.format(cls.get_executable() or "/bin/sh", script)
if container_img:
args = context.get("singularity_args", "")
cmd = singularity.shellcmd(
container_img,
cmd,
args,
envvars=None,
shell_executable=cls._process_args["executable"],
container_workdir=shadow_dir,
)
logger.info(
"Activating singularity image {}".format(container_img))
if conda_env:
logger.info("Activating conda environment: {}".format(conda_env))
threads = str(context.get("threads", 1))
# environment variable lists for linear algebra libraries taken from:
# https://stackoverflow.com/a/53224849/2352071
# https://github.com/xianyi/OpenBLAS/tree/59243d49ab8e958bb3872f16a7c0ef8c04067c0a#setting-the-number-of-threads-using-environment-variables
envvars = dict(os.environ)
envvars["OMP_NUM_THREADS"] = threads
envvars["GOTO_NUM_THREADS"] = threads
envvars["OPENBLAS_NUM_THREADS"] = threads
envvars["MKL_NUM_THREADS"] = threads
envvars["VECLIB_MAXIMUM_THREADS"] = threads
envvars["NUMEXPR_NUM_THREADS"] = threads
if conda_env and cls.conda_block_conflicting_envvars:
# remove envvars that conflict with conda
for var in ["R_LIBS", "PYTHONPATH", "PERLLIB", "PERL5LIB"]:
try:
del envvars[var]
except KeyError:
pass
use_shell = True
if ON_WINDOWS and cls.get_executable():
# If executable is set on Windows shell mode can not be used
# and the executable should be prepended the command together
# with a command prefix (e.g. -c for bash).
use_shell = False
cmd = '"{}" {} {}'.format(cls.get_executable(),
cls._win_command_prefix, argvquote(cmd))
proc = sp.Popen(
cmd,
bufsize=-1,
shell=use_shell,
stdout=stdout,
universal_newlines=iterable or read or None,
close_fds=close_fds,
**cls._process_args,
env=envvars,
)
if jobid is not None:
with cls._lock:
cls._processes[jobid] = proc
ret = None
if iterable:
return cls.iter_stdout(proc, cmd, tmpdir)
if read:
ret = proc.stdout.read()
if bench_record is not None:
from snakemake.benchmark import benchmarked
with benchmarked(proc.pid, bench_record):
retcode = proc.wait()
else:
retcode = proc.wait()
if tmpdir:
shutil.rmtree(tmpdir)
if jobid is not None:
with cls._lock:
del cls._processes[jobid]
if retcode:
raise sp.CalledProcessError(retcode, cmd)
return ret
def _get_cmd(self, cmd):
if self.singularity_img:
return singularity.shellcmd(self.singularity_img, cmd)
return cmd
