def _check(self):
from snakemake.shell import shell
frontends = ["conda"]
if self.frontend == "mamba":
frontends = ["mamba", "conda"]
for frontend in frontends:
# Use type here since conda now is a function.
# type allows to check for both functions and regular commands.
if not ON_WINDOWS or shell.get_executable():
locate_cmd = f"type {frontend}"
else:
locate_cmd = f"where {frontend}"
try:
shell.check_output(self._get_cmd(locate_cmd),
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
if self.container_img:
msg = (f"The '{frontend}' command is not "
"available inside "
"your singularity container "
"image. Snakemake mounts "
"your conda installation "
"into singularity. "
"Sometimes, this can fail "
"because of shell restrictions. "
"It has been tested to work "
"with docker://ubuntu, but "
"it e.g. fails with "
"docker://bash ")
else:
msg = (f"The '{frontend}' command is not "
"available in the "
f"shell {shell.get_executable()} that will be "
"used by Snakemake. You have "
"to ensure that it is in your "
"PATH, e.g., first activating "
"the conda base environment "
"with `conda activate base`.")
if frontend == "mamba":
msg += (
"The mamba package manager (https://github.com/mamba-org/mamba) is a "
"fast and robust conda replacement. "
"It is the recommended way of using Snakemake's conda integration. "
"It can be installed with `conda install -n base -c conda-forge mamba`. "
"If you still prefer to use conda, you can enforce that by setting "
"`--conda-frontend conda`.", )
raise CreateCondaEnvironmentException(msg)
try:
self._check_version()
self._check_condarc()
except subprocess.CalledProcessError as e:
raise CreateCondaEnvironmentException(
f"Unable to check conda installation:"
"\n" + e.stderr.decode())
def _check(self):
from snakemake.shell import shell
# Use type here since conda now is a function.
# type allows to check for both functions and regular commands.
if not ON_WINDOWS or shell.get_executable():
locate_cmd = "type conda"
else:
locate_cmd = "where conda"
try:
shell.check_output(self._get_cmd(locate_cmd), stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
if self.container_img:
raise CreateCondaEnvironmentException(
"The 'conda' command is not "
"available inside "
"your singularity container "
"image. Snakemake mounts "
"your conda installation "
"into singularity. "
"Sometimes, this can fail "
"because of shell restrictions. "
"It has been tested to work "
"with docker://ubuntu, but "
"it e.g. fails with "
"docker://bash "
)
else:
raise CreateCondaEnvironmentException(
"The 'conda' command is not "
"available in the "
"shell {} that will be "
"used by Snakemake. You have "
"to ensure that it is in your "
"PATH, e.g., first activating "
"the conda base environment "
"with `conda activate base`.".format(shell.get_executable())
)
try:
version = shell.check_output(
self._get_cmd("conda --version"),
stderr=subprocess.STDOUT,
universal_newlines=True,
)
version = version.split()[1]
if StrictVersion(version) < StrictVersion("4.2"):
raise CreateCondaEnvironmentException(
"Conda must be version 4.2 or later, found version {}.".format(
version
)
)
except subprocess.CalledProcessError as e:
raise CreateCondaEnvironmentException(
"Unable to check conda version:\n" + e.output.decode()
)
def shellcmd(env_path):
from snakemake.shell import shell
try:
shell.check_output("type conda", stderr=subprocess.STDOUT)
except subprocess.CalledProcessError:
raise CreateCondaEnvironmentException("The 'conda' command is not "
"available ")
try:
version = shell.check_output("conda --version",
stderr=subprocess.STDOUT).decode() \
.split()[1]
if StrictVersion(version) < StrictVersion("4.5.12"):
return "source activate '{}';".format(env_path)
else:
return "source ~/.bashrc && conda activate {};".format(env_path)
except subprocess.CalledProcessError as e:
raise CreateCondaEnvironmentException(
"Unable to check conda version:\n" + e.output.decode()
)
def check_conda(singularity_img=None):
def get_cmd(cmd, singularity_img=None):
if singularity_img:
return singularity.shellcmd(self.singularity_img.path, cmd)
return cmd
if subprocess.check_output(get_cmd("which conda"),
shell=True,
stderr=subprocess.STDOUT) is None:
raise CreateCondaEnvironmentException(
"The 'conda' command is not available in $PATH.")
try:
version = subprocess.check_output(get_cmd("conda --version"),
shell=True,
stderr=subprocess.STDOUT).decode() \
.split()[1]
if StrictVersion(version) < StrictVersion("4.2"):
raise CreateCondaEnvironmentException(
"Conda must be version 4.2 or later.")
except subprocess.CalledProcessError as e:
raise CreateCondaEnvironmentException(
"Unable to check conda version:\n" + e.output.decode())
def _check_version(self):
from snakemake.shell import shell
version = shell.check_output(
self._get_cmd("conda --version"),
stderr=subprocess.PIPE,
universal_newlines=True,
)
version_matches = re.findall("\d+.\d+.\d+", version)
if len(version_matches) != 1:
raise WorkflowError(
f"Unable to determine conda version. 'conda --version' returned {version}"
)
else:
version = version_matches[0]
if StrictVersion(version) < StrictVersion("4.2"):
raise CreateCondaEnvironmentException(
"Conda must be version 4.2 or later, found version {}.".format(
version))
def _check_condarc(self):
if self.container_img:
# Do not check for strict priorities when running conda in an image
# Instead, we set priorities to strict ourselves in the image.
return
from snakemake.shell import shell
res = json.loads(
shell.check_output(
self._get_cmd("conda config --get channel_priority --json"),
universal_newlines=True,
stderr=subprocess.PIPE,
))
if res["get"].get("channel_priority") != "strict":
raise CreateCondaEnvironmentException(
"Your conda installation is not configured to use strict channel priorities. "
"This is however crucial for having robust and correct environments (for details, "
"see https://conda-forge.org/docs/user/tipsandtricks.html). "
"Please configure strict priorities by executing 'conda config --set channel_priority strict'."
)
def create(self, dryrun=False):
""" Create the conda enviroment."""
from snakemake.shell import shell
# Read env file and create hash.
env_file = self.file
tmp_file = None
url_scheme, *_ = urlparse(env_file)
if (url_scheme and not url_scheme == "file") or (
not url_scheme and env_file.startswith("git+file:/")):
with tempfile.NamedTemporaryFile(delete=False,
suffix=".yaml") as tmp:
tmp.write(self.content)
env_file = tmp.name
tmp_file = tmp.name
env_hash = self.hash
env_path = self.path
# Check for broken environment
if os.path.exists(os.path.join(
env_path, "env_setup_start")) and not os.path.exists(
os.path.join(env_path, "env_setup_done")):
if dryrun:
logger.info(
"Incomplete Conda environment {} will be recreated.".
format(utils.simplify_path(self.file)))
else:
logger.info(
"Removing incomplete Conda environment {}...".format(
utils.simplify_path(self.file)))
shutil.rmtree(env_path, ignore_errors=True)
# Create environment if not already present.
if not os.path.exists(env_path):
if dryrun:
logger.info("Conda environment {} will be created.".format(
utils.simplify_path(self.file)))
return env_path
conda = Conda(self._container_img)
logger.info("Creating conda environment {}...".format(
utils.simplify_path(self.file)))
# Check if env archive exists. Use that if present.
env_archive = self.archive_file
try:
# Touch "start" flag file
os.makedirs(env_path, exist_ok=True)
with open(os.path.join(env_path, "env_setup_start"), "a") as f:
pass
if os.path.exists(env_archive):
logger.info("Installing archived conda packages.")
pkg_list = os.path.join(env_archive, "packages.txt")
if os.path.exists(pkg_list):
# read pacakges in correct order
# this is for newer env archives where the package list
# was stored
packages = [
os.path.join(env_archive, pkg.rstrip())
for pkg in open(pkg_list)
]
else:
# guess order
packages = glob(os.path.join(env_archive, "*.tar.bz2"))
# install packages manually from env archive
cmd = " ".join([
"conda", "create", "--copy", "--prefix '{}'".format(
env_path)
] + packages)
if self._container_img:
cmd = singularity.shellcmd(
self._container_img.path,
cmd,
envvars=self.get_singularity_envvars(),
)
out = shell.check_output(cmd, stderr=subprocess.STDOUT)
else:
# Copy env file to env_path (because they can be on
# different volumes and singularity should only mount one).
# In addition, this allows to immediately see what an
# environment in .snakemake/conda contains.
target_env_file = env_path + ".yaml"
shutil.copy(env_file, target_env_file)
logger.info("Downloading and installing remote packages.")
cmd = " ".join([
"conda",
"env",
"create",
"--file '{}'".format(target_env_file),
"--prefix '{}'".format(env_path),
])
if self._container_img:
cmd = singularity.shellcmd(
self._container_img.path,
cmd,
envvars=self.get_singularity_envvars(),
)
out = shell.check_output(cmd, stderr=subprocess.STDOUT)
# Touch "done" flag file
with open(os.path.join(env_path, "env_setup_done"), "a") as f:
pass
logger.debug(out.decode())
logger.info("Environment for {} created (location: {})".format(
os.path.relpath(env_file), os.path.relpath(env_path)))
except subprocess.CalledProcessError as e:
# remove potential partially installed environment
shutil.rmtree(env_path, ignore_errors=True)
raise CreateCondaEnvironmentException(
"Could not create conda environment from {}:\n".format(
env_file) + e.output.decode())
if tmp_file:
# temporary file was created
os.remove(tmp_file)
return env_path
def create(self, dryrun=False):
"""Create the conda enviroment."""
from snakemake.shell import shell
# Read env file and create hash.
env_file = self.file
tmp_file = None
if not isinstance(env_file, LocalSourceFile) or isinstance(
env_file, LocalGitFile):
with tempfile.NamedTemporaryFile(delete=False,
suffix=".yaml") as tmp:
# write to temp file such that conda can open it
tmp.write(self.content)
env_file = tmp.name
tmp_file = tmp.name
else:
env_file = env_file.get_path_or_uri()
env_hash = self.hash
env_path = self.path
if self.is_containerized:
if not dryrun:
try:
shell.check_output(
singularity.shellcmd(
self._container_img.path,
"[ -d '{}' ]".format(env_path),
args=self._singularity_args,
envvars=self.get_singularity_envvars(),
quiet=True,
),
stderr=subprocess.PIPE,
)
except subprocess.CalledProcessError as e:
raise WorkflowError(
"Unable to find environment in container image. "
"Maybe a conda environment was modified without containerizing again "
"(see snakemake --containerize)?\nDetails:\n{}\n{}".
format(e, e.stderr.decode()))
return env_path
else:
# env should be present in the container
return env_path
# Check for broken environment
if os.path.exists(os.path.join(
env_path, "env_setup_start")) and not os.path.exists(
os.path.join(env_path, "env_setup_done")):
if dryrun:
logger.info(
"Incomplete Conda environment {} will be recreated.".
format(self.file.simplify_path()))
else:
logger.info(
"Removing incomplete Conda environment {}...".format(
self.file.simplify_path()))
shutil.rmtree(env_path, ignore_errors=True)
# Create environment if not already present.
if not os.path.exists(env_path):
if dryrun:
logger.info("Conda environment {} will be created.".format(
self.file.simplify_path()))
return env_path
conda = Conda(self._container_img)
logger.info("Creating conda environment {}...".format(
self.file.simplify_path()))
# Check if env archive exists. Use that if present.
env_archive = self.archive_file
try:
# Touch "start" flag file
os.makedirs(env_path, exist_ok=True)
with open(os.path.join(env_path, "env_setup_start"), "a") as f:
pass
if os.path.exists(env_archive):
logger.info("Installing archived conda packages.")
pkg_list = os.path.join(env_archive, "packages.txt")
if os.path.exists(pkg_list):
# read pacakges in correct order
# this is for newer env archives where the package list
# was stored
packages = [
os.path.join(env_archive, pkg.rstrip())
for pkg in open(pkg_list)
]
else:
# guess order
packages = glob(os.path.join(env_archive, "*.tar.bz2"))
# install packages manually from env archive
cmd = " ".join([
"conda",
"create",
"--quiet",
"--yes",
"--prefix '{}'".format(env_path),
] + packages)
if self._container_img:
cmd = singularity.shellcmd(
self._container_img.path,
cmd,
args=self._singularity_args,
envvars=self.get_singularity_envvars(),
)
out = shell.check_output(cmd,
stderr=subprocess.STDOUT,
universal_newlines=True)
else:
# Copy env file to env_path (because they can be on
# different volumes and singularity should only mount one).
# In addition, this allows to immediately see what an
# environment in .snakemake/conda contains.
target_env_file = env_path + ".yaml"
shutil.copy(env_file, target_env_file)
logger.info("Downloading and installing remote packages.")
cmd = " ".join([
self.frontend,
"env",
"create",
"--quiet",
'--file "{}"'.format(target_env_file),
'--prefix "{}"'.format(env_path),
])
if self._container_img:
cmd = singularity.shellcmd(
self._container_img.path,
cmd,
args=self._singularity_args,
envvars=self.get_singularity_envvars(),
)
out = shell.check_output(cmd,
stderr=subprocess.STDOUT,
universal_newlines=True)
# cleanup if requested
if self._cleanup is CondaCleanupMode.tarballs:
logger.info("Cleaning up conda package tarballs.")
shell.check_output("conda clean -y --tarballs")
elif self._cleanup is CondaCleanupMode.cache:
logger.info(
"Cleaning up conda package tarballs and package cache."
)
shell.check_output(
"conda clean -y --tarballs --packages")
# Touch "done" flag file
with open(os.path.join(env_path, "env_setup_done"), "a") as f:
pass
logger.debug(out)
logger.info("Environment for {} created (location: {})".format(
os.path.relpath(env_file), os.path.relpath(env_path)))
except subprocess.CalledProcessError as e:
# remove potential partially installed environment
shutil.rmtree(env_path, ignore_errors=True)
raise CreateCondaEnvironmentException(
"Could not create conda environment from {}:\n".format(
env_file) + e.output)
if tmp_file:
# temporary file was created
os.remove(tmp_file)
return env_path
def create(self, dryrun=False):
"""Create the conda enviroment."""
from snakemake.shell import shell
self.check_is_file_based()
# Read env file and create hash.
env_file = self.file
deploy_file = None
pin_file = None
tmp_env_file = None
tmp_deploy_file = None
tmp_pin_file = None
if not isinstance(env_file, LocalSourceFile) or isinstance(
env_file, LocalGitFile):
with tempfile.NamedTemporaryFile(delete=False,
suffix=".yaml") as tmp:
# write to temp file such that conda can open it
tmp.write(self.content)
env_file = tmp.name
tmp_env_file = tmp.name
if self.post_deploy_file:
with tempfile.NamedTemporaryFile(
delete=False, suffix=".post-deploy.sh") as tmp:
# write to temp file such that conda can open it
tmp.write(self.content_deploy)
deploy_file = tmp.name
tmp_deploy_file = tmp.name
if self.pin_file:
with tempfile.NamedTemporaryFile(delete=False,
suffix="pin.txt") as tmp:
tmp.write(self.content_pin)
pin_file = tmp.name
tmp_pin_file = tmp.name
else:
env_file = env_file.get_path_or_uri()
deploy_file = self.post_deploy_file
pin_file = self.pin_file
env_path = self.address
if self.is_containerized:
if not dryrun:
try:
shell.check_output(
singularity.shellcmd(
self._container_img.path,
"[ -d '{}' ]".format(env_path),
args=self._singularity_args,
envvars=self.get_singularity_envvars(),
quiet=True,
),
stderr=subprocess.PIPE,
)
except subprocess.CalledProcessError as e:
raise WorkflowError(
"Unable to find environment in container image. "
"Maybe a conda environment was modified without containerizing again "
"(see snakemake --containerize)?\nDetails:\n{}\n{}".
format(e, e.stderr.decode()))
return env_path
else:
# env should be present in the container
return env_path
# Check for broken environment
if os.path.exists(os.path.join(
env_path, "env_setup_start")) and not os.path.exists(
os.path.join(env_path, "env_setup_done")):
if dryrun:
logger.info(
"Incomplete Conda environment {} will be recreated.".
format(self.file.simplify_path()))
else:
logger.info(
"Removing incomplete Conda environment {}...".format(
self.file.simplify_path()))
shutil.rmtree(env_path, ignore_errors=True)
# Create environment if not already present.
if not os.path.exists(env_path):
if dryrun:
logger.info("Conda environment {} will be created.".format(
self.file.simplify_path()))
return env_path
logger.info("Creating conda environment {}...".format(
self.file.simplify_path()))
env_archive = self.archive_file
try:
# Touch "start" flag file
os.makedirs(env_path, exist_ok=True)
with open(os.path.join(env_path, "env_setup_start"), "a") as f:
pass
# Check if env archive exists. Use that if present.
if os.path.exists(env_archive):
logger.info("Installing archived conda packages.")
pkg_list = os.path.join(env_archive, "packages.txt")
if os.path.exists(pkg_list):
# read pacakges in correct order
# this is for newer env archives where the package list
# was stored
packages = [
os.path.join(env_archive, pkg.rstrip())
for pkg in open(pkg_list)
]
else:
# guess order
packages = glob(os.path.join(env_archive, "*.tar.bz2"))
# install packages manually from env archive
cmd = " ".join([
"conda",
"create",
"--quiet",
"--yes",
"--prefix '{}'".format(env_path),
] + packages)
if self._container_img:
cmd = singularity.shellcmd(
self._container_img.path,
cmd,
args=self._singularity_args,
envvars=self.get_singularity_envvars(),
)
out = shell.check_output(cmd,
stderr=subprocess.STDOUT,
universal_newlines=True)
else:
def create_env(env_file, filetype="yaml"):
# Copy env file to env_path (because they can be on
# different volumes and singularity should only mount one).
# In addition, this allows to immediately see what an
# environment in .snakemake/conda contains.
target_env_file = env_path + f".{filetype}"
shutil.copy(env_file, target_env_file)
logger.info(
"Downloading and installing remote packages.")
strict_priority = ([
"conda config --set channel_priority strict &&"
] if self._container_img else [])
subcommand = [self.frontend]
yes_flag = ["--yes"]
if filetype == "yaml":
subcommand.append("env")
yes_flag = []
cmd = (strict_priority + subcommand + [
"create",
"--quiet",
'--file "{}"'.format(target_env_file),
'--prefix "{}"'.format(env_path),
] + yes_flag)
cmd = " ".join(cmd)
if self._container_img:
cmd = singularity.shellcmd(
self._container_img.path,
cmd,
args=self._singularity_args,
envvars=self.get_singularity_envvars(),
)
out = shell.check_output(cmd,
stderr=subprocess.STDOUT,
universal_newlines=True)
# cleanup if requested
if self._cleanup is CondaCleanupMode.tarballs:
logger.info("Cleaning up conda package tarballs.")
shell.check_output("conda clean -y --tarballs")
elif self._cleanup is CondaCleanupMode.cache:
logger.info(
"Cleaning up conda package tarballs and package cache."
)
shell.check_output(
"conda clean -y --tarballs --packages")
return out
if pin_file is not None:
try:
logger.info(
f"Using pinnings from {self.pin_file.get_path_or_uri()}."
)
out = create_env(pin_file, filetype="pin.txt")
except subprocess.CalledProcessError as e:
# remove potential partially installed environment
shutil.rmtree(env_path, ignore_errors=True)
advice = ""
if isinstance(self.file, LocalSourceFile):
advice = (
" If that works, make sure to update the pin file with "
f"'snakedeploy pin-conda-env {self.file.get_path_or_uri()}'."
)
logger.warning(
f"Failed to install conda environment from pin file ({self.pin_file.get_path_or_uri()}). "
f"Trying regular environment definition file.{advice}"
)
out = create_env(env_file, filetype="yaml")
else:
out = create_env(env_file, filetype="yaml")
# Execute post-deplay script if present
if deploy_file:
target_deploy_file = env_path + ".post-deploy.sh"
shutil.copy(deploy_file, target_deploy_file)
self.execute_deployment_script(env_file,
target_deploy_file)
# Touch "done" flag file
with open(os.path.join(env_path, "env_setup_done"), "a") as f:
pass
logger.debug(out)
logger.info(
f"Environment for {self.file.get_path_or_uri()} created (location: {os.path.relpath(env_path)})"
)
except subprocess.CalledProcessError as e:
# remove potential partially installed environment
shutil.rmtree(env_path, ignore_errors=True)
raise CreateCondaEnvironmentException(
f"Could not create conda environment from {env_file}:\nCommand:\n{e.cmd}\nOutput:\n{e.output}"
)
if tmp_env_file:
# temporary file was created
os.remove(tmp_env_file)
if tmp_deploy_file:
os.remove(tmp_deploy_file)
return env_path
def create(self, dryrun=False):
""" Create the conda enviroment."""
if self._singularity_img:
check_conda(self._singularity_img)
# Read env file and create hash.
env_file = self.file
tmp_file = None
url_scheme, *_ = urlparse(env_file)
if url_scheme and not url_scheme == 'file':
with tempfile.NamedTemporaryFile(delete=False,
suffix=".yaml") as tmp:
tmp.write(self.content)
env_file = tmp.name
tmp_file = tmp.name
env_hash = self.hash
env_path = self.path
# Create environment if not already present.
if not os.path.exists(env_path):
if dryrun:
logger.info("Conda environment {} will be created.".format(
utils.simplify_path(self.file)))
return env_path
logger.info("Creating conda environment {}...".format(
utils.simplify_path(self.file)))
# Check if env archive exists. Use that if present.
env_archive = self.archive_file
try:
if os.path.exists(env_archive):
logger.info("Using archived local conda packages.")
# install packages manually from env archive
cmd = " ".join(
["conda", "create", "--copy", "--prefix", env_path] +
glob(os.path.join(env_archive, "*.tar.bz2")))
if self._singularity_img:
cmd = singularity.shellcmd(self._singularity_img.path,
cmd)
out = subprocess.check_output(cmd,
shell=True,
stderr=subprocess.STDOUT)
else:
# Copy env file to env_path (because they can be on
# different volumes and singularity should only mount one).
# In addition, this allows to immediately see what an
# environment in .snakemake/conda contains.
target_env_file = env_path + ".yaml"
shutil.copy(env_file, target_env_file)
logger.info("Downloading remote packages.")
cmd = " ".join([
"conda", "env", "create", "--file", target_env_file,
"--prefix", env_path
])
if self._singularity_img:
cmd = singularity.shellcmd(self._singularity_img.path,
cmd)
out = subprocess.check_output(cmd,
shell=True,
stderr=subprocess.STDOUT)
logger.debug(out.decode())
logger.info("Environment for {} created (location: {})".format(
os.path.relpath(env_file), os.path.relpath(env_path)))
except subprocess.CalledProcessError as e:
# remove potential partially installed environment
shutil.rmtree(env_path, ignore_errors=True)
raise CreateCondaEnvironmentException(
"Could not create conda environment from {}:\n".format(
env_file) + e.output.decode())
if tmp_file:
# temporary file was created
os.remove(tmp_file)
return env_path
