def _check(self):
from snakemake.shell import shell
frontends = ["conda"]
if self.frontend == "mamba":
frontends = ["mamba", "conda"]
for frontend in frontends:
# Use type here since conda now is a function.
# type allows to check for both functions and regular commands.
if not ON_WINDOWS or shell.get_executable():
locate_cmd = f"type {frontend}"
else:
locate_cmd = f"where {frontend}"
try:
shell.check_output(self._get_cmd(locate_cmd),
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
if self.container_img:
msg = (f"The '{frontend}' command is not "
"available inside "
"your singularity container "
"image. Snakemake mounts "
"your conda installation "
"into singularity. "
"Sometimes, this can fail "
"because of shell restrictions. "
"It has been tested to work "
"with docker://ubuntu, but "
"it e.g. fails with "
"docker://bash ")
else:
msg = (f"The '{frontend}' command is not "
"available in the "
f"shell {shell.get_executable()} that will be "
"used by Snakemake. You have "
"to ensure that it is in your "
"PATH, e.g., first activating "
"the conda base environment "
"with `conda activate base`.")
if frontend == "mamba":
msg += (
"The mamba package manager (https://github.com/mamba-org/mamba) is a "
"fast and robust conda replacement. "
"It is the recommended way of using Snakemake's conda integration. "
"It can be installed with `conda install -n base -c conda-forge mamba`. "
"If you still prefer to use conda, you can enforce that by setting "
"`--conda-frontend conda`.", )
raise CreateCondaEnvironmentException(msg)
try:
self._check_version()
self._check_condarc()
except subprocess.CalledProcessError as e:
raise CreateCondaEnvironmentException(
f"Unable to check conda installation:"
"\n" + e.stderr.decode())
def _check(self):
from snakemake.shell import shell
# Use type here since conda now is a function.
# type allows to check for both functions and regular commands.
if not ON_WINDOWS or shell.get_executable():
locate_cmd = "type conda"
else:
locate_cmd = "where conda"
try:
shell.check_output(self._get_cmd(locate_cmd), stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
if self.container_img:
raise CreateCondaEnvironmentException(
"The 'conda' command is not "
"available inside "
"your singularity container "
"image. Snakemake mounts "
"your conda installation "
"into singularity. "
"Sometimes, this can fail "
"because of shell restrictions. "
"It has been tested to work "
"with docker://ubuntu, but "
"it e.g. fails with "
"docker://bash "
)
else:
raise CreateCondaEnvironmentException(
"The 'conda' command is not "
"available in the "
"shell {} that will be "
"used by Snakemake. You have "
"to ensure that it is in your "
"PATH, e.g., first activating "
"the conda base environment "
"with `conda activate base`.".format(shell.get_executable())
)
try:
version = shell.check_output(
self._get_cmd("conda --version"),
stderr=subprocess.STDOUT,
universal_newlines=True,
)
version = version.split()[1]
if StrictVersion(version) < StrictVersion("4.2"):
raise CreateCondaEnvironmentException(
"Conda must be version 4.2 or later, found version {}.".format(
version
)
)
except subprocess.CalledProcessError as e:
raise CreateCondaEnvironmentException(
"Unable to check conda version:\n" + e.output.decode()
)
def create_env(env_file, filetype="yaml"):
# Copy env file to env_path (because they can be on
# different volumes and singularity should only mount one).
# In addition, this allows to immediately see what an
# environment in .snakemake/conda contains.
target_env_file = env_path + f".{filetype}"
shutil.copy(env_file, target_env_file)
logger.info(
"Downloading and installing remote packages.")
strict_priority = ([
"conda config --set channel_priority strict &&"
] if self._container_img else [])
subcommand = [self.frontend]
yes_flag = ["--yes"]
if filetype == "yaml":
subcommand.append("env")
yes_flag = []
cmd = (strict_priority + subcommand + [
"create",
"--quiet",
'--file "{}"'.format(target_env_file),
'--prefix "{}"'.format(env_path),
] + yes_flag)
cmd = " ".join(cmd)
if self._container_img:
cmd = singularity.shellcmd(
self._container_img.path,
cmd,
args=self._singularity_args,
envvars=self.get_singularity_envvars(),
)
out = shell.check_output(cmd,
stderr=subprocess.STDOUT,
universal_newlines=True)
# cleanup if requested
if self._cleanup is CondaCleanupMode.tarballs:
logger.info("Cleaning up conda package tarballs.")
shell.check_output("conda clean -y --tarballs")
elif self._cleanup is CondaCleanupMode.cache:
logger.info(
"Cleaning up conda package tarballs and package cache."
)
shell.check_output(
"conda clean -y --tarballs --packages")
return out
def execute_deployment_script(self, env_file, deploy_file):
"""Execute post-deployment script if present"""
from snakemake.shell import shell
if ON_WINDOWS:
raise WorkflowError(
"Post deploy script {} provided for conda env {} but unsupported on windows."
.format(deploy_file, env_file))
logger.info("Running post-deploy script {}...".format(
os.path.relpath(path=deploy_file, start=os.getcwd())))
shell.check_output(
self.conda.shellcmd(self.address, "sh {}".format(deploy_file)),
stderr=subprocess.STDOUT,
)
def __init__(self,
container_img=None,
prefix_path=None,
frontend=None,
check=False):
if not self.is_initialized: # avoid superfluous init calls
from snakemake.deployment import singularity
from snakemake.shell import shell
if isinstance(container_img, singularity.Image):
container_img = container_img.path
self.container_img = container_img
self.frontend = frontend
self.info = json.loads(
shell.check_output(
self._get_cmd(f"conda info --json"),
universal_newlines=True,
))
if prefix_path is None or container_img is not None:
self.prefix_path = self.info["conda_prefix"]
else:
self.prefix_path = prefix_path
self.platform = self.info["platform"]
# check conda installation
if check:
if frontend is None:
raise ValueError(
"Frontend must be specified if check is True.")
self._check()
def __init__(self, singularity_img=None):
from snakemake.shell import shell
from snakemake import singularity
if isinstance(singularity_img, singularity.Image):
singularity_img = singularity_img.path
self.singularity_img = singularity_img
self._check()
self.info = json.loads(shell.check_output(self._get_cmd("conda info --json")))
def __init__(self, container_img=None):
from snakemake.shell import shell
from snakemake.deployment import singularity
if isinstance(container_img, singularity.Image):
container_img = container_img.path
self.container_img = container_img
self._check()
self.info = json.loads(shell.check_output(self._get_cmd("conda info --json")))
def shellcmd(env_path):
from snakemake.shell import shell
try:
shell.check_output("type conda", stderr=subprocess.STDOUT)
except subprocess.CalledProcessError:
raise CreateCondaEnvironmentException("The 'conda' command is not "
"available ")
try:
version = shell.check_output("conda --version",
stderr=subprocess.STDOUT).decode() \
.split()[1]
if StrictVersion(version) < StrictVersion("4.5.12"):
return "source activate '{}';".format(env_path)
else:
return "source ~/.bashrc && conda activate {};".format(env_path)
except subprocess.CalledProcessError as e:
raise CreateCondaEnvironmentException(
"Unable to check conda version:\n" + e.output.decode()
)
def create_archive(self):
"""Create self-contained archive of environment."""
from snakemake.shell import shell
try:
import yaml
except ImportError:
raise WorkflowError("Error importing PyYAML. "
"Please install PyYAML to archive workflows.")
# importing requests locally because it interferes with instantiating conda environments
import requests
env_archive = self.archive_file
if os.path.exists(env_archive):
return env_archive
try:
# Download
logger.info("Downloading packages for conda environment {}...".format(self.file))
os.makedirs(env_archive, exist_ok=True)
try:
out = shell.check_output(
"conda list --explicit --prefix '{}'".format(self.path),
stderr=subprocess.STDOUT)
logger.debug(out.decode())
except subprocess.CalledProcessError as e:
raise WorkflowError("Error exporting conda packages:\n" +
e.output.decode())
with open(os.path.join(env_archive,
"packages.txt"), "w") as pkg_list:
for l in out.decode().split("\n"):
if l and not l.startswith("#") and not l.startswith("@"):
pkg_url = l
logger.info(pkg_url)
parsed = urlparse(pkg_url)
pkg_name = os.path.basename(parsed.path)
# write package name to list
print(pkg_name, file=pkg_list)
# download package
pkg_path = os.path.join(env_archive, pkg_name)
with open(pkg_path, "wb") as copy:
r = requests.get(pkg_url)
r.raise_for_status()
copy.write(r.content)
try:
tarfile.open(pkg_path)
except:
raise WorkflowError("Package is invalid tar archive: {}".format(pkg_url))
except (requests.exceptions.ChunkedEncodingError, requests.exceptions.HTTPError) as e:
shutil.rmtree(env_archive)
raise WorkflowError("Error downloading conda package {}.".format(pkg_url))
except (Exception, BaseException) as e:
shutil.rmtree(env_archive)
raise e
return env_archive
def _get_content(self):
if self.is_named:
from snakemake.shell import shell
content = shell.check_output(
"conda env export {}".format(self.address_argument),
stderr=subprocess.STDOUT,
universal_newlines=True,
)
return content.encode()
else:
return self.workflow.sourcecache.open(self.file, "rb").read()
def __new__(cls, container_img=None):
if container_img not in cls.instances:
from snakemake.shell import shell
inst = super().__new__(cls)
inst.__init__(container_img=container_img)
cls.instances[container_img] = inst
inst._check()
inst.info = json.loads(
shell.check_output(inst._get_cmd("conda info --json")))
return inst
else:
return cls.instances[container_img]
def main(fname_info, accession, restart_times, threads, logfiles):
outdir = Path(os.path.dirname(fname_info))
tmpdir = Path(os.path.join(outdir, f"tmp.{accession}"))
counter = 0
while True:
try:
shell(
"fasterq-dump --threads {threads} --outdir {outdir} --temp {tmpdir} {accession} > >(tee {logfiles.outfile}) 2>&1"
)
except subprocess.CalledProcessError:
print("Download process crashed, hopefully this is just a fluke...")
time.sleep(100)
# make sure the files were actually created (network issues...)
available_files = list(outdir.glob(f"{accession}*.fastq"))
if len(available_files) in (1, 2, 3):
# downloaded SE, PE, varying read number per spot
os.rmdir(tmpdir)
# TODO: maybe check all files for >SE reads
read_len = int(
shell.check_output(
f"bioawk -c fastx '{{{{ bases += length($seq); count++ }}}} END{{{{print int(bases/count)}}}}' {available_files[0]}"
).rstrip()
)
# TODO: how to get date
with open(fname_info, "w") as fd:
fd.write(f"{accession}\t19700101\t{read_len}\n")
# rename files to make V-pipe recognize them (_R3 is not used)
for entry in available_files:
fname_new = (
str(entry)
.replace("_1.fastq", "_R1.fastq")
.replace("_2.fastq", "_R2.fastq")
.replace("_3.fastq", "_R3.fastq")
)
entry.rename(fname_new)
break
# no files were downloaded, retry...
shell('echo "Download failed, restarting" >> {logfiles.errfile}')
counter += 1
if counter > restart_times:
raise RuntimeError(f"Download failed {counter} times")
def __init__(self, container_img=None, prefix_path=None):
if not self.is_initialized: # avoid superfluous init calls
from snakemake.deployment import singularity
from snakemake.shell import shell
if isinstance(container_img, singularity.Image):
container_img = container_img.path
self.container_img = container_img
if prefix_path is None or container_img is not None:
self.prefix_path = json.loads(
shell.check_output(
self._get_cmd("conda info --json")))["conda_prefix"]
else:
self.prefix_path = prefix_path
# check conda installation
self._check()
def _check_version(self):
from snakemake.shell import shell
version = shell.check_output(
self._get_cmd("conda --version"),
stderr=subprocess.PIPE,
universal_newlines=True,
)
version_matches = re.findall("\d+.\d+.\d+", version)
if len(version_matches) != 1:
raise WorkflowError(
f"Unable to determine conda version. 'conda --version' returned {version}"
)
else:
version = version_matches[0]
if StrictVersion(version) < StrictVersion("4.2"):
raise CreateCondaEnvironmentException(
"Conda must be version 4.2 or later, found version {}.".format(
version))
def _check_condarc(self):
if self.container_img:
# Do not check for strict priorities when running conda in an image
# Instead, we set priorities to strict ourselves in the image.
return
from snakemake.shell import shell
res = json.loads(
shell.check_output(
self._get_cmd("conda config --get channel_priority --json"),
universal_newlines=True,
stderr=subprocess.PIPE,
))
if res["get"].get("channel_priority") != "strict":
logger.warning(
"Your conda installation is not configured to use strict channel priorities. "
"This is however crucial for having robust and correct environments (for details, "
"see https://conda-forge.org/docs/user/tipsandtricks.html). "
"Please consider to configure strict priorities by executing 'conda config --set channel_priority strict'."
)
def create(self, dryrun=False):
"""Create the conda enviroment."""
from snakemake.shell import shell
# Read env file and create hash.
env_file = self.file
tmp_file = None
if not isinstance(env_file, LocalSourceFile) or isinstance(
env_file, LocalGitFile):
with tempfile.NamedTemporaryFile(delete=False,
suffix=".yaml") as tmp:
# write to temp file such that conda can open it
tmp.write(self.content)
env_file = tmp.name
tmp_file = tmp.name
else:
env_file = env_file.get_path_or_uri()
env_hash = self.hash
env_path = self.path
if self.is_containerized:
if not dryrun:
try:
shell.check_output(
singularity.shellcmd(
self._container_img.path,
"[ -d '{}' ]".format(env_path),
args=self._singularity_args,
envvars=self.get_singularity_envvars(),
quiet=True,
),
stderr=subprocess.PIPE,
)
except subprocess.CalledProcessError as e:
raise WorkflowError(
"Unable to find environment in container image. "
"Maybe a conda environment was modified without containerizing again "
"(see snakemake --containerize)?\nDetails:\n{}\n{}".
format(e, e.stderr.decode()))
return env_path
else:
# env should be present in the container
return env_path
# Check for broken environment
if os.path.exists(os.path.join(
env_path, "env_setup_start")) and not os.path.exists(
os.path.join(env_path, "env_setup_done")):
if dryrun:
logger.info(
"Incomplete Conda environment {} will be recreated.".
format(self.file.simplify_path()))
else:
logger.info(
"Removing incomplete Conda environment {}...".format(
self.file.simplify_path()))
shutil.rmtree(env_path, ignore_errors=True)
# Create environment if not already present.
if not os.path.exists(env_path):
if dryrun:
logger.info("Conda environment {} will be created.".format(
self.file.simplify_path()))
return env_path
conda = Conda(self._container_img)
logger.info("Creating conda environment {}...".format(
self.file.simplify_path()))
# Check if env archive exists. Use that if present.
env_archive = self.archive_file
try:
# Touch "start" flag file
os.makedirs(env_path, exist_ok=True)
with open(os.path.join(env_path, "env_setup_start"), "a") as f:
pass
if os.path.exists(env_archive):
logger.info("Installing archived conda packages.")
pkg_list = os.path.join(env_archive, "packages.txt")
if os.path.exists(pkg_list):
# read pacakges in correct order
# this is for newer env archives where the package list
# was stored
packages = [
os.path.join(env_archive, pkg.rstrip())
for pkg in open(pkg_list)
]
else:
# guess order
packages = glob(os.path.join(env_archive, "*.tar.bz2"))
# install packages manually from env archive
cmd = " ".join([
"conda",
"create",
"--quiet",
"--yes",
"--prefix '{}'".format(env_path),
] + packages)
if self._container_img:
cmd = singularity.shellcmd(
self._container_img.path,
cmd,
args=self._singularity_args,
envvars=self.get_singularity_envvars(),
)
out = shell.check_output(cmd,
stderr=subprocess.STDOUT,
universal_newlines=True)
else:
# Copy env file to env_path (because they can be on
# different volumes and singularity should only mount one).
# In addition, this allows to immediately see what an
# environment in .snakemake/conda contains.
target_env_file = env_path + ".yaml"
shutil.copy(env_file, target_env_file)
logger.info("Downloading and installing remote packages.")
cmd = " ".join([
self.frontend,
"env",
"create",
"--quiet",
'--file "{}"'.format(target_env_file),
'--prefix "{}"'.format(env_path),
])
if self._container_img:
cmd = singularity.shellcmd(
self._container_img.path,
cmd,
args=self._singularity_args,
envvars=self.get_singularity_envvars(),
)
out = shell.check_output(cmd,
stderr=subprocess.STDOUT,
universal_newlines=True)
# cleanup if requested
if self._cleanup is CondaCleanupMode.tarballs:
logger.info("Cleaning up conda package tarballs.")
shell.check_output("conda clean -y --tarballs")
elif self._cleanup is CondaCleanupMode.cache:
logger.info(
"Cleaning up conda package tarballs and package cache."
)
shell.check_output(
"conda clean -y --tarballs --packages")
# Touch "done" flag file
with open(os.path.join(env_path, "env_setup_done"), "a") as f:
pass
logger.debug(out)
logger.info("Environment for {} created (location: {})".format(
os.path.relpath(env_file), os.path.relpath(env_path)))
except subprocess.CalledProcessError as e:
# remove potential partially installed environment
shutil.rmtree(env_path, ignore_errors=True)
raise CreateCondaEnvironmentException(
"Could not create conda environment from {}:\n".format(
env_file) + e.output)
if tmp_file:
# temporary file was created
os.remove(tmp_file)
return env_path
def create(self, dryrun=False):
""" Create the conda enviroment."""
from snakemake.shell import shell
# Read env file and create hash.
env_file = self.file
tmp_file = None
url_scheme, *_ = urlparse(env_file)
if (url_scheme and not url_scheme == "file") or (
not url_scheme and env_file.startswith("git+file:/")):
with tempfile.NamedTemporaryFile(delete=False,
suffix=".yaml") as tmp:
tmp.write(self.content)
env_file = tmp.name
tmp_file = tmp.name
env_hash = self.hash
env_path = self.path
# Check for broken environment
if os.path.exists(os.path.join(
env_path, "env_setup_start")) and not os.path.exists(
os.path.join(env_path, "env_setup_done")):
if dryrun:
logger.info(
"Incomplete Conda environment {} will be recreated.".
format(utils.simplify_path(self.file)))
else:
logger.info(
"Removing incomplete Conda environment {}...".format(
utils.simplify_path(self.file)))
shutil.rmtree(env_path, ignore_errors=True)
# Create environment if not already present.
if not os.path.exists(env_path):
if dryrun:
logger.info("Conda environment {} will be created.".format(
utils.simplify_path(self.file)))
return env_path
conda = Conda(self._container_img)
logger.info("Creating conda environment {}...".format(
utils.simplify_path(self.file)))
# Check if env archive exists. Use that if present.
env_archive = self.archive_file
try:
# Touch "start" flag file
os.makedirs(env_path, exist_ok=True)
with open(os.path.join(env_path, "env_setup_start"), "a") as f:
pass
if os.path.exists(env_archive):
logger.info("Installing archived conda packages.")
pkg_list = os.path.join(env_archive, "packages.txt")
if os.path.exists(pkg_list):
# read pacakges in correct order
# this is for newer env archives where the package list
# was stored
packages = [
os.path.join(env_archive, pkg.rstrip())
for pkg in open(pkg_list)
]
else:
# guess order
packages = glob(os.path.join(env_archive, "*.tar.bz2"))
# install packages manually from env archive
cmd = " ".join([
"conda", "create", "--copy", "--prefix '{}'".format(
env_path)
] + packages)
if self._container_img:
cmd = singularity.shellcmd(
self._container_img.path,
cmd,
envvars=self.get_singularity_envvars(),
)
out = shell.check_output(cmd, stderr=subprocess.STDOUT)
else:
# Copy env file to env_path (because they can be on
# different volumes and singularity should only mount one).
# In addition, this allows to immediately see what an
# environment in .snakemake/conda contains.
target_env_file = env_path + ".yaml"
shutil.copy(env_file, target_env_file)
logger.info("Downloading and installing remote packages.")
cmd = " ".join([
"conda",
"env",
"create",
"--file '{}'".format(target_env_file),
"--prefix '{}'".format(env_path),
])
if self._container_img:
cmd = singularity.shellcmd(
self._container_img.path,
cmd,
envvars=self.get_singularity_envvars(),
)
out = shell.check_output(cmd, stderr=subprocess.STDOUT)
# Touch "done" flag file
with open(os.path.join(env_path, "env_setup_done"), "a") as f:
pass
logger.debug(out.decode())
logger.info("Environment for {} created (location: {})".format(
os.path.relpath(env_file), os.path.relpath(env_path)))
except subprocess.CalledProcessError as e:
# remove potential partially installed environment
shutil.rmtree(env_path, ignore_errors=True)
raise CreateCondaEnvironmentException(
"Could not create conda environment from {}:\n".format(
env_file) + e.output.decode())
if tmp_file:
# temporary file was created
os.remove(tmp_file)
return env_path
def create(self, dryrun=False):
"""Create the conda enviroment."""
from snakemake.shell import shell
self.check_is_file_based()
# Read env file and create hash.
env_file = self.file
deploy_file = None
pin_file = None
tmp_env_file = None
tmp_deploy_file = None
tmp_pin_file = None
if not isinstance(env_file, LocalSourceFile) or isinstance(
env_file, LocalGitFile):
with tempfile.NamedTemporaryFile(delete=False,
suffix=".yaml") as tmp:
# write to temp file such that conda can open it
tmp.write(self.content)
env_file = tmp.name
tmp_env_file = tmp.name
if self.post_deploy_file:
with tempfile.NamedTemporaryFile(
delete=False, suffix=".post-deploy.sh") as tmp:
# write to temp file such that conda can open it
tmp.write(self.content_deploy)
deploy_file = tmp.name
tmp_deploy_file = tmp.name
if self.pin_file:
with tempfile.NamedTemporaryFile(delete=False,
suffix="pin.txt") as tmp:
tmp.write(self.content_pin)
pin_file = tmp.name
tmp_pin_file = tmp.name
else:
env_file = env_file.get_path_or_uri()
deploy_file = self.post_deploy_file
pin_file = self.pin_file
env_path = self.address
if self.is_containerized:
if not dryrun:
try:
shell.check_output(
singularity.shellcmd(
self._container_img.path,
"[ -d '{}' ]".format(env_path),
args=self._singularity_args,
envvars=self.get_singularity_envvars(),
quiet=True,
),
stderr=subprocess.PIPE,
)
except subprocess.CalledProcessError as e:
raise WorkflowError(
"Unable to find environment in container image. "
"Maybe a conda environment was modified without containerizing again "
"(see snakemake --containerize)?\nDetails:\n{}\n{}".
format(e, e.stderr.decode()))
return env_path
else:
# env should be present in the container
return env_path
# Check for broken environment
if os.path.exists(os.path.join(
env_path, "env_setup_start")) and not os.path.exists(
os.path.join(env_path, "env_setup_done")):
if dryrun:
logger.info(
"Incomplete Conda environment {} will be recreated.".
format(self.file.simplify_path()))
else:
logger.info(
"Removing incomplete Conda environment {}...".format(
self.file.simplify_path()))
shutil.rmtree(env_path, ignore_errors=True)
# Create environment if not already present.
if not os.path.exists(env_path):
if dryrun:
logger.info("Conda environment {} will be created.".format(
self.file.simplify_path()))
return env_path
logger.info("Creating conda environment {}...".format(
self.file.simplify_path()))
env_archive = self.archive_file
try:
# Touch "start" flag file
os.makedirs(env_path, exist_ok=True)
with open(os.path.join(env_path, "env_setup_start"), "a") as f:
pass
# Check if env archive exists. Use that if present.
if os.path.exists(env_archive):
logger.info("Installing archived conda packages.")
pkg_list = os.path.join(env_archive, "packages.txt")
if os.path.exists(pkg_list):
# read pacakges in correct order
# this is for newer env archives where the package list
# was stored
packages = [
os.path.join(env_archive, pkg.rstrip())
for pkg in open(pkg_list)
]
else:
# guess order
packages = glob(os.path.join(env_archive, "*.tar.bz2"))
# install packages manually from env archive
cmd = " ".join([
"conda",
"create",
"--quiet",
"--yes",
"--prefix '{}'".format(env_path),
] + packages)
if self._container_img:
cmd = singularity.shellcmd(
self._container_img.path,
cmd,
args=self._singularity_args,
envvars=self.get_singularity_envvars(),
)
out = shell.check_output(cmd,
stderr=subprocess.STDOUT,
universal_newlines=True)
else:
def create_env(env_file, filetype="yaml"):
# Copy env file to env_path (because they can be on
# different volumes and singularity should only mount one).
# In addition, this allows to immediately see what an
# environment in .snakemake/conda contains.
target_env_file = env_path + f".{filetype}"
shutil.copy(env_file, target_env_file)
logger.info(
"Downloading and installing remote packages.")
strict_priority = ([
"conda config --set channel_priority strict &&"
] if self._container_img else [])
subcommand = [self.frontend]
yes_flag = ["--yes"]
if filetype == "yaml":
subcommand.append("env")
yes_flag = []
cmd = (strict_priority + subcommand + [
"create",
"--quiet",
'--file "{}"'.format(target_env_file),
'--prefix "{}"'.format(env_path),
] + yes_flag)
cmd = " ".join(cmd)
if self._container_img:
cmd = singularity.shellcmd(
self._container_img.path,
cmd,
args=self._singularity_args,
envvars=self.get_singularity_envvars(),
)
out = shell.check_output(cmd,
stderr=subprocess.STDOUT,
universal_newlines=True)
# cleanup if requested
if self._cleanup is CondaCleanupMode.tarballs:
logger.info("Cleaning up conda package tarballs.")
shell.check_output("conda clean -y --tarballs")
elif self._cleanup is CondaCleanupMode.cache:
logger.info(
"Cleaning up conda package tarballs and package cache."
)
shell.check_output(
"conda clean -y --tarballs --packages")
return out
if pin_file is not None:
try:
logger.info(
f"Using pinnings from {self.pin_file.get_path_or_uri()}."
)
out = create_env(pin_file, filetype="pin.txt")
except subprocess.CalledProcessError as e:
# remove potential partially installed environment
shutil.rmtree(env_path, ignore_errors=True)
advice = ""
if isinstance(self.file, LocalSourceFile):
advice = (
" If that works, make sure to update the pin file with "
f"'snakedeploy pin-conda-env {self.file.get_path_or_uri()}'."
)
logger.warning(
f"Failed to install conda environment from pin file ({self.pin_file.get_path_or_uri()}). "
f"Trying regular environment definition file.{advice}"
)
out = create_env(env_file, filetype="yaml")
else:
out = create_env(env_file, filetype="yaml")
# Execute post-deplay script if present
if deploy_file:
target_deploy_file = env_path + ".post-deploy.sh"
shutil.copy(deploy_file, target_deploy_file)
self.execute_deployment_script(env_file,
target_deploy_file)
# Touch "done" flag file
with open(os.path.join(env_path, "env_setup_done"), "a") as f:
pass
logger.debug(out)
logger.info(
f"Environment for {self.file.get_path_or_uri()} created (location: {os.path.relpath(env_path)})"
)
except subprocess.CalledProcessError as e:
# remove potential partially installed environment
shutil.rmtree(env_path, ignore_errors=True)
raise CreateCondaEnvironmentException(
f"Could not create conda environment from {env_file}:\nCommand:\n{e.cmd}\nOutput:\n{e.output}"
)
if tmp_env_file:
# temporary file was created
os.remove(tmp_env_file)
if tmp_deploy_file:
os.remove(tmp_deploy_file)
return env_path
