def remote(self, value, *args, provider_kws=None, **kwargs):
if isinstance(value, str):
values = [value]
elif isinstance(value, collections.abc.Iterable):
values = value
else:
raise TypeError(f"Invalid type ({type(value)}) passed to remote: {value}")
provider_remote_list = []
for value in values:
# select provider
o = parse_uri(value)
Provider = self.protocol_mapping.get(o.scheme)
if Provider is None:
raise TypeError(f"Could not find remote provider for: {value}")
# use provider's remote
provider_kws = {} if provider_kws is None else provider_kws.copy()
provider_remote_list.append(
Provider(**provider_kws).remote(value, *args, **kwargs)
)
return (
provider_remote_list[0]
if len(provider_remote_list) == 1
else provider_remote_list
)
def conda_env_file(self):
if self._conda_env_file is None:
expanded_env = self.rule.expand_conda_env(self.wildcards_dict)
if expanded_env is not None:
# Normalize 'file:///my/path.yml' to '/my/path.yml'
if is_local_file(expanded_env):
self._conda_env_file = parse_uri(expanded_env).uri_path
else:
self._conda_env_file = expanded_env
return self._conda_env_file
def create_archive(self):
"""Create self-contained archive of environment."""
from snakemake.shell import shell
try:
import yaml
except ImportError:
raise WorkflowError("Error importing PyYAML. "
"Please install PyYAML to archive workflows.")
# importing requests locally because it interferes with instantiating conda environments
import requests
env_archive = self.archive_file
if os.path.exists(env_archive):
return env_archive
try:
# Download
logger.info(
"Downloading packages for conda environment {}...".format(
self.file.get_path_or_uri()))
os.makedirs(env_archive, exist_ok=True)
try:
out = shell.check_output(
"conda list --explicit --prefix '{}'".format(self.path),
stderr=subprocess.STDOUT,
universal_newlines=True,
)
logger.debug(out)
except subprocess.CalledProcessError as e:
raise WorkflowError("Error exporting conda packages:\n" +
e.output)
with open(os.path.join(env_archive, "packages.txt"),
"w") as pkg_list:
for l in out.split("\n"):
if l and not l.startswith("#") and not l.startswith("@"):
pkg_url = l
logger.info(pkg_url)
parsed = parse_uri(pkg_url)
pkg_name = os.path.basename(parsed.uri_path)
# write package name to list
print(pkg_name, file=pkg_list)
# download package
pkg_path = os.path.join(env_archive, pkg_name)
with open(pkg_path, "wb") as copy:
r = requests.get(pkg_url)
r.raise_for_status()
copy.write(r.content)
try:
if pkg_path.endswith(".conda"):
assert zipfile.ZipFile(
pkg_path).testzip() is None
else:
tarfile.open(pkg_path)
except:
raise WorkflowError(
"Package is invalid tar/zip archive: {}".
format(pkg_url))
except (
requests.exceptions.ChunkedEncodingError,
requests.exceptions.HTTPError,
) as e:
shutil.rmtree(env_archive)
raise WorkflowError(
"Error downloading conda package {}.".format(pkg_url))
except (Exception, BaseException) as e:
shutil.rmtree(env_archive)
raise e
return env_archive
def path(self):
if self.is_local:
return parse_uri(self.url).uri_path
return os.path.join(self._img_dir, self.hash) + ".simg"
def apply_wildcards(self, wildcards, rule):
filepath = self.file.apply_wildcards(wildcards)
if is_local_file(filepath):
# Normalize 'file:///my/path.yml' to '/my/path.yml'
filepath = parse_uri(filepath).uri_path
return CondaEnvFileSpec(filepath, rule)