def get_source(path, basedir=".", wildcards=None, params=None):
source = None
if not path.startswith("http") and not path.startswith("git+file"):
if path.startswith("file://"):
path = path[7:]
elif path.startswith("file:"):
path = path[5:]
if not os.path.isabs(path):
path = smart_join(basedir, path, abspath=True)
if is_local_file(path):
path = "file://" + path
if wildcards is not None and params is not None:
# Format path if wildcards are given.
path = format(path, wildcards=wildcards, params=params)
if path.startswith("file://"):
sourceurl = "file:" + pathname2url(path[7:])
elif path.startswith("git+file"):
source = git_content(path).encode()
(root_path, file_path, version) = split_git_path(path)
path = path.rstrip("@" + version)
else:
sourceurl = path
if source is None:
with urlopen(sourceurl) as source:
source = source.read()
language = get_language(path, source)
return path, source, language
def infer_source_file(path_or_uri, basedir: SourceFile = None):
if isinstance(path_or_uri, SourceFile):
if basedir is None or isinstance(path_or_uri, HostingProviderFile):
return path_or_uri
else:
path_or_uri = path_or_uri.get_path_or_uri()
if isinstance(path_or_uri, Path):
path_or_uri = str(path_or_uri)
if not isinstance(path_or_uri, str):
raise SourceFileError(
"must be given as Python string or one of the predefined source file marker types (see docs)"
)
if is_local_file(path_or_uri):
# either local file or relative to some remote basedir
for schema in ("file://", "file:"):
if path_or_uri.startswith(schema):
path_or_uri = path_or_uri[len(schema):]
break
if not os.path.isabs(path_or_uri) and basedir is not None:
return basedir.join(path_or_uri)
return LocalSourceFile(path_or_uri)
if path_or_uri.startswith("git+file:"):
try:
root_path, file_path, ref = split_git_path(path_or_uri)
except Exception as e:
raise WorkflowError(
f"Failed to read source {path_or_uri} from git repo.", e)
return LocalGitFile(root_path, file_path, ref=ref)
# something else
return GenericSourceFile(path_or_uri)
def conda_env_file(self):
if self._conda_env_file is None:
expanded_env = self.rule.expand_conda_env(self.wildcards_dict)
if expanded_env is not None:
# Normalize 'file:///my/path.yml' to '/my/path.yml'
if is_local_file(expanded_env):
self._conda_env_file = parse_uri(expanded_env).uri_path
else:
self._conda_env_file = expanded_env
return self._conda_env_file
def get_resource_as_string(path_or_uri):
import requests
if is_local_file(path_or_uri):
return open(Path(__file__).parent.parent / "template" /
path_or_uri).read()
else:
r = requests.get(path_or_uri)
if r.status_code == requests.codes.ok:
return r.text
raise WorkflowError("Failed to download resource needed for "
"report: {}".format(path_or_uri))
def outputs_older_than_script_or_notebook(self):
"""return output that's older than script, i.e. script has changed"""
path = self.rule.script or self.rule.notebook
if not path:
return
if self.rule.basedir:
# needed if rule is included from another subdirectory
path = self.rule.basedir.join(path).get_path_or_uri()
if is_local_file(path):
assert os.path.exists(path), "cannot find {0}".format(path)
script_mtime = os.lstat(path).st_mtime
for f in self.expanded_output:
if f.exists:
if not f.is_newer(script_mtime):
yield f
def expand_conda_env(self, wildcards, params=None, input=None):
from snakemake.common import is_local_file
from snakemake.sourcecache import SourceFile, infer_source_file
from snakemake.deployment.conda import (
is_conda_env_file,
CondaEnvFileSpec,
CondaEnvNameSpec,
)
conda_env = self._conda_env
if callable(conda_env):
conda_env, _ = self.apply_input_function(conda_env,
wildcards=wildcards,
params=params,
input=input)
if conda_env is None:
return None
if is_conda_env_file(conda_env):
if not isinstance(conda_env, SourceFile):
if is_local_file(conda_env) and not os.path.isabs(conda_env):
# Conda env file paths are considered to be relative to the directory of the Snakefile
# hence we adjust the path accordingly.
# This is not necessary in case of receiving a SourceFile.
conda_env = self.basedir.join(conda_env)
else:
# infer source file from unmodified uri or path
conda_env = infer_source_file(conda_env)
conda_env = CondaEnvFileSpec(conda_env, rule=self)
else:
conda_env = CondaEnvNameSpec(conda_env)
conda_env = conda_env.apply_wildcards(wildcards, self)
conda_env.check()
return conda_env
def is_persistently_cacheable(self, path_or_uri):
# TODO remove special git url handling once included in smart_open
if path_or_uri.startswith("git+file:"):
return False
return is_local_file(path_or_uri) and self.cacheable_prefixes.match(
path_or_uri)
def is_local(self):
return is_local_file(self.url)
def apply_wildcards(self, wildcards, rule):
filepath = self.file.apply_wildcards(wildcards)
if is_local_file(filepath):
# Normalize 'file:///my/path.yml' to '/my/path.yml'
filepath = parse_uri(filepath).uri_path
return CondaEnvFileSpec(filepath, rule)
def notebook(
path,
basedir,
input,
output,
params,
wildcards,
threads,
resources,
log,
config,
rulename,
conda_env,
conda_base_path,
container_img,
singularity_args,
env_modules,
bench_record,
jobid,
bench_iteration,
cleanup_scripts,
shadow_dir,
edit,
runtime_sourcecache_path,
):
"""
Load a script from the given basedir + path and execute it.
"""
draft = False
if edit is not None:
if is_local_file(path):
if not os.path.isabs(path):
local_path = os.path.join(basedir, path)
else:
local_path = path
if not os.path.exists(local_path):
# draft the notebook, it does not exist yet
language = None
draft = True
path = "file://{}".format(os.path.abspath(local_path))
if path.endswith(".py.ipynb"):
language = "jupyter_python"
elif path.endswith(".r.ipynb"):
language = "jupyter_r"
else:
raise WorkflowError(
"Notebook to edit has to end on .py.ipynb or .r.ipynb in order "
"to decide which programming language shall be used.")
else:
raise WorkflowError(
"Notebook {} is not local, but edit mode is only allowed for "
"local notebooks.".format(path))
if not draft:
path, source, language, is_local = get_source(
path, SourceCache(runtime_sourcecache_path), basedir, wildcards,
params)
else:
source = None
is_local = True
exec_class = get_exec_class(language)
executor = exec_class(
path,
source,
basedir,
input,
output,
params,
wildcards,
threads,
resources,
log,
config,
rulename,
conda_env,
conda_base_path,
container_img,
singularity_args,
env_modules,
bench_record,
jobid,
bench_iteration,
cleanup_scripts,
shadow_dir,
is_local,
)
if draft:
executor.draft(listen=edit)
else:
executor.evaluate(edit=edit)
def notebook(
path,
basedir,
input,
output,
params,
wildcards,
threads,
resources,
log,
config,
rulename,
conda_env,
conda_base_path,
container_img,
singularity_args,
env_modules,
bench_record,
jobid,
bench_iteration,
cleanup_scripts,
shadow_dir,
edit,
runtime_sourcecache_path,
):
"""
Load a script from the given basedir + path and execute it.
"""
draft = False
if edit is not None:
if is_local_file(path):
if not os.path.isabs(path):
local_path = os.path.join(basedir, path)
else:
local_path = path
if not os.path.exists(local_path):
# draft the notebook, it does not exist yet
language = None
draft = True
path = "file://{}".format(os.path.abspath(local_path))
if path.endswith(".py.ipynb"):
language = "jupyter_python"
elif path.endswith(".r.ipynb"):
language = "jupyter_r"
else:
raise WorkflowError(
"Notebook to edit has to end on .py.ipynb or .r.ipynb in order "
"to decide which programming language shall be used.")
else:
raise WorkflowError(
"Notebook {} is not local, but edit mode is only allowed for "
"local notebooks.".format(path))
if not draft:
path, source, language, is_local = get_source(
path, SourceCache(runtime_sourcecache_path), basedir, wildcards,
params)
else:
source = None
is_local = True
path = infer_source_file(path)
exec_class = get_exec_class(language)
executor = exec_class(
path,
source,
basedir,
input,
output,
params,
wildcards,
threads,
resources,
log,
config,
rulename,
conda_env,
conda_base_path,
container_img,
singularity_args,
env_modules,
bench_record,
jobid,
bench_iteration,
cleanup_scripts,
shadow_dir,
is_local,
)
if edit is None:
executor.evaluate(edit=edit)
elif edit.draft_only:
executor.draft()
msg = "Generated skeleton notebook:\n{} ".format(path)
if conda_env and not container_img:
msg += (
"\n\nEditing with VSCode:\nOpen notebook, run command 'Select notebook kernel' (Ctrl+Shift+P or Cmd+Shift+P), and choose:"
"\n{}\n".format(
str(
Path(conda_env) / "bin" /
executor.get_interpreter_exec())))
msg += ("\nEditing with Jupyter CLI:"
"\nconda activate {}\njupyter notebook {}\n".format(
conda_env, path))
logger.info(msg)
elif draft:
executor.draft_and_edit(listen=edit)
else:
executor.evaluate(edit=edit)
def create(self, dryrun=False):
"""Create the conda enviroment."""
from snakemake.shell import shell
# Read env file and create hash.
env_file = self.file
tmp_file = None
if not is_local_file(env_file) or env_file.startswith("git+file:/"):
with tempfile.NamedTemporaryFile(delete=False, suffix=".yaml") as tmp:
tmp.write(self.content)
env_file = tmp.name
tmp_file = tmp.name
env_hash = self.hash
env_path = self.path
if self.is_containerized:
if not dryrun:
try:
shell.check_output(
singularity.shellcmd(
self._container_img.path,
"[ -d '{}' ]".format(env_path),
args=self._singularity_args,
envvars=self.get_singularity_envvars(),
quiet=True,
),
stderr=subprocess.PIPE,
)
except subprocess.CalledProcessError as e:
raise WorkflowError(
"Unable to find environment in container image. "
"Maybe a conda environment was modified without containerizing again "
"(see snakemake --containerize)?\nDetails:\n{}\n{}".format(
e, e.stderr.decode()
)
)
return env_path
else:
# env should be present in the container
return env_path
# Check for broken environment
if os.path.exists(
os.path.join(env_path, "env_setup_start")
) and not os.path.exists(os.path.join(env_path, "env_setup_done")):
if dryrun:
logger.info(
"Incomplete Conda environment {} will be recreated.".format(
utils.simplify_path(self.file)
)
)
else:
logger.info(
"Removing incomplete Conda environment {}...".format(
utils.simplify_path(self.file)
)
)
shutil.rmtree(env_path, ignore_errors=True)
# Create environment if not already present.
if not os.path.exists(env_path):
if dryrun:
logger.info(
"Conda environment {} will be created.".format(
utils.simplify_path(self.file)
)
)
return env_path
conda = Conda(self._container_img)
logger.info(
"Creating conda environment {}...".format(
utils.simplify_path(self.file)
)
)
# Check if env archive exists. Use that if present.
env_archive = self.archive_file
try:
# Touch "start" flag file
os.makedirs(env_path, exist_ok=True)
with open(os.path.join(env_path, "env_setup_start"), "a") as f:
pass
if os.path.exists(env_archive):
logger.info("Installing archived conda packages.")
pkg_list = os.path.join(env_archive, "packages.txt")
if os.path.exists(pkg_list):
# read pacakges in correct order
# this is for newer env archives where the package list
# was stored
packages = [
os.path.join(env_archive, pkg.rstrip())
for pkg in open(pkg_list)
]
else:
# guess order
packages = glob(os.path.join(env_archive, "*.tar.bz2"))
# install packages manually from env archive
cmd = " ".join(
[
"conda",
"create",
"--quiet",
"--yes",
"--prefix '{}'".format(env_path),
]
+ packages
)
if self._container_img:
cmd = singularity.shellcmd(
self._container_img.path,
cmd,
args=self._singularity_args,
envvars=self.get_singularity_envvars(),
)
out = shell.check_output(
cmd, stderr=subprocess.STDOUT, universal_newlines=True
)
else:
# Copy env file to env_path (because they can be on
# different volumes and singularity should only mount one).
# In addition, this allows to immediately see what an
# environment in .snakemake/conda contains.
target_env_file = env_path + ".yaml"
shutil.copy(env_file, target_env_file)
logger.info("Downloading and installing remote packages.")
cmd = " ".join(
[
self.frontend,
"env",
"create",
"--quiet",
'--file "{}"'.format(target_env_file),
'--prefix "{}"'.format(env_path),
]
)
if self._container_img:
cmd = singularity.shellcmd(
self._container_img.path,
cmd,
args=self._singularity_args,
envvars=self.get_singularity_envvars(),
)
out = shell.check_output(
cmd, stderr=subprocess.STDOUT, universal_newlines=True
)
# cleanup if requested
if self._cleanup is CondaCleanupMode.tarballs:
logger.info("Cleaning up conda package tarballs.")
shell.check_output("conda clean -y --tarballs")
elif self._cleanup is CondaCleanupMode.cache:
logger.info(
"Cleaning up conda package tarballs and package cache."
)
shell.check_output("conda clean -y --tarballs --packages")
# Touch "done" flag file
with open(os.path.join(env_path, "env_setup_done"), "a") as f:
pass
logger.debug(out)
logger.info(
"Environment for {} created (location: {})".format(
os.path.relpath(env_file), os.path.relpath(env_path)
)
)
except subprocess.CalledProcessError as e:
# remove potential partially installed environment
shutil.rmtree(env_path, ignore_errors=True)
raise CreateCondaEnvironmentException(
"Could not create conda environment from {}:\n".format(env_file)
+ e.output
)
if tmp_file:
# temporary file was created
os.remove(tmp_file)
return env_path
def validate(data, schema, set_default=True):
"""Validate data with JSON schema at given path.
Args:
data (object): data to validate. Can be a config dict or a pandas data frame.
schema (str): Path to JSON schema used for validation. The schema can also be
in YAML format. If validating a pandas data frame, the schema has to
describe a row record (i.e., a dict with column names as keys pointing
to row values). See https://json-schema.org. The path is interpreted
relative to the Snakefile when this function is called.
set_default (bool): set default values defined in schema. See
https://python-jsonschema.readthedocs.io/en/latest/faq/ for more
information
"""
frame = inspect.currentframe().f_back
workflow = frame.f_globals.get("workflow")
if workflow and workflow.modifier.skip_validation:
# skip if a corresponding modifier has been defined
return
try:
import jsonschema
from jsonschema import validators, RefResolver
except ImportError:
raise WorkflowError(
"The Python 3 package jsonschema must be installed "
"in order to use the validate directive.")
schemafile = schema
if not os.path.isabs(schemafile):
frame = inspect.currentframe().f_back
# if workflow object is not available this has not been started from a workflow
if workflow:
schemafile = smart_join(workflow.current_basedir, schemafile)
source = workflow.sourcecache.open(schemafile) if workflow else schemafile
schema = _load_configfile(source, filetype="Schema")
if is_local_file(schemafile):
resolver = RefResolver(
urljoin("file:", schemafile),
schema,
handlers={
"file":
lambda uri: _load_configfile(re.sub("^file://", "", uri))
},
)
else:
resolver = RefResolver(
schemafile,
schema,
)
# Taken from https://python-jsonschema.readthedocs.io/en/latest/faq/
def extend_with_default(validator_class):
validate_properties = validator_class.VALIDATORS["properties"]
def set_defaults(validator, properties, instance, schema):
for property, subschema in properties.items():
if "default" in subschema:
instance.setdefault(property, subschema["default"])
for error in validate_properties(validator, properties, instance,
schema):
yield error
return validators.extend(validator_class, {"properties": set_defaults})
Validator = validators.validator_for(schema)
if Validator.META_SCHEMA["$schema"] != schema["$schema"]:
logger.warning(
"No validator found for JSON Schema version identifier '{}'".
format(schema["$schema"]))
logger.warning(
"Defaulting to validator for JSON Schema version '{}'".format(
Validator.META_SCHEMA["$schema"]))
logger.warning("Note that schema file may not be validated correctly.")
DefaultValidator = extend_with_default(Validator)
if not isinstance(data, dict):
try:
import pandas as pd
recordlist = []
if isinstance(data, pd.DataFrame):
for i, record in enumerate(data.to_dict("records")):
record = {
k: v
for k, v in record.items() if not pd.isnull(v)
}
try:
if set_default:
DefaultValidator(
schema, resolver=resolver).validate(record)
recordlist.append(record)
else:
jsonschema.validate(record,
schema,
resolver=resolver)
except jsonschema.exceptions.ValidationError as e:
raise WorkflowError(
"Error validating row {} of data frame.".format(i),
e)
if set_default:
newdata = pd.DataFrame(recordlist, data.index)
newcol = ~newdata.columns.isin(data.columns)
n = len(data.columns)
for col in newdata.loc[:, newcol].columns:
data.insert(n, col, newdata.loc[:, col])
n = n + 1
return
except ImportError:
pass
raise WorkflowError("Unsupported data type for validation.")
else:
try:
if set_default:
DefaultValidator(schema, resolver=resolver).validate(data)
else:
jsonschema.validate(data, schema, resolver=resolver)
except jsonschema.exceptions.ValidationError as e:
raise WorkflowError("Error validating config file.", e)
