def generate_preamble(
path,
source,
basedir,
input_,
output,
params,
wildcards,
threads,
resources,
log,
config,
rulename,
conda_env,
singularity_img,
singularity_args,
env_modules,
bench_record,
jobid,
bench_iteration,
cleanup_scripts,
shadow_dir,
preamble_addendum="",
):
wrapper_path = path[7:] if path.startswith("file://") else path
snakemake = Snakemake(
input_,
output,
params,
wildcards,
threads,
resources,
log,
config,
rulename,
bench_iteration,
os.path.dirname(wrapper_path),
)
snakemake = pickle.dumps(snakemake)
# Obtain search path for current snakemake module.
# The module is needed for unpickling in the script.
# We append it at the end (as a fallback).
searchpath = SNAKEMAKE_SEARCHPATH
if singularity_img is not None:
searchpath = singularity.SNAKEMAKE_MOUNTPOINT
searchpath = '"{}"'.format(searchpath)
# For local scripts, add their location to the path in case they use path-based imports
if path.startswith("file://"):
searchpath += ', "{}"'.format(os.path.dirname(path[7:]))
return textwrap.dedent("""
######## Snakemake header ########
import sys; sys.path.extend([{searchpath}]); import pickle; snakemake = pickle.loads({snakemake}); from snakemake.logging import logger; logger.printshellcmds = {printshellcmds}; {preamble_addendum}
######## Original script #########
""").format(
searchpath=escape_backslash(searchpath),
snakemake=snakemake,
printshellcmds=logger.printshellcmds,
preamble_addendum=preamble_addendum,
)
def args(self):
# basedir
yield ', "{}"'.format(
escape_backslash(
os.path.abspath(os.path.dirname(self.snakefile.path))))
# other args
yield (", input, output, params, wildcards, threads, resources, log, "
"config, rule, use_singularity, bench_record, jobid")
def args(self):
# basedir
yield ', "{}"'.format(
escape_backslash(
os.path.abspath(os.path.dirname(self.snakefile.path))))
# other args
yield (", input, output, params, wildcards, threads, resources, log, "
"config, rule, conda_env, singularity_img, singularity_args, "
"bench_record, jobid, bench_iteration, shadow_dir")
def script(
path,
basedir,
input,
output,
params,
wildcards,
threads,
resources,
log,
config,
rulename,
conda_env,
singularity_img,
singularity_args,
bench_record,
jobid,
bench_iteration,
shadow_dir,
):
"""
Load a script from the given basedir + path and execute it.
Supports Python 3 and R.
"""
f = None
try:
path, source, language = get_source(path, basedir)
if language == "python":
wrapper_path = path[7:] if path.startswith("file://") else path
snakemake = Snakemake(
input,
output,
params,
wildcards,
threads,
resources,
log,
config,
rulename,
bench_iteration,
os.path.dirname(wrapper_path),
)
snakemake = pickle.dumps(snakemake)
# Obtain search path for current snakemake module.
# The module is needed for unpickling in the script.
# We append it at the end (as a fallback).
searchpath = SNAKEMAKE_SEARCHPATH
if singularity_img is not None:
searchpath = singularity.SNAKEMAKE_MOUNTPOINT
searchpath = '"{}"'.format(searchpath)
# For local scripts, add their location to the path in case they use path-based imports
if path.startswith("file://"):
searchpath += ', "{}"'.format(os.path.dirname(path[7:]))
preamble = textwrap.dedent("""
######## Snakemake header ########
import sys; sys.path.extend([{searchpath}]); import pickle; snakemake = pickle.loads({snakemake}); from snakemake.logging import logger; logger.printshellcmds = {printshellcmds}; __real_file__ = __file__; __file__ = {file_override};
######## Original script #########
""").format(
searchpath=escape_backslash(searchpath),
snakemake=snakemake,
printshellcmds=logger.printshellcmds,
file_override=repr(os.path.realpath(wrapper_path)),
)
elif language == "r" or language == "rmarkdown":
preamble = textwrap.dedent("""
######## Snakemake header ########
library(methods)
Snakemake <- setClass(
"Snakemake",
slots = c(
input = "list",
output = "list",
params = "list",
wildcards = "list",
threads = "numeric",
log = "list",
resources = "list",
config = "list",
rule = "character",
bench_iteration = "numeric",
scriptdir = "character",
source = "function"
)
)
snakemake <- Snakemake(
input = {},
output = {},
params = {},
wildcards = {},
threads = {},
log = {},
resources = {},
config = {},
rule = {},
bench_iteration = {},
scriptdir = {},
source = function(...){{
wd <- getwd()
setwd(snakemake@scriptdir)
source(...)
setwd(wd)
}}
)
######## Original script #########
""").format(
REncoder.encode_namedlist(input),
REncoder.encode_namedlist(output),
REncoder.encode_namedlist(params),
REncoder.encode_namedlist(wildcards),
threads,
REncoder.encode_namedlist(log),
REncoder.encode_namedlist({
name: value
for name, value in resources.items()
if name != "_cores" and name != "_nodes"
}),
REncoder.encode_dict(config),
REncoder.encode_value(rulename),
REncoder.encode_numeric(bench_iteration),
REncoder.encode_value(
os.path.dirname(path[7:]) if path.
startswith("file://") else os.path.dirname(path)),
)
elif language == "julia":
preamble = textwrap.dedent("""
######## Snakemake header ########
struct Snakemake
input::Dict
output::Dict
params::Dict
wildcards::Dict
threads::Int64
log::Dict
resources::Dict
config::Dict
rule::String
bench_iteration
scriptdir::String
#source::Any
end
snakemake = Snakemake(
{}, #input::Dict
{}, #output::Dict
{}, #params::Dict
{}, #wildcards::Dict
{}, #threads::Int64
{}, #log::Dict
{}, #resources::Dict
{}, #config::Dict
{}, #rule::String
{}, #bench_iteration::Int64
{}, #scriptdir::String
#, #source::Any
)
######## Original script #########
""".format(
JuliaEncoder.encode_namedlist(input),
JuliaEncoder.encode_namedlist(output),
JuliaEncoder.encode_namedlist(params),
JuliaEncoder.encode_namedlist(wildcards),
JuliaEncoder.encode_value(threads),
JuliaEncoder.encode_namedlist(log),
JuliaEncoder.encode_namedlist({
name: value
for name, value in resources.items()
if name != "_cores" and name != "_nodes"
}),
JuliaEncoder.encode_dict(config),
JuliaEncoder.encode_value(rulename),
JuliaEncoder.encode_value(bench_iteration),
JuliaEncoder.encode_value(
os.path.dirname(path[7:]) if path.
startswith("file://") else os.path.dirname(path)),
).replace("'", '"'))
else:
raise ValueError(
"Unsupported script: Expecting either Python (.py), R (.R), RMarkdown (.Rmd) or Julia (.jl) script."
)
dir = ".snakemake/scripts"
os.makedirs(dir, exist_ok=True)
with tempfile.NamedTemporaryFile(suffix="." + os.path.basename(path),
dir=dir,
delete=False) as f:
if not language == "rmarkdown":
f.write(preamble.encode())
f.write(source)
else:
# Insert Snakemake object after the RMarkdown header
code = source.decode()
pos = next(islice(re.finditer(r"---\n", code), 1,
2)).start() + 3
f.write(str.encode(code[:pos]))
preamble = textwrap.dedent("""
```{r, echo=FALSE, message=FALSE, warning=FALSE}
%s
```
""" % preamble)
f.write(preamble.encode())
f.write(str.encode(code[pos:]))
if language == "python":
py_exec = sys.executable
if conda_env is not None:
py = os.path.join(conda_env, "bin", "python")
if os.path.exists(py):
out = subprocess.check_output(
[py, "--version"],
stderr=subprocess.STDOUT,
universal_newlines=True,
)
ver = tuple(
map(int,
PY_VER_RE.match(out).group("ver_min").split(".")))
if ver >= MIN_PY_VERSION:
# Python version is new enough, make use of environment
# to execute script
py_exec = "python"
else:
logger.warning(
"Conda environment defines Python "
"version < {0}.{1}. Using Python of the "
"master process to execute "
"script. Note that this cannot be avoided, "
"because the script uses data structures from "
"Snakemake which are Python >={0}.{1} "
"only.".format(*MIN_PY_VERSION))
if singularity_img is not None:
# use python from image
py_exec = "python"
# use the same Python as the running process or the one from the environment
shell("{py_exec} {f.name:q}", bench_record=bench_record)
elif language == "r":
if conda_env is not None and "R_LIBS" in os.environ:
logger.warning("R script job uses conda environment but "
"R_LIBS environment variable is set. This "
"is likely not intended, as R_LIBS can "
"interfere with R packages deployed via "
"conda. Consider running `unset R_LIBS` or "
"remove it entirely before executing "
"Snakemake.")
shell("Rscript --vanilla {f.name:q}", bench_record=bench_record)
elif language == "rmarkdown":
if len(output) != 1:
raise WorkflowError(
"RMarkdown scripts (.Rmd) may only have a single output file."
)
out = os.path.abspath(output[0])
shell(
'Rscript --vanilla -e \'rmarkdown::render("{f.name}", output_file="{out}", quiet=TRUE, knit_root_dir = "{workdir}", params = list(rmd="{f.name}"))\'',
bench_record=bench_record,
workdir=os.getcwd(),
)
elif language == "julia":
shell("julia {f.name:q}", bench_record=bench_record)
except URLError as e:
raise WorkflowError(e)
finally:
if f:
os.remove(f.name)
def script(path, basedir, input, output, params, wildcards, threads, resources,
log, config, rulename, conda_env, singularity_img, singularity_args,
bench_record, jobid):
"""
Load a script from the given basedir + path and execute it.
Supports Python 3 and R.
"""
if not path.startswith("http"):
if path.startswith("file://"):
path = path[7:]
elif path.startswith("file:"):
path = path[5:]
if not os.path.isabs(path):
path = os.path.abspath(os.path.join(basedir, path))
path = "file://" + path
path = format(path, stepout=1)
if path.startswith("file://"):
sourceurl = "file:" + pathname2url(path[7:])
else:
sourceurl = path
f = None
try:
with urlopen(sourceurl) as source:
if path.endswith(".py"):
snakemake = Snakemake(input, output, params, wildcards,
threads, resources, log, config,
rulename)
snakemake = pickle.dumps(snakemake)
# Obtain search path for current snakemake module.
# The module is needed for unpickling in the script.
# We append it at the end (as a fallback).
searchpath = os.path.dirname(os.path.dirname(__file__))
preamble = textwrap.dedent("""
######## Snakemake header ########
import sys; sys.path.append("{}"); import pickle; snakemake = pickle.loads({}); from snakemake.logging import logger; logger.printshellcmds = {}
######## Original script #########
""").format(escape_backslash(searchpath), snakemake,
logger.printshellcmds)
elif path.endswith(".R") or path.endswith(".Rmd"):
preamble = textwrap.dedent("""
######## Snakemake header ########
library(methods)
Snakemake <- setClass(
"Snakemake",
slots = c(
input = "list",
output = "list",
params = "list",
wildcards = "list",
threads = "numeric",
log = "list",
resources = "list",
config = "list",
rule = "character"
)
)
snakemake <- Snakemake(
input = {},
output = {},
params = {},
wildcards = {},
threads = {},
log = {},
resources = {},
config = {},
rule = {}
)
######## Original script #########
""").format(
REncoder.encode_namedlist(input),
REncoder.encode_namedlist(output),
REncoder.encode_namedlist(params),
REncoder.encode_namedlist(wildcards), threads,
REncoder.encode_namedlist(log),
REncoder.encode_namedlist({
name: value
for name, value in resources.items()
if name != "_cores" and name != "_nodes"
}), REncoder.encode_dict(config),
REncoder.encode_value(rulename))
else:
raise ValueError(
"Unsupported script: Expecting either Python (.py), R (.R) or RMarkdown (.Rmd) script."
)
if path.startswith("file://"):
# in case of local path, use the same directory
dir = os.path.dirname(path)[7:]
prefix = ".snakemake."
else:
dir = ".snakemake/scripts"
prefix = ""
os.makedirs(dir, exist_ok=True)
with tempfile.NamedTemporaryFile(suffix="." +
os.path.basename(path),
prefix=prefix,
dir=dir,
delete=False) as f:
if not path.endswith(".Rmd"):
f.write(preamble.encode())
f.write(source.read())
else:
# Insert Snakemake object after the RMarkdown header
code = source.read().decode()
pos = code.rfind("---")
f.write(str.encode(code[:pos + 3]))
preamble = textwrap.dedent("""
```{r, echo=FALSE, message=FALSE, warning=FALSE}
%s
```
""" % preamble)
f.write(preamble.encode())
f.write(str.encode(code[pos + 3:]))
if path.endswith(".py"):
py_exec = sys.executable
if conda_env is not None:
py = os.path.join(conda_env, "bin", "python")
if os.path.exists(py):
out = subprocess.check_output([py, "--version"],
stderr=subprocess.STDOUT,
universal_newlines=True)
ver = tuple(
map(
int,
PY_VER_RE.match(out).group("ver_min").split(
".")))
if ver >= MIN_PY_VERSION:
# Python version is new enough, make use of environment
# to execute script
py_exec = "python"
else:
logger.warning(
"Conda environment defines Python "
"version < {0}.{1}. Using Python of the "
"master process to execute "
"script. Note that this cannot be avoided, "
"because the script uses data structures from "
"Snakemake which are Python >={0}.{1} "
"only.".format(*MIN_PY_VERSION))
if singularity_img is not None:
# use python from image
py_exec = "python"
# use the same Python as the running process or the one from the environment
shell("{py_exec} {f.name}", bench_record=bench_record)
elif path.endswith(".R"):
shell("Rscript {f.name}", bench_record=bench_record)
elif path.endswith(".Rmd"):
if len(output) != 1:
raise WorkflowError(
"RMarkdown scripts (.Rmd) may only have a single output file."
)
out = os.path.abspath(output[0])
shell(
"Rscript -e 'rmarkdown::render(\"{f.name}\", output_file=\"{out}\", quiet=TRUE, params = list(rmd=\"{f.name}\"))'",
bench_record=bench_record)
except URLError as e:
raise WorkflowError(e)
finally:
if f:
os.remove(f.name)