def test_list_option_rendering():
o = Option("test", nargs="*")
o.value = ["A", "B"]
from jip.templates import render_template
assert render_template("${o}", o=o) == "A B"
assert render_template("${o|join(',')}", o=o) == "A,B"
def test_render_boolean_option():
@jip.tool("simple")
class SimpleTool(object):
"""\
Test tool
Usage:
tool [-t]
Options:
-t A bool option
"""
pass
tool = jip.find("simple")
assert render_template("${t|arg}", tool=tool, test="1") == ""
tool.parse_args(['-t'])
assert render_template("${t|arg}", tool=tool, test="1") == "-t"
def test_render_boolean_option():
@jip.tool("simple")
class SimpleTool(object):
"""\
Test tool
Usage:
tool [-t]
Options:
-t A bool option
"""
pass
tool = jip.find("simple")
assert render_template("${t|arg}", tool=tool, test="1") == ""
tool.parse_args(['-t'])
assert render_template("${t|arg}", tool=tool, test="1") == "-t"
def test_render_value_option():
@jip.tool("simple")
class SimpleTool(object):
"""\
Test tool
Usage:
tool [-t <in>]
Options:
-t <in> A value option
"""
pass
tool = jip.find("simple")
tool.parse_args(["-t", "infile"])
assert render_template("${t|arg}", tool=tool, test="1") == "-t infile"
assert render_template("${t|arg('-o ')}",
tool=tool, test="1") == "-o infile"
def _render(self, job, name):
ctx = {}
for o in job.tool.options:
ctx[o.name] = o
ctx['name'] = self.name
ctx['job'] = self
return render_template(
"%s%s" % ("" if not self.prefix else self.prefix, name), **ctx)
def test_render_value_option():
@jip.tool("simple")
class SimpleTool(object):
"""\
Test tool
Usage:
tool [-t <in>]
Options:
-t <in> A value option
"""
pass
tool = jip.find("simple")
tool.parse_args(["-t", "infile"])
assert render_template("${t|arg}", tool=tool, test="1") == "-t infile"
assert render_template("${t|arg('-o ')}", tool=tool,
test="1") == "-o infile"
def _render(self, job, name):
ctx = {}
for o in job.tool.options:
ctx[o.name] = o
ctx['name'] = self.name
ctx['job'] = self
return render_template(
"%s%s" % ("" if not self.prefix else self.prefix, name), **ctx
)
def _render_job_name(self, job):
ctx = {}
for o in job.tool.options:
ctx[o.name] = o
name = job.name
if not name:
name = self.name
if not name:
name = job.tool.name
return render_template(
"%s%s" % ("" if not self.prefix else self.prefix, name), **ctx)
def _render_job_name(self, job):
ctx = {}
for o in job.tool.options:
ctx[o.name] = o
name = job.name
if not name:
name = self.name
if not name:
name = job.tool.name
return render_template(
"%s%s" % ("" if not self.prefix else self.prefix, name), **ctx
)
def apply_overwrite(self, job):
"""Apply the profile and overwrite all settings that are set
in this profile
"""
log.debug("Profiles | Overwriting job profile to %s", job)
if self.name:
job.name = self._render_job_name(job)
if self.threads:
job.threads = int(self.threads)
if self.nodes is not None:
job.nodes = self.nodes
if self.tasks is not None:
job.tasks = self.tasks
if self.tasks_per_node is not None:
job.tasks_per_node = self.tasks_per_node
if self.environment is not None:
job.environment = self.environment
if self.queue is not None:
job.queue = self.queue
if self.priority is not None:
job.priority = self.priority
if self.time is not None:
job.max_time = jip.utils.parse_time(self.time)
if self.mem is not None:
job.max_memory = jip.utils.parse_mem(self.mem)
if self.log is not None:
job.stderr = self._render(job, self.log)
if self.out is not None:
job.stdout = self._render(job, self.out)
if self.account is not None:
job.account = self.account
if self.temp is not None:
job.temp = self.temp
if self.extra is not None:
job.extra = self.extra
if self.working_dir is not None:
job.working_directory = os.path.abspath(self.working_dir)
# make log files absolute
if job.stdout and not job.stdout.startswith("/"):
job.stdout = os.path.join(job.working_directory, job.stdout)
if job.stderr and not job.stderr.startswith("/"):
job.stderr = os.path.join(job.working_directory, job.stderr)
# load environment
if self.env:
current = os.environ.copy()
if job.env:
current.update(job.env)
rendered = {}
for k, v in iteritems(self.env):
rendered[k] = render_template(v, **current)
job.env.update(rendered)
if hasattr(job, 'pipe_to'):
for child in job.pipe_to:
self.apply_overwrite(child)
# check specs
for spec_name, spec in iteritems(self.specs):
if fnmatch.fnmatch(job.name, spec_name):
spec.apply_overwrite(job)
def __init__(self,
name=None,
threads=None,
nodes=None,
tasks=None,
tasks_per_node=None,
environment=None,
time=None,
queue=None,
priority=None,
log=None,
out=None,
account=None,
mem=0,
extra=None,
profile=None,
prefix=None,
temp=False,
_load=True,
env=None,
tool_name=None,
working_dir=None,
description=None,
specs=None,
_name=None,
**kwargs):
self._name = name if not _name else _name # render_template(name)
self.environment = render_template(environment)
self.nodes = render_template(nodes)
self.threads = render_template(threads)
self.tasks = render_template(tasks)
self.tasks_per_node = render_template(tasks_per_node)
self.profile = render_template(profile)
self.queue = render_template(queue)
self.time = render_template(time)
self.mem = render_template(mem)
self.priority = render_template(priority)
self.log = log
self.out = out
self.account = render_template(account)
self.prefix = render_template(prefix)
self.description = description
self.env = env
self.temp = temp
self.extra = extra
self.tool_name = tool_name
self.working_dir = working_dir
if self.working_dir is None and kwargs.get('dir', None):
self.working_dir = kwargs['dir']
self.specs = specs if specs else {}
if profile is not None and _load:
self.load(profile)
def test_render_variable():
assert render_template("${test}", test="result") == "result"
def test_render_unknonw_variable():
assert render_template("${unknown}") == "${unknown}"
def apply(self, job, pipeline=False, overwrite=False):
"""Apply this profile to the given job."""
log.debug("Profiles | Applying job profile to %s", job)
if overwrite:
self.apply_overwrite(job)
return
# set the job name or the pipeline name
# if this is a job or a pipeline
if not pipeline:
job.name = self._render_job_name(job)
elif self.name is not None:
log.info("Apply pipeline name to job: %s %s", job, self.name)
job.pipeline = self._render(job, self.name)
if self.threads and job.threads is None:
job.threads = int(self.threads)
if self.nodes is not None and job.nodes is None:
job.nodes = self.nodes
if self.tasks is not None and job.tasks is None:
job.tasks = self.tasks
if self.tasks_per_node is not None and job.tasts_per_node is None:
job.tasks_per_node = self.tasks_per_node
if self.environment is not None and job.environment is None:
job.environment = self.environment
if self.queue is not None and job.queue is None:
job.queue = self.queue
if self.priority is not None and job.priority is None:
job.priority = self.priority
if self.time is not None and job.max_time is None:
job.max_time = jip.utils.parse_time(self.time)
if self.mem is not None and job.max_memory is None:
job.max_memory = jip.utils.parse_mem(self.mem)
if self.log is not None and job.stderr is None:
job.stderr = self._render(job, self.log)
if self.out is not None and job.stdout is None:
job.stdout = self._render(job, self.out)
if self.account is not None and job.account is None:
job.account = self.account
if self.temp is not None and job.temp is None:
job.temp = self.temp
if self.extra is not None and job.extra is None:
job.extra = self.extra
if self.working_dir is not None and job.working_directory is None:
job.working_directory = os.path.abspath(self.working_dir)
# make log files absolute
if job.stdout and not job.stdout.startswith("/"):
job.stdout = os.path.join(job.working_directory, job.stdout)
if job.stderr and not job.stderr.startswith("/"):
job.stderr = os.path.join(job.working_directory, job.stderr)
# load environment
if self.env:
current = os.environ.copy()
if job.env:
current.update(job.env)
rendered = {}
for k, v in self.env.iteritems():
rendered[k] = render_template(v, **current)
job.env.update(rendered)
if hasattr(job, 'pipe_to'):
for child in job.pipe_to:
self.apply(child)
def apply_overwrite(self, job):
"""Apply the profile and overwrite all settings that are set
in this profile
"""
log.debug("Profiles | Overwriting job profile to %s", job)
if self.name:
job.name = self._render_job_name(job)
if self.threads:
job.threads = int(self.threads)
if self.nodes is not None:
job.nodes = self.nodes
if self.tasks is not None:
job.tasks = self.tasks
if self.tasks_per_node is not None:
job.tasks_per_node = self.tasks_per_node
if self.environment is not None:
job.environment = self.environment
if self.queue is not None:
job.queue = self.queue
if self.priority is not None:
job.priority = self.priority
if self.time is not None:
job.max_time = jip.utils.parse_time(self.time)
if self.mem is not None:
job.max_memory = jip.utils.parse_mem(self.mem)
if self.log is not None:
job.stderr = self._render(job, self.log)
if self.out is not None:
job.stdout = self._render(job, self.out)
if self.account is not None:
job.account = self.account
if self.temp is not None:
job.temp = self.temp
if self.extra is not None:
job.extra = self.extra
if self.working_dir is not None:
job.working_directory = os.path.abspath(self.working_dir)
# make log files absolute
if job.stdout and not job.stdout.startswith("/"):
job.stdout = os.path.join(job.working_directory, job.stdout)
if job.stderr and not job.stderr.startswith("/"):
job.stderr = os.path.join(job.working_directory, job.stderr)
# load environment
if self.env:
current = os.environ.copy()
if job.env:
current.update(job.env)
rendered = {}
for k, v in self.env.iteritems():
rendered[k] = render_template(v, **current)
job.env.update(rendered)
if hasattr(job, 'pipe_to'):
for child in job.pipe_to:
self.apply_overwrite(child)
# check specs
for spec_name, spec in self.specs.iteritems():
if fnmatch.fnmatch(job.name, spec_name):
spec.apply_overwrite(job)
def apply(self, job, pipeline=False, overwrite=False):
"""Apply this profile to the given job."""
log.debug("Profiles | Applying job profile to %s", job)
if overwrite:
self.apply_overwrite(job)
return
# set the job name or the pipeline name
# if this is a job or a pipeline
if not pipeline:
job.name = self._render_job_name(job)
elif self.name is not None:
log.info("Apply pipeline name to job: %s %s", job, self.name)
job.pipeline = self._render(job, self.name)
if self.threads and job.threads is None:
job.threads = int(self.threads)
if self.nodes is not None and job.nodes is None:
job.nodes = self.nodes
if self.tasks is not None and job.tasks is None:
job.tasks = self.tasks
if self.tasks_per_node is not None and job.tasts_per_node is None:
job.tasks_per_node = self.tasks_per_node
if self.environment is not None and job.environment is None:
job.environment = self.environment
if self.queue is not None and job.queue is None:
job.queue = self.queue
if self.priority is not None and job.priority is None:
job.priority = self.priority
if self.time is not None and job.max_time is None:
job.max_time = jip.utils.parse_time(self.time)
if self.mem is not None:
if job.max_memory is None:
job.max_memory = 0
job.max_memory += jip.utils.parse_mem(self.mem)
if self.log is not None and job.stderr is None:
job.stderr = self._render(job, self.log)
if self.out is not None and job.stdout is None:
job.stdout = self._render(job, self.out)
if self.account is not None and job.account is None:
job.account = self.account
if self.temp is not None and job.temp is None:
job.temp = self.temp
if self.extra is not None and job.extra is None:
job.extra = self.extra
if self.working_dir is not None and job.working_directory is None:
job.working_directory = os.path.abspath(self.working_dir)
# make log files absolute
if job.stdout and not job.stdout.startswith("/"):
job.stdout = os.path.join(job.working_directory, job.stdout)
if job.stderr and not job.stderr.startswith("/"):
job.stderr = os.path.join(job.working_directory, job.stderr)
# load environment
if self.env:
current = os.environ.copy()
if job.env:
current.update(job.env)
rendered = {}
for k, v in iteritems(self.env):
rendered[k] = render_template(v, **current)
job.env.update(rendered)
if hasattr(job, 'pipe_to'):
for child in job.pipe_to:
self.apply(child)
def test_render_unknonw_variable():
assert render_template("${unknown}") == "${unknown}"
def test_ext_filter():
assert render_template('${f|ext}', f='my.file.txt') == 'my.file'
assert render_template('${f|ext(all=True)}', f='my.file.txt') == 'my'
def test_render_variable():
assert render_template("${test}", test="result") == "result"
def test_list_option_rendering():
o = Option('test', nargs="*")
o.value = ["A", "B"]
from jip.templates import render_template
assert render_template("${o}", o=o) == 'A B'
assert render_template("${o|join(',')}", o=o) == 'A,B'
def test_ext_filter():
assert render_template('${f|ext}', f='my.file.txt') == 'my.file'
assert render_template('${f|ext(all=True)}', f='my.file.txt') == 'my'
def __init__(self, name=None, threads=None, nodes=None, tasks=None,
tasks_per_node=None, environment=None, time=None, queue=None,
priority=None, log=None, out=None, account=None, mem=0,
extra=None, profile=None, prefix=None, temp=False, _load=True,
env=None, tool_name=None, working_dir=None, description=None,
specs=None, _name=None, **kwargs):
self._name = name if not _name else _name # render_template(name)
self.environment = render_template(environment)
self.nodes = render_template(nodes)
self.threads = render_template(threads)
self.tasks = render_template(tasks)
self.tasks_per_node = render_template(tasks_per_node)
self.profile = render_template(profile)
self.queue = render_template(queue)
self.time = render_template(time)
self.mem = render_template(mem)
self.priority = render_template(priority)
self.log = log
self.out = out
self.account = render_template(account)
self.prefix = render_template(prefix)
self.description = description
self.env = env
self.temp = temp
self.extra = extra
self.tool_name = tool_name
self.working_dir = working_dir
if self.working_dir is None and kwargs.get('dir', None):
self.working_dir = kwargs['dir']
self.specs = specs if specs else {}
if profile is not None and _load:
self.load(profile)
