def run(self, context):
src = self.getarg('src', context)
dst = self.getarg('dst', context)
self.log_info(f'Render {src} --> {dst}')
# The template search path:
# 1. .
# 2. ./templates
# 3. <ocwd>
# 4. <ocwd>/templates
# where <ocwd> is the original working directory at the time when the
# se script was called
search_path = ['.', 'templates']
if '_se_cmd_cwd' in context:
search_path.extend([
context['_se_cmd_cwd'],
os.path.join(context['_se_cmd_cwd'], 'templates')
])
self.log_debug(f'Search path for template: {search_path}')
loader = jinja2.FileSystemLoader(search_path)
environment = jinja2.Environment(loader=loader)
for name, function in j2filters().items():
environment.filters[name] = function
template = environment.get_template(src)
output_text = j2render(template.render(context), context)
with open(dst, 'w') as output_file:
output_file.write(f'{output_text}\n')
def run(self, context):
src = self.getarg("src", context)
dst = Path(self.getarg("dst", context))
self.log_info(f"Jinja2 render: {src} --> {dst}")
# Prepare the Jinja render environment with the proper search path
search_path = (
Path("."),
Path(".") / "templates",
Path(context["se"]["cli"]["cwd"]),
Path(context["se"]["cli"]["cwd"]) / "templates",
)
self.log_debug(f"Template search path: {tuple(map(str, search_path))}")
j2loader = jinja2.FileSystemLoader(search_path)
j2env = jinja2.Environment(loader=j2loader)
for name, function in j2filters().items():
j2env.filters[name] = function
output = j2render(j2env.get_template(src).render(context), context)
with dst.open(mode="w") as f:
f.write(output + "\n")
if self.getarg("executable", context, default=False):
umask = os.umask(0)
os.umask(umask)
dst.chmod(dst.stat().st_mode
| (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) & ~umask)
def when(self, context):
try:
return self._when is None or j2render(
self._when, context, boolean=True)
except ScriptEngineParseJinjaError:
self.log_error("Error while parsing (Jinja2) invalid when clause "
f'"{self._when}" with context "{context}"')
raise ScriptEngineJobParseError
def loop(self, context):
loop_iter = j2render(self._loop_iter, context)
if isinstance(loop_iter, str):
try:
loop_iter = ast.literal_eval(loop_iter)
except SyntaxError:
raise ScriptEngineParseError(
'Syntax error while evaluating loop expression '
f'"{loop_iter}" in job with id {self.id}')
return self._loop_var, loop_iter
def run(self, context):
# No recursive job submission by default: check if we're already in a job
if is_batch_job() and not self.getarg('submit_from_batch_job',
default=False):
self.log_debug(
f'{self.__class__.__name__} task from within a batch '
'job and "submit_from_batch_job" is not set or false: '
'not submitting new job')
return
# Append all opts,args that should go to the submit command to a list
opt_args = []
for opt, arg in self.__dict__.items():
if opt in ('scripts',
'submit_from_batch_job') or opt.startswith('_'):
continue
opt_args.append(f'--{opt}')
if arg:
opt_args.append(j2render(arg, context))
# Build the submit command line
submit_cmd = [_SUBMIT_CMD]
submit_cmd.extend(map(str, opt_args))
scripts = self.getarg('scripts', default=None)
if scripts:
submit_cmd.append('se')
submit_cmd.extend(
map(str, scripts if isinstance(scripts, list) else [scripts]))
else:
# If no scripts were given, use the original SE command line
submit_cmd.extend(sys.argv)
self.log_debug(f'Command line for submitting job: {submit_cmd}')
# Run submit command, with handling of errors
try:
subprocess.run(submit_cmd, check=True)
except subprocess.CalledProcessError as e:
self.log_error(f'Submit command returns error: {e}')
raise ScriptEngineTaskRunError
else:
self.log_info(
'Requesting stop after submitting batch job to SLURM')
raise ScriptEngineStopException
def loop_spec(self, context):
try:
iter = j2render(self._loop, context)
except ScriptEngineParseJinjaError:
self.log_error(
"Error while parsing (Jinja2) invalid loop expression "
f'"{self._loop}" with context "{context}"')
raise ScriptEngineJobParseError
if isinstance(iter, str):
try:
iter = ast.literal_eval(iter or "None")
except (SyntaxError, ValueError):
self.log_error(
"Error while evaluating (AST) invalid loop expression "
f'"{iter}" with context "{context}"')
raise ScriptEngineJobParseError
if isinstance(iter, dict):
iter = iter.items()
vars = self._loop_vars or ("key", "value")
else:
vars = self._loop_vars or ("item", )
return iter, vars
def loop(self, context):
if self._loop:
iter, vars = self.loop_spec(context)
if iter:
for items in iter:
parsed_items = tuple(
j2render(i, context)
for i in _listy(items, type_=tuple))
vars_tuple = _listy(vars, type_=tuple)
if len(vars_tuple) == 1 and len(parsed_items) > 1:
# If only one loop variable is given and items is listy,
# assign the whole list to the variable (see #59)
yield {vars_tuple[0]: parsed_items}
else:
# Otherwise, map (zip) loop vars with items
# Note that extra vars or items are ignored!
yield dict(zip(vars_tuple, parsed_items))
else:
self.log_warning(
"Null loop after parsing loop descriptor, job is not run!")
raise StopIteration
else:
yield {}
def when(self, context):
return (self._when is None
or j2render(self._when, context, boolean=True))