def serialize(self, value):
""""""
if not value:
return ""
else:
# ensure uniqueness
if self._unique:
value = make_unique(value)
# check min_len and max_len
self._check_len(value)
return ",".join(str(self._inst.serialize(elem)) for elem in value)
def _check_choices(self, value):
if not self._choices:
return
unknown = []
for v in value:
if v not in self._choices:
unknown.append(v)
if unknown:
str_repr = lambda value: ",".join(str(v) for v in value)
raise ValueError("invalid parameter value(s) '{}', valid choices are '{}'".format(
str_repr(make_unique(unknown)), str_repr(self._choices)))
def parse(self, inp):
""""""
if not inp:
ret = tuple()
elif isinstance(inp, (tuple, list)) or is_lazy_iterable(inp):
ret = make_tuple(inp)
else:
ret = tuple(self._inst.parse(elem) for elem in inp.split(","))
# ensure uniqueness
if self._unique:
ret = make_unique(ret)
# check min_len and max_len
self._check_len(ret)
return ret
def create(self, postfix=None, render_variables=None, **kwargs):
# merge kwargs and instance attributes
c = self.get_config(kwargs)
# some sanity checks
if not c.file_name:
raise ValueError("file_name must not be empty")
elif not c.executable:
raise ValueError("executable must not be empty")
# default render variables
if not render_variables:
render_variables = {}
# add postfix to render variables
if postfix and "file_postfix" not in render_variables:
render_variables["file_postfix"] = postfix
# add output_uri to render variables
if c.output_uri and "output_uri" not in render_variables:
render_variables["output_uri"] = c.output_uri
# linearize render variables
render_variables = self.linearize_render_variables(render_variables)
# prepare the job file
job_file = self.postfix_file(os.path.join(c.dir, c.file_name), postfix)
# prepare input files
def prepare_input(tpl):
# consider strings to be the base filename and use an identical source with no options
if isinstance(tpl, six.string_types):
tpl = (os.path.basename(tpl), tpl, "")
path, src, opts = (tpl + ("", ""))[:3]
path = self.postfix_file(path, postfix)
if src and get_scheme(src) in ("file", None):
src = self.provide_input(os.path.abspath(src), postfix, c.dir,
render_variables)
if not c.absolute_paths:
src = os.path.basename(src)
if src == path:
src = ""
return (path, src, opts) if opts else (path, src)
c.input_files = list(map(prepare_input, c.input_files))
# postfix the executable
pf_executable = self.postfix_file(os.path.basename(c.executable),
postfix)
executable_is_file = pf_executable in [
os.path.basename(tpl[0]) for tpl in c.input_files
]
if executable_is_file:
c.executable = pf_executable
# ensure that log files are contained in the output files
if c.log and c.log not in c.output_files:
c.output_files.append(c.log)
if c.stdout and c.stdout not in c.output_files:
c.output_files.append(c.stdout)
if c.stderr and c.stderr not in c.output_files:
c.output_files.append(c.stderr)
# ensure a correct format of output files
def prepare_output(tpl):
# consider strings to be the filename and when output_uri is set, use it
# as the URL, otherwise it's also empty
if isinstance(tpl, six.string_types):
dst = os.path.join(
c.output_uri,
os.path.basename(tpl)) if c.output_uri else ""
tpl = (tpl, dst)
path, dst, opts = (tpl + ("", ""))[:3]
if c.postfix_output_files:
path = self.postfix_file(path, postfix)
if dst:
dst = self.postfix_file(dst, postfix)
if c.overwrite_output_files and "overwrite" not in opts:
opts += (";" if opts else "") + "overwrite=yes"
return (path, dst, opts) if opts else (path, dst)
c.output_files = map(prepare_output, c.output_files)
# also postfix log files
if c.postfix_output_files:
c.log = c.log and self.postfix_file(c.log, postfix)
c.stdout = c.stdout and self.postfix_file(c.stdout, postfix)
c.stderr = c.stderr and self.postfix_file(c.stderr, postfix)
# custom log file
if c.custom_log_file:
c.output_files.append(prepare_output(c.custom_log_file))
c.custom_log_file = self.postfix_file(c.custom_log_file, postfix)
# job file content
content = []
content.append(("executable", c.executable))
if c.arguments:
content.append(("arguments", c.arguments))
if c.job_name:
content.append(("jobName", c.job_name))
if c.input_files:
content.append(("inputFiles", make_unique(c.input_files)))
if c.output_files:
content.append(("outputFiles", make_unique(c.output_files)))
if c.log:
content.append(("gmlog", c.log))
if c.stdout:
content.append(("stdout", c.stdout))
if c.stderr:
content.append(("stderr", c.stderr))
# add custom content
if c.custom_content:
content += c.custom_content
# write the job file
with open(job_file, "w") as f:
f.write("&\n")
for key, value in content:
line = self.create_line(key, value)
f.write(line + "\n")
logger.debug("created glite job file at '{}'".format(job_file))
return job_file, c
def _check_unique(self, value):
if not self._unique:
return value
return make_unique(value)
def create(self, postfix=None, **kwargs):
# merge kwargs and instance attributes
c = self.get_config(kwargs)
# some sanity checks
if not c.file_name:
raise ValueError("file_name must not be empty")
if not c.command and not c.executable:
raise ValueError("either command or executable must not be empty")
# ensure that all log files are output files
for attr in ["log", "stdout", "stderr", "custom_log_file"]:
if c[attr] and c[attr] not in c.output_files:
c.output_files.append(c[attr])
# postfix certain output files
if c.postfix_output_files:
c.output_files = [
self.postfix_output_file(path, postfix)
for path in c.output_files
]
for attr in ["log", "stdout", "stderr", "custom_log_file"]:
if c[attr]:
c[attr] = self.postfix_output_file(c[attr], postfix)
# ensure that all input files are JobInputFile's
c.input_files = {
key: JobInputFile(f)
for key, f in c.input_files.items()
}
# special case: remote input files must never be copied
for f in c.input_files.values:
if f.is_remote:
f.copy = False
# ensure that the executable is an input file, remember the key to access it
if c.executable:
executable_keys = [
k for k, v in c.input_files.items() if v == c.executable
]
if executable_keys:
executable_key = executable_keys[0]
else:
executable_key = "executable_file"
c.input_files[executable_key] = JobInputFile(c.executable)
# prepare input files
def prepare_input(f):
if f.is_remote:
return f.path
# when not copied, just return the absolute, original path
abs_path = os.path.abspath(f.path)
if not f.copy:
return abs_path
# copy the file
abs_path = self.provide_input(abs_path,
postfix if f.postfix else None,
c.dir)
return abs_path
abs_input_paths = {
key: prepare_input(f)
for key, f in c.input_files.items()
}
# convert to basenames, relative to the submission or initial dir
maybe_basename = lambda path: path if c.absolute_paths else os.path.basename(
path)
rel_input_paths_sub = {
key:
maybe_basename(abs_path) if c.input_files[key].copy else abs_path
for key, abs_path in abs_input_paths.items()
}
# convert to basenames as seen by the job
rel_input_paths_job = {
key:
os.path.basename(abs_path) if c.input_files[key].copy else abs_path
for key, abs_path in abs_input_paths.items()
}
# add all input files to render variables
c.render_variables.update(rel_input_paths_job)
c.render_variables["input_files"] = " ".join(
rel_input_paths_job.values())
# add the custom log file to render variables
if c.custom_log_file:
c.render_variables["log_file"] = c.custom_log_file
# add the file postfix to render variables
if postfix and "file_postfix" not in c.render_variables:
c.render_variables["file_postfix"] = postfix
# add output_uri to render variables
if c.output_uri and "output_uri" not in c.render_variables:
c.render_variables["output_uri"] = c.output_uri
# linearize render variables
render_variables = self.linearize_render_variables(c.render_variables)
# prepare the job file
job_file = self.postfix_input_file(os.path.join(c.dir, c.file_name),
postfix)
# render copied input files
for key, abs_path in abs_input_paths.items():
if c.input_files[key].copy and c.input_files[key].render:
self.render_file(abs_path,
abs_path,
render_variables,
postfix=postfix)
# create arc-style input file pairs
input_file_pairs = [(os.path.basename(rel_input_paths_sub[key]),
"" if f.copy else abs_input_paths[key])
for key, f in c.input_files.items()]
# prepare the executable when given
if c.executable:
c.executable = rel_input_paths_job[executable_key]
# make the file executable for the user and group
path = os.path.join(c.dir, os.path.basename(c.executable))
if os.path.exists(path):
os.chmod(path,
os.stat(path).st_mode | stat.S_IXUSR | stat.S_IXGRP)
# ensure a correct format of output files
def prepare_output(path):
# consider strings to be the filename and when output_uri is set, use it
# as the URL, otherwise it's also empty
if c.postfix_output_files:
path = self.postfix_output_file(path, postfix)
dst = os.path.join(c.output_uri,
os.path.basename(path)) if c.output_uri else ""
opts = "overwrite=yes" if c.overwrite_output_files else None
return (path, dst, opts) if opts else (path, dst)
c.output_files = list(map(prepare_output, c.output_files))
# job file content
content = []
if c.command:
cmd = quote_cmd(c.command) if isinstance(c.command,
(list,
tuple)) else c.command
content.append(("executable", cmd))
elif c.executable:
content.append(("executable", c.executable))
if c.arguments:
args = quote_cmd(c.arguments) if isinstance(
c.arguments, (list, tuple)) else c.arguments
content.append(("arguments", args))
if c.job_name:
content.append(("jobName", c.job_name))
if c.input_files:
content.append(("inputFiles", make_unique(input_file_pairs)))
if c.output_files:
content.append(("outputFiles", make_unique(c.output_files)))
if c.log:
content.append(("gmlog", c.log))
if c.stdout:
content.append(("stdout", c.stdout))
if c.stderr:
content.append(("stderr", c.stderr))
# add custom content
if c.custom_content:
content += c.custom_content
# write the job file
with open(job_file, "w") as f:
f.write("&\n")
for key, value in content:
line = self.create_line(key, value)
f.write(line + "\n")
logger.debug("created glite job file at '{}'".format(job_file))
return job_file, c
def create(self, postfix=None, render_variables=None, **kwargs):
# merge kwargs and instance attributes
c = self.get_config(kwargs)
# some sanity checks
if not c.file_name:
raise ValueError("file_name must not be empty")
elif not c.executable:
raise ValueError("executable must not be empty")
# default render variables
if not render_variables:
render_variables = {}
# add postfix to render variables
if postfix and "file_postfix" not in render_variables:
render_variables["file_postfix"] = postfix
# add output_uri to render variables
if c.output_uri and "output_uri" not in render_variables:
render_variables["output_uri"] = c.output_uri
# linearize render variables
render_variables = self.linearize_render_variables(render_variables)
# prepare the job file and the executable
job_file = self.postfix_file(os.path.join(c.dir, c.file_name), postfix)
executable_is_file = c.executable in map(os.path.basename,
c.input_files)
if executable_is_file:
c.executable = self.postfix_file(c.executable, postfix)
# prepare input files
def prepare_input(path):
path = self.provide_input(os.path.abspath(path), postfix, c.dir,
render_variables)
path = add_scheme(
path, "file") if c.absolute_paths else os.path.basename(path)
return path
c.input_files = list(map(prepare_input, c.input_files))
# ensure that log files are contained in the output files
if c.stdout and c.stdout not in c.output_files:
c.output_files.append(c.stdout)
if c.stderr and c.stderr not in c.output_files:
c.output_files.append(c.stderr)
# postfix output files
if c.postfix_output_files:
c.output_files = [
self.postfix_file(path, postfix) for path in c.output_files
]
c.stdout = c.stdout and self.postfix_file(c.stdout, postfix)
c.stderr = c.stderr and self.postfix_file(c.stderr, postfix)
# custom log file
if c.custom_log_file:
c.custom_log_file = self.postfix_file(c.custom_log_file, postfix)
c.output_files.append(c.custom_log_file)
# job file content
content = []
content.append(("Executable", c.executable))
if c.arguments:
content.append(("Arguments", c.arguments))
if c.input_files:
content.append(("InputSandbox", make_unique(c.input_files)))
if c.output_files:
content.append(("OutputSandbox", make_unique(c.output_files)))
if c.output_uri:
content.append(("OutputSandboxBaseDestUri", c.output_uri))
if c.vo:
content.append(("VirtualOrganisation", c.vo))
if c.stdout:
content.append(("StdOutput", c.stdout))
if c.stderr:
content.append(("StdError", c.stderr))
# add custom content
if c.custom_content:
content += c.custom_content
# write the job file
with open(job_file, "w") as f:
f.write("[\n")
for key, value in content:
f.write(self.create_line(key, value) + "\n")
f.write("]\n")
logger.debug("created glite job file at '{}'".format(job_file))
return job_file, c
def create(self, postfix=None, render_variables=None, **kwargs):
# merge kwargs and instance attributes
c = self.get_config(kwargs)
# some sanity checks
if not c.file_name:
raise ValueError("file_name must not be empty")
elif not c.universe:
raise ValueError("universe must not be empty")
elif not c.executable:
raise ValueError("executable must not be empty")
# default render variables
if not render_variables:
render_variables = {}
# add postfix to render variables
if postfix and "file_postfix" not in render_variables:
render_variables["file_postfix"] = postfix
# linearize render variables
render_variables = self.linearize_render_variables(render_variables)
# prepare the job file and the executable
job_file = self.postfix_file(os.path.join(c.dir, c.file_name), postfix)
executable_is_file = c.executable in map(os.path.basename, c.input_files)
if executable_is_file:
c.executable = self.postfix_file(os.path.basename(c.executable), postfix)
# prepare input files
def prepare_input(path):
path = self.provide_input(os.path.abspath(path), postfix, c.dir, render_variables)
path = path if c.absolute_paths else os.path.basename(path)
return path
c.input_files = list(map(prepare_input, c.input_files))
# make the executable file executable for the user
if executable_is_file:
for input_file in c.input_files:
if os.path.basename(input_file) == c.executable:
if not c.absolute_paths:
input_file = os.path.join(c.dir, input_file)
if not os.path.exists(input_file):
raise IOError("could not find input file '{}'".format(input_file))
os.chmod(input_file, os.stat(input_file).st_mode | stat.S_IXUSR)
break
# output files
if c.postfix_output_files:
c.output_files = [self.postfix_file(path, postfix) for path in c.output_files]
c.log = c.log and self.postfix_file(c.log, postfix)
c.stdout = c.stdout and self.postfix_file(c.stdout, postfix)
c.stderr = c.stdout and self.postfix_file(c.stderr, postfix)
# custom log file
if c.custom_log_file:
c.custom_log_file = self.postfix_file(c.custom_log_file, postfix)
c.output_files.append(c.custom_log_file)
# job file content
content = []
content.append(("universe", c.universe))
content.append(("executable", c.executable))
if c.log:
content.append(("log", c.log))
if c.stdout:
content.append(("output", c.stdout))
if c.stderr:
content.append(("error", c.stderr))
if c.input_files or c.output_files:
content.append(("should_transfer_files", "YES"))
if c.input_files:
content.append(("transfer_input_files", make_unique(c.input_files)))
if c.output_files:
content.append(("transfer_output_files", make_unique(c.output_files)))
content.append(("when_to_transfer_output", "ON_EXIT"))
if c.notification:
content.append(("notification", c.notification))
# add custom content
if c.custom_content:
content += c.custom_content
# finally arguments and queuing statements
if c.arguments:
for _arguments in make_list(c.arguments):
content.append(("arguments", _arguments))
content.append("queue")
else:
content.append("queue")
# write the job file
with open(job_file, "w") as f:
for obj in content:
line = self.create_line(*make_list(obj))
f.write(line + "\n")
logger.debug("created htcondor job file at '{}'".format(job_file))
return job_file, c
def create(self, postfix=None, **kwargs):
# merge kwargs and instance attributes
c = self.get_config(kwargs)
# some sanity checks
if not c.file_name:
raise ValueError("file_name must not be empty")
if not c.command and not c.executable:
raise ValueError("either command or executable must not be empty")
if not c.shell:
raise ValueError("shell must not be empty")
# ensure that the custom log file is an output file
if c.custom_log_file and c.custom_log_file not in c.output_files:
c.output_files.append(c.custom_log_file)
# postfix certain output files
if c.postfix_output_files:
c.output_files = [
path if path.startswith("/dev/") else self.postfix_output_file(
path, postfix) for path in c.output_files
]
for attr in ["stdout", "stderr", "custom_log_file"]:
if c[attr] and not c[attr].startswith("/dev/"):
c[attr] = self.postfix_output_file(c[attr], postfix)
# ensure that all input files are JobInputFile's
c.input_files = {
key: JobInputFile(f)
for key, f in c.input_files.items()
}
# ensure that the executable is an input file, remember the key to access it
if c.executable:
executable_keys = [
k for k, v in c.input_files.items() if v == c.executable
]
if executable_keys:
executable_key = executable_keys[0]
else:
executable_key = "executable_file"
c.input_files[executable_key] = JobInputFile(c.executable)
# prepare input files
def prepare_input(f):
# when not copied, just return the absolute, original path
abs_path = os.path.abspath(f.path)
if not f.copy:
return abs_path
# copy the file
abs_path = self.provide_input(abs_path,
postfix if f.postfix else None,
c.dir)
return abs_path
abs_input_paths = {
key: prepare_input(f)
for key, f in c.input_files.items()
}
# convert to basenames, relative to the submission or initial dir
maybe_basename = lambda path: path if c.absolute_paths else os.path.basename(
path)
rel_input_paths_sub = {
key:
maybe_basename(abs_path) if c.input_files[key].copy else abs_path
for key, abs_path in abs_input_paths.items()
}
# convert to basenames as seen by the job
rel_input_paths_job = {
key:
os.path.basename(abs_path) if c.input_files[key].copy else abs_path
for key, abs_path in abs_input_paths.items()
}
# add all input files to render variables
c.render_variables.update(rel_input_paths_job)
c.render_variables["input_files"] = " ".join(
rel_input_paths_job.values())
# add the custom log file to render variables
if c.custom_log_file:
c.render_variables["log_file"] = c.custom_log_file
# add the file postfix to render variables
if postfix and "file_postfix" not in c.render_variables:
c.render_variables["file_postfix"] = postfix
# linearize render variables
render_variables = self.linearize_render_variables(c.render_variables)
# prepare the job file
job_file = self.postfix_input_file(os.path.join(c.dir, c.file_name),
postfix)
# render copied input files
for key, abs_path in abs_input_paths.items():
if c.input_files[key].copy and c.input_files[key].render:
self.render_file(abs_path,
abs_path,
render_variables,
postfix=postfix)
# prepare the executable when given
if c.executable:
c.executable = rel_input_paths_job[executable_key]
# make the file executable for the user and group
path = os.path.join(c.dir, os.path.basename(c.executable))
if os.path.exists(path):
os.chmod(path,
os.stat(path).st_mode | stat.S_IXUSR | stat.S_IXGRP)
# job file content
content = []
content.append("#!/usr/bin/env {}".format(c.shell))
if c.job_name:
content.append(("-J", c.job_name))
if c.queue:
content.append(("-q", c.queue))
if c.cwd:
content.append(("-cwd", c.cwd))
if c.stdout:
content.append(("-o", c.stdout))
if c.stderr:
content.append(("-e", c.stderr))
if c.emails:
content.append(("-N", ))
if c.custom_content:
content += c.custom_content
if not c.manual_stagein:
for path in make_unique(rel_input_paths_sub.values()):
content.append(
("-f", "\"{} > {}\"".format(path, os.path.basename(path))))
if not c.manual_stageout:
for path in make_unique(c.output_files):
content.append(
("-f", "\"{} < {}\"".format(path, os.path.basename(path))))
if c.manual_stagein:
tmpl = "cp " + ("{}" if c.absolute_paths else
"$LS_EXECCWD/{}") + " $PWD/{}"
for path in make_unique(rel_input_paths_sub.values()):
content.append(tmpl.format(path, os.path.basename(path)))
if c.command:
content.append(c.command)
else:
content.append("./" + rel_input_paths_job[executable_key])
if c.arguments:
args = quote_cmd(c.arguments) if isinstance(
c.arguments, (list, tuple)) else c.arguments
content[-1] += " {}".format(args)
if c.manual_stageout:
tmpl = "cp $PWD/{} $LS_EXECCWD/{}"
for path in c.output_files:
content.append(tmpl.format(path, path))
# write the job file
with open(job_file, "w") as f:
for line in content:
if not isinstance(line, six.string_types):
line = self.create_line(*make_list(line))
f.write(line + "\n")
logger.debug("created lsf job file at '{}'".format(job_file))
return job_file, c
def create(self, postfix=None, **kwargs):
# merge kwargs and instance attributes
c = self.get_config(kwargs)
# some sanity checks
if not c.file_name:
raise ValueError("file_name must not be empty")
if not c.command and not c.executable:
raise ValueError("either command or executable must not be empty")
if not c.universe:
raise ValueError("universe must not be empty")
# ensure that the custom log file is an output file
if c.custom_log_file and c.custom_log_file not in c.output_files:
c.output_files.append(c.custom_log_file)
# postfix certain output files
if c.postfix_output_files:
c.output_files = [
path if path.startswith("/dev/") else self.postfix_output_file(
path, postfix) for path in c.output_files
]
for attr in ["log", "stdout", "stderr", "custom_log_file"]:
if c[attr] and not c[attr].startswith("/dev/"):
c[attr] = self.postfix_output_file(c[attr], postfix)
# ensure that all input files are JobInputFile's
c.input_files = {
key: JobInputFile(f)
for key, f in c.input_files.items()
}
# ensure that the executable is an input file, remember the key to access it
if c.executable:
executable_keys = [
k for k, v in c.input_files.items() if v == c.executable
]
if executable_keys:
executable_key = executable_keys[0]
else:
executable_key = "executable_file"
c.input_files[executable_key] = JobInputFile(c.executable)
# prepare input files
def prepare_input(f):
# when not copied, just return the absolute, original path
abs_path = os.path.abspath(f.path)
if not f.copy:
return abs_path
# copy the file
abs_path = self.provide_input(abs_path,
postfix if f.postfix else None,
c.dir)
return abs_path
abs_input_paths = {
key: prepare_input(f)
for key, f in c.input_files.items()
}
# convert to basenames, relative to the submission or initial dir
maybe_basename = lambda path: path if c.absolute_paths else os.path.basename(
path)
rel_input_paths_sub = {
key:
maybe_basename(abs_path) if c.input_files[key].copy else abs_path
for key, abs_path in abs_input_paths.items()
}
# convert to basenames as seen by the job
rel_input_paths_job = {
key:
os.path.basename(abs_path) if c.input_files[key].copy else abs_path
for key, abs_path in abs_input_paths.items()
}
# add all input files to render variables
c.render_variables.update(rel_input_paths_job)
c.render_variables["input_files"] = " ".join(
rel_input_paths_job.values())
# add the custom log file to render variables
if c.custom_log_file:
c.render_variables["log_file"] = c.custom_log_file
# add the file postfix to render variables
if postfix and "file_postfix" not in c.render_variables:
c.render_variables["file_postfix"] = postfix
# linearize render variables
render_variables = self.linearize_render_variables(c.render_variables)
# prepare the job file
job_file = self.postfix_input_file(os.path.join(c.dir, c.file_name),
postfix)
# render copied input files
for key, abs_path in abs_input_paths.items():
if c.input_files[key].copy and c.input_files[key].render:
self.render_file(abs_path,
abs_path,
render_variables,
postfix=postfix)
# prepare the executable when given
if c.executable:
c.executable = rel_input_paths_job[executable_key]
# make the file executable for the user and group
path = os.path.join(c.dir, os.path.basename(c.executable))
if os.path.exists(path):
os.chmod(path,
os.stat(path).st_mode | stat.S_IXUSR | stat.S_IXGRP)
# job file content
content = []
content.append(("universe", c.universe))
if c.command:
cmd = quote_cmd(c.command) if isinstance(c.command,
(list,
tuple)) else c.command
content.append(("executable", cmd))
elif c.executable:
content.append(("executable", c.executable))
if c.log:
content.append(("log", c.log))
if c.stdout:
content.append(("output", c.stdout))
if c.stderr:
content.append(("error", c.stderr))
if c.input_files or c.output_files:
content.append(("should_transfer_files", "YES"))
if c.input_files:
content.append(("transfer_input_files",
make_unique(rel_input_paths_sub.values())))
if c.output_files:
content.append(
("transfer_output_files", make_unique(c.output_files)))
content.append(("when_to_transfer_output", "ON_EXIT"))
if c.notification:
content.append(("notification", c.notification))
# add custom content
if c.custom_content:
content += c.custom_content
# finally arguments and queuing statements
if c.arguments:
for _arguments in make_list(c.arguments):
content.append(("arguments", _arguments))
content.append("queue")
else:
content.append("queue")
# write the job file
with open(job_file, "w") as f:
for obj in content:
line = self.create_line(*make_list(obj))
f.write(line + "\n")
logger.debug("created htcondor job file at '{}'".format(job_file))
return job_file, c
def create(self, postfix=None, render_variables=None, **kwargs):
# merge kwargs and instance attributes
c = self.get_config(kwargs)
# some sanity checks
if not c.file_name:
raise ValueError("file_name must not be empty")
if not c.command and not c.executable:
raise ValueError("either command or executable must not be empty")
# ensure that all log files are output files
for attr in ["stdout", "stderr", "custom_log_file"]:
if c[attr] and c[attr] not in c.output_files:
c.output_files.append(c[attr])
# postfix certain output files
if c.postfix_output_files:
c.output_files = [
self.postfix_output_file(path, postfix)
for path in c.output_files
]
for attr in ["stdout", "stderr", "custom_log_file"]:
if c[attr]:
c[attr] = self.postfix_output_file(c[attr], postfix)
# ensure that all input files are JobInputFile's
c.input_files = {
key: JobInputFile(f)
for key, f in c.input_files.items()
}
# ensure that the executable is an input file, remember the key to access it
if c.executable:
executable_keys = [
k for k, v in c.input_files.items() if v == c.executable
]
if executable_keys:
executable_key = executable_keys[0]
else:
executable_key = "executable_file"
c.input_files[executable_key] = JobInputFile(c.executable)
# prepare input files
def prepare_input(f):
# when not copied, just return the absolute, original path
abs_path = os.path.abspath(f.path)
if not f.copy:
return abs_path
# copy the file
abs_path = self.provide_input(abs_path,
postfix if f.postfix else None,
c.dir)
return abs_path
abs_input_paths = {
key: prepare_input(f)
for key, f in c.input_files.items()
}
# convert to basenames, relative to the submission or initial dir
maybe_basename = lambda path: path if c.absolute_paths else os.path.basename(
path)
rel_input_paths_sub = {
key:
maybe_basename(abs_path) if c.input_files[key].copy else abs_path
for key, abs_path in abs_input_paths.items()
}
# convert to basenames as seen by the job
rel_input_paths_job = {
key:
os.path.basename(abs_path) if c.input_files[key].copy else abs_path
for key, abs_path in abs_input_paths.items()
}
# add all input files to render variables
c.render_variables.update(rel_input_paths_job)
c.render_variables["input_files"] = " ".join(
rel_input_paths_job.values())
# add the custom log file to render variables
if c.custom_log_file:
c.render_variables["log_file"] = c.custom_log_file
# add the file postfix to render variables
if postfix and "file_postfix" not in c.render_variables:
c.render_variables["file_postfix"] = postfix
# add output_uri to render variables
if c.output_uri and "output_uri" not in c.render_variables:
c.render_variables["output_uri"] = c.output_uri
# linearize render variables
render_variables = self.linearize_render_variables(c.render_variables)
# prepare the job file
job_file = self.postfix_input_file(os.path.join(c.dir, c.file_name),
postfix)
# render copied input files
for key, abs_path in abs_input_paths.items():
if c.input_files[key].copy and c.input_files[key].render:
self.render_file(abs_path,
abs_path,
render_variables,
postfix=postfix)
# prepare the executable when given
if c.executable:
c.executable = rel_input_paths_job[executable_key]
# make the file executable for the user and group
path = os.path.join(c.dir, os.path.basename(c.executable))
if os.path.exists(path):
os.chmod(path,
os.stat(path).st_mode | stat.S_IXUSR | stat.S_IXGRP)
# job file content
content = []
if c.command:
cmd = quote_cmd(c.command) if isinstance(c.command,
(list,
tuple)) else c.command
content.append(("Executable", cmd))
elif c.executable:
content.append(("Executable", c.executable))
if c.arguments:
args = quote_cmd(c.arguments) if isinstance(
c.arguments, (list, tuple)) else c.arguments
content.append(("Arguments", args))
if c.input_files:
content.append(
("InputSandbox", make_unique(rel_input_paths_sub.values())))
if c.output_files:
content.append(("OutputSandbox", make_unique(c.output_files)))
if c.output_uri:
content.append(("OutputSandboxBaseDestUri", c.output_uri))
if c.vo:
content.append(("VirtualOrganisation", c.vo))
if c.stdout:
content.append(("StdOutput", c.stdout))
if c.stderr:
content.append(("StdError", c.stderr))
# add custom content
if c.custom_content:
content += c.custom_content
# write the job file
with open(job_file, "w") as f:
f.write("[\n")
for key, value in content:
f.write(self.create_line(key, value) + "\n")
f.write("]\n")
logger.debug("created glite job file at '{}'".format(job_file))
return job_file, c
def create(self, postfix=None, render_variables=None, **kwargs):
# merge kwargs and instance attributes
c = self.get_config(kwargs)
# some sanity checks
if not c.file_name:
raise ValueError("file_name must not be empty")
elif not c.command:
raise ValueError("command must not be empty")
elif not c.shell:
raise ValueError("shell must not be empty")
# default render variables
if not render_variables:
render_variables = {}
# add postfix to render variables
if postfix and "file_postfix" not in render_variables:
render_variables["file_postfix"] = postfix
# linearize render variables
render_variables = self.linearize_render_variables(render_variables)
# prepare paths
job_file = self.postfix_file(os.path.join(c.dir, c.file_name), postfix)
# prepare input files
def prepare_input(path):
path = self.provide_input(os.path.abspath(path), postfix, c.dir,
render_variables)
path = path if c.absolute_paths else os.path.basename(path)
return path
c.input_files = list(map(prepare_input, c.input_files))
# output files
if c.postfix_output_files:
c.output_files = [
self.postfix_file(path, postfix) for path in c.output_files
]
c.stdout = c.stdout and self.postfix_file(c.stdout, postfix)
c.stderr = c.stdout and self.postfix_file(c.stderr, postfix)
# custom log file
if c.custom_log_file:
c.custom_log_file = self.postfix_file(c.custom_log_file, postfix)
c.output_files.append(c.custom_log_file)
# job file content
content = []
content.append("#!/usr/bin/env {}".format(c.shell))
if c.job_name:
content.append(("-J", c.job_name))
if c.queue:
content.append(("-q", c.queue))
if c.cwd:
content.append(("-cwd", c.cwd))
if c.stdout:
content.append(("-o", c.stdout))
if c.stderr:
content.append(("-e", c.stderr))
if c.emails:
content.append(("-N", ))
if c.custom_content:
content += c.custom_content
if not c.manual_stagein:
for input_file in make_unique(c.input_files):
content.append(
("-f", "\"{} > {}\"".format(input_file,
os.path.basename(input_file))))
if not c.manual_stageout:
for output_file in make_unique(c.output_files):
content.append(
("-f",
"\"{} < {}\"".format(output_file,
os.path.basename(output_file))))
if c.manual_stagein:
tmpl = "cp " + ("{}" if c.absolute_paths else
"$LS_EXECCWD/{}") + " $( pwd )/{}"
for input_file in make_unique(c.input_files):
content.append(
tmpl.format(input_file, os.path.basename(input_file)))
content.append(c.command)
if c.manual_stageout:
tmpl = "cp $( pwd )/{} $LS_EXECCWD/{}"
for output_file in c.output_files:
content.append(tmpl.format(output_file, output_file))
# write the job file
with open(job_file, "w") as f:
for line in content:
if not isinstance(line, six.string_types):
line = self.create_line(*make_list(line))
f.write(line + "\n")
logger.debug("created lsf job file at '{}'".format(job_file))
return job_file, c
