def serialize(self, value): """""" if not value: return "" else: # ensure uniqueness if self._unique: value = make_unique(value) # check min_len and max_len self._check_len(value) return ",".join(str(self._inst.serialize(elem)) for elem in value)
def _check_choices(self, value): if not self._choices: return unknown = [] for v in value: if v not in self._choices: unknown.append(v) if unknown: str_repr = lambda value: ",".join(str(v) for v in value) raise ValueError("invalid parameter value(s) '{}', valid choices are '{}'".format( str_repr(make_unique(unknown)), str_repr(self._choices)))
def parse(self, inp): """""" if not inp: ret = tuple() elif isinstance(inp, (tuple, list)) or is_lazy_iterable(inp): ret = make_tuple(inp) else: ret = tuple(self._inst.parse(elem) for elem in inp.split(",")) # ensure uniqueness if self._unique: ret = make_unique(ret) # check min_len and max_len self._check_len(ret) return ret
def create(self, postfix=None, render_variables=None, **kwargs): # merge kwargs and instance attributes c = self.get_config(kwargs) # some sanity checks if not c.file_name: raise ValueError("file_name must not be empty") elif not c.executable: raise ValueError("executable must not be empty") # default render variables if not render_variables: render_variables = {} # add postfix to render variables if postfix and "file_postfix" not in render_variables: render_variables["file_postfix"] = postfix # add output_uri to render variables if c.output_uri and "output_uri" not in render_variables: render_variables["output_uri"] = c.output_uri # linearize render variables render_variables = self.linearize_render_variables(render_variables) # prepare the job file job_file = self.postfix_file(os.path.join(c.dir, c.file_name), postfix) # prepare input files def prepare_input(tpl): # consider strings to be the base filename and use an identical source with no options if isinstance(tpl, six.string_types): tpl = (os.path.basename(tpl), tpl, "") path, src, opts = (tpl + ("", ""))[:3] path = self.postfix_file(path, postfix) if src and get_scheme(src) in ("file", None): src = self.provide_input(os.path.abspath(src), postfix, c.dir, render_variables) if not c.absolute_paths: src = os.path.basename(src) if src == path: src = "" return (path, src, opts) if opts else (path, src) c.input_files = list(map(prepare_input, c.input_files)) # postfix the executable pf_executable = self.postfix_file(os.path.basename(c.executable), postfix) executable_is_file = pf_executable in [ os.path.basename(tpl[0]) for tpl in c.input_files ] if executable_is_file: c.executable = pf_executable # ensure that log files are contained in the output files if c.log and c.log not in c.output_files: c.output_files.append(c.log) if c.stdout and c.stdout not in c.output_files: c.output_files.append(c.stdout) if c.stderr and c.stderr not in c.output_files: c.output_files.append(c.stderr) # ensure a correct format of output files def prepare_output(tpl): # consider strings to be the filename and when output_uri is set, use it # as the URL, otherwise it's also empty if isinstance(tpl, six.string_types): dst = os.path.join( c.output_uri, os.path.basename(tpl)) if c.output_uri else "" tpl = (tpl, dst) path, dst, opts = (tpl + ("", ""))[:3] if c.postfix_output_files: path = self.postfix_file(path, postfix) if dst: dst = self.postfix_file(dst, postfix) if c.overwrite_output_files and "overwrite" not in opts: opts += (";" if opts else "") + "overwrite=yes" return (path, dst, opts) if opts else (path, dst) c.output_files = map(prepare_output, c.output_files) # also postfix log files if c.postfix_output_files: c.log = c.log and self.postfix_file(c.log, postfix) c.stdout = c.stdout and self.postfix_file(c.stdout, postfix) c.stderr = c.stderr and self.postfix_file(c.stderr, postfix) # custom log file if c.custom_log_file: c.output_files.append(prepare_output(c.custom_log_file)) c.custom_log_file = self.postfix_file(c.custom_log_file, postfix) # job file content content = [] content.append(("executable", c.executable)) if c.arguments: content.append(("arguments", c.arguments)) if c.job_name: content.append(("jobName", c.job_name)) if c.input_files: content.append(("inputFiles", make_unique(c.input_files))) if c.output_files: content.append(("outputFiles", make_unique(c.output_files))) if c.log: content.append(("gmlog", c.log)) if c.stdout: content.append(("stdout", c.stdout)) if c.stderr: content.append(("stderr", c.stderr)) # add custom content if c.custom_content: content += c.custom_content # write the job file with open(job_file, "w") as f: f.write("&\n") for key, value in content: line = self.create_line(key, value) f.write(line + "\n") logger.debug("created glite job file at '{}'".format(job_file)) return job_file, c
def _check_unique(self, value): if not self._unique: return value return make_unique(value)
def create(self, postfix=None, **kwargs): # merge kwargs and instance attributes c = self.get_config(kwargs) # some sanity checks if not c.file_name: raise ValueError("file_name must not be empty") if not c.command and not c.executable: raise ValueError("either command or executable must not be empty") # ensure that all log files are output files for attr in ["log", "stdout", "stderr", "custom_log_file"]: if c[attr] and c[attr] not in c.output_files: c.output_files.append(c[attr]) # postfix certain output files if c.postfix_output_files: c.output_files = [ self.postfix_output_file(path, postfix) for path in c.output_files ] for attr in ["log", "stdout", "stderr", "custom_log_file"]: if c[attr]: c[attr] = self.postfix_output_file(c[attr], postfix) # ensure that all input files are JobInputFile's c.input_files = { key: JobInputFile(f) for key, f in c.input_files.items() } # special case: remote input files must never be copied for f in c.input_files.values: if f.is_remote: f.copy = False # ensure that the executable is an input file, remember the key to access it if c.executable: executable_keys = [ k for k, v in c.input_files.items() if v == c.executable ] if executable_keys: executable_key = executable_keys[0] else: executable_key = "executable_file" c.input_files[executable_key] = JobInputFile(c.executable) # prepare input files def prepare_input(f): if f.is_remote: return f.path # when not copied, just return the absolute, original path abs_path = os.path.abspath(f.path) if not f.copy: return abs_path # copy the file abs_path = self.provide_input(abs_path, postfix if f.postfix else None, c.dir) return abs_path abs_input_paths = { key: prepare_input(f) for key, f in c.input_files.items() } # convert to basenames, relative to the submission or initial dir maybe_basename = lambda path: path if c.absolute_paths else os.path.basename( path) rel_input_paths_sub = { key: maybe_basename(abs_path) if c.input_files[key].copy else abs_path for key, abs_path in abs_input_paths.items() } # convert to basenames as seen by the job rel_input_paths_job = { key: os.path.basename(abs_path) if c.input_files[key].copy else abs_path for key, abs_path in abs_input_paths.items() } # add all input files to render variables c.render_variables.update(rel_input_paths_job) c.render_variables["input_files"] = " ".join( rel_input_paths_job.values()) # add the custom log file to render variables if c.custom_log_file: c.render_variables["log_file"] = c.custom_log_file # add the file postfix to render variables if postfix and "file_postfix" not in c.render_variables: c.render_variables["file_postfix"] = postfix # add output_uri to render variables if c.output_uri and "output_uri" not in c.render_variables: c.render_variables["output_uri"] = c.output_uri # linearize render variables render_variables = self.linearize_render_variables(c.render_variables) # prepare the job file job_file = self.postfix_input_file(os.path.join(c.dir, c.file_name), postfix) # render copied input files for key, abs_path in abs_input_paths.items(): if c.input_files[key].copy and c.input_files[key].render: self.render_file(abs_path, abs_path, render_variables, postfix=postfix) # create arc-style input file pairs input_file_pairs = [(os.path.basename(rel_input_paths_sub[key]), "" if f.copy else abs_input_paths[key]) for key, f in c.input_files.items()] # prepare the executable when given if c.executable: c.executable = rel_input_paths_job[executable_key] # make the file executable for the user and group path = os.path.join(c.dir, os.path.basename(c.executable)) if os.path.exists(path): os.chmod(path, os.stat(path).st_mode | stat.S_IXUSR | stat.S_IXGRP) # ensure a correct format of output files def prepare_output(path): # consider strings to be the filename and when output_uri is set, use it # as the URL, otherwise it's also empty if c.postfix_output_files: path = self.postfix_output_file(path, postfix) dst = os.path.join(c.output_uri, os.path.basename(path)) if c.output_uri else "" opts = "overwrite=yes" if c.overwrite_output_files else None return (path, dst, opts) if opts else (path, dst) c.output_files = list(map(prepare_output, c.output_files)) # job file content content = [] if c.command: cmd = quote_cmd(c.command) if isinstance(c.command, (list, tuple)) else c.command content.append(("executable", cmd)) elif c.executable: content.append(("executable", c.executable)) if c.arguments: args = quote_cmd(c.arguments) if isinstance( c.arguments, (list, tuple)) else c.arguments content.append(("arguments", args)) if c.job_name: content.append(("jobName", c.job_name)) if c.input_files: content.append(("inputFiles", make_unique(input_file_pairs))) if c.output_files: content.append(("outputFiles", make_unique(c.output_files))) if c.log: content.append(("gmlog", c.log)) if c.stdout: content.append(("stdout", c.stdout)) if c.stderr: content.append(("stderr", c.stderr)) # add custom content if c.custom_content: content += c.custom_content # write the job file with open(job_file, "w") as f: f.write("&\n") for key, value in content: line = self.create_line(key, value) f.write(line + "\n") logger.debug("created glite job file at '{}'".format(job_file)) return job_file, c
def create(self, postfix=None, render_variables=None, **kwargs): # merge kwargs and instance attributes c = self.get_config(kwargs) # some sanity checks if not c.file_name: raise ValueError("file_name must not be empty") elif not c.executable: raise ValueError("executable must not be empty") # default render variables if not render_variables: render_variables = {} # add postfix to render variables if postfix and "file_postfix" not in render_variables: render_variables["file_postfix"] = postfix # add output_uri to render variables if c.output_uri and "output_uri" not in render_variables: render_variables["output_uri"] = c.output_uri # linearize render variables render_variables = self.linearize_render_variables(render_variables) # prepare the job file and the executable job_file = self.postfix_file(os.path.join(c.dir, c.file_name), postfix) executable_is_file = c.executable in map(os.path.basename, c.input_files) if executable_is_file: c.executable = self.postfix_file(c.executable, postfix) # prepare input files def prepare_input(path): path = self.provide_input(os.path.abspath(path), postfix, c.dir, render_variables) path = add_scheme( path, "file") if c.absolute_paths else os.path.basename(path) return path c.input_files = list(map(prepare_input, c.input_files)) # ensure that log files are contained in the output files if c.stdout and c.stdout not in c.output_files: c.output_files.append(c.stdout) if c.stderr and c.stderr not in c.output_files: c.output_files.append(c.stderr) # postfix output files if c.postfix_output_files: c.output_files = [ self.postfix_file(path, postfix) for path in c.output_files ] c.stdout = c.stdout and self.postfix_file(c.stdout, postfix) c.stderr = c.stderr and self.postfix_file(c.stderr, postfix) # custom log file if c.custom_log_file: c.custom_log_file = self.postfix_file(c.custom_log_file, postfix) c.output_files.append(c.custom_log_file) # job file content content = [] content.append(("Executable", c.executable)) if c.arguments: content.append(("Arguments", c.arguments)) if c.input_files: content.append(("InputSandbox", make_unique(c.input_files))) if c.output_files: content.append(("OutputSandbox", make_unique(c.output_files))) if c.output_uri: content.append(("OutputSandboxBaseDestUri", c.output_uri)) if c.vo: content.append(("VirtualOrganisation", c.vo)) if c.stdout: content.append(("StdOutput", c.stdout)) if c.stderr: content.append(("StdError", c.stderr)) # add custom content if c.custom_content: content += c.custom_content # write the job file with open(job_file, "w") as f: f.write("[\n") for key, value in content: f.write(self.create_line(key, value) + "\n") f.write("]\n") logger.debug("created glite job file at '{}'".format(job_file)) return job_file, c
def create(self, postfix=None, render_variables=None, **kwargs): # merge kwargs and instance attributes c = self.get_config(kwargs) # some sanity checks if not c.file_name: raise ValueError("file_name must not be empty") elif not c.universe: raise ValueError("universe must not be empty") elif not c.executable: raise ValueError("executable must not be empty") # default render variables if not render_variables: render_variables = {} # add postfix to render variables if postfix and "file_postfix" not in render_variables: render_variables["file_postfix"] = postfix # linearize render variables render_variables = self.linearize_render_variables(render_variables) # prepare the job file and the executable job_file = self.postfix_file(os.path.join(c.dir, c.file_name), postfix) executable_is_file = c.executable in map(os.path.basename, c.input_files) if executable_is_file: c.executable = self.postfix_file(os.path.basename(c.executable), postfix) # prepare input files def prepare_input(path): path = self.provide_input(os.path.abspath(path), postfix, c.dir, render_variables) path = path if c.absolute_paths else os.path.basename(path) return path c.input_files = list(map(prepare_input, c.input_files)) # make the executable file executable for the user if executable_is_file: for input_file in c.input_files: if os.path.basename(input_file) == c.executable: if not c.absolute_paths: input_file = os.path.join(c.dir, input_file) if not os.path.exists(input_file): raise IOError("could not find input file '{}'".format(input_file)) os.chmod(input_file, os.stat(input_file).st_mode | stat.S_IXUSR) break # output files if c.postfix_output_files: c.output_files = [self.postfix_file(path, postfix) for path in c.output_files] c.log = c.log and self.postfix_file(c.log, postfix) c.stdout = c.stdout and self.postfix_file(c.stdout, postfix) c.stderr = c.stdout and self.postfix_file(c.stderr, postfix) # custom log file if c.custom_log_file: c.custom_log_file = self.postfix_file(c.custom_log_file, postfix) c.output_files.append(c.custom_log_file) # job file content content = [] content.append(("universe", c.universe)) content.append(("executable", c.executable)) if c.log: content.append(("log", c.log)) if c.stdout: content.append(("output", c.stdout)) if c.stderr: content.append(("error", c.stderr)) if c.input_files or c.output_files: content.append(("should_transfer_files", "YES")) if c.input_files: content.append(("transfer_input_files", make_unique(c.input_files))) if c.output_files: content.append(("transfer_output_files", make_unique(c.output_files))) content.append(("when_to_transfer_output", "ON_EXIT")) if c.notification: content.append(("notification", c.notification)) # add custom content if c.custom_content: content += c.custom_content # finally arguments and queuing statements if c.arguments: for _arguments in make_list(c.arguments): content.append(("arguments", _arguments)) content.append("queue") else: content.append("queue") # write the job file with open(job_file, "w") as f: for obj in content: line = self.create_line(*make_list(obj)) f.write(line + "\n") logger.debug("created htcondor job file at '{}'".format(job_file)) return job_file, c
def create(self, postfix=None, **kwargs): # merge kwargs and instance attributes c = self.get_config(kwargs) # some sanity checks if not c.file_name: raise ValueError("file_name must not be empty") if not c.command and not c.executable: raise ValueError("either command or executable must not be empty") if not c.shell: raise ValueError("shell must not be empty") # ensure that the custom log file is an output file if c.custom_log_file and c.custom_log_file not in c.output_files: c.output_files.append(c.custom_log_file) # postfix certain output files if c.postfix_output_files: c.output_files = [ path if path.startswith("/dev/") else self.postfix_output_file( path, postfix) for path in c.output_files ] for attr in ["stdout", "stderr", "custom_log_file"]: if c[attr] and not c[attr].startswith("/dev/"): c[attr] = self.postfix_output_file(c[attr], postfix) # ensure that all input files are JobInputFile's c.input_files = { key: JobInputFile(f) for key, f in c.input_files.items() } # ensure that the executable is an input file, remember the key to access it if c.executable: executable_keys = [ k for k, v in c.input_files.items() if v == c.executable ] if executable_keys: executable_key = executable_keys[0] else: executable_key = "executable_file" c.input_files[executable_key] = JobInputFile(c.executable) # prepare input files def prepare_input(f): # when not copied, just return the absolute, original path abs_path = os.path.abspath(f.path) if not f.copy: return abs_path # copy the file abs_path = self.provide_input(abs_path, postfix if f.postfix else None, c.dir) return abs_path abs_input_paths = { key: prepare_input(f) for key, f in c.input_files.items() } # convert to basenames, relative to the submission or initial dir maybe_basename = lambda path: path if c.absolute_paths else os.path.basename( path) rel_input_paths_sub = { key: maybe_basename(abs_path) if c.input_files[key].copy else abs_path for key, abs_path in abs_input_paths.items() } # convert to basenames as seen by the job rel_input_paths_job = { key: os.path.basename(abs_path) if c.input_files[key].copy else abs_path for key, abs_path in abs_input_paths.items() } # add all input files to render variables c.render_variables.update(rel_input_paths_job) c.render_variables["input_files"] = " ".join( rel_input_paths_job.values()) # add the custom log file to render variables if c.custom_log_file: c.render_variables["log_file"] = c.custom_log_file # add the file postfix to render variables if postfix and "file_postfix" not in c.render_variables: c.render_variables["file_postfix"] = postfix # linearize render variables render_variables = self.linearize_render_variables(c.render_variables) # prepare the job file job_file = self.postfix_input_file(os.path.join(c.dir, c.file_name), postfix) # render copied input files for key, abs_path in abs_input_paths.items(): if c.input_files[key].copy and c.input_files[key].render: self.render_file(abs_path, abs_path, render_variables, postfix=postfix) # prepare the executable when given if c.executable: c.executable = rel_input_paths_job[executable_key] # make the file executable for the user and group path = os.path.join(c.dir, os.path.basename(c.executable)) if os.path.exists(path): os.chmod(path, os.stat(path).st_mode | stat.S_IXUSR | stat.S_IXGRP) # job file content content = [] content.append("#!/usr/bin/env {}".format(c.shell)) if c.job_name: content.append(("-J", c.job_name)) if c.queue: content.append(("-q", c.queue)) if c.cwd: content.append(("-cwd", c.cwd)) if c.stdout: content.append(("-o", c.stdout)) if c.stderr: content.append(("-e", c.stderr)) if c.emails: content.append(("-N", )) if c.custom_content: content += c.custom_content if not c.manual_stagein: for path in make_unique(rel_input_paths_sub.values()): content.append( ("-f", "\"{} > {}\"".format(path, os.path.basename(path)))) if not c.manual_stageout: for path in make_unique(c.output_files): content.append( ("-f", "\"{} < {}\"".format(path, os.path.basename(path)))) if c.manual_stagein: tmpl = "cp " + ("{}" if c.absolute_paths else "$LS_EXECCWD/{}") + " $PWD/{}" for path in make_unique(rel_input_paths_sub.values()): content.append(tmpl.format(path, os.path.basename(path))) if c.command: content.append(c.command) else: content.append("./" + rel_input_paths_job[executable_key]) if c.arguments: args = quote_cmd(c.arguments) if isinstance( c.arguments, (list, tuple)) else c.arguments content[-1] += " {}".format(args) if c.manual_stageout: tmpl = "cp $PWD/{} $LS_EXECCWD/{}" for path in c.output_files: content.append(tmpl.format(path, path)) # write the job file with open(job_file, "w") as f: for line in content: if not isinstance(line, six.string_types): line = self.create_line(*make_list(line)) f.write(line + "\n") logger.debug("created lsf job file at '{}'".format(job_file)) return job_file, c
def create(self, postfix=None, **kwargs): # merge kwargs and instance attributes c = self.get_config(kwargs) # some sanity checks if not c.file_name: raise ValueError("file_name must not be empty") if not c.command and not c.executable: raise ValueError("either command or executable must not be empty") if not c.universe: raise ValueError("universe must not be empty") # ensure that the custom log file is an output file if c.custom_log_file and c.custom_log_file not in c.output_files: c.output_files.append(c.custom_log_file) # postfix certain output files if c.postfix_output_files: c.output_files = [ path if path.startswith("/dev/") else self.postfix_output_file( path, postfix) for path in c.output_files ] for attr in ["log", "stdout", "stderr", "custom_log_file"]: if c[attr] and not c[attr].startswith("/dev/"): c[attr] = self.postfix_output_file(c[attr], postfix) # ensure that all input files are JobInputFile's c.input_files = { key: JobInputFile(f) for key, f in c.input_files.items() } # ensure that the executable is an input file, remember the key to access it if c.executable: executable_keys = [ k for k, v in c.input_files.items() if v == c.executable ] if executable_keys: executable_key = executable_keys[0] else: executable_key = "executable_file" c.input_files[executable_key] = JobInputFile(c.executable) # prepare input files def prepare_input(f): # when not copied, just return the absolute, original path abs_path = os.path.abspath(f.path) if not f.copy: return abs_path # copy the file abs_path = self.provide_input(abs_path, postfix if f.postfix else None, c.dir) return abs_path abs_input_paths = { key: prepare_input(f) for key, f in c.input_files.items() } # convert to basenames, relative to the submission or initial dir maybe_basename = lambda path: path if c.absolute_paths else os.path.basename( path) rel_input_paths_sub = { key: maybe_basename(abs_path) if c.input_files[key].copy else abs_path for key, abs_path in abs_input_paths.items() } # convert to basenames as seen by the job rel_input_paths_job = { key: os.path.basename(abs_path) if c.input_files[key].copy else abs_path for key, abs_path in abs_input_paths.items() } # add all input files to render variables c.render_variables.update(rel_input_paths_job) c.render_variables["input_files"] = " ".join( rel_input_paths_job.values()) # add the custom log file to render variables if c.custom_log_file: c.render_variables["log_file"] = c.custom_log_file # add the file postfix to render variables if postfix and "file_postfix" not in c.render_variables: c.render_variables["file_postfix"] = postfix # linearize render variables render_variables = self.linearize_render_variables(c.render_variables) # prepare the job file job_file = self.postfix_input_file(os.path.join(c.dir, c.file_name), postfix) # render copied input files for key, abs_path in abs_input_paths.items(): if c.input_files[key].copy and c.input_files[key].render: self.render_file(abs_path, abs_path, render_variables, postfix=postfix) # prepare the executable when given if c.executable: c.executable = rel_input_paths_job[executable_key] # make the file executable for the user and group path = os.path.join(c.dir, os.path.basename(c.executable)) if os.path.exists(path): os.chmod(path, os.stat(path).st_mode | stat.S_IXUSR | stat.S_IXGRP) # job file content content = [] content.append(("universe", c.universe)) if c.command: cmd = quote_cmd(c.command) if isinstance(c.command, (list, tuple)) else c.command content.append(("executable", cmd)) elif c.executable: content.append(("executable", c.executable)) if c.log: content.append(("log", c.log)) if c.stdout: content.append(("output", c.stdout)) if c.stderr: content.append(("error", c.stderr)) if c.input_files or c.output_files: content.append(("should_transfer_files", "YES")) if c.input_files: content.append(("transfer_input_files", make_unique(rel_input_paths_sub.values()))) if c.output_files: content.append( ("transfer_output_files", make_unique(c.output_files))) content.append(("when_to_transfer_output", "ON_EXIT")) if c.notification: content.append(("notification", c.notification)) # add custom content if c.custom_content: content += c.custom_content # finally arguments and queuing statements if c.arguments: for _arguments in make_list(c.arguments): content.append(("arguments", _arguments)) content.append("queue") else: content.append("queue") # write the job file with open(job_file, "w") as f: for obj in content: line = self.create_line(*make_list(obj)) f.write(line + "\n") logger.debug("created htcondor job file at '{}'".format(job_file)) return job_file, c
def create(self, postfix=None, render_variables=None, **kwargs): # merge kwargs and instance attributes c = self.get_config(kwargs) # some sanity checks if not c.file_name: raise ValueError("file_name must not be empty") if not c.command and not c.executable: raise ValueError("either command or executable must not be empty") # ensure that all log files are output files for attr in ["stdout", "stderr", "custom_log_file"]: if c[attr] and c[attr] not in c.output_files: c.output_files.append(c[attr]) # postfix certain output files if c.postfix_output_files: c.output_files = [ self.postfix_output_file(path, postfix) for path in c.output_files ] for attr in ["stdout", "stderr", "custom_log_file"]: if c[attr]: c[attr] = self.postfix_output_file(c[attr], postfix) # ensure that all input files are JobInputFile's c.input_files = { key: JobInputFile(f) for key, f in c.input_files.items() } # ensure that the executable is an input file, remember the key to access it if c.executable: executable_keys = [ k for k, v in c.input_files.items() if v == c.executable ] if executable_keys: executable_key = executable_keys[0] else: executable_key = "executable_file" c.input_files[executable_key] = JobInputFile(c.executable) # prepare input files def prepare_input(f): # when not copied, just return the absolute, original path abs_path = os.path.abspath(f.path) if not f.copy: return abs_path # copy the file abs_path = self.provide_input(abs_path, postfix if f.postfix else None, c.dir) return abs_path abs_input_paths = { key: prepare_input(f) for key, f in c.input_files.items() } # convert to basenames, relative to the submission or initial dir maybe_basename = lambda path: path if c.absolute_paths else os.path.basename( path) rel_input_paths_sub = { key: maybe_basename(abs_path) if c.input_files[key].copy else abs_path for key, abs_path in abs_input_paths.items() } # convert to basenames as seen by the job rel_input_paths_job = { key: os.path.basename(abs_path) if c.input_files[key].copy else abs_path for key, abs_path in abs_input_paths.items() } # add all input files to render variables c.render_variables.update(rel_input_paths_job) c.render_variables["input_files"] = " ".join( rel_input_paths_job.values()) # add the custom log file to render variables if c.custom_log_file: c.render_variables["log_file"] = c.custom_log_file # add the file postfix to render variables if postfix and "file_postfix" not in c.render_variables: c.render_variables["file_postfix"] = postfix # add output_uri to render variables if c.output_uri and "output_uri" not in c.render_variables: c.render_variables["output_uri"] = c.output_uri # linearize render variables render_variables = self.linearize_render_variables(c.render_variables) # prepare the job file job_file = self.postfix_input_file(os.path.join(c.dir, c.file_name), postfix) # render copied input files for key, abs_path in abs_input_paths.items(): if c.input_files[key].copy and c.input_files[key].render: self.render_file(abs_path, abs_path, render_variables, postfix=postfix) # prepare the executable when given if c.executable: c.executable = rel_input_paths_job[executable_key] # make the file executable for the user and group path = os.path.join(c.dir, os.path.basename(c.executable)) if os.path.exists(path): os.chmod(path, os.stat(path).st_mode | stat.S_IXUSR | stat.S_IXGRP) # job file content content = [] if c.command: cmd = quote_cmd(c.command) if isinstance(c.command, (list, tuple)) else c.command content.append(("Executable", cmd)) elif c.executable: content.append(("Executable", c.executable)) if c.arguments: args = quote_cmd(c.arguments) if isinstance( c.arguments, (list, tuple)) else c.arguments content.append(("Arguments", args)) if c.input_files: content.append( ("InputSandbox", make_unique(rel_input_paths_sub.values()))) if c.output_files: content.append(("OutputSandbox", make_unique(c.output_files))) if c.output_uri: content.append(("OutputSandboxBaseDestUri", c.output_uri)) if c.vo: content.append(("VirtualOrganisation", c.vo)) if c.stdout: content.append(("StdOutput", c.stdout)) if c.stderr: content.append(("StdError", c.stderr)) # add custom content if c.custom_content: content += c.custom_content # write the job file with open(job_file, "w") as f: f.write("[\n") for key, value in content: f.write(self.create_line(key, value) + "\n") f.write("]\n") logger.debug("created glite job file at '{}'".format(job_file)) return job_file, c
def create(self, postfix=None, render_variables=None, **kwargs): # merge kwargs and instance attributes c = self.get_config(kwargs) # some sanity checks if not c.file_name: raise ValueError("file_name must not be empty") elif not c.command: raise ValueError("command must not be empty") elif not c.shell: raise ValueError("shell must not be empty") # default render variables if not render_variables: render_variables = {} # add postfix to render variables if postfix and "file_postfix" not in render_variables: render_variables["file_postfix"] = postfix # linearize render variables render_variables = self.linearize_render_variables(render_variables) # prepare paths job_file = self.postfix_file(os.path.join(c.dir, c.file_name), postfix) # prepare input files def prepare_input(path): path = self.provide_input(os.path.abspath(path), postfix, c.dir, render_variables) path = path if c.absolute_paths else os.path.basename(path) return path c.input_files = list(map(prepare_input, c.input_files)) # output files if c.postfix_output_files: c.output_files = [ self.postfix_file(path, postfix) for path in c.output_files ] c.stdout = c.stdout and self.postfix_file(c.stdout, postfix) c.stderr = c.stdout and self.postfix_file(c.stderr, postfix) # custom log file if c.custom_log_file: c.custom_log_file = self.postfix_file(c.custom_log_file, postfix) c.output_files.append(c.custom_log_file) # job file content content = [] content.append("#!/usr/bin/env {}".format(c.shell)) if c.job_name: content.append(("-J", c.job_name)) if c.queue: content.append(("-q", c.queue)) if c.cwd: content.append(("-cwd", c.cwd)) if c.stdout: content.append(("-o", c.stdout)) if c.stderr: content.append(("-e", c.stderr)) if c.emails: content.append(("-N", )) if c.custom_content: content += c.custom_content if not c.manual_stagein: for input_file in make_unique(c.input_files): content.append( ("-f", "\"{} > {}\"".format(input_file, os.path.basename(input_file)))) if not c.manual_stageout: for output_file in make_unique(c.output_files): content.append( ("-f", "\"{} < {}\"".format(output_file, os.path.basename(output_file)))) if c.manual_stagein: tmpl = "cp " + ("{}" if c.absolute_paths else "$LS_EXECCWD/{}") + " $( pwd )/{}" for input_file in make_unique(c.input_files): content.append( tmpl.format(input_file, os.path.basename(input_file))) content.append(c.command) if c.manual_stageout: tmpl = "cp $( pwd )/{} $LS_EXECCWD/{}" for output_file in c.output_files: content.append(tmpl.format(output_file, output_file)) # write the job file with open(job_file, "w") as f: for line in content: if not isinstance(line, six.string_types): line = self.create_line(*make_list(line)) f.write(line + "\n") logger.debug("created lsf job file at '{}'".format(job_file)) return job_file, c