def run(self):
"""Method submits command list to shell.
"""
self._build_shell_script()
system("bash %s" % self._shell_script)
logger.info("WorkFlow SUBMIT: %s" % self._shell_script)
def __init__(self, job_list=[], lsf=True, slurm=False, name=None, additionalmodules={}):
"""Initialize WorkFlow.
Method sets job lists and environment. Depending on the environment,
job names are checked before submission.
"""
if not((not slurm and lsf) or (slurm and not lsf)):
raise RuntimeError('You can only choose LSF or SLURM',
'Can not process a workflow as only LSF or SLURM may be invoked')
self.jobs = job_list
#LSF for the moment overides SLURM
if lsf and not slurm:
self.lsf = True
self.slurm = False
elif not lsf and slurm:
self.lsf = False
self.slurm = True
else:
assert False
self._check_jobnames()
self.additionalmodules = additionalmodules
now = datetime.now()
if not name:
self._shell_script = '%s_tfpipe_workflow.sh' % \
now.strftime("%Y%m%d%H%M%S")
else:
self._shell_script = name
logger.info("WorkFlow created")
def show(self):
"""Method prints out the shell script to stdout
"""
submit_str = self._build_shell_script_to_text()
print submit_str
logger.info("WorkFlow SHOW: %s" % submit_str)
def _io_flag_input(self, value):
"""Get job's input file from previous job output.
"""
self.input_file = value
logger.info("%s: input_file attribute '%s' set for %s" %
(self.name, value, self.cmd))
def _io_flag_output(self, value):
"""Set output_file attribute.
"""
self.output_file = value
logger.info("%s: output_file attribute '%s' set for %s" %
(self.name, value, self.cmd))
def add_bsub_argument(self, arg, value=None):
"""Method adds command line arguments to future bsub command.
"""
self.bsub_args[arg] = True and value or ''
logger.info("%s: argument '%s %s' added to %s" %
(self.name, arg, self.bsub_args[arg], self.cmd))
def __init__(self, **inputs):
"""Initialize Job.
Objects that inherit this class receive Job methods and attributes.
Those parent objects can also override the cmd attribute.
Parameters:
cmd, args, pos_args, name, dep, dep_str
"""
message = "Illegal input argument pass during init."
self._check_valid_input_options(self.init_options,
inputs.keys(),
message)
if not hasattr(self, '_cmd'):
raise InvalidObjectCall, "This object cannot be called directly."
if hasattr(self,'_memory_req_slurm'):
self._memory_req_slurm = inputs.get('_memory_req_slurm', self._memory_req_slurm)
else:
self._memory_req_slurm = None
if hasattr(self,'_memory_req_lsf'):
self._memory_req_lsf = inputs.get('_memory_req_lsf', self._memory_req_lsf)
else:
self._memory_req_lsf = None
self.cmd = inputs.get('cmd', self._cmd)
self.args = inputs.get('args', {})
self.pos_args = inputs.get('pos_args', [])
self.name = self._initialize_name(inputs)
# These store values in the string form of the job control system in question
self._dep_str_lsf = None
self._dep_str_slurm = None
self._time_str_slurm = '"06:00:00"'
# TODO REFACTOR - This is old code when you could pass dependencies at initialization.
self.dep = {}
self.redirect_output_file = ''
self.append_output_file = ''
self.redirect_error_file = ''
self.input_file = None
self.output_file = None
self.error_file = None
self.queue = None
self.hoststospan = 1
self.numberofprocesses = 1
self.job_output_file = "%s.out" % (self.name)
self.io_flag_handler = {'input': self._io_flag_input,
'output': self._io_flag_output,
None: None}
jobobj = jobid.Instance()
self._jobid = jobobj.getjobid()
#Deal with the memory requirements for seperate job controllers
if inputs.get('module'):
self._module = inputs.get('module')
if inputs.get('module_slurm'):
self._module_slurm = inputs.get('module_slurm')
logger.info("%s: initialized with '%s' arguments and command: %s " %
(self.name, self._parse_args(), self.cmd))
def show(self):
"""Method shows all job submission strings and lists in WorkFlow.
"""
for job in self.jobs:
submit_str = self._create_submit_str(job)
print submit_str
logger.info("WorkFlow SHOW: %s" % submit_str)
def run(self):
"""Method submits command list to shell.
"""
with open(self._shell_script, 'w') as f:
f.write(self._build_shell_script_to_text())
system("bash %s" % self._shell_script)
logger.info("WorkFlow SUBMIT: %s" % self._shell_script)
def add_jobname(self, jobname):
"""Add name to current job.
"""
tmp = self.name
self.name = str(jobname)
logger.info("%s: replacing jobname with '%s'" %
(tmp, self.name))
def add_positional_argument(self, arg, io_flag=None):
"""Method adds positional arguments to object.
"""
handler = self.io_flag_handler.get(io_flag)
if handler:
handler(arg)
self.pos_args.append(arg)
logger.info("%s: argument '%s' added to %s" %
(self.name, arg, self.cmd))
def _check_jobnames(self):
"""Method to check job names for duplicates.
WorkFlow terminates if duplicate is found in LSF mode.
"""
job_names = [job.name for job in self.jobs]
if (len(set(job_names)) == len(job_names)) and self.lsf:
logger.info("WorkFlow job names are unique.")
elif self.lsf:
DuplicateJobNames("WARNING: WorkFlow job names are NOT unique.")
def redirect_error(self, errorfile, io_flag=None):
"""Method to redirect error in the Unix sense.
Has ability to set error file as file to be referenced.
"""
handler = self.io_flag_handler.get(io_flag)
if handler:
handler(errorfile)
self.redirect_error_file = errorfile
logger.info("%s: error_file attribute '%s' set for %s" %
(self.name, self.error_file, self.cmd))
def add_argument(self, arg, value=None, io_flag=None):
"""Method adds command line arguments to object.
Argument value should be either a string or some Job instance. io_flag
can only be 'input' or 'output' and will set the respective file
attributes.
"""
handler = self.io_flag_handler.get(io_flag)
if handler:
handler(value)
self.args[arg] = True and value or ''
logger.info("%s: argument '%s %s' added to %s" %
(self.name, arg, self.args[arg], self.cmd))
def append_output(self, outputfile, io_flag=None):
"""Method to append output in the Unix sense.
Has ability to set output as output file to be referenced.
"""
if self.redirect_output_file:
error = "Cannot redirect and append output"
logger.error(error)
exit(error)
handler = self.io_flag_handler.get(io_flag)
if handler:
handler(outputfile)
self.append_output_file = outputfile
logger.info("%s: output_file attribute '%s' set for %s" %
(self.name, self.output_file, self.cmd))
def __init__(self, job_list=[], lsf=True, name=None, additionalmodules={}):
"""Initialize WorkFlow.
Method sets job lists and environment. Depending on the environment,
job names are checked before submission.
"""
self.jobs = job_list
self.lsf = lsf
self._check_jobnames()
self.additionalmodules = additionalmodules
now = datetime.now()
if not name:
self._shell_script = '%s_tfpipe_workflow.sh' % \
now.strftime("%Y%m%d%H%M%S")
else:
self._shell_script = name
logger.info("WorkFlow created")
def __init__(self, **inputs):
"""Initialize Job.
Objects that inherit this class receive Job methods and attributes.
Those parent objects can also override the cmd attribute.
Parameters:
cmd, args, pos_args, name, dep, dep_str
"""
message = "Illegal input argument pass during init."
self._check_valid_input_options(self.init_options,
inputs.keys(),
message)
if not hasattr(self, '_cmd'):
raise InvalidObjectCall, "This object cannot be called directly."
self.cmd = inputs.get('cmd', self._cmd)
self.args = inputs.get('args', {})
self.pos_args = inputs.get('pos_args', [])
self.name = self._initialize_name(inputs)
self.dep_str = inputs.get('dep_str', '')
self.dep_str_at_init = bool(self.dep_str)
self.dep = self._initialize_dependencies(inputs)
self.bsub_args = inputs.get('bsub_args', {})
self.redirect_output_file = ''
self.append_output_file = ''
self.redirect_error_file = ''
self.input_file = None
self.output_file = None
self.error_file = None
self.job_output_file = "%s.out" % (self.name)
self.io_flag_handler = {'input': self._io_flag_input,
'output': self._io_flag_output,
None: None}
if inputs.get('module'):
self._module = inputs.get('module')
logger.info("%s: initialized with '%s' arguments and command: %s " %
(self.name, self._parse_args(), self.cmd))
def _build_shell_script(self):
""" """
mods = []
with open(self._shell_script, 'w') as f:
f.write("#!/bin/bash\n")
if self.lsf:
f.write(". /nas02/apps/Modules/default/init/bash\n")
for job in self.jobs:
try:
if job._module not in mods:
f.write("module load %s\n" % job._module)
mods.append(job._module)
except AttributeError:
pass
for module in self.additionalmodules:
try:
if module not in mods:
f.write("module load %s\n" % module)
mods.append(module)
except AttributeError:
pass
for job in self.jobs:
f.write("%s\n" % self._create_submit_str(job))
logger.info("WorkFlow Submission Script Created")
def add_job(self, newjob):
"""Add job to list.
"""
self.jobs.append(newjob)
logger.info("WorkFlow ADD: %s" % newjob)