def __init__(self, terminal = None, command = None, working_directory=None, dependencies=None, identifier = None, exitcode_zero = True, **kwargs):
self.verbose = False
if terminal is None:
terminal = current_machine()
elif isinstance(terminal, str):
self.terminal_name = terminal
terminal =getattr(current_machine(), terminal)
self.working_directory = working_directory
self.terminal = terminal
self.command = command
self.has_compression = False
self.is_compressed = False
if hasattr(self,"additional_arguments"):
self.additional_arguments.update( kwargs )
else:
self.additional_arguments = kwargs
self._state = self.STATE.NOJOB
self.dependencies = [] if dependencies is None else dependencies
self._identifier = self.generate_identifier() if identifier is None else identifier
self._identifier_filename = ".batchq.%s"%self._identifier
self._ret = ""
self._exitcode = -1
self._was_executed = False
self.exitcode_zero = exitcode_zero
def __init__(self, terminal, command, working_directory=None, dependencies=None,identifier = None, **kwargs):
if not hasattr(self,"original_command"):
self.original_command = command
self.additional_arguments = {'processes': 1, 'time': -1, 'mpi': False, 'threads': 1, 'memory':-1, 'diskspace': -1}
# setting the terminal variables
self.machine = current_machine()
super(PBSScript,self).__init__(terminal, command, working_directory, dependencies,identifier, **kwargs)
prepend = ""
qsubparams ="-l "
if self.additional_arguments['processes'] != 1 or self.additional_arguments['threads'] != 1:
qsubparams += "nodes=%s" % self.additional_arguments['processes']
if self.additional_arguments['threads'] != 1:
qsubparams += ":ppn=%s" % self.additional_arguments['threads']
if self.additional_arguments['time'] !=-1: qsubparams += ",walltime=%s" % str(self.additional_arguments['time'])
if self.additional_arguments['memory'] !=-1: qsubparams += ",mem=%s" % str(self.additional_arguments['memory'])
if self.additional_arguments['diskspace'] !=-1: qsubparams += ",file=%s" % str(self.additional_arguments['diskspace'])
if qsubparams == "-l ":
qsubparams = ''
# PBS Pro does not have this
#qsubparams+=" -d ."
self.batch_arguments = ". \"%s\" \"%s\" \"%s\" \"%s\""%(self._identifier,self.original_command, prepend,qsubparams)
self.command = "batchq pbs_submit_script %s"%self.batch_arguments
self.has_compression = True
self.is_compressed = False
self.job_info = None
def __init__(self,
terminal=None,
command=None,
working_directory=None,
dependencies=None,
identifier=None,
**kwargs):
self.cache_timeout = 600
self.machine = current_machine()
if not hasattr(self, "original_command"):
self.original_command = command
self._pid = None
super(Subshell, self).__init__(terminal, command, working_directory,
dependencies, identifier, **kwargs)
self.command = "".join([
"(touch ", self._identifier_filename, ".submitted ; touch ",
self._identifier_filename, ".running ; ( ( ",
self.original_command, " ) 1> ", self._identifier_filename,
".running 2> ", self._identifier_filename, ".error ; echo $? > ",
self._identifier_filename, ".finished ; if [ $(cat ",
self._identifier_filename, ".finished) -ne 0 ] ; then mv ",
self._identifier_filename, ".finished ", self._identifier_filename,
".failed ; fi ) & echo $! > ", self._identifier_filename,
".pid )"
])
def __init__(self, terminal= None, command=None, working_directory=None, dependencies=None,identifier = None, **kwargs):
self.cache_timeout = 5
self.machine = current_machine()
if not hasattr(self,"original_command"):
self.original_command = command
self._pid = None
super(Subshell,self).__init__(terminal, command, working_directory, dependencies,identifier,**kwargs)
self.command = "".join( ["(touch ",self._identifier_filename, ".submitted ; touch ",self._identifier_filename,".running ; ( ( ", self.original_command, " ) 1> ",self._identifier_filename,".running 2> ",self._identifier_filename,".error ; echo $? > ",self._identifier_filename,".finished ; if [ $(cat ",self._identifier_filename,".finished) -ne 0 ] ; then mv ",self._identifier_filename,".finished ",self._identifier_filename,".failed ; fi ) & echo $! > ",self._identifier_filename,".pid )"] )
def __init__(self,
terminal=None,
command=None,
working_directory=None,
dependencies=None,
identifier=None,
exitcode_zero=True,
**kwargs):
self.verbose = False
if terminal is None:
terminal = current_machine()
elif isinstance(terminal, str):
self.terminal_name = terminal
terminal = getattr(current_machine(), terminal)
self.working_directory = working_directory
self.terminal = terminal
self.command = command
self.has_compression = False
self.is_compressed = False
if hasattr(self, "additional_arguments"):
self.additional_arguments.update(kwargs)
else:
self.additional_arguments = kwargs
self._state = self.STATE.NOJOB
self.dependencies = [] if dependencies is None else dependencies
self._identifier = self.generate_identifier(
) if identifier is None else identifier
self._identifier_filename = ".batchq.%s" % self._identifier
self._ret = ""
self._exitcode = -1
self._was_executed = False
self.exitcode_zero = exitcode_zero
def __init__(self,
terminal,
command,
working_directory=None,
dependencies=None,
identifier=None,
**kwargs):
if not hasattr(self, "original_command"):
self.original_command = command
self.additional_arguments = {
'processes': 1,
'time': -1,
'mpi': False,
'threads': 1,
'memory': -1,
'diskspace': -1
}
# setting the terminal variables
self.machine = current_machine()
super(PBSScript, self).__init__(terminal, command, working_directory,
dependencies, identifier, **kwargs)
prepend = ""
qsubparams = "-l "
if self.additional_arguments[
'processes'] != 1 or self.additional_arguments['threads'] != 1:
qsubparams += "nodes=%s" % self.additional_arguments['processes']
if self.additional_arguments['threads'] != 1:
qsubparams += ":ppn=%s" % self.additional_arguments['threads']
if self.additional_arguments['time'] != -1:
qsubparams += ",walltime=%s" % str(
self.additional_arguments['time'])
if self.additional_arguments['memory'] != -1:
qsubparams += ",mem=%s" % str(self.additional_arguments['memory'])
if self.additional_arguments['diskspace'] != -1:
qsubparams += ",file=%s" % str(
self.additional_arguments['diskspace'])
if qsubparams == "-l ":
qsubparams = ''
# PBS Pro does not have this
#qsubparams+=" -d ."
self.batch_arguments = ". \"%s\" \"%s\" \"%s\" \"%s\"" % (
self._identifier, self.original_command, prepend, qsubparams)
self.command = "batchq pbs_submit_script %s" % self.batch_arguments
self.has_compression = True
self.is_compressed = False
self.job_info = None
def __init__(self,
terminal,
command,
working_directory=None,
dependencies=None,
identifier=None,
**kwargs):
if not hasattr(self, "original_command"):
self.original_command = command
self.additional_arguments = {
'processes': 1,
'time': -1,
'mpi': False,
'threads': 1,
'memory': -1,
'diskspace': -1
}
# setting the terminal variables
self.machine = current_machine()
super(LSF, self).__init__(terminal, command, working_directory,
dependencies, identifier, **kwargs)
prepend = ""
if self.additional_arguments['mpi']:
prepend = "mpirun -np %d " % self.additional_arguments['processes']
bsubparams = "-n %d " % self.additional_arguments['processes']
if self.additional_arguments['time'] != -1:
bsubparams += "".join(
["-W ", str(self.additional_arguments['time']), " "])
if self.additional_arguments['memory'] != -1:
bsubparams += "".join([
"-R \"rusage[mem=",
str(self.additional_arguments['memory']), "]\" "
])
if self.additional_arguments['diskspace'] != -1:
bsubparams += "".join([
"-R \"rusage[scratch=",
str(self.additional_arguments['diskspace']), "]\" "
])
self.batch_arguments = ". \"%s\" \"%s\" \"%s\" \"%s\"" % (
self._identifier, self.original_command, prepend, bsubparams)
self.command = "batchq lsf_submit_job %s" % self.batch_arguments
self.has_compression = True
self.is_compressed = False
def compute(self):
if not self.hasInputFromPort('machine'):
raise ModuleError(self, "No machine specified")
if not self.hasInputFromPort('command'):
raise ModuleError(self, "No command specified")
command = self.getInputFromPort('command').strip()
machine = self.getInputFromPort('machine').machine
jm = JobMonitor.getInstance()
cache = jm.getCache(self.signature)
if cache:
result = cache['result']
else:
## This indicates that the coming commands submitted on the machine
# trick to select machine without initializing every time
use_machine(machine)
m = current_machine()
result = m.remote.send_command(command)
end_machine()
cache = jm.setCache(self.signature, {'result':result})
self.setResult("output", result)
self.setResult("machine", self.getInputFromPort('machine'))
def compute(self):
if not self.hasInputFromPort('machine'):
raise ModuleError(self, "No machine specified")
if not self.hasInputFromPort('command'):
raise ModuleError(self, "No command specified")
command = self.getInputFromPort('command').strip()
machine = self.getInputFromPort('machine').machine
jm = JobMonitor.getInstance()
cache = jm.getCache(self.signature)
if cache:
result = cache['result']
else:
## This indicates that the coming commands submitted on the machine
# trick to select machine without initializing every time
use_machine(machine)
m = current_machine()
result = m.remote.send_command(command)
end_machine()
cache = jm.setCache(self.signature, {'result': result})
self.setResult("output", result)
self.setResult("machine", self.getInputFromPort('machine'))
def __init__(self, terminal, command, working_directory=None, dependencies=None,identifier = None, **kwargs):
if not hasattr(self,"original_command"):
self.original_command = command
self.additional_arguments = {'processes': 1, 'time': -1, 'mpi': False, 'threads': 1, 'memory':-1, 'diskspace': -1}
# setting the terminal variables
self.machine = current_machine()
super(LSF,self).__init__(terminal, command, working_directory, dependencies,identifier, **kwargs)
prepend = ""
if self.additional_arguments['mpi']: prepend = "mpirun -np %d " % self.additional_arguments['processes']
bsubparams ="-n %d " % self.additional_arguments['processes']
if self.additional_arguments['time'] !=-1: bsubparams+="".join(["-W ",str(self.additional_arguments['time']), " "])
if self.additional_arguments['memory'] !=-1: bsubparams+="".join(["-R \"rusage[mem=",str(self.additional_arguments['memory']), "]\" "])
if self.additional_arguments['diskspace'] !=-1: bsubparams+="".join(["-R \"rusage[scratch=",str(self.additional_arguments['diskspace']), "]\" "])
self.batch_arguments = ". \"%s\" \"%s\" \"%s\" \"%s\""%(self._identifier,self.original_command, prepend,bsubparams)
self.command = "batchq lsf_submit_job %s"%self.batch_arguments
self.has_compression = True
self.is_compressed = False
