def __init__(self, terminal = None, command = None, working_directory=None, dependencies=None, identifier = None, exitcode_zero = True, **kwargs): self.verbose = False if terminal is None: terminal = current_machine() elif isinstance(terminal, str): self.terminal_name = terminal terminal =getattr(current_machine(), terminal) self.working_directory = working_directory self.terminal = terminal self.command = command self.has_compression = False self.is_compressed = False if hasattr(self,"additional_arguments"): self.additional_arguments.update( kwargs ) else: self.additional_arguments = kwargs self._state = self.STATE.NOJOB self.dependencies = [] if dependencies is None else dependencies self._identifier = self.generate_identifier() if identifier is None else identifier self._identifier_filename = ".batchq.%s"%self._identifier self._ret = "" self._exitcode = -1 self._was_executed = False self.exitcode_zero = exitcode_zero
def __init__(self, terminal, command, working_directory=None, dependencies=None,identifier = None, **kwargs): if not hasattr(self,"original_command"): self.original_command = command self.additional_arguments = {'processes': 1, 'time': -1, 'mpi': False, 'threads': 1, 'memory':-1, 'diskspace': -1} # setting the terminal variables self.machine = current_machine() super(PBSScript,self).__init__(terminal, command, working_directory, dependencies,identifier, **kwargs) prepend = "" qsubparams ="-l " if self.additional_arguments['processes'] != 1 or self.additional_arguments['threads'] != 1: qsubparams += "nodes=%s" % self.additional_arguments['processes'] if self.additional_arguments['threads'] != 1: qsubparams += ":ppn=%s" % self.additional_arguments['threads'] if self.additional_arguments['time'] !=-1: qsubparams += ",walltime=%s" % str(self.additional_arguments['time']) if self.additional_arguments['memory'] !=-1: qsubparams += ",mem=%s" % str(self.additional_arguments['memory']) if self.additional_arguments['diskspace'] !=-1: qsubparams += ",file=%s" % str(self.additional_arguments['diskspace']) if qsubparams == "-l ": qsubparams = '' # PBS Pro does not have this #qsubparams+=" -d ." self.batch_arguments = ". \"%s\" \"%s\" \"%s\" \"%s\""%(self._identifier,self.original_command, prepend,qsubparams) self.command = "batchq pbs_submit_script %s"%self.batch_arguments self.has_compression = True self.is_compressed = False self.job_info = None
def __init__(self, terminal=None, command=None, working_directory=None, dependencies=None, identifier=None, **kwargs): self.cache_timeout = 600 self.machine = current_machine() if not hasattr(self, "original_command"): self.original_command = command self._pid = None super(Subshell, self).__init__(terminal, command, working_directory, dependencies, identifier, **kwargs) self.command = "".join([ "(touch ", self._identifier_filename, ".submitted ; touch ", self._identifier_filename, ".running ; ( ( ", self.original_command, " ) 1> ", self._identifier_filename, ".running 2> ", self._identifier_filename, ".error ; echo $? > ", self._identifier_filename, ".finished ; if [ $(cat ", self._identifier_filename, ".finished) -ne 0 ] ; then mv ", self._identifier_filename, ".finished ", self._identifier_filename, ".failed ; fi ) & echo $! > ", self._identifier_filename, ".pid )" ])
def __init__(self, terminal= None, command=None, working_directory=None, dependencies=None,identifier = None, **kwargs): self.cache_timeout = 5 self.machine = current_machine() if not hasattr(self,"original_command"): self.original_command = command self._pid = None super(Subshell,self).__init__(terminal, command, working_directory, dependencies,identifier,**kwargs) self.command = "".join( ["(touch ",self._identifier_filename, ".submitted ; touch ",self._identifier_filename,".running ; ( ( ", self.original_command, " ) 1> ",self._identifier_filename,".running 2> ",self._identifier_filename,".error ; echo $? > ",self._identifier_filename,".finished ; if [ $(cat ",self._identifier_filename,".finished) -ne 0 ] ; then mv ",self._identifier_filename,".finished ",self._identifier_filename,".failed ; fi ) & echo $! > ",self._identifier_filename,".pid )"] )
def __init__(self, terminal=None, command=None, working_directory=None, dependencies=None, identifier=None, exitcode_zero=True, **kwargs): self.verbose = False if terminal is None: terminal = current_machine() elif isinstance(terminal, str): self.terminal_name = terminal terminal = getattr(current_machine(), terminal) self.working_directory = working_directory self.terminal = terminal self.command = command self.has_compression = False self.is_compressed = False if hasattr(self, "additional_arguments"): self.additional_arguments.update(kwargs) else: self.additional_arguments = kwargs self._state = self.STATE.NOJOB self.dependencies = [] if dependencies is None else dependencies self._identifier = self.generate_identifier( ) if identifier is None else identifier self._identifier_filename = ".batchq.%s" % self._identifier self._ret = "" self._exitcode = -1 self._was_executed = False self.exitcode_zero = exitcode_zero
def __init__(self, terminal, command, working_directory=None, dependencies=None, identifier=None, **kwargs): if not hasattr(self, "original_command"): self.original_command = command self.additional_arguments = { 'processes': 1, 'time': -1, 'mpi': False, 'threads': 1, 'memory': -1, 'diskspace': -1 } # setting the terminal variables self.machine = current_machine() super(PBSScript, self).__init__(terminal, command, working_directory, dependencies, identifier, **kwargs) prepend = "" qsubparams = "-l " if self.additional_arguments[ 'processes'] != 1 or self.additional_arguments['threads'] != 1: qsubparams += "nodes=%s" % self.additional_arguments['processes'] if self.additional_arguments['threads'] != 1: qsubparams += ":ppn=%s" % self.additional_arguments['threads'] if self.additional_arguments['time'] != -1: qsubparams += ",walltime=%s" % str( self.additional_arguments['time']) if self.additional_arguments['memory'] != -1: qsubparams += ",mem=%s" % str(self.additional_arguments['memory']) if self.additional_arguments['diskspace'] != -1: qsubparams += ",file=%s" % str( self.additional_arguments['diskspace']) if qsubparams == "-l ": qsubparams = '' # PBS Pro does not have this #qsubparams+=" -d ." self.batch_arguments = ". \"%s\" \"%s\" \"%s\" \"%s\"" % ( self._identifier, self.original_command, prepend, qsubparams) self.command = "batchq pbs_submit_script %s" % self.batch_arguments self.has_compression = True self.is_compressed = False self.job_info = None
def __init__(self, terminal, command, working_directory=None, dependencies=None, identifier=None, **kwargs): if not hasattr(self, "original_command"): self.original_command = command self.additional_arguments = { 'processes': 1, 'time': -1, 'mpi': False, 'threads': 1, 'memory': -1, 'diskspace': -1 } # setting the terminal variables self.machine = current_machine() super(LSF, self).__init__(terminal, command, working_directory, dependencies, identifier, **kwargs) prepend = "" if self.additional_arguments['mpi']: prepend = "mpirun -np %d " % self.additional_arguments['processes'] bsubparams = "-n %d " % self.additional_arguments['processes'] if self.additional_arguments['time'] != -1: bsubparams += "".join( ["-W ", str(self.additional_arguments['time']), " "]) if self.additional_arguments['memory'] != -1: bsubparams += "".join([ "-R \"rusage[mem=", str(self.additional_arguments['memory']), "]\" " ]) if self.additional_arguments['diskspace'] != -1: bsubparams += "".join([ "-R \"rusage[scratch=", str(self.additional_arguments['diskspace']), "]\" " ]) self.batch_arguments = ". \"%s\" \"%s\" \"%s\" \"%s\"" % ( self._identifier, self.original_command, prepend, bsubparams) self.command = "batchq lsf_submit_job %s" % self.batch_arguments self.has_compression = True self.is_compressed = False
def compute(self): if not self.hasInputFromPort('machine'): raise ModuleError(self, "No machine specified") if not self.hasInputFromPort('command'): raise ModuleError(self, "No command specified") command = self.getInputFromPort('command').strip() machine = self.getInputFromPort('machine').machine jm = JobMonitor.getInstance() cache = jm.getCache(self.signature) if cache: result = cache['result'] else: ## This indicates that the coming commands submitted on the machine # trick to select machine without initializing every time use_machine(machine) m = current_machine() result = m.remote.send_command(command) end_machine() cache = jm.setCache(self.signature, {'result':result}) self.setResult("output", result) self.setResult("machine", self.getInputFromPort('machine'))
def compute(self): if not self.hasInputFromPort('machine'): raise ModuleError(self, "No machine specified") if not self.hasInputFromPort('command'): raise ModuleError(self, "No command specified") command = self.getInputFromPort('command').strip() machine = self.getInputFromPort('machine').machine jm = JobMonitor.getInstance() cache = jm.getCache(self.signature) if cache: result = cache['result'] else: ## This indicates that the coming commands submitted on the machine # trick to select machine without initializing every time use_machine(machine) m = current_machine() result = m.remote.send_command(command) end_machine() cache = jm.setCache(self.signature, {'result': result}) self.setResult("output", result) self.setResult("machine", self.getInputFromPort('machine'))
def __init__(self, terminal, command, working_directory=None, dependencies=None,identifier = None, **kwargs): if not hasattr(self,"original_command"): self.original_command = command self.additional_arguments = {'processes': 1, 'time': -1, 'mpi': False, 'threads': 1, 'memory':-1, 'diskspace': -1} # setting the terminal variables self.machine = current_machine() super(LSF,self).__init__(terminal, command, working_directory, dependencies,identifier, **kwargs) prepend = "" if self.additional_arguments['mpi']: prepend = "mpirun -np %d " % self.additional_arguments['processes'] bsubparams ="-n %d " % self.additional_arguments['processes'] if self.additional_arguments['time'] !=-1: bsubparams+="".join(["-W ",str(self.additional_arguments['time']), " "]) if self.additional_arguments['memory'] !=-1: bsubparams+="".join(["-R \"rusage[mem=",str(self.additional_arguments['memory']), "]\" "]) if self.additional_arguments['diskspace'] !=-1: bsubparams+="".join(["-R \"rusage[scratch=",str(self.additional_arguments['diskspace']), "]\" "]) self.batch_arguments = ". \"%s\" \"%s\" \"%s\" \"%s\""%(self._identifier,self.original_command, prepend,bsubparams) self.command = "batchq lsf_submit_job %s"%self.batch_arguments self.has_compression = True self.is_compressed = False