def _launch(self, classname, funcname, hosts='all'): unix.mkdir(PATH.SYSTEM) nodes = math.ceil(PAR.NTASK / float(PAR.NODESIZE)) ncpus = PAR.NPROC mpiprocs = PAR.NPROC hours = PAR.STEPTIME / 60 minutes = PAR.STEPTIME % 60 walltime = 'walltime=%02d:%02d:00 ' % (hours, minutes) # submit job with open(PATH.SYSTEM + '/' + 'job_id', 'w') as f: call( 'qsub ' + '%s ' % PAR.PBSARGS + '-l select=%d:ncpus=%d:mpiprocs=%d ' (nodes, ncpus, mpiprocs) + '-l %s ' % walltime + '-J 0-%s ' % (PAR.NTASK - 1) + '-N %s ' % PAR.TITLE + '-o %s ' % (PATH.SUBMIT + '/' + 'output.pbs/' + '$PBS_ARRAYID') + '-r y ' + '-j oe ' + '-V ' + self.launch_args(hosts) + PATH.OUTPUT + ' ' + classname + ' ' + funcname + ' ' + findpath('seisflows.system'), stdout=f) # retrieve job ids with open(PATH.SYSTEM + '/' + 'job_id', 'r') as f: line = f.readline() job = line.split()[-1].strip() if hosts == 'all' and PAR.NTASK > 1: nn = range(PAR.NTASK) job0 = job.strip('[].sdb') return [job0 + '[' + str(ii) + '].sdb' for ii in nn] else: return [job]
def submit(self, workflow): """Submits job """ unix.mkdir(PATH.OUTPUT) unix.cd(PATH.OUTPUT) # save current state self.checkpoint() # construct resource list nodes = int(PAR.NTASK / PAR.NODESIZE) cores = PAR.NTASK % PAR.NODESIZE hours = int(PAR.WALLTIME / 60) minutes = PAR.WALLTIME % 60 resources = 'walltime=%02d:%02d:00' % (hours, minutes) if nodes == 0: resources += ',mem=%dgb,nodes=1:ppn=%d' % (PAR.MEMORY, cores) elif cores == 0: resources += ',mem=%dgb,nodes=%d:ppn=%d' % (PAR.MEMORY, nodes, PAR.NODESIZE) else: resources += ',mem=%dgb,nodes=%d:ppn=%d+1:ppn=%d' % ( PAR.MEMORY, nodes, PAR.NODESIZE, cores) # construct arguments list call('qsub ' + '%s ' % PAR.PBSARGS + '-N %s ' % PAR.TITLE + '-o %s ' % (PATH.SUBMIT + '/' + 'output.log') + '-l %s ' % resources + '-j %s ' % 'oe' + findpath('seisflows.system') + '/' + 'wrappers/submit ' + '-F %s ' % PATH.OUTPUT)
def submit(self, workflow): """ Submits workflow """ unix.mkdir(PATH.OUTPUT) unix.cd(PATH.OUTPUT) unix.mkdir(PATH.SUBMIT+'/'+'output.pbs') self.checkpoint() hours = PAR.WALLTIME/60 minutes = PAR.WALLTIME%60 walltime = 'walltime=%02d:%02d:00 ' % (hours, minutes) ncpus = PAR.NODESIZE mpiprocs = PAR.NODESIZE # prepare qsub arguments call( 'qsub ' + '%s ' % PAR.PBSARGS + '-l select=1:ncpus=%d:mpiprocs=%d ' % (ncpus, mpiprocs) + '-l %s ' % walltime + '-N %s ' % PAR.TITLE + '-j %s '%'oe' + '-o %s ' % (PATH.SUBMIT+'/'+'output.log') + '-V ' + ' -- ' + findpath('seisflows.system') +'/'+ 'wrappers/submit ' + PATH.OUTPUT)
def launch(self, classname, funcname, hosts='all'): unix.mkdir(PATH.SYSTEM) # submit job with open(PATH.SYSTEM + '/' + 'job_id', 'w') as f: call('bsub ' + '%s ' % PAR.LSFARGS + '-n %d ' % PAR.NPROC + '-R "span[ptile=%d]" ' % PAR.NODESIZE + '-W %d:00 ' % PAR.STEPTIME + '-J "%s' % PAR.TITLE + self.launch_args(hosts) + findpath('seisflows.system') + '/' + 'wrapper/run ' + PATH.OUTPUT + ' ' + classname + ' ' + funcname + ' ', stdout=f) # retrieve job ids with open(PATH.SYSTEM + '/' + 'job_id', 'r') as f: # reads one entire line from the file line = f.readline() job_buf = line.split()[1].strip() job = job_buf[1:-1] if hosts == 'all' and PAR.NSRC > 1: nn = range(1, PAR.NSRC + 1) #return [job+'_'+str(ii) for ii in nn] return [job + '[' + str(ii) + ']' for ii in nn] else: return [job]
def _launch(self, classname, funcname, hosts='all'): unix.mkdir(PATH.SYSTEM) with open(PATH.SYSTEM+'/'+'job_id', 'w') as f: call('sbatch ' + '%s ' % PAR.SLURMARGS + '--job-name=%s ' % PAR.TITLE + '--nodes=%d ' % math.ceil(PAR.NPROC/float(PAR.NODESIZE)) + '--ntasks-per-node=%d ' % PAR.NODESIZE + '--ntasks=%d ' % PAR.NPROC + '--time=%d ' % PAR.STEPTIME + self._launch_args(hosts) + findpath('seisflows.system') +'/'+ 'wrappers/run ' + PATH.OUTPUT + ' ' + classname + ' ' + funcname + ' ', stdout=f) # retrieve job ids with open(PATH.SYSTEM+'/'+'job_id', 'r') as f: line = f.readline() job = line.split()[-1].strip() if hosts == 'all' and PAR.NTASK > 1: return [job+'_'+str(ii) for ii in range(PAR.NTASK)] else: return [job]
def run(self, classname, funcname, hosts='all', **kwargs): """ Runs tasks in serial or parallel on specified hosts """ self.checkpoint() self.save_kwargs(classname, funcname, kwargs) if hosts == 'all': # run on all available nodes call('srun ' + '--wait=0 ' + join(findpath('seisflows.system'), 'wrappers/run ') + PATH.OUTPUT + ' ' + classname + ' ' + funcname) elif hosts == 'head': # run on head node call('srun ' + '--wait=0 ' + join(findpath('seisflows.system'), 'wrappers/run_head ') + PATH.OUTPUT + ' ' + classname + ' ' + funcname) else: raise(KeyError('Hosts parameter not set/recognized.'))
def launch(self, classname, funcname, hosts='all'): unix.mkdir(PATH.SYSTEM) # submit job with open(PATH.SYSTEM+'/'+'job_id', 'w') as f: call('bsub ' + '%s ' % PAR.LSFARGS + '-n %d ' % PAR.NPROC + '-R "span[ptile=%d]" ' % PAR.NODESIZE + '-W %d:00 ' % PAR.STEPTIME + '-J "%s' %PAR.TITLE + self.launch_args(hosts) + findpath('seisflows.system') +'/'+ 'wrapper/run ' + PATH.OUTPUT + ' ' + classname + ' ' + funcname + ' ', stdout=f) # retrieve job ids with open(PATH.SYSTEM+'/'+'job_id', 'r') as f: # reads one entire line from the file line = f.readline() job_buf = line.split()[1].strip() job = job_buf[1:-1] if hosts == 'all' and PAR.NSRC > 1: nn = range(1,PAR.NSRC+1) #return [job+'_'+str(ii) for ii in nn] return [job+'['+str(ii)+']' for ii in nn] else: return [job]
def getstate(self, jobid): """ Retrives job state from LSF database """ with open(PATH.SYSTEM+'/'+'job_status', 'w') as f: call('bjobs -a -d "' + jobid + '"', stdout=f) with open(PATH.SYSTEM+'/'+'job_status', 'r') as f: lines = f.readlines() state = lines[1].split()[2].strip() return state
def getstate(self, jobid): """ Retrives job state from LSF database """ with open(PATH.SYSTEM + '/' + 'job_status', 'w') as f: call('bjobs -a -d "' + jobid + '"', stdout=f) with open(PATH.SYSTEM + '/' + 'job_status', 'r') as f: lines = f.readlines() state = lines[1].split()[2].strip() return state
def _query(self, jobid): """ Queries job state from SLURM database """ with open(PATH.SYSTEM+'/'+'job_status', 'w') as f: call('sacct -n -o state -j '+jobid, stdout=f) with open(PATH.SYSTEM+'/'+'job_status', 'r') as f: line = f.readline() state = line.strip() return state
def _query(self, jobid): """ Queries job state from PBS database """ # TODO: replace shell utilities with native Python with open(PATH.SYSTEM + '/' + 'job_status', 'w') as f: call('qstat -x -tJ ' + jobid + ' | ' + 'tail -n 1 ' + ' | ' + 'awk \'{print $5}\'', stdout=f) with open(PATH.SYSTEM + '/' + 'job_status', 'r') as f: line = f.readline() state = line.strip() return state
def submit(self, workflow): """ Submits workflow """ unix.mkdir(PATH.OUTPUT) unix.cd(PATH.OUTPUT) self.checkpoint() # submit workflow call('sbatch ' + '%s ' % PAR.SLURMARGS + '--job-name=%s ' % PAR.TITLE + '--output=%s ' % (PATH.SUBMIT + '/' + 'output.log') + '--cpus-per-task=%d ' % PAR.NPROC + '--ntasks=%d ' % PAR.NTASK + '--time=%d ' % PAR.WALLTIME + findpath('seisflows.system') + '/' + 'wrappers/submit ' + PATH.OUTPUT)
def submit(self, workflow): """ Submits workflow """ unix.mkdir(PATH.OUTPUT) unix.cd(PATH.OUTPUT) unix.mkdir(PATH.SUBMIT + '/' + 'output.slurm') self.checkpoint() # prepare sbatch arguments call('sbatch ' + '%s ' % PAR.SLURMARGS + '--job-name=%s ' % PAR.TITLE + '--output %s ' % (PATH.SUBMIT + '/' + 'output.log') + '--ntasks-per-node=%d ' % PAR.NODESIZE + '--nodes=%d ' % 1 + '--time=%d ' % PAR.WALLTIME + findpath('seisflows.system') + '/' + 'wrappers/submit ' + PATH.OUTPUT)
def submit(self, workflow): """ Submits workflow """ unix.mkdir(PATH.OUTPUT) unix.cd(PATH.OUTPUT) unix.mkdir(PATH.SUBMIT + '/' + 'output.lsf') self.checkpoint() # prepare bsub arguments call('bsub ' + '%s ' % PAR.LSFARGS + '-J %s ' % PAR.TITLE + '-o %s ' % (PATH.SUBMIT + '/' + 'output.log') + '-n %d ' % PAR.NTASK + '-e %s ' % (PATH.SUBMIT + '/' + 'error.log') + '-W %d:00 ' % PAR.WALLTIME + findpath('seisflows.system') + '/' + 'wrappers/submit ' + PATH.OUTPUT)
def _query(self, jobid): """ Queries job state from PBS database """ # TODO: replace shell utilities with native Python with open(PATH.SYSTEM+'/'+'job_status', 'w') as f: call('qstat -x -tJ ' + jobid + ' | ' + 'tail -n 1 ' + ' | ' + 'awk \'{print $5}\'', stdout=f) with open(PATH.SYSTEM+'/'+'job_status', 'r') as f: line = f.readline() state = line.strip() return state
def submit(self, workflow): """ Submits workflow """ unix.mkdir(PATH.OUTPUT) unix.cd(PATH.OUTPUT) self.checkpoint() # submit workflow call('sbatch ' + '%s ' % PAR.SLURMARGS + '--job-name=%s '%PAR.TITLE + '--output=%s '%(PATH.SUBMIT +'/'+ 'output.log') + '--cpus-per-task=%d '%PAR.NPROC + '--ntasks=%d '%PAR.NTASK + '--time=%d '%PAR.WALLTIME + findpath('seisflows.system') +'/'+ 'wrappers/submit ' + PATH.OUTPUT)
def submit(self, workflow): """ Submits workflow """ unix.mkdir(PATH.OUTPUT) unix.cd(PATH.OUTPUT) unix.mkdir(PATH.SUBMIT+'/'+'output.slurm') self.checkpoint() # prepare sbatch arguments call('sbatch ' + '%s ' % PAR.SLURMARGS + '--job-name=%s ' % PAR.TITLE + '--output %s ' % (PATH.SUBMIT+'/'+'output.log') + '--ntasks-per-node=%d ' % PAR.NODESIZE + '--nodes=%d ' % 1 + '--time=%d ' % PAR.WALLTIME + findpath('seisflows.system') +'/'+ 'wrappers/submit ' + PATH.OUTPUT)
def submit(self, workflow): """ Submits workflow """ unix.mkdir(PATH.OUTPUT) unix.cd(PATH.OUTPUT) unix.mkdir(PATH.SUBMIT+'/'+'output.lsf') self.checkpoint() # prepare bsub arguments call('bsub ' + '%s ' % PAR.LSFARGS + '-J %s ' % PAR.TITLE + '-o %s ' % (PATH.SUBMIT+'/'+'output.log') + '-n %d ' % PAR.NTASK + '-e %s ' % (PATH.SUBMIT+'/'+'error.log') + '-W %d:00 ' % PAR.WALLTIME + findpath('seisflows.system') +'/'+ 'wrappers/submit ' + PATH.OUTPUT)
def run(self, classname, funcname, hosts='all', **kwargs): """ Runs tasks in serial or parallel on specified hosts """ self.checkpoint() self.save_kwargs(classname, funcname, kwargs) if hosts == 'all': # run on all available nodes call('srun ' + '--wait=0 ' + join(findpath('seisflows.system'), 'wrappers/run ') + PATH.OUTPUT + ' ' + classname + ' ' + funcname) elif hosts == 'head': # run on head node call('srun ' + '--wait=0 ' + join(findpath('seisflows.system'), 'wrappers/run_head ') + PATH.OUTPUT + ' ' + classname + ' ' + funcname) else: raise (KeyError('Hosts parameter not set/recognized.'))
def submit(self, workflow): """ Submits workflow """ unix.mkdir(PATH.OUTPUT) unix.cd(PATH.OUTPUT) self.checkpoint() if not exists(PATH.SUBMIT + '/' + 'scratch'): unix.ln(PATH.SCRATCH, PATH.SUBMIT + '/' + 'scratch') call('sbatch ' + '--job-name=%s ' % PAR.SUBTITLE + '--output=%s ' % (PATH.SUBMIT +'/'+ 'output.log') + '--nodes 1 ' + '--ntasks=% ' % PAR.NGPU + '--ntasks-per-socket=%d ' % PAR.NGPU + '--gres=gpu:%d ' % PAR.NGPU + '--time=%d ' % PAR.WALLTIME + findpath('seisflows.system') +'/'+ 'wrappers/submit ' + PATH.OUTPUT)
def _launch(self, classname, funcname, hosts='all'): unix.mkdir(PATH.SYSTEM) nodes = math.ceil(PAR.NTASK/float(PAR.NODESIZE)) ncpus = PAR.NPROC mpiprocs = PAR.NPROC hours = PAR.STEPTIME/60 minutes = PAR.STEPTIME%60 walltime = 'walltime=%02d:%02d:00 '%(hours, minutes) # submit job with open(PATH.SYSTEM+'/'+'job_id', 'w') as f: call('qsub ' + '%s ' % PAR.PBSARGS + '-l select=%d:ncpus=%d:mpiprocs=%d ' (nodes, ncpus, mpiprocs) + '-l %s ' % walltime + '-J 0-%s ' % (PAR.NTASK-1) + '-N %s ' % PAR.TITLE + '-o %s ' % (PATH.SUBMIT+'/'+'output.pbs/' + '$PBS_ARRAYID') + '-r y ' + '-j oe ' + '-V ' + self.launch_args(hosts) + PATH.OUTPUT + ' ' + classname + ' ' + funcname + ' ' + findpath('seisflows.system'), stdout=f) # retrieve job ids with open(PATH.SYSTEM+'/'+'job_id', 'r') as f: line = f.readline() job = line.split()[-1].strip() if hosts == 'all' and PAR.NTASK > 1: nn = range(PAR.NTASK) job0 = job.strip('[].sdb') return [job0+'['+str(ii)+'].sdb' for ii in nn] else: return [job]
def run(self, classname, funcname, hosts='all', **kwargs): """ Runs tasks in serial or parallel on specified hosts """ self.checkpoint() self.save_kwargs(classname, funcname, kwargs) if hosts == 'all': call('srun ' + '--wait=0 ' + join(findpath('seisflows.system'), 'wrappers/run ') + PATH.OUTPUT + ' ' + classname + ' ' + funcname) elif hosts == 'head': # run on head node call('srun ' + '--wait=0 ' + join(findpath('seisflows.system'), 'wrappers/run_head ') + PATH.OUTPUT + ' ' + classname + ' ' + funcname)
def submit(self, workflow): """ Submits workflow """ unix.mkdir(PATH.OUTPUT) unix.cd(PATH.OUTPUT) unix.mkdir(PATH.SUBMIT + '/' + 'output.pbs') self.checkpoint() hours = PAR.WALLTIME / 60 minutes = PAR.WALLTIME % 60 walltime = 'walltime=%02d:%02d:00 ' % (hours, minutes) ncpus = PAR.NODESIZE mpiprocs = PAR.NODESIZE # prepare qsub arguments call('qsub ' + '%s ' % PAR.PBSARGS + '-l select=1:ncpus=%d:mpiprocs=%d ' % (ncpus, mpiprocs) + '-l %s ' % walltime + '-N %s ' % PAR.TITLE + '-j %s ' % 'oe' + '-o %s ' % (PATH.SUBMIT + '/' + 'output.log') + '-V ' + ' -- ' + findpath('seisflows.system') + '/' + 'wrappers/submit ' + PATH.OUTPUT)
def run(self, classname, funcname, hosts='all', **kwargs): """ Runs tasks in serial or parallel on specified hosts """ self.checkpoint() self.save_kwargs(classname, funcname, kwargs) if hosts == 'all': # run on all available nodes call('pbsdsh ' + join(findpath('seisflows.system'), 'wrappers/export_paths.sh ') + os.getenv('PATH') + ' ' + os.getenv('LD_LIBRARY_PATH') + ' ' + join(findpath('seisflows.system'), 'wrappers/run_pbsdsh ') + PATH.OUTPUT + ' ' + classname + ' ' + funcname + ' ' + dirname(findpath('seisflows'))) elif hosts == 'head': # run on head node call('pbsdsh ' + join(findpath('seisflows.system'), 'wrappers/export_paths.sh ') + os.getenv('PATH') + ' ' + os.getenv('LD_LIBRARY_PATH') + ' ' + join(findpath('seisflows.system'), 'wrappers/run_pbsdsh_head ') + PATH.OUTPUT + ' ' + classname + ' ' + funcname + ' ' + dirname(findpath('seisflows')))
def submit(self, workflow): """Submits job """ unix.mkdir(PATH.OUTPUT) unix.cd(PATH.OUTPUT) # save current state self.checkpoint() # construct resource list resources = [] nodes = int(PAR.NTASK / PAR.NODESIZE) cores = PAR.NTASK % PAR.NODESIZE hours = int(PAR.WALLTIME / 60) minutes = PAR.WALLTIME % 60 if PAR.WALLTIME: resources += ['walltime=%02d:%02d:00'%(hours, minutes)] if PAR.MEMORY: resources += ['mem=%dgb' % PAR.MEMORY] if nodes == 0: resources += ['nodes=1:ppn=%d'%(cores)] elif cores == 0: resources += ['nodes=%d:ppn=%d'%(nodes, PAR.NODESIZE)] else: resources += ['nodes=%d:ppn=%d+1:ppn=%d'%(nodes, PAR.NODESIZE, cores)] # construct arguments list call('qsub ' + '%s ' % PAR.PBSARGS + '-N %s '%PAR.TITLE + '-o %s '%(PATH.SUBMIT +'/'+ 'output.log') + '-l %s '%resources.join(',') + '-j %s '%'oe' + findpath('seisflows.system') +'/'+ 'wrappers/submit ' + '-F %s '%PATH.OUTPUT)