def submit(self, workflow): """ Submits workflow """ # create scratch directories unix.mkdir(PATH.SCRATCH) unix.mkdir(PATH.SYSTEM) # create output directories unix.mkdir(PATH.OUTPUT) unix.mkdir(PATH.WORKDIR+'/'+'output.pbs') workflow.checkpoint() hours = PAR.WALLTIME/60 minutes = PAR.WALLTIME%60 walltime = 'walltime=%02d:%02d:00 ' % (hours, minutes) ncpus = PAR.NODESIZE mpiprocs = PAR.NODESIZE # prepare qsub arguments call( 'qsub ' + '%s ' % PAR.PBSARGS + '-l select=1:ncpus=%d:mpiprocs=%d ' % (ncpus, mpiprocs) + '-l %s ' % walltime + '-N %s ' % PAR.TITLE + '-j %s '%'oe' + '-o %s ' % (PATH.WORKDIR+'/'+'output.log') + '-V ' + ' -- ' + findpath('seisflows.system') +'/'+ 'wrappers/submit ' + PATH.OUTPUT)
def submit(self, workflow): """ Submits workflow """ # create scratch directories unix.mkdir(PATH.SCRATCH) unix.mkdir(PATH.SYSTEM) # create output directories unix.mkdir(PATH.OUTPUT) unix.mkdir(PATH.WORKDIR+'/'+'output.pbs') self.checkpoint() hours = PAR.WALLTIME/60 minutes = PAR.WALLTIME%60 walltime = 'walltime=%02d:%02d:00 ' % (hours, minutes) ncpus = PAR.NODESIZE mpiprocs = PAR.NODESIZE # prepare qsub arguments call( 'qsub ' + '%s ' % PAR.PBSARGS + '-l select=1:ncpus=%d:mpiprocs=%d ' % (ncpus, mpiprocs) + '-l %s ' % walltime + '-N %s ' % PAR.TITLE + '-j %s '%'oe' + '-o %s ' % (PATH.WORKDIR+'/'+'output.log') + '-V ' + ' -- ' + findpath('seisflows.system') +'/'+ 'wrappers/submit ' + PATH.OUTPUT)
def run(self, classname, funcname, hosts='all', **kwargs): """ Runs task(s) on specified hosts """ self.checkpoint() if hosts == 'all': # run all tasks call(findpath('seisflows.system') +'/'+'wrappers/dsh ' + ','.join(self.hostlist()) + ' ' + PATH.OUTPUT + ' ' + classname + ' ' + funcname + ' ' + findpath('seisflows.system') +'/'+'wrappers/run ' + 'PYTHONPATH='+findpath('seisflows.system'),+',' + PAR.ENVIRONS) elif hosts == 'head': # run a single task call('ssh ' + self.hostlist()[0] + ' ' + '"' + 'export SEISFLOWS_TASK_ID=0; ' + join(findpath('seisflows.system'), 'wrappers/run ') + PATH.OUTPUT + ' ' + classname + ' ' + funcname + ' ' + 'PYTHONPATH='+findpath('seisflows.system'),+',' + PAR.ENVIRONS +'"') else: raise KeyError('Bad keyword argument: system.run: hosts')
def submit(self, *args, **kwargs): """ Submits job """ # create scratch directories if not exists(PATH.SCRATCH): path = '/scratch/gpfs' + '/' + getuser( ) + '/' + 'seisflows' + '/' + str(uuid4()) unix.mkdir(path) unix.ln(path, PATH.SCRATCH) unix.mkdir(PATH.SYSTEM) # create output directories unix.mkdir(PATH.OUTPUT) unix.mkdir(PATH.WORKDIR + '/' + 'output.slurm') self.checkpoint() if not exists(PATH.SCRATCH): path = '/scratch/gpfs' + '/' + getuser( ) + '/' + 'seisflows' + '/' + str(uuid4()) unix.mkdir(path) unix.ln(path, PATH.SCRATCH) call('sbatch ' + '%s ' % PAR.SLURMARGS + '--job-name=%s ' % PAR.TITLE + '--output=%s ' % (PATH.WORKDIR + '/' + 'output.log') + '--ntasks-per-node=%d ' % 28 + '--gres=gpu:%d ' % 4 + '--nodes=%d ' % 1 + '--time=%d ' % PAR.WALLTIME + pkgpath('seisflows') + '/' + 'system/wrappers/submit ' + PATH.OUTPUT)
def run(self, classname, funcname, hosts='all', **kwargs): """ Runs tasks in serial or parallel on specified hosts """ self.checkpoint() self.save_kwargs(classname, funcname, kwargs) if hosts == 'all': call(findpath('seisflows.system') +'/'+'wrappers/dsh ' + str(PAR.NTASK) + ' ' + findpath('seisflows.system') +'/'+'wrappers/run ' + PATH.OUTPUT + ' ' + classname + ' ' + funcname + ' ' + PAR.ENVIRONS) #call('srun ' # + '--wait=0 ' # + '--exclusive ' # + join(findpath('seisflows.system'), 'wrappers/run ') # + PATH.OUTPUT + ' ' # + classname + ' ' # + funcname + ' ' # + PAR.ENVIRONS) elif hosts == 'head': # run on head node call(findpath('seisflows.system') +'/'+'wrappers/run ' + PATH.OUTPUT + ' ' + classname + ' ' + funcname + ' ' + PAR.ENVIRONS)
def run(self, classname, funcname, hosts='all', **kwargs): """ Runs tasks in serial or parallel on specified hosts """ self.checkpoint() self.save_kwargs(classname, funcname, kwargs) if hosts == 'all': # run on all available nodes call('srun ' + '--wait=0 ' + join(findpath('seisflows.system'), 'wrappers/run ') + PATH.OUTPUT + ' ' + classname + ' ' + funcname + ' ' + PAR.ENVIRONS) elif hosts == 'head': # run on head node call('srun ' + '--wait=0 ' + '--ntasks=1 ' + '--nodes=1 ' + join(findpath('seisflows.system'), 'wrappers/run ') + PATH.OUTPUT + ' ' + classname + ' ' + funcname + ' ' + PAR.ENVIRONS) else: raise KeyError('Bad keyword argument: system.run: hosts')
def run(self, classname, method, hosts='all', **kwargs): """ Executes the following task: classname.method(*args, **kwargs) """ self.checkpoint() self.save_kwargs(classname, method, kwargs) if hosts == 'all': # run on all available nodes call(join(pkgpath('seisflows-hpc'), 'system/wrappers/dsh_tigergpu') + ' ' + join(pkgpath('seisflows'), 'system/wrappers/run') + ' ' + PATH.OUTPUT + ' ' + classname + ' ' + method + ' ' + str(PAR.NTASK) + ' ' + PAR.ENVIRONS) elif hosts == 'head': # run on head node call('srun ' + '--ntasks=1 ' + '--nodes=1 ' + join(pkgpath('seisflows'), 'system/wrappers/run') + PATH.OUTPUT + ' ' + classname + ' ' + method + ' ' + PAR.ENVIRONS)
def run(self, classname, method, hosts='all', **kwargs): """ Executes the following task: classname.method(*args, **kwargs) """ self.checkpoint() if hosts == 'all': # run all tasks call( findpath('seisflows.system') + '/' + 'wrappers/dsh ' + ','.join(self.hostlist()) + ' ' + findpath('seisflows.system') + '/' + 'wrappers/run ' + PATH.OUTPUT + ' ' + classname + ' ' + method + ' ' + 'PYTHONPATH=' + findpath('seisflows'), + ',' + PAR.ENVIRONS) elif hosts == 'head': # run a single task call( 'ssh ' + self.hostlist()[0] + ' ' + '"' + 'export SEISFLOWS_TASK_ID=0; ' + join(findpath('seisflows.system'), 'wrappers/run ') + PATH.OUTPUT + ' ' + classname + ' ' + method + ' ' + 'PYTHONPATH=' + findpath('seisflows'), + ',' + PAR.ENVIRONS + '"') else: raise KeyError('Bad keyword argument: system.run: hosts')
def submit(self, workflow): """ Submits workflow """ # create scratch directories if not exists(PATH.SCRATCH): path = '/scratch/gpfs'+'/'+getuser()+'/'+'seisflows'+'/'+str(uuid4()) unix.mkdir(path) unix.ln(path, PATH.SCRATCH) unix.mkdir(PATH.SYSTEM) # create output directories unix.mkdir(PATH.OUTPUT) unix.mkdir(PATH.WORKDIR+'/'+'output.slurm') self.checkpoint() # prepare sbatch arguments call('sbatch ' + '%s ' % PAR.SLURMARGS + '--job-name=%s ' % PAR.TITLE + '--output %s ' % (PATH.WORKDIR+'/'+'output.log') + '--ntasks-per-node=28 ' + '--ntasks=28 ' + '--gres=gpu:4 ' + '--nodes=%d ' % 1 + '--time=%d ' % PAR.WALLTIME + findpath('seisflows.system') +'/'+ 'wrappers/submit ' + PATH.OUTPUT)
def run(self, classname, method, hosts='all', **kwargs): """ Runs embarrassingly parallel tasks Executes the following multiple times: classname.method(*args, **kwargs) system.taskid serves to provide each running task a unique identifier """ self.checkpoint(PATH.OUTPUT, classname, method, args, kwargs) if hosts == 'all': # run all tasks call( findpath('seisflows.system') + '/' + 'wrappers/dsh ' + ','.join(self.hostlist()) + ' ' + findpath('seisflows.system') + '/' + 'wrappers/run ' + PATH.OUTPUT + ' ' + classname + ' ' + method + ' ' + 'PYTHONPATH=' + findpath('seisflows'), + ',' + PAR.ENVIRONS) elif hosts == 'head': # run a single task call( 'ssh ' + self.hostlist()[0] + ' ' + '"' + 'export SEISFLOWS_TASK_ID=0; ' + join(findpath('seisflows.system'), 'wrappers/run ') + PATH.OUTPUT + ' ' + classname + ' ' + method + ' ' + 'PYTHONPATH=' + findpath('seisflows'), + ',' + PAR.ENVIRONS + '"') else: raise KeyError('Bad keyword argument: system.run: hosts')
def run_single(self, classname, method, *args, **kwargs): """ Runs task a single time """ self.checkpoint(PATH.OUTPUT, classname, method, args, kwargs) call('srun ' + '--wait=0 ' + '--ntasks=1 ' + '--nodes=1 ' + '%s ' % join(findpath('seisflows.system'), 'wrappers/run ') + '%s ' % PATH.OUTPUT + '%s ' % classname + '%s ' % method + '%s ' % PAR.ENVIRONS)
def run(self, classname, method, *args, **kwargs): """ Runs task multiple times in embarrassingly parallel fasion """ self.checkpoint(PATH.OUTPUT, classname, method, args, kwargs) call('srun ' + '--wait=0 ' + '%s ' % join(findpath('seisflows.system'), 'wrappers/run ') + '%s ' % PATH.OUTPUT + '%s ' % classname + '%s ' % method + '%s ' % PAR.ENVIRONS)
def _query(self, jobid): """ Retrives job state from LSF database """ with open(PATH.SYSTEM+'/'+'job_status', 'w') as f: call('bjobs -a -d "' + jobid + '"', stdout=f) with open(PATH.SYSTEM+'/'+'job_status', 'r') as f: lines = f.readlines() state = lines[1].split()[2].strip() return state
def _query(self, jobid): """ Retrives job state from LSF database """ with open(PATH.SYSTEM+'/'+'job_status', 'w') as f: call('bjobs -a -d "' + jobid + '"', stdout=f) with open(PATH.SYSTEM+'/'+'job_status', 'r') as f: lines = f.readlines() state = lines[1].split()[2].strip() return state
def _query(self, jobid): """ Queries job state from SLURM database """ with open(PATH.SYSTEM + '/' + 'job_status', 'w') as f: call('sacct -n -o state -j ' + jobid, stdout=f) with open(PATH.SYSTEM + '/' + 'job_status', 'r') as f: line = f.readline() state = line.strip() return state
def _query(self, job): """ Queries job state from SLURM database """ with open(PATH.SYSTEM + '/' + 'job_status', 'w') as file: call('sacct -n -o jobid,state -j ' + job.split('_')[0], stdout=file) state = '' with open(PATH.SYSTEM + '/' + 'job_status', 'r') as file: for line in file.readlines(): if line.split()[0] == job: state = line.split()[1] return state
def run(self, classname, method, *args, **kwargs): """ Runs task multiple times in embarrassingly parallel fasion """ self.checkpoint(PATH.OUTPUT, classname, method, args, kwargs) call('srun ' + '--wait=0 ' + '%s ' % join(findpath('seisflows.system'), 'wrappers/run ') + '%s ' % PATH.OUTPUT + '%s ' % classname + '%s ' % method + '%s ' % PAR.ENVIRONS)
def submit_job_array(self, classname, funcname, hosts='all'): with open(PATH.SYSTEM + '/' + 'job_id', 'w') as f: call(self.job_array_cmd(classname, funcname, hosts), stdout=f) # retrieve job ids with open(PATH.SYSTEM + '/' + 'job_id', 'r') as f: line = f.readline() job = line.split()[-1].strip() if hosts == 'all' and PAR.NTASK > 1: return [job + '_' + str(ii) for ii in range(PAR.NTASK)] else: return [job]
def resubmit_failed_job(self, classname, method, jobs, taskid): with open(PATH.SYSTEM + '/' + 'job_id', 'w') as file: call(self.resubmit_cmd(classname, method, taskid), stdout=file) with open(PATH.SYSTEM + '/' + 'job_id', 'r') as file: line = file.readline() jobid = line.split()[-1].strip() # remove failed job from list jobs.pop(taskid) # add resubmitted job to list jobs.insert(taskid, jobid) return jobs
def submit_job_array(self, classname, funcname, hosts='all'): with open(PATH.SYSTEM + '/' + 'array_id', 'w') as file: call(self.job_array_cmd(classname, funcname, hosts), stdout=file) with open(PATH.SYSTEM + '/' + 'array_id', 'r') as file: line = file.readline() id = line.split()[-1].strip() if hosts == 'all': tasks = range(PAR.NTASK) jobs = [id + '_' + str(task) for task in tasks] else: jobs = [id + '_0'] return jobs
def _query(self, jobid): """ Queries job state from PBS database """ # TODO: replace shell utilities with native Python with open(PATH.SYSTEM + '/' + 'job_status', 'w') as f: call('qstat -x -tJ ' + jobid + ' | ' + 'tail -n 1 ' + ' | ' + 'awk \'{print $5}\'', stdout=f) with open(PATH.SYSTEM + '/' + 'job_status', 'r') as f: line = f.readline() state = line.strip() return state
def submit_job_array(self, classname, method, hosts='all'): with open(PATH.SYSTEM + '/' + 'job_id', 'w') as f: call(self.job_array_cmd(classname, method, hosts), stdout=f) # retrieve job ids with open(PATH.SYSTEM + '/' + 'job_id', 'r') as f: line = f.readline() job = line.split()[-1].strip() if hosts == 'all' and PAR.NTASK > 1: nn = range(PAR.NTASK) job0 = job.strip('[].sdb') return [job0 + '[' + str(ii) + '].sdb' for ii in nn] else: return [job]
def run_single(self, classname, method, *args, **kwargs): """ Runs task a single time """ self.checkpoint(PATH.OUTPUT, classname, method, args, kwargs) call('srun ' + '--wait=0 ' + '--ntasks=1 ' + '--nodes=1 ' + '%s ' % join(findpath('seisflows.system'), 'wrappers/run ') + '%s ' % PATH.OUTPUT + '%s ' % classname + '%s ' % method + '%s ' % PAR.ENVIRONS)
def submit_job_array(self, classname, method, hosts='all'): with open(PATH.SYSTEM+'/'+'job_id', 'w') as f: call(self.job_array_cmd(classname, method, hosts), stdout=f) # retrieve job ids with open(PATH.SYSTEM+'/'+'job_id', 'r') as f: line = f.readline() job = line.split()[-1].strip() if hosts == 'all' and PAR.NTASK > 1: nn = range(PAR.NTASK) job0 = job.strip('[].sdb') return [job0+'['+str(ii)+'].sdb' for ii in nn] else: return [job]
def _query(self, jobid): """ Queries job state from PBS database """ # TODO: replace shell utilities with native Python with open(PATH.SYSTEM+'/'+'job_status', 'w') as f: call('qstat -x -tJ ' + jobid + ' | ' + 'tail -n 1 ' + ' | ' + 'awk \'{print $5}\'', stdout=f) with open(PATH.SYSTEM+'/'+'job_status', 'r') as f: line = f.readline() state = line.strip() return state
def submit(self, workflow): """ Submits workflow """ unix.cd(PATH.WORKDIR) if not exists('./scratch'): unix.ln(PATH.SCRATCH, PATH.WORKDIR + '/' + 'scratch') unix.mkdir(PATH.OUTPUT) workflow.checkpoint() # prepare sbatch arguments call('sbatch ' + '%s ' % PAR.SLURMARGS + '--partition=%s ' % PAR.PARTITION + '--job-name=%s ' % PAR.TITLE + '--output %s ' % (PATH.WORKDIR + '/' + 'output.log') + '--cpus-per-task=%d ' % PAR.NPROC + '--ntasks=%d ' % PAR.NTASK + '--time=%d ' % PAR.WALLTIME + findpath('seisflows.system') + '/' + 'wrappers/submit ' + PATH.OUTPUT)
def submit_job_array(self, classname, funcname, hosts='all'): # submit job with open(PATH.SYSTEM+'/'+'job_id', 'w') as f: call(self.job_array_cmd(classname, funcname, hosts), stdout=f) # retrieve job ids with open(PATH.SYSTEM+'/'+'job_id', 'r') as f: # reads one entire line from the file line = f.readline() job_buf = line.split()[1].strip() job = job_buf[1:-1] if hosts == 'all' and PAR.NSRC > 1: nn = range(1,PAR.NSRC+1) #return [job+'_'+str(ii) for ii in nn] return [job+'['+str(ii)+']' for ii in nn] else: return [job]
def submit(self, workflow): """ Submits workflow """ unix.mkdir(PATH.OUTPUT) unix.cd(PATH.OUTPUT) self.checkpoint() # if not exists(PATH.SUBMIT + '/' + 'scratch'): # unix.ln(PATH.SCRATCH, PATH.SUBMIT + '/' + 'scratch') nnodes = 1 #PAR.NTASK / PAR.NGPU call('sbatch ' + '%s ' % PAR.SLURMARGS + '--job-name=%s ' % PAR.TITLE + '--output=%s ' % (PATH.WORKDIR + '/' + 'output.log') + '--nodes %d ' % nnodes + '--ntasks-per-node=%d ' % PAR.NGPU + '--ntasks-per-socket=%d ' % PAR.NGPU + '--gres=gpu:%d ' % PAR.NGPU + '--time=%d ' % PAR.WALLTIME + findpath('seisflows.system') + '/' + 'wrappers/submit ' + PATH.OUTPUT)
def submit(self, workflow): """ Submits workflow """ # create scratch directories unix.mkdir(PATH.SCRATCH) unix.mkdir(PATH.SYSTEM) # create output directories unix.mkdir(PATH.OUTPUT) self.checkpoint() # submit workflow call('sbatch ' + '%s ' % PAR.SLURMARGS + '--job-name=%s ' % PAR.TITLE + '--output=%s ' % (PATH.WORKDIR + '/' + 'output.log') + '--cpus-per-task=%d ' % PAR.NPROC + '--ntasks=%d ' % PAR.NTASK + '--time=%d ' % PAR.WALLTIME + findpath('seisflows.system') + '/' + 'wrappers/submit ' + PATH.OUTPUT)
def submit(self, workflow): """ Submits workflow """ # create scratch directories unix.mkdir(PATH.SCRATCH) unix.mkdir(PATH.SYSTEM) # create output directories unix.mkdir(PATH.OUTPUT) unix.mkdir(PATH.WORKDIR + '/' + 'output.slurm') workflow.checkpoint() # prepare sbatch arguments call('sbatch ' + '%s ' % PAR.SLURMARGS + '--job-name=%s ' % PAR.TITLE + '--output %s ' % (PATH.WORKDIR + '/' + 'output.log') + '--ntasks-per-node=%d ' % PAR.NODESIZE + '--nodes=%d ' % 1 + '--time=%d ' % PAR.WALLTIME + findpath('seisflows.system') + '/' + 'wrappers/submit ' + PATH.OUTPUT)
def submit(self, workflow): """ Submits workflow """ # create scratch directories unix.mkdir(PATH.SCRATCH) unix.mkdir(PATH.SYSTEM) # create output directories unix.mkdir(PATH.OUTPUT) unix.mkdir(PATH.WORKDIR + '/' + 'output.lsf') self.checkpoint() # prepare bsub arguments call('bsub ' + '%s ' % PAR.LSFARGS + '-J %s ' % PAR.TITLE + '-o %s ' % (PATH.WORKDIR + '/' + 'output.log') + '-n %d ' % PAR.NODESIZE + '-e %s ' % (PATH.WORKDIR + '/' + 'error.log') + '-R "span[ptile=%d]" ' % PAR.NODESIZE + '-W %d:00 ' % PAR.WALLTIME + findpath('seisflows.system') + '/' + 'wrappers/submit ' + PATH.OUTPUT)
def submit(self, workflow): """ Submits workflow """ # create scratch directories unix.mkdir(PATH.SCRATCH) unix.mkdir(PATH.SYSTEM) # create output directories unix.mkdir(PATH.OUTPUT) workflow.checkpoint() # submit workflow call('sbatch ' + '%s ' % PAR.SLURMARGS + '--job-name=%s '%PAR.TITLE + '--output=%s '%(PATH.WORKDIR +'/'+ 'output.log') + '--cpus-per-task=%d '%PAR.NPROC + '--ntasks=%d '%PAR.NTASK + '--time=%d '%PAR.WALLTIME + '%s ' % findpath('seisflows.system') +'/'+ 'wrappers/submit' + '%s ' % PATH.OUTPUT)
def detect_edges(self): path_input = PATH.MUMFORD_SHAH_INPUT path_output = PATH.MUMFORD_SHAH_OUTPUT path_run = PATH.SUBMIT unix.cp(glob(PATH.GRAD+'/'+'model/*'), path_input) unix.mkdir(path_output) unix.cd(path_run) # writes damping term to disk with open('mumford_shah.log', 'w') as fileobj: call('srun -n 1 -N 1 ' +PATH.MUMFORD_SHAH_BIN+'/'+'psemimage ' + PATH.MUMFORD_SHAH_CONFIG + ' -ksp_type fgmres ' + ' -pc_type asm ' + ' -ksp_gmres_restart 300 ', shell=True, stdout=fileobj) for parameter in solver.parameters: self.apply_smoothing(parameter) self.write_damping_term(parameter)
def submit(self, workflow): """ Submits workflow """ # create scratch directories unix.mkdir(PATH.SCRATCH) unix.mkdir(PATH.SYSTEM) # create output directories unix.mkdir(PATH.OUTPUT) unix.mkdir(PATH.WORKDIR+'/'+'output.slurm') workflow.checkpoint() # prepare sbatch arguments call('sbatch ' + '%s ' % PAR.SLURMARGS + '--job-name=%s ' % PAR.TITLE + '--output %s ' % (PATH.WORKDIR+'/'+'output.log') + '--ntasks-per-node=%d ' % PAR.NODESIZE + '--nodes=%d ' % 1 + '--time=%d ' % PAR.WALLTIME + findpath('seisflows.system') +'/'+ 'wrappers/submit ' + PATH.OUTPUT)
def run(self, classname, method, hosts='all', **kwargs): """ Executes the following task: classname.method(*args, **kwargs) """ self.checkpoint() self.save_kwargs(classname, method, kwargs) if hosts == 'all': # run on all available nodes call('srun ' + '--wait=0 ' + join(findpath('seisflows.system'), 'wrappers/run ') + PATH.OUTPUT + ' ' + classname + ' ' + method + ' ' + PAR.ENVIRONS) elif hosts == 'head': # run on head node call('srun ' + '--wait=0 ' + '--ntasks=1 ' + '--nodes=1 ' + join(findpath('seisflows.system'), 'wrappers/run ') + PATH.OUTPUT + ' ' + classname + ' ' + method + ' ' + PAR.ENVIRONS) else: raise KeyError('Bad keyword argument: system.run: hosts')
def submit(self, workflow): """ Submits workflow """ # create scratch directories unix.mkdir(PATH.SCRATCH) unix.mkdir(PATH.SYSTEM) # create output directories unix.mkdir(PATH.OUTPUT) self.checkpoint() # submit workflow call('sbatch ' + '%s ' % PAR.SLURMARGS + '--job-name=%s '%PAR.TITLE + '--output=%s '%(PATH.WORKDIR +'/'+ 'output.log') + '--ntasks=%d '%PAR.NTASK + '--nodes=%d '%ceil(PAR.NTASK/4.) + '--ntasks-per-node=%d '%min(PAR.NTASK,4) + '--gres=gpu:%d '%min(PAR.NTASK,4) + '--time=%d '%PAR.WALLTIME + pkgpath('seisflows') +'/'+ 'system/wrappers/submit ' + PATH.OUTPUT)
def submit(self, workflow): """ Submits workflow """ # create scratch directories unix.mkdir(PATH.SCRATCH) unix.mkdir(PATH.SYSTEM) # create output directories unix.mkdir(PATH.OUTPUT) unix.mkdir(PATH.WORKDIR+'/'+'output.lsf') workflow.checkpoint() # prepare bsub arguments call('bsub ' + '%s ' % PAR.LSFARGS + '-J %s ' % PAR.TITLE + '-o %s ' % (PATH.WORKDIR+'/'+'output.log') + '-n %d ' % PAR.NODESIZE + '-e %s ' % (PATH.WORKDIR+'/'+'error.log') + '-R "span[ptile=%d]" ' % PAR.NODESIZE + '-W %d:00 ' % PAR.WALLTIME + findpath('seisflows.system') +'/'+ 'wrappers/submit ' + PATH.OUTPUT)
def run(self, classname, method, hosts='all', **kwargs): """ Runs embarrassingly parallel tasks Executes the following multiple times: classname.method(*args, **kwargs) system.taskid serves to provide each running task a unique identifier """ self.checkpoint(PATH.OUTPUT, classname, method, args, kwargs) if hosts == 'all': # run all tasks call(findpath('seisflows.system') +'/'+'wrappers/dsh ' + ','.join(self.hostlist()) + ' ' + findpath('seisflows.system') +'/'+'wrappers/run ' + PATH.OUTPUT + ' ' + classname + ' ' + method + ' ' + 'PYTHONPATH='+findpath('seisflows'),+',' + PAR.ENVIRONS) elif hosts == 'head': # run a single task call('ssh ' + self.hostlist()[0] + ' ' + '"' + 'export SEISFLOWS_TASK_ID=0; ' + join(findpath('seisflows.system'), 'wrappers/run ') + PATH.OUTPUT + ' ' + classname + ' ' + method + ' ' + 'PYTHONPATH='+findpath('seisflows'),+',' + PAR.ENVIRONS +'"') else: raise KeyError('Bad keyword argument: system.run: hosts')
def hostlist(self): with open(PATH.SYSTEM+'/'+'hostlist', 'w') as f: call('scontrol show hostname $SLURM_JOB_NODEFILE', stdout=f) with open(PATH.SYSTEM+'/'+'hostlist', 'r') as f: return [line.strip() for line in f.readlines()]
def resubmit_failed_job(self, job): call('scontrol requeue ' + job)