def _launch(self, classname, funcname, hosts='all'):
unix.mkdir(PATH.SYSTEM)
nodes = math.ceil(PAR.NTASK / float(PAR.NODESIZE))
ncpus = PAR.NPROC
mpiprocs = PAR.NPROC
hours = PAR.STEPTIME / 60
minutes = PAR.STEPTIME % 60
walltime = 'walltime=%02d:%02d:00 ' % (hours, minutes)
# submit job
with open(PATH.SYSTEM + '/' + 'job_id', 'w') as f:
call(
'qsub ' + '%s ' % PAR.PBSARGS +
'-l select=%d:ncpus=%d:mpiprocs=%d ' (nodes, ncpus, mpiprocs) +
'-l %s ' % walltime + '-J 0-%s ' % (PAR.NTASK - 1) +
'-N %s ' % PAR.TITLE + '-o %s ' %
(PATH.SUBMIT + '/' + 'output.pbs/' + '$PBS_ARRAYID') +
'-r y ' + '-j oe ' + '-V ' + self.launch_args(hosts) +
PATH.OUTPUT + ' ' + classname + ' ' + funcname + ' ' +
findpath('seisflows.system'),
stdout=f)
# retrieve job ids
with open(PATH.SYSTEM + '/' + 'job_id', 'r') as f:
line = f.readline()
job = line.split()[-1].strip()
if hosts == 'all' and PAR.NTASK > 1:
nn = range(PAR.NTASK)
job0 = job.strip('[].sdb')
return [job0 + '[' + str(ii) + '].sdb' for ii in nn]
else:
return [job]
def submit(self, workflow):
"""Submits job
"""
unix.mkdir(PATH.OUTPUT)
unix.cd(PATH.OUTPUT)
# save current state
self.checkpoint()
# construct resource list
nodes = int(PAR.NTASK / PAR.NODESIZE)
cores = PAR.NTASK % PAR.NODESIZE
hours = int(PAR.WALLTIME / 60)
minutes = PAR.WALLTIME % 60
resources = 'walltime=%02d:%02d:00' % (hours, minutes)
if nodes == 0:
resources += ',mem=%dgb,nodes=1:ppn=%d' % (PAR.MEMORY, cores)
elif cores == 0:
resources += ',mem=%dgb,nodes=%d:ppn=%d' % (PAR.MEMORY, nodes,
PAR.NODESIZE)
else:
resources += ',mem=%dgb,nodes=%d:ppn=%d+1:ppn=%d' % (
PAR.MEMORY, nodes, PAR.NODESIZE, cores)
# construct arguments list
call('qsub ' + '%s ' % PAR.PBSARGS + '-N %s ' % PAR.TITLE + '-o %s ' %
(PATH.SUBMIT + '/' + 'output.log') + '-l %s ' % resources +
'-j %s ' % 'oe' + findpath('seisflows.system') + '/' +
'wrappers/submit ' + '-F %s ' % PATH.OUTPUT)
def submit(self, workflow):
""" Submits workflow
"""
unix.mkdir(PATH.OUTPUT)
unix.cd(PATH.OUTPUT)
unix.mkdir(PATH.SUBMIT+'/'+'output.pbs')
self.checkpoint()
hours = PAR.WALLTIME/60
minutes = PAR.WALLTIME%60
walltime = 'walltime=%02d:%02d:00 ' % (hours, minutes)
ncpus = PAR.NODESIZE
mpiprocs = PAR.NODESIZE
# prepare qsub arguments
call( 'qsub '
+ '%s ' % PAR.PBSARGS
+ '-l select=1:ncpus=%d:mpiprocs=%d ' % (ncpus, mpiprocs)
+ '-l %s ' % walltime
+ '-N %s ' % PAR.TITLE
+ '-j %s '%'oe'
+ '-o %s ' % (PATH.SUBMIT+'/'+'output.log')
+ '-V '
+ ' -- ' + findpath('seisflows.system') +'/'+ 'wrappers/submit '
+ PATH.OUTPUT)
def launch(self, classname, funcname, hosts='all'):
unix.mkdir(PATH.SYSTEM)
# submit job
with open(PATH.SYSTEM + '/' + 'job_id', 'w') as f:
call('bsub ' + '%s ' % PAR.LSFARGS + '-n %d ' % PAR.NPROC +
'-R "span[ptile=%d]" ' % PAR.NODESIZE +
'-W %d:00 ' % PAR.STEPTIME + '-J "%s' % PAR.TITLE +
self.launch_args(hosts) + findpath('seisflows.system') + '/' +
'wrapper/run ' + PATH.OUTPUT + ' ' + classname + ' ' +
funcname + ' ',
stdout=f)
# retrieve job ids
with open(PATH.SYSTEM + '/' + 'job_id', 'r') as f:
# reads one entire line from the file
line = f.readline()
job_buf = line.split()[1].strip()
job = job_buf[1:-1]
if hosts == 'all' and PAR.NSRC > 1:
nn = range(1, PAR.NSRC + 1)
#return [job+'_'+str(ii) for ii in nn]
return [job + '[' + str(ii) + ']' for ii in nn]
else:
return [job]
def _launch(self, classname, funcname, hosts='all'):
unix.mkdir(PATH.SYSTEM)
with open(PATH.SYSTEM+'/'+'job_id', 'w') as f:
call('sbatch '
+ '%s ' % PAR.SLURMARGS
+ '--job-name=%s ' % PAR.TITLE
+ '--nodes=%d ' % math.ceil(PAR.NPROC/float(PAR.NODESIZE))
+ '--ntasks-per-node=%d ' % PAR.NODESIZE
+ '--ntasks=%d ' % PAR.NPROC
+ '--time=%d ' % PAR.STEPTIME
+ self._launch_args(hosts)
+ findpath('seisflows.system') +'/'+ 'wrappers/run '
+ PATH.OUTPUT + ' '
+ classname + ' '
+ funcname + ' ',
stdout=f)
# retrieve job ids
with open(PATH.SYSTEM+'/'+'job_id', 'r') as f:
line = f.readline()
job = line.split()[-1].strip()
if hosts == 'all' and PAR.NTASK > 1:
return [job+'_'+str(ii) for ii in range(PAR.NTASK)]
else:
return [job]
def run(self, classname, funcname, hosts='all', **kwargs):
""" Runs tasks in serial or parallel on specified hosts
"""
self.checkpoint()
self.save_kwargs(classname, funcname, kwargs)
if hosts == 'all':
# run on all available nodes
call('srun '
+ '--wait=0 '
+ join(findpath('seisflows.system'), 'wrappers/run ')
+ PATH.OUTPUT + ' '
+ classname + ' '
+ funcname)
elif hosts == 'head':
# run on head node
call('srun '
+ '--wait=0 '
+ join(findpath('seisflows.system'), 'wrappers/run_head ')
+ PATH.OUTPUT + ' '
+ classname + ' '
+ funcname)
else:
raise(KeyError('Hosts parameter not set/recognized.'))
def _launch(self, classname, funcname, hosts='all'):
unix.mkdir(PATH.SYSTEM)
with open(PATH.SYSTEM+'/'+'job_id', 'w') as f:
call('sbatch '
+ '%s ' % PAR.SLURMARGS
+ '--job-name=%s ' % PAR.TITLE
+ '--nodes=%d ' % math.ceil(PAR.NPROC/float(PAR.NODESIZE))
+ '--ntasks-per-node=%d ' % PAR.NODESIZE
+ '--ntasks=%d ' % PAR.NPROC
+ '--time=%d ' % PAR.STEPTIME
+ self._launch_args(hosts)
+ findpath('seisflows.system') +'/'+ 'wrappers/run '
+ PATH.OUTPUT + ' '
+ classname + ' '
+ funcname + ' ',
stdout=f)
# retrieve job ids
with open(PATH.SYSTEM+'/'+'job_id', 'r') as f:
line = f.readline()
job = line.split()[-1].strip()
if hosts == 'all' and PAR.NTASK > 1:
return [job+'_'+str(ii) for ii in range(PAR.NTASK)]
else:
return [job]
def launch(self, classname, funcname, hosts='all'):
unix.mkdir(PATH.SYSTEM)
# submit job
with open(PATH.SYSTEM+'/'+'job_id', 'w') as f:
call('bsub '
+ '%s ' % PAR.LSFARGS
+ '-n %d ' % PAR.NPROC
+ '-R "span[ptile=%d]" ' % PAR.NODESIZE
+ '-W %d:00 ' % PAR.STEPTIME
+ '-J "%s' %PAR.TITLE
+ self.launch_args(hosts)
+ findpath('seisflows.system') +'/'+ 'wrapper/run '
+ PATH.OUTPUT + ' '
+ classname + ' '
+ funcname + ' ',
stdout=f)
# retrieve job ids
with open(PATH.SYSTEM+'/'+'job_id', 'r') as f:
# reads one entire line from the file
line = f.readline()
job_buf = line.split()[1].strip()
job = job_buf[1:-1]
if hosts == 'all' and PAR.NSRC > 1:
nn = range(1,PAR.NSRC+1)
#return [job+'_'+str(ii) for ii in nn]
return [job+'['+str(ii)+']' for ii in nn]
else:
return [job]
def getstate(self, jobid):
""" Retrives job state from LSF database
"""
with open(PATH.SYSTEM+'/'+'job_status', 'w') as f:
call('bjobs -a -d "' + jobid + '"', stdout=f)
with open(PATH.SYSTEM+'/'+'job_status', 'r') as f:
lines = f.readlines()
state = lines[1].split()[2].strip()
return state
def getstate(self, jobid):
""" Retrives job state from LSF database
"""
with open(PATH.SYSTEM + '/' + 'job_status', 'w') as f:
call('bjobs -a -d "' + jobid + '"', stdout=f)
with open(PATH.SYSTEM + '/' + 'job_status', 'r') as f:
lines = f.readlines()
state = lines[1].split()[2].strip()
return state
def _query(self, jobid):
""" Queries job state from SLURM database
"""
with open(PATH.SYSTEM+'/'+'job_status', 'w') as f:
call('sacct -n -o state -j '+jobid, stdout=f)
with open(PATH.SYSTEM+'/'+'job_status', 'r') as f:
line = f.readline()
state = line.strip()
return state
def _query(self, jobid):
""" Queries job state from SLURM database
"""
with open(PATH.SYSTEM+'/'+'job_status', 'w') as f:
call('sacct -n -o state -j '+jobid, stdout=f)
with open(PATH.SYSTEM+'/'+'job_status', 'r') as f:
line = f.readline()
state = line.strip()
return state
def _query(self, jobid):
""" Queries job state from PBS database
"""
# TODO: replace shell utilities with native Python
with open(PATH.SYSTEM + '/' + 'job_status', 'w') as f:
call('qstat -x -tJ ' + jobid + ' | ' + 'tail -n 1 ' + ' | ' +
'awk \'{print $5}\'',
stdout=f)
with open(PATH.SYSTEM + '/' + 'job_status', 'r') as f:
line = f.readline()
state = line.strip()
return state
def submit(self, workflow):
""" Submits workflow
"""
unix.mkdir(PATH.OUTPUT)
unix.cd(PATH.OUTPUT)
self.checkpoint()
# submit workflow
call('sbatch ' + '%s ' % PAR.SLURMARGS + '--job-name=%s ' % PAR.TITLE +
'--output=%s ' % (PATH.SUBMIT + '/' + 'output.log') +
'--cpus-per-task=%d ' % PAR.NPROC + '--ntasks=%d ' % PAR.NTASK +
'--time=%d ' % PAR.WALLTIME + findpath('seisflows.system') + '/' +
'wrappers/submit ' + PATH.OUTPUT)
def submit(self, workflow):
""" Submits workflow
"""
unix.mkdir(PATH.OUTPUT)
unix.cd(PATH.OUTPUT)
unix.mkdir(PATH.SUBMIT + '/' + 'output.slurm')
self.checkpoint()
# prepare sbatch arguments
call('sbatch ' + '%s ' % PAR.SLURMARGS + '--job-name=%s ' % PAR.TITLE +
'--output %s ' % (PATH.SUBMIT + '/' + 'output.log') +
'--ntasks-per-node=%d ' % PAR.NODESIZE + '--nodes=%d ' % 1 +
'--time=%d ' % PAR.WALLTIME + findpath('seisflows.system') + '/' +
'wrappers/submit ' + PATH.OUTPUT)
def submit(self, workflow):
""" Submits workflow
"""
unix.mkdir(PATH.OUTPUT)
unix.cd(PATH.OUTPUT)
unix.mkdir(PATH.SUBMIT + '/' + 'output.lsf')
self.checkpoint()
# prepare bsub arguments
call('bsub ' + '%s ' % PAR.LSFARGS + '-J %s ' % PAR.TITLE + '-o %s ' %
(PATH.SUBMIT + '/' + 'output.log') + '-n %d ' % PAR.NTASK +
'-e %s ' % (PATH.SUBMIT + '/' + 'error.log') +
'-W %d:00 ' % PAR.WALLTIME + findpath('seisflows.system') + '/' +
'wrappers/submit ' + PATH.OUTPUT)
def _query(self, jobid):
""" Queries job state from PBS database
"""
# TODO: replace shell utilities with native Python
with open(PATH.SYSTEM+'/'+'job_status', 'w') as f:
call('qstat -x -tJ ' + jobid + ' | '
+ 'tail -n 1 ' + ' | '
+ 'awk \'{print $5}\'',
stdout=f)
with open(PATH.SYSTEM+'/'+'job_status', 'r') as f:
line = f.readline()
state = line.strip()
return state
def submit(self, workflow):
""" Submits workflow
"""
unix.mkdir(PATH.OUTPUT)
unix.cd(PATH.OUTPUT)
self.checkpoint()
# submit workflow
call('sbatch '
+ '%s ' % PAR.SLURMARGS
+ '--job-name=%s '%PAR.TITLE
+ '--output=%s '%(PATH.SUBMIT +'/'+ 'output.log')
+ '--cpus-per-task=%d '%PAR.NPROC
+ '--ntasks=%d '%PAR.NTASK
+ '--time=%d '%PAR.WALLTIME
+ findpath('seisflows.system') +'/'+ 'wrappers/submit '
+ PATH.OUTPUT)
def submit(self, workflow):
""" Submits workflow
"""
unix.mkdir(PATH.OUTPUT)
unix.cd(PATH.OUTPUT)
unix.mkdir(PATH.SUBMIT+'/'+'output.slurm')
self.checkpoint()
# prepare sbatch arguments
call('sbatch '
+ '%s ' % PAR.SLURMARGS
+ '--job-name=%s ' % PAR.TITLE
+ '--output %s ' % (PATH.SUBMIT+'/'+'output.log')
+ '--ntasks-per-node=%d ' % PAR.NODESIZE
+ '--nodes=%d ' % 1
+ '--time=%d ' % PAR.WALLTIME
+ findpath('seisflows.system') +'/'+ 'wrappers/submit '
+ PATH.OUTPUT)
def submit(self, workflow):
""" Submits workflow
"""
unix.mkdir(PATH.OUTPUT)
unix.cd(PATH.OUTPUT)
unix.mkdir(PATH.SUBMIT+'/'+'output.lsf')
self.checkpoint()
# prepare bsub arguments
call('bsub '
+ '%s ' % PAR.LSFARGS
+ '-J %s ' % PAR.TITLE
+ '-o %s ' % (PATH.SUBMIT+'/'+'output.log')
+ '-n %d ' % PAR.NTASK
+ '-e %s ' % (PATH.SUBMIT+'/'+'error.log')
+ '-W %d:00 ' % PAR.WALLTIME
+ findpath('seisflows.system') +'/'+ 'wrappers/submit '
+ PATH.OUTPUT)
def run(self, classname, funcname, hosts='all', **kwargs):
""" Runs tasks in serial or parallel on specified hosts
"""
self.checkpoint()
self.save_kwargs(classname, funcname, kwargs)
if hosts == 'all':
# run on all available nodes
call('srun ' + '--wait=0 ' +
join(findpath('seisflows.system'), 'wrappers/run ') +
PATH.OUTPUT + ' ' + classname + ' ' + funcname)
elif hosts == 'head':
# run on head node
call('srun ' + '--wait=0 ' +
join(findpath('seisflows.system'), 'wrappers/run_head ') +
PATH.OUTPUT + ' ' + classname + ' ' + funcname)
else:
raise (KeyError('Hosts parameter not set/recognized.'))
def submit(self, workflow):
""" Submits workflow
"""
unix.mkdir(PATH.OUTPUT)
unix.cd(PATH.OUTPUT)
self.checkpoint()
if not exists(PATH.SUBMIT + '/' + 'scratch'):
unix.ln(PATH.SCRATCH, PATH.SUBMIT + '/' + 'scratch')
call('sbatch '
+ '--job-name=%s ' % PAR.SUBTITLE
+ '--output=%s ' % (PATH.SUBMIT +'/'+ 'output.log')
+ '--nodes 1 '
+ '--ntasks=% ' % PAR.NGPU
+ '--ntasks-per-socket=%d ' % PAR.NGPU
+ '--gres=gpu:%d ' % PAR.NGPU
+ '--time=%d ' % PAR.WALLTIME
+ findpath('seisflows.system') +'/'+ 'wrappers/submit '
+ PATH.OUTPUT)
def _launch(self, classname, funcname, hosts='all'):
unix.mkdir(PATH.SYSTEM)
nodes = math.ceil(PAR.NTASK/float(PAR.NODESIZE))
ncpus = PAR.NPROC
mpiprocs = PAR.NPROC
hours = PAR.STEPTIME/60
minutes = PAR.STEPTIME%60
walltime = 'walltime=%02d:%02d:00 '%(hours, minutes)
# submit job
with open(PATH.SYSTEM+'/'+'job_id', 'w') as f:
call('qsub '
+ '%s ' % PAR.PBSARGS
+ '-l select=%d:ncpus=%d:mpiprocs=%d ' (nodes, ncpus, mpiprocs)
+ '-l %s ' % walltime
+ '-J 0-%s ' % (PAR.NTASK-1)
+ '-N %s ' % PAR.TITLE
+ '-o %s ' % (PATH.SUBMIT+'/'+'output.pbs/' + '$PBS_ARRAYID')
+ '-r y '
+ '-j oe '
+ '-V '
+ self.launch_args(hosts)
+ PATH.OUTPUT + ' '
+ classname + ' '
+ funcname + ' '
+ findpath('seisflows.system'),
stdout=f)
# retrieve job ids
with open(PATH.SYSTEM+'/'+'job_id', 'r') as f:
line = f.readline()
job = line.split()[-1].strip()
if hosts == 'all' and PAR.NTASK > 1:
nn = range(PAR.NTASK)
job0 = job.strip('[].sdb')
return [job0+'['+str(ii)+'].sdb' for ii in nn]
else:
return [job]
def run(self, classname, funcname, hosts='all', **kwargs):
""" Runs tasks in serial or parallel on specified hosts
"""
self.checkpoint()
self.save_kwargs(classname, funcname, kwargs)
if hosts == 'all':
call('srun '
+ '--wait=0 '
+ join(findpath('seisflows.system'), 'wrappers/run ')
+ PATH.OUTPUT + ' '
+ classname + ' '
+ funcname)
elif hosts == 'head':
# run on head node
call('srun '
+ '--wait=0 '
+ join(findpath('seisflows.system'), 'wrappers/run_head ')
+ PATH.OUTPUT + ' '
+ classname + ' '
+ funcname)
def submit(self, workflow):
""" Submits workflow
"""
unix.mkdir(PATH.OUTPUT)
unix.cd(PATH.OUTPUT)
unix.mkdir(PATH.SUBMIT + '/' + 'output.pbs')
self.checkpoint()
hours = PAR.WALLTIME / 60
minutes = PAR.WALLTIME % 60
walltime = 'walltime=%02d:%02d:00 ' % (hours, minutes)
ncpus = PAR.NODESIZE
mpiprocs = PAR.NODESIZE
# prepare qsub arguments
call('qsub ' + '%s ' % PAR.PBSARGS +
'-l select=1:ncpus=%d:mpiprocs=%d ' % (ncpus, mpiprocs) +
'-l %s ' % walltime + '-N %s ' % PAR.TITLE + '-j %s ' % 'oe' +
'-o %s ' % (PATH.SUBMIT + '/' + 'output.log') + '-V ' + ' -- ' +
findpath('seisflows.system') + '/' + 'wrappers/submit ' +
PATH.OUTPUT)
def run(self, classname, funcname, hosts='all', **kwargs):
""" Runs tasks in serial or parallel on specified hosts
"""
self.checkpoint()
self.save_kwargs(classname, funcname, kwargs)
if hosts == 'all':
# run on all available nodes
call('pbsdsh ' + join(findpath('seisflows.system'),
'wrappers/export_paths.sh ') +
os.getenv('PATH') + ' ' + os.getenv('LD_LIBRARY_PATH') + ' ' +
join(findpath('seisflows.system'), 'wrappers/run_pbsdsh ') +
PATH.OUTPUT + ' ' + classname + ' ' + funcname + ' ' +
dirname(findpath('seisflows')))
elif hosts == 'head':
# run on head node
call('pbsdsh ' + join(findpath('seisflows.system'),
'wrappers/export_paths.sh ') +
os.getenv('PATH') + ' ' + os.getenv('LD_LIBRARY_PATH') + ' ' +
join(findpath('seisflows.system'),
'wrappers/run_pbsdsh_head ') + PATH.OUTPUT + ' ' +
classname + ' ' + funcname + ' ' +
dirname(findpath('seisflows')))
def submit(self, workflow):
"""Submits job
"""
unix.mkdir(PATH.OUTPUT)
unix.cd(PATH.OUTPUT)
# save current state
self.checkpoint()
# construct resource list
resources = []
nodes = int(PAR.NTASK / PAR.NODESIZE)
cores = PAR.NTASK % PAR.NODESIZE
hours = int(PAR.WALLTIME / 60)
minutes = PAR.WALLTIME % 60
if PAR.WALLTIME:
resources += ['walltime=%02d:%02d:00'%(hours, minutes)]
if PAR.MEMORY:
resources += ['mem=%dgb' % PAR.MEMORY]
if nodes == 0:
resources += ['nodes=1:ppn=%d'%(cores)]
elif cores == 0:
resources += ['nodes=%d:ppn=%d'%(nodes, PAR.NODESIZE)]
else:
resources += ['nodes=%d:ppn=%d+1:ppn=%d'%(nodes, PAR.NODESIZE, cores)]
# construct arguments list
call('qsub '
+ '%s ' % PAR.PBSARGS
+ '-N %s '%PAR.TITLE
+ '-o %s '%(PATH.SUBMIT +'/'+ 'output.log')
+ '-l %s '%resources.join(',')
+ '-j %s '%'oe'
+ findpath('seisflows.system') +'/'+ 'wrappers/submit '
+ '-F %s '%PATH.OUTPUT)
