def run(self, classname, funcname, hosts='all', **kwargs):
""" Runs tasks in serial or parallel on specified hosts
"""
self.checkpoint()
self.save_kwargs(classname, funcname, kwargs)
if hosts == 'all':
# run on all available nodes
unix.run('srun '
+ '--wait=0 '
+ join(findpath('seisflows.system'), 'wrappers/run ')
+ PATH.OUTPUT + ' '
+ classname + ' '
+ funcname)
elif hosts == 'head':
# run on head node
unix.run('srun '
+ '--wait=0 '
+ join(findpath('seisflows.system'), 'wrappers/run_head ')
+ PATH.OUTPUT + ' '
+ classname + ' '
+ funcname)
else:
raise(KeyError('Hosts parameter not set/recognized.'))
def run(self, classname, funcname, hosts='all', **kwargs):
""" Runs tasks in serial or parallel on specified hosts
"""
# to avoid cryptic MPI messages, use "--mca_warn_on_fork 0" as the
# default value for MPIARGS, and use subprocess.call rather than
# call_catch to invoke mpiexec
self.checkpoint()
self.save_kwargs(classname, funcname, kwargs)
if hosts == 'all':
unix.cd(join(findpath('seisflows.system'), 'wrappers'))
subprocess.call(PAR.MPIEXEC + ' '
+ '-n %d ' % PAR.NTASK
+ PAR.MPIARGS + ' '
+ 'run_mpi' + ' '
+ PATH.OUTPUT + ' '
+ classname + ' '
+ funcname,
shell=True)
elif hosts == 'head':
unix.cd(join(findpath('seisflows.system'), 'wrappers'))
subprocess.call(PAR.MPIEXEC + ' '
+ '-n 1 '
+ PAR.MPIARGS + ' '
+ 'run_mpi_head' + ' '
+ PATH.OUTPUT + ' '
+ classname + ' '
+ funcname,
shell=True)
else:
raise(KeyError('Hosts parameter not set/recognized.'))
def run(self, classname, funcname, hosts='all', **kwargs):
""" Runs tasks in serial or parallel on specified hosts
"""
# to avoid cryptic MPI messages, use "--mca_warn_on_fork 0" as the
# default value for MPIARGS, and use subprocess.call rather than
# call_catch to invoke mpiexec
self.checkpoint()
self.save_kwargs(classname, funcname, kwargs)
if hosts == 'all':
unix.cd(join(findpath('seisflows.system'), 'wrappers'))
subprocess.call(PAR.MPIEXEC + ' ' + '-n %d ' % PAR.NTASK +
PAR.MPIARGS + ' ' + 'run_mpi' + ' ' + PATH.OUTPUT +
' ' + classname + ' ' + funcname,
shell=True)
elif hosts == 'head':
unix.cd(join(findpath('seisflows.system'), 'wrappers'))
subprocess.call(PAR.MPIEXEC + ' ' + '-n 1 ' + PAR.MPIARGS + ' ' +
'run_mpi_head' + ' ' + PATH.OUTPUT + ' ' +
classname + ' ' + funcname,
shell=True)
else:
raise (KeyError('Hosts parameter not set/recognized.'))
def run(self, classname, funcname, hosts='all', **kwargs):
""" Runs tasks in serial or parallel on specified hosts
"""
self.checkpoint()
self.save_kwargs(classname, funcname, kwargs)
if hosts == 'all':
unix.cd(join(findpath('seisflows.system'), 'wrappers'))
unix.run('mpiexec -n {} '.format(PAR.NTASK)
+ PAR.MPIARGS + ' '
+ 'run_mpi' + ' '
+ PATH.OUTPUT + ' '
+ classname + ' '
+ funcname)
elif hosts == 'head':
unix.cd(join(findpath('seisflows.system'), 'wrappers'))
unix.run('mpiexec -n 1 '
+ PAR.MPIARGS + ' '
+ 'run_mpi_head' + ' '
+ PATH.OUTPUT + ' '
+ classname + ' '
+ funcname)
else:
raise(KeyError('Hosts parameter not set/recognized.'))
def launch_args(self, hosts):
if hosts == 'all':
arg = ('-J 0-%s ' % (PAR.NTASK-1)
+'-o %s ' % (PATH.SUBMIT+'/'+'output.pbs/' + '$PBS_ARRAYID')
+ ' -- ' + findpath('seisflows.system') +'/'+ 'wrappers/run_pbsdsh ')
elif hosts == 'head':
arg = ('-J 0-0 '
+'-o %s ' % (PATH.SUBMIT+'/'+'output.pbs/' + '$PBS_JOBID')
+ ' -- ' + findpath('seisflows.system') +'/'+ 'wrappers/run_pbsdsh_head ')
return arg
def launch_args(self, hosts):
if hosts == 'all':
arg = ('-J 0-%s ' % (PAR.NTASK - 1) + '-o %s ' %
(PATH.SUBMIT + '/' + 'output.pbs/' + '$PBS_ARRAYID') +
' -- ' + findpath('seisflows.system') + '/' +
'wrappers/run_pbsdsh ')
elif hosts == 'head':
arg = ('-J 0-0 ' + '-o %s ' %
(PATH.SUBMIT + '/' + 'output.pbs/' + '$PBS_JOBID') +
' -- ' + findpath('seisflows.system') + '/' +
'wrappers/run_pbsdsh_head ')
return arg
def launch(self, classname, funcname, hosts='all'):
unix.mkdir(PATH.SYSTEM)
# submit job
with open(PATH.SYSTEM+'/'+'job_id', 'w') as f:
subprocess.call('bsub '
+ '%s ' % PAR.LSFARGS
+ '-n %d ' % PAR.NPROC
+ '-R "span[ptile=%d]" ' % PAR.NODESIZE
+ '-W %d:00 ' % PAR.STEPTIME
+ '-J "%s' %PAR.TITLE
+ self.launch_args(hosts)
+ findpath('seisflows.system') +'/'+ 'wrapper/run '
+ PATH.OUTPUT + ' '
+ classname + ' '
+ funcname + ' ',
shell=True,
stdout=f)
# retrieve job ids
with open(PATH.SYSTEM+'/'+'job_id', 'r') as f:
# reads one entire line from the file
line = f.readline()
job_buf = line.split()[1].strip()
job = job_buf[1:-1]
if hosts == 'all' and PAR.NSRC > 1:
nn = range(1,PAR.NSRC+1)
#return [job+'_'+str(ii) for ii in nn]
return [job+'['+str(ii)+']' for ii in nn]
else:
return [job]
def write_parameters(par, version='git-devel'):
""" Writes parameters to text file
"""
# read template
file = findpath('sesiflows.seistools') + '/' + 'specfem2d/par-' + version
with open(file, 'r') as f:
lines = f.readlines()
lines[-1] = ' '.join(['1', str(par.NX), '1', str(par.NZ), '1'])
# write parameter file
file = 'DATA/Par_file'
_writelines(file, lines)
setpar('xmin', str(par.XMIN))
setpar('xmax', str(par.XMAX))
setpar('nx', str(par.NX))
setpar('nt', str(par.NT))
setpar('deltat', str(par.DT))
setpar('nsources', str(1))
# write interfaces file
file = 'DATA/interfaces.dat'
lines = []
lines.extend('2\n')
lines.extend('2\n')
lines.extend('%f %f\n' % (par.XMIN, par.ZMIN))
lines.extend('%f %f\n' % (par.XMAX, par.ZMIN))
lines.extend('2\n')
lines.extend('%f %f\n' % (par.XMIN, par.ZMAX))
lines.extend('%f %f\n' % (par.XMAX, par.ZMAX))
lines.extend(str(par.NZ))
_writelines(file, lines)
def _launch(self, classname, funcname, hosts='all'):
unix.mkdir(PATH.SYSTEM)
with open(PATH.SYSTEM+'/'+'job_id', 'w') as f:
call('sbatch '
+ '%s ' % PAR.SLURMARGS
+ '--job-name=%s ' % PAR.TITLE
+ '--nodes=%d ' % math.ceil(PAR.NPROC/float(PAR.NODESIZE))
+ '--ntasks-per-node=%d ' % PAR.NODESIZE
+ '--ntasks=%d ' % PAR.NPROC
+ '--time=%d ' % PAR.STEPTIME
+ self._launch_args(hosts)
+ findpath('seisflows.system') +'/'+ 'wrappers/run '
+ PATH.OUTPUT + ' '
+ classname + ' '
+ funcname + ' ',
stdout=f)
# retrieve job ids
with open(PATH.SYSTEM+'/'+'job_id', 'r') as f:
line = f.readline()
job = line.split()[-1].strip()
if hosts == 'all' and PAR.NTASK > 1:
return [job+'_'+str(ii) for ii in range(PAR.NTASK)]
else:
return [job]
def _launch(self, classname, funcname, hosts='all'):
unix.mkdir(PATH.SYSTEM)
with open(PATH.SYSTEM+'/'+'job_id', 'w') as f:
subprocess.call('sbatch '
+ '%s ' % PAR.SLURMARGS
+ '--job-name=%s ' % PAR.TITLE
+ '--nodes=%d ' % math.ceil(PAR.NPROC/float(PAR.NODESIZE))
+ '--ntasks-per-node=%d ' % PAR.NODESIZE
+ '--ntasks=%d ' % PAR.NPROC
+ '--time=%d ' % PAR.STEPTIME
+ self._launch_args(hosts)
+ findpath('seisflows.system') +'/'+ 'wrappers/run '
+ PATH.OUTPUT + ' '
+ classname + ' '
+ funcname + ' ',
shell=1,
stdout=f)
# retrieve job ids
with open(PATH.SYSTEM+'/'+'job_id', 'r') as f:
line = f.readline()
job = line.split()[-1].strip()
if hosts == 'all' and PAR.NTASK > 1:
return [job+'_'+str(ii) for ii in range(PAR.NTASK)]
else:
return [job]
def launch(self, classname, funcname, hosts='all'):
unix.mkdir(PATH.SYSTEM)
# submit job
with open(PATH.SYSTEM + '/' + 'job_id', 'w') as f:
call('bsub ' + '%s ' % PAR.LSFARGS + '-n %d ' % PAR.NPROC +
'-R "span[ptile=%d]" ' % PAR.NODESIZE +
'-W %d:00 ' % PAR.STEPTIME + '-J "%s' % PAR.TITLE +
self.launch_args(hosts) + findpath('seisflows.system') + '/' +
'wrapper/run ' + PATH.OUTPUT + ' ' + classname + ' ' +
funcname + ' ',
stdout=f)
# retrieve job ids
with open(PATH.SYSTEM + '/' + 'job_id', 'r') as f:
# reads one entire line from the file
line = f.readline()
job_buf = line.split()[1].strip()
job = job_buf[1:-1]
if hosts == 'all' and PAR.NSRC > 1:
nn = range(1, PAR.NSRC + 1)
#return [job+'_'+str(ii) for ii in nn]
return [job + '[' + str(ii) + ']' for ii in nn]
else:
return [job]
def write_parameters(par, version='git-devel'):
""" Writes parameters to text file
"""
# read template
file = findpath('sesiflows.seistools') + '/' + 'specfem2d/par-' + version
with open(file, 'r') as f:
lines = f.readlines()
lines[-1] = ' '.join(['1', str(par.NX), '1', str(par.NZ), '1'])
# write parameter file
file = 'DATA/Par_file'
_writelines(file, lines)
setpar('xmin', str(par.XMIN))
setpar('xmax', str(par.XMAX))
setpar('nx', str(par.NX))
setpar('nt', str(par.NT))
setpar('deltat', str(par.DT))
setpar('nsources', str(1))
# write interfaces file
file = 'DATA/interfaces.dat'
lines = []
lines.extend('2\n')
lines.extend('2\n')
lines.extend('%f %f\n'%(par.XMIN, par.ZMIN))
lines.extend('%f %f\n'%(par.XMAX, par.ZMIN))
lines.extend('2\n')
lines.extend('%f %f\n'%(par.XMIN, par.ZMAX))
lines.extend('%f %f\n'%(par.XMAX, par.ZMAX))
lines.extend(str(par.NZ))
_writelines(file, lines)
def submit(self, workflow):
""" Submits workflow
"""
unix.mkdir(PATH.OUTPUT)
unix.cd(PATH.OUTPUT)
unix.mkdir(PATH.SUBMIT+'/'+'output.pbs')
self.checkpoint()
hours = PAR.WALLTIME/60
minutes = PAR.WALLTIME%60
walltime = 'walltime=%02d:%02d:00 ' % (hours, minutes)
ncpus = PAR.NODESIZE
mpiprocs = PAR.NODESIZE
# prepare qsub arguments
unix.run( 'qsub '
+ '%s ' % PAR.PBSARGS
+ '-l select=1:ncpus=%d:mpiprocs=%d ' % (ncpus, mpiprocs)
+ '-l %s ' % walltime
+ '-N %s ' % PAR.TITLE
+ '-j %s '%'oe'
+ '-o %s ' % (PATH.SUBMIT+'/'+'output.log')
+ '-V '
+ ' -- ' + findpath('seisflows.system') +'/'+ 'wrappers/submit '
+ PATH.OUTPUT)
def submit(self, workflow):
"""Submits job
"""
unix.mkdir(PATH.OUTPUT)
unix.cd(PATH.OUTPUT)
# save current state
self.checkpoint()
# construct resource list
nodes = int(PAR.NTASK / PAR.NODESIZE)
cores = PAR.NTASK % PAR.NODESIZE
hours = int(PAR.WALLTIME / 60)
minutes = PAR.WALLTIME % 60
resources = 'walltime=%02d:%02d:00' % (hours, minutes)
if nodes == 0:
resources += ',mem=%dgb,nodes=1:ppn=%d' % (PAR.MEMORY, cores)
elif cores == 0:
resources += ',mem=%dgb,nodes=%d:ppn=%d' % (PAR.MEMORY, nodes,
PAR.NODESIZE)
else:
resources += ',mem=%dgb,nodes=%d:ppn=%d+1:ppn=%d' % (
PAR.MEMORY, nodes, PAR.NODESIZE, cores)
# construct arguments list
call('qsub ' + '%s ' % PAR.PBSARGS + '-N %s ' % PAR.TITLE + '-o %s ' %
(PATH.SUBMIT + '/' + 'output.log') + '-l %s ' % resources +
'-j %s ' % 'oe' + findpath('seisflows.system') + '/' +
'wrappers/submit ' + '-F %s ' % PATH.OUTPUT)
def _launch(self, classname, funcname, hosts='all'):
unix.mkdir(PATH.SYSTEM)
nodes = math.ceil(PAR.NTASK / float(PAR.NODESIZE))
ncpus = PAR.NPROC
mpiprocs = PAR.NPROC
hours = PAR.STEPTIME / 60
minutes = PAR.STEPTIME % 60
walltime = 'walltime=%02d:%02d:00 ' % (hours, minutes)
# submit job
with open(PATH.SYSTEM + '/' + 'job_id', 'w') as f:
call(
'qsub ' + '%s ' % PAR.PBSARGS +
'-l select=%d:ncpus=%d:mpiprocs=%d ' (nodes, ncpus, mpiprocs) +
'-l %s ' % walltime + '-J 0-%s ' % (PAR.NTASK - 1) +
'-N %s ' % PAR.TITLE + '-o %s ' %
(PATH.SUBMIT + '/' + 'output.pbs/' + '$PBS_ARRAYID') +
'-r y ' + '-j oe ' + '-V ' + self.launch_args(hosts) +
PATH.OUTPUT + ' ' + classname + ' ' + funcname + ' ' +
findpath('seisflows.system'),
stdout=f)
# retrieve job ids
with open(PATH.SYSTEM + '/' + 'job_id', 'r') as f:
line = f.readline()
job = line.split()[-1].strip()
if hosts == 'all' and PAR.NTASK > 1:
nn = range(PAR.NTASK)
job0 = job.strip('[].sdb')
return [job0 + '[' + str(ii) + '].sdb' for ii in nn]
else:
return [job]
def run(self, classname, funcname, hosts='all', **kwargs):
""" Runs tasks in serial or parallel on specified hosts
"""
self.checkpoint()
self.save_kwargs(classname, funcname, kwargs)
if hosts == 'all':
# run on all available nodes
call('srun ' + '--wait=0 ' +
join(findpath('seisflows.system'), 'wrappers/run ') +
PATH.OUTPUT + ' ' + classname + ' ' + funcname)
elif hosts == 'head':
# run on head node
call('srun ' + '--wait=0 ' +
join(findpath('seisflows.system'), 'wrappers/run_head ') +
PATH.OUTPUT + ' ' + classname + ' ' + funcname)
else:
raise (KeyError('Hosts parameter not set/recognized.'))
def _launch(self, classname, funcname, itask=0):
self.progress(itask)
env = os.environ.copy().items()
env += [['SEISFLOWS_TASKID', str(itask)]]
p = Popen(findpath('seisflows.system') + '/' + 'wrappers/run ' +
PATH.OUTPUT + ' ' + classname + ' ' + funcname,
shell=True,
env=dict(env))
return p
def submit(self, workflow):
""" Submits workflow
"""
unix.mkdir(PATH.OUTPUT)
unix.cd(PATH.OUTPUT)
self.checkpoint()
# submit workflow
call('sbatch ' + '%s ' % PAR.SLURMARGS + '--job-name=%s ' % PAR.TITLE +
'--output=%s ' % (PATH.SUBMIT + '/' + 'output.log') +
'--cpus-per-task=%d ' % PAR.NPROC + '--ntasks=%d ' % PAR.NTASK +
'--time=%d ' % PAR.WALLTIME + findpath('seisflows.system') + '/' +
'wrappers/submit ' + PATH.OUTPUT)
def run(self, classname, funcname, hosts='all', **kwargs):
""" Runs tasks in serial or parallel on specified hosts
"""
self.checkpoint()
self.save_kwargs(classname, funcname, kwargs)
if hosts == 'all':
call('srun '
+ '--wait=0 '
+ join(findpath('seisflows.system'), 'wrappers/run ')
+ PATH.OUTPUT + ' '
+ classname + ' '
+ funcname)
elif hosts == 'head':
# run on head node
call('srun '
+ '--wait=0 '
+ join(findpath('seisflows.system'), 'wrappers/run_head ')
+ PATH.OUTPUT + ' '
+ classname + ' '
+ funcname)
def _launch(self, classname, funcname, itask=0):
self.progress(itask)
env = os.environ.copy().items()
env += [['SEISFLOWS_TASKID', str(itask)]]
p = Popen(
findpath('seisflows.system') +'/'+ 'wrappers/run '
+ PATH.OUTPUT + ' '
+ classname + ' '
+ funcname,
shell=True,
env=dict(env))
return p
def submit(self, workflow):
""" Submits workflow
"""
unix.mkdir(PATH.OUTPUT)
unix.cd(PATH.OUTPUT)
unix.mkdir(PATH.SUBMIT + '/' + 'output.lsf')
self.checkpoint()
# prepare bsub arguments
call('bsub ' + '%s ' % PAR.LSFARGS + '-J %s ' % PAR.TITLE + '-o %s ' %
(PATH.SUBMIT + '/' + 'output.log') + '-n %d ' % PAR.NTASK +
'-e %s ' % (PATH.SUBMIT + '/' + 'error.log') +
'-W %d:00 ' % PAR.WALLTIME + findpath('seisflows.system') + '/' +
'wrappers/submit ' + PATH.OUTPUT)
def submit(self, workflow):
""" Submits workflow
"""
unix.mkdir(PATH.OUTPUT)
unix.cd(PATH.OUTPUT)
unix.mkdir(PATH.SUBMIT + '/' + 'output.slurm')
self.checkpoint()
# prepare sbatch arguments
call('sbatch ' + '%s ' % PAR.SLURMARGS + '--job-name=%s ' % PAR.TITLE +
'--output %s ' % (PATH.SUBMIT + '/' + 'output.log') +
'--ntasks-per-node=%d ' % PAR.NODESIZE + '--nodes=%d ' % 1 +
'--time=%d ' % PAR.WALLTIME + findpath('seisflows.system') + '/' +
'wrappers/submit ' + PATH.OUTPUT)
def write_sources(PAR, h, path='.'):
""" Writes source information to text file
"""
file = findpath('sesiflows.seistools') + '/' + 'specfem3d/FORCESOLUTION'
with open(file, 'r') as f:
lines = f.readlines()
file = 'DATA/FORCESOURCE'
_writelines(file, lines)
# adjust coordinates
setpar('xs', h.sx[0], file)
setpar('zs', h.sz[0], file)
setpar('ts', h.ts, file)
# adjust wavelet
setpar('f0', PAR['F0'], file)
def write_sources(PAR, h, path='.'):
""" Writes source information to text file
"""
file = findpath('sesiflows.seistools') + '/' + 'specfem3d/FORCESOLUTION'
with open(file, 'r') as f:
lines = f.readlines()
file = 'DATA/FORCESOURCE'
_writelines(file, lines)
# adjust coordinates
setpar('xs', h.sx[0], file)
setpar('zs', h.sz[0], file)
setpar('ts', h.ts, file)
# adjust wavelet
setpar('f0', PAR['F0'], file)
def submit(self, workflow):
""" Submits workflow
"""
unix.mkdir(PATH.OUTPUT)
unix.cd(PATH.OUTPUT)
self.checkpoint()
# submit workflow
call('sbatch '
+ '%s ' % PAR.SLURMARGS
+ '--job-name=%s '%PAR.TITLE
+ '--output=%s '%(PATH.SUBMIT +'/'+ 'output.log')
+ '--cpus-per-task=%d '%PAR.NPROC
+ '--ntasks=%d '%PAR.NTASK
+ '--time=%d '%PAR.WALLTIME
+ findpath('seisflows.system') +'/'+ 'wrappers/submit '
+ PATH.OUTPUT)
def submit(self, workflow):
""" Submits workflow
"""
unix.mkdir(PATH.OUTPUT)
unix.cd(PATH.OUTPUT)
unix.mkdir(PATH.SUBMIT+'/'+'output.lsf')
self.checkpoint()
# prepare bsub arguments
call('bsub '
+ '%s ' % PAR.LSFARGS
+ '-J %s ' % PAR.TITLE
+ '-o %s ' % (PATH.SUBMIT+'/'+'output.log')
+ '-n %d ' % PAR.NTASK
+ '-e %s ' % (PATH.SUBMIT+'/'+'error.log')
+ '-W %d:00 ' % PAR.WALLTIME
+ findpath('seisflows.system') +'/'+ 'wrappers/submit '
+ PATH.OUTPUT)
def submit(self, workflow):
""" Submits workflow
"""
unix.mkdir(PATH.OUTPUT)
unix.cd(PATH.OUTPUT)
unix.mkdir(PATH.SUBMIT+'/'+'output.slurm')
self.checkpoint()
# prepare sbatch arguments
unix.run('sbatch '
+ '%s ' % PAR.SLURMARGS
+ '--job-name=%s ' % PAR.TITLE
+ '--output %s ' % (PATH.SUBMIT+'/'+'output.log')
+ '--ntasks-per-node=%d ' % PAR.NODESIZE
+ '--nodes=%d ' % 1
+ '--time=%d ' % PAR.WALLTIME
+ findpath('seisflows.system') +'/'+ 'wrappers/submit '
+ PATH.OUTPUT)
def write_sources(coords, path='.', ws=1., suffix=''):
""" Writes source information to text file
"""
sx, sy, sz = coords
filename = findpath('seisflows.seistools') + '/' + 'specfem2d/SOURCE'
with open(filename, 'r') as f:
lines = f.readlines()
filename = 'DATA/SOURCE' + suffix
with open(filename, 'w') as f:
f.writelines(lines)
# adjust source coordinates
setpar('xs', sx, filename)
setpar('zs', sy, filename)
#setpar('ts', ts[0], filename)
# adjust source amplitude
try:
fs = float(getpar('factor', filename))
fs *= ws
setpar('factor', str(fs), filename)
except:
pass
# adjust source wavelet
if 1:
# Ricker wavelet
setpar('time_function_type', 1, filename)
elif 0:
# first derivative of Gaussian
setpar('time_function_type', 2, filename)
elif 0:
# Gaussian
setpar('time_function_type', 3, filename)
elif 0:
# Dirac
setpar('time_function_type', 4, filename)
elif 0:
# Heaviside
setpar('time_function_type', 5, filename)
def _launch(self, classname, funcname, hosts='all'):
unix.mkdir(PATH.SYSTEM)
nodes = math.ceil(PAR.NTASK/float(PAR.NODESIZE))
ncpus = PAR.NPROC
mpiprocs = PAR.NPROC
hours = PAR.STEPTIME/60
minutes = PAR.STEPTIME%60
walltime = 'walltime=%02d:%02d:00 '%(hours, minutes)
# submit job
with open(PATH.SYSTEM+'/'+'job_id', 'w') as f:
subprocess.call('qsub '
+ '%s ' % PAR.PBSARGS
+ '-l select=%d:ncpus=%d:mpiprocs=%d ' (nodes, ncpus, mpiprocs)
+ '-l %s ' % walltime
+ '-J 0-%s ' % (PAR.NTASK-1)
+ '-N %s ' % PAR.TITLE
+ '-o %s ' % (PATH.SUBMIT+'/'+'output.pbs/' + '$PBS_ARRAYID')
+ '-r y '
+ '-j oe '
+ '-V '
+ self.launch_args(hosts)
+ PATH.OUTPUT + ' '
+ classname + ' '
+ funcname + ' '
+ findpath('seisflows'),
stdout=f,
shell=True)
# retrieve job ids
with open(PATH.SYSTEM+'/'+'job_id', 'r') as f:
line = f.readline()
job = line.split()[-1].strip()
if hosts == 'all' and PAR.NTASK > 1:
nn = range(PAR.NTASK)
job0 = job.strip('[].sdb')
return [job0+'['+str(ii)+'].sdb' for ii in nn]
else:
return [job]
def submit(self, workflow):
""" Submits workflow
"""
unix.mkdir(PATH.OUTPUT)
unix.cd(PATH.OUTPUT)
self.checkpoint()
if not exists(PATH.SUBMIT + '/' + 'scratch'):
unix.ln(PATH.SCRATCH, PATH.SUBMIT + '/' + 'scratch')
call('sbatch '
+ '--job-name=%s ' % PAR.SUBTITLE
+ '--output=%s ' % (PATH.SUBMIT +'/'+ 'output.log')
+ '--nodes 1 '
+ '--ntasks=% ' % PAR.NGPU
+ '--ntasks-per-socket=%d ' % PAR.NGPU
+ '--gres=gpu:%d ' % PAR.NGPU
+ '--time=%d ' % PAR.WALLTIME
+ findpath('seisflows.system') +'/'+ 'wrappers/submit '
+ PATH.OUTPUT)
def write_sources(par, hdr, path='.', suffix=''):
""" Writes source information to text file
"""
file = findpath('sesiflows.seistools') + '/' + 'specfem2d/SOURCE'
with open(file, 'r') as f:
lines = f.readlines()
file = path + '/' + 'DATA/SOURCE' + suffix
_writelines(file, lines)
# adjust source coordinates
setpar('xs', hdr.sx[0], file)
setpar('zs', hdr.sy[0], file)
setpar('ts', hdr.ts, file)
# adjust source amplitude
try:
fs = float(getpar('factor', file))
setpar('factor', str(fs*hdr.fs), file)
except:
pass
# adjust source wavelet
if 1:
# Ricker wavelet
setpar('time_function_type', 1, file)
elif 0:
# first derivative of Gaussian
setpar('time_function_type', 2, file)
elif 0:
# Gaussian
setpar('time_function_type', 3, file)
elif 0:
# Dirac
setpar('time_function_type', 4, file)
elif 0:
# Heaviside
setpar('time_function_type', 5, file)
setpar('f0', par['F0'], file)
def write_sources(par, hdr, path='.', suffix=''):
""" Writes source information to text file
"""
file = findpath('sesiflows.seistools') + '/' + 'specfem2d/SOURCE'
with open(file, 'r') as f:
lines = f.readlines()
file = path + '/' + 'DATA/SOURCE' + suffix
_writelines(file, lines)
# adjust source coordinates
setpar('xs', hdr.sx[0], file)
setpar('zs', hdr.sy[0], file)
setpar('ts', hdr.ts, file)
# adjust source amplitude
try:
fs = float(getpar('factor', file))
setpar('factor', str(fs * hdr.fs), file)
except:
pass
# adjust source wavelet
if 1:
# Ricker wavelet
setpar('time_function_type', 1, file)
elif 0:
# first derivative of Gaussian
setpar('time_function_type', 2, file)
elif 0:
# Gaussian
setpar('time_function_type', 3, file)
elif 0:
# Dirac
setpar('time_function_type', 4, file)
elif 0:
# Heaviside
setpar('time_function_type', 5, file)
setpar('f0', par['F0'], file)
def submit(self, workflow):
""" Submits workflow
"""
unix.mkdir(PATH.OUTPUT)
unix.cd(PATH.OUTPUT)
unix.mkdir(PATH.SUBMIT + '/' + 'output.pbs')
self.checkpoint()
hours = PAR.WALLTIME / 60
minutes = PAR.WALLTIME % 60
walltime = 'walltime=%02d:%02d:00 ' % (hours, minutes)
ncpus = PAR.NODESIZE
mpiprocs = PAR.NODESIZE
# prepare qsub arguments
call('qsub ' + '%s ' % PAR.PBSARGS +
'-l select=1:ncpus=%d:mpiprocs=%d ' % (ncpus, mpiprocs) +
'-l %s ' % walltime + '-N %s ' % PAR.TITLE + '-j %s ' % 'oe' +
'-o %s ' % (PATH.SUBMIT + '/' + 'output.log') + '-V ' + ' -- ' +
findpath('seisflows.system') + '/' + 'wrappers/submit ' +
PATH.OUTPUT)
def submit(self, workflow):
"""Submits job
"""
unix.mkdir(PATH.OUTPUT)
unix.cd(PATH.OUTPUT)
# save current state
self.checkpoint()
# construct resource list
resources = []
nodes = int(PAR.NTASK / PAR.NODESIZE)
cores = PAR.NTASK % PAR.NODESIZE
hours = int(PAR.WALLTIME / 60)
minutes = PAR.WALLTIME % 60
if PAR.WALLTIME:
resources += ['walltime=%02d:%02d:00'%(hours, minutes)]
if PAR.MEMORY:
resources += ['mem=%dgb' % PAR.MEMORY]
if nodes == 0:
resources += ['nodes=1:ppn=%d'%(cores)]
elif cores == 0:
resources += ['nodes=%d:ppn=%d'%(nodes, PAR.NODESIZE)]
else:
resources += ['nodes=%d:ppn=%d+1:ppn=%d'%(nodes, PAR.NODESIZE, cores)]
# construct arguments list
call('qsub '
+ '%s ' % PAR.PBSARGS
+ '-N %s '%PAR.TITLE
+ '-o %s '%(PATH.SUBMIT +'/'+ 'output.log')
+ '-l %s '%resources.join(',')
+ '-j %s '%'oe'
+ findpath('seisflows.system') +'/'+ 'wrappers/submit '
+ '-F %s '%PATH.OUTPUT)
def run(self, classname, funcname, hosts='all', **kwargs):
""" Runs tasks in serial or parallel on specified hosts
"""
self.checkpoint()
self.save_kwargs(classname, funcname, kwargs)
if hosts == 'all':
# run on all available nodes
call('pbsdsh ' + join(findpath('seisflows.system'),
'wrappers/export_paths.sh ') +
os.getenv('PATH') + ' ' + os.getenv('LD_LIBRARY_PATH') + ' ' +
join(findpath('seisflows.system'), 'wrappers/run_pbsdsh ') +
PATH.OUTPUT + ' ' + classname + ' ' + funcname + ' ' +
dirname(findpath('seisflows')))
elif hosts == 'head':
# run on head node
call('pbsdsh ' + join(findpath('seisflows.system'),
'wrappers/export_paths.sh ') +
os.getenv('PATH') + ' ' + os.getenv('LD_LIBRARY_PATH') + ' ' +
join(findpath('seisflows.system'),
'wrappers/run_pbsdsh_head ') + PATH.OUTPUT + ' ' +
classname + ' ' + funcname + ' ' +
dirname(findpath('seisflows')))
