def run(self, classname, funcname, hosts='all', **kwargs):
""" Runs tasks in serial or parallel on specified hosts
"""
self.checkpoint()
self.save_kwargs(classname, funcname, kwargs)
if hosts == 'all':
# run on all available nodes
unix.run('srun '
+ '--wait=0 '
+ join(findpath('system'), 'slurm/wrapper_pbsdsh ')
+ PATH.OUTPUT + ' '
+ classname + ' '
+ funcname)
elif hosts == 'head':
# run on head node
unix.run('srun '
+ '--wait=0 '
+ join(findpath('system'), 'slurm/wrapper_pbsdsh_head ')
+ PATH.OUTPUT + ' '
+ classname + ' '
+ funcname + ' '
+ findpath('seisflows'),
shell=True)
def run(self, classname, funcname, hosts='all', **kwargs):
""" Runs tasks in serial or parallel on specified hosts
"""
name = task.__name__
if PAR.VERBOSE >= 2:
print 'running', name
# save current state
save_objects(join(PATH.OUTPUT, 'SeisflowsObjects'))
# save keyword arguments
kwargspath = join(PATH.OUTPUT, 'SeisflowsObjects',
classname + '_kwargs')
kwargsfile = join(kwargspath, funcname + '.p')
unix.mkdir(kwargspath)
saveobj(kwargsfile, kwargs)
if hosts == 'all':
# run on all available nodes
args = ('pbsdsh ' + findpath('system') + '/' +
'pbs/wrapper_pbsdsh ' + PATH.OUTPUT + ' ' + classname +
' ' + funcname)
elif hosts == 'head':
# run on head node
args = ('pbsdsh ' + findpath('system') + '/' +
'slurm/wrapper_pbsdsh ' + PATH.OUTPUT + ' ' + classname +
' ' + funcname)
else:
raise Exception
subprocess.call(args, shell=1)
def _launch(self, classname, funcname, hosts='all'):
unix.mkdir(PATH.SYSTEM)
# prepare sbatch arguments
if hosts == 'all':
args = ('--array=%d-%d ' % (0, PAR.NTASK - 1) + '--output %s ' %
(PATH.SUBMIT + '/' + 'output.slurm/' + '%A_%a'))
elif hosts == 'head':
args = ('--array=%d-%d ' % (0, 0) + '--output=%s ' %
(PATH.SUBMIT + '/' + 'output.slurm/' + '%j'))
#+('--export=SEISFLOWS_TASK_ID=%s ' % 0
# submit job
with open(PATH.SYSTEM + '/' + 'job_id', 'w') as f:
subprocess.call(
'sbatch ' + '--job-name=%s ' % PAR.TITLE +
'--nodes=%d ' % math.ceil(PAR.NPROC / float(PAR.NODESIZE)) +
'--ntasks-per-node=%d ' % PAR.NODESIZE +
'--time=%d ' % PAR.STEPTIME + args + findpath('system') + '/' +
'slurm/wrapper_srun ' + PATH.OUTPUT + ' ' + classname + ' ' +
funcname + ' ',
shell=1,
stdout=f)
# retrieve job ids
with open(PATH.SYSTEM + '/' + 'job_id', 'r') as f:
line = f.readline()
job = line.split()[-1].strip()
if hosts == 'all' and PAR.NTASK > 1:
nn = range(PAR.NTASK)
return [job + '_' + str(ii) for ii in nn]
else:
return [job]
def _launch(self, classname, funcname, hosts='all'):
unix.mkdir(PATH.SYSTEM)
# submit job
with open(PATH.SYSTEM+'/'+'job_id', 'w') as f:
hours = PAR.WALLTIME/60
minutes = PAR.WALLTIME%60
resources = 'walltime=%02d:%02d:00 '%(hours, minutes)
args = ('/opt/pbs/12.1.1.131502/bin/qsub '
+ '-l select=1:ncpus=32:mpiprocs=32 '
+ '-l %s '%resources
+ '-q standard '
+ '-A ERDCH38424KSC '
+ '-J 0-%s ' % PAR.NTASK
+ '-N %s ' % PAR.TITLE
+ ' -- ' + findpath('system') +'/'+ 'wrappers/runpbsdsh '
+ PATH.OUTPUT + ' '
+ classname + ' '
+ funcname + ' ')
subprocess.call(args, shell=1, stdout=f)
# retrieve job ids
with open(PATH.SYSTEM+'/'+'job_id', 'r') as f:
line = f.readline()
job = line.split()[-1].strip()
if hosts == 'all' and PAR.NTASK > 1:
nn = range(PAR.NTASK)
return [job+'_'+str(ii) for ii in nn]
else:
return [job]
def submit(self, workflow):
"""Submits job
"""
unix.mkdir(PATH.OUTPUT)
unix.cd(PATH.OUTPUT)
# save current state
self.checkpoint()
# construct resource list
nodes = PAR.NTASK / PAR.NODESIZE
cores = PAR.NTASK % PAR.NODESIZE
hours = PAR.WALLTIME / 60
minutes = PAR.WALLTIME % 60
resources = 'walltime=%02d:%02d:00 ' % (hours, minutes)
if nodes == 0:
resources += ',nodes=1:ppn=%d' % cores
elif cores == 0:
resources += ',nodes=%d:ppn=16' % nodes
else:
resources += ',nodes=%d:ppn=16+1:ppn=%d' % (nodes, cores)
# construct arguments list
unix.run('qsub ' + '-N %s ' % PAR.TITLE + '-o %s ' %
(PATH.SUBMIT + '/' + 'output.log') + '-l %s ' % resources +
'-j %s ' % 'oe' + findpath('system') + '/' +
'pbs/wrapper_qsub ' + PATH.OUTPUT,
shell=True)
def write_parameters(par, version='git-devel'):
""" Writes parameters to text file
"""
# read template
file = findpath('seistools') + '/' + 'specfem2d/par-' + version
with open(file, 'r') as f:
lines = f.readlines()
lines[-1] = ' '.join(['1', str(par.NX), '1', str(par.NZ), '1'])
# write parameter file
file = 'DATA/Par_file'
_writelines(file, lines)
setpar('xmin', str(par.XMIN))
setpar('xmax', str(par.XMAX))
setpar('nx', str(par.NX))
setpar('nt', str(par.NT))
setpar('deltat', str(par.DT))
setpar('nsources', str(1))
# write interfaces file
file = 'DATA/interfaces.dat'
lines = []
lines.extend('2\n')
lines.extend('2\n')
lines.extend('%f %f\n' % (par.XMIN, par.ZMIN))
lines.extend('%f %f\n' % (par.XMAX, par.ZMIN))
lines.extend('2\n')
lines.extend('%f %f\n' % (par.XMIN, par.ZMAX))
lines.extend('%f %f\n' % (par.XMAX, par.ZMAX))
lines.extend(str(par.NZ))
_writelines(file, lines)
def submit(self, workflow):
"""Submits job
"""
unix.mkdir(PATH.OUTPUT)
unix.cd(PATH.OUTPUT)
# save current state
save_objects('SeisflowsObjects')
save_parameters('SeisflowsParameters.json')
save_paths('SeisflowsPaths.json')
nodes = PAR.NTASK / 16
cores = PAR.NTASK % 16
hours = PAR.WALLTIME / 60
minutes = PAR.WALLTIME % 60
# construct resource list
resources = 'walltime=%02d:%02d:00 ' % (hours, minutes)
if nodes == 0:
resources += ',nodes=1:ppn=%d' % cores
elif cores == 0:
resources += ',nodes=%d:ppn=16' % nodes
else:
resources += ',nodes=%d:ppn=16+1:ppn=%d' % (nodes, cores)
args = ('qsub ' + '-N %s ' % PAR.TITLE + '-o %s ' %
(PATH.SUBMIT + '/' + 'output.log') + '-l %s ' % resources +
'-j %s ' % 'oe' + findpath('system') + '/' +
'pbs/wrapper_qsub ' + PATH.OUTPUT)
print args # DEBUG
subprocess.call(args, shell=1)
def submit(self, workflow):
"""Submits job
"""
unix.mkdir(PATH.OUTPUT)
unix.cd(PATH.OUTPUT)
# save current state
self.checkpoint()
# construct resource list
nodes = PAR.NTASK/PAR.NODESIZE
cores = PAR.NTASK%PAR.NODESIZE
hours = PAR.WALLTIME/60
minutes = PAR.WALLTIME%60
resources = 'walltime=%02d:%02d:00 '%(hours, minutes)
if nodes == 0:
resources += ',nodes=1:ppn=%d'%cores
elif cores == 0:
resources += ',nodes=%d:ppn=16'%nodes
else:
resources += ',nodes=%d:ppn=16+1:ppn=%d'%(nodes, cores)
# construct arguments list
unix.run('qsub '
+ '-N %s '%PAR.TITLE
+ '-o %s '%(PATH.SUBMIT +'/'+ 'output.log')
+ '-l %s '%resources
+ '-j %s '%'oe'
+ findpath('system') +'/'+ 'pbs/wrapper_qsub '
+ PATH.OUTPUT,
shell=True)
def submit(self, workflow):
"""Submits job
"""
unix.mkdir(PATH.OUTPUT)
unix.cd(PATH.OUTPUT)
# save current state
self.checkpoint()
# construct resource list
nodes = int(PAR.NTASK / PAR.NODESIZE)
cores = PAR.NTASK % PAR.NODESIZE
hours = int(PAR.WALLTIME / 60)
minutes = PAR.WALLTIME % 60
resources = 'walltime=%02d:%02d:00'%(hours, minutes)
if nodes == 0:
resources += ',mem=%dgb,nodes=1:ppn=%d'%(PAR.MEMORY, cores)
elif cores == 0:
resources += ',mem=%dgb,nodes=%d:ppn=%d'%(PAR.MEMORY, nodes, PAR.NODESIZE)
else:
resources += ',mem=%dgb,nodes=%d:ppn=%d+1:ppn=%d'%(PAR.MEMORY, nodes, PAR.NODESIZE, cores)
# construct arguments list
unix.run('qsub '
+ '-N %s '%PAR.TITLE
+ '-o %s '%(PATH.SUBMIT +'/'+ 'output.log')
+ '-l %s '%resources
+ '-j %s '%'oe'
+ findpath('system') +'/'+ 'wrappers/submit '
+ '-F %s '%PATH.OUTPUT)
def write_parameters(par, version="git-devel"):
""" Writes parameters to text file
"""
# read template
file = findpath("seistools") + "/" + "specfem2d/par-" + version
with open(file, "r") as f:
lines = f.readlines()
lines[-1] = " ".join(["1", str(par.NX), "1", str(par.NZ), "1"])
# write parameter file
file = "DATA/Par_file"
_writelines(file, lines)
setpar("xmin", str(par.XMIN))
setpar("xmax", str(par.XMAX))
setpar("nx", str(par.NX))
setpar("nt", str(par.NT))
setpar("deltat", str(par.DT))
setpar("nsources", str(1))
# write interfaces file
file = "DATA/interfaces.dat"
lines = []
lines.extend("2\n")
lines.extend("2\n")
lines.extend("%f %f\n" % (par.XMIN, par.ZMIN))
lines.extend("%f %f\n" % (par.XMAX, par.ZMIN))
lines.extend("2\n")
lines.extend("%f %f\n" % (par.XMIN, par.ZMAX))
lines.extend("%f %f\n" % (par.XMAX, par.ZMAX))
lines.extend(str(par.NZ))
_writelines(file, lines)
def run(self, classname, funcname, hosts='all', **kwargs):
""" Runs tasks in serial or parallel on specified hosts
"""
self.checkpoint()
self.save_kwargs(classname, funcname, kwargs)
if hosts == 'all':
# run on all available nodes
unix.run('srun ' + '--wait=0 ' +
join(findpath('system'), 'slurm/wrapper_srun ') +
PATH.OUTPUT + ' ' + classname + ' ' + funcname)
elif hosts == 'head':
# run on head node
unix.run('srun ' + '--wait=0 ' +
join(findpath('system'), 'slurm/wrapper_srun_head ') +
PATH.OUTPUT + ' ' + classname + ' ' + funcname)
def run(self, classname, funcname, hosts='all', **kwargs):
""" Runs tasks in serial or parallel on specified hosts
"""
self.checkpoint()
self.save_kwargs(classname, funcname, kwargs)
if hosts == 'all':
# run on all available nodes
unix.run('pbsdsh '
+ join(findpath('system'), 'wrapper/export_paths.sh ')
+ os.getenv('PATH') + ' '
+ os.getenv('LD_LIBRARY_PATH') + ' '
+ join(findpath('system'), 'wrapper/run_pbsdsh ')
+ PATH.OUTPUT + ' '
+ classname + ' '
+ funcname + ' '
+ dirname(findpath('seisflows')))
elif hosts == 'head':
# run on head node
unix.run('pbsdsh '
+ join(findpath('system'), 'wrapper/export_paths.sh ')
+ os.getenv('PATH') + ' '
+ os.getenv('LD_LIBRARY_PATH') + ' '
+ join(findpath('system'), 'wrapper/run_pbsdsh_head ')
+ PATH.OUTPUT + ' '
+ classname + ' '
+ funcname + ' '
+ dirname(findpath('seisflows')))
def submit(self, workflow):
""" Submits workflow
"""
unix.mkdir(PATH.OUTPUT)
unix.cd(PATH.OUTPUT)
self.checkpoint()
# submit workflow
unix.run('sbatch ' + '--job-name=%s ' % PAR.SUBTITLE + '--output=%s ' %
(PATH.SUBMIT + '/' + 'output.log') +
'--cpus-per-task=%d ' % PAR.NPROC +
'--ntasks=%d ' % PAR.NTASK + '--time=%d ' % PAR.WALLTIME +
findpath('system') + '/' + 'slurm/wrapper_sbatch ' +
PATH.OUTPUT)
def submit(self, workflow):
""" Submits workflow
"""
unix.mkdir(PATH.OUTPUT)
unix.cd(PATH.OUTPUT)
unix.mkdir(PATH.SUBMIT + '/' + 'output.slurm')
self.checkpoint()
# prepare sbatch arguments
unix.run('sbatch ' + '--job-name=%s ' % PAR.SUBTITLE + '--output %s ' %
(PATH.SUBMIT + '/' + 'output.log') +
'--ntasks-per-node=%d ' % PAR.NODESIZE + '--nodes=%d ' % 1 +
'--time=%d ' % PAR.WALLTIME + findpath('system') + '/' +
'slurm/wrapper_sbatch ' + PATH.OUTPUT)
def _launch(self, classname, funcname, itask=0):
self.progress(itask)
env = os.environ.copy().items()
env += [['SEISFLOWS_TASKID', str(itask)]]
p = Popen(
findpath('system') +'/'+ 'slurm/wrapper_srun '
+ PATH.OUTPUT + ' '
+ classname + ' '
+ funcname,
shell=True,
env=dict(env))
return p
def submit(self, workflow):
""" Submits workflow
"""
unix.mkdir(PATH.OUTPUT)
unix.cd(PATH.OUTPUT)
self.checkpoint()
# submit workflow
unix.run('sbatch '
+ '--job-name=%s '%PAR.SUBTITLE
+ '--output=%s '%(PATH.SUBMIT +'/'+ 'output.log')
+ '--cpus-per-task=%d '%PAR.NPROC
+ '--ntasks=%d '%PAR.NTASK
+ '--time=%d '%PAR.WALLTIME
+ findpath('system') +'/'+ 'wrappers/submit '
+ PATH.OUTPUT)
def write_sources(PAR, h, path='.'):
""" Writes source information to text file
"""
file = findpath('seistools') + '/' + 'specfem3d/FORCESOLUTION'
with open(file, 'r') as f:
lines = f.readlines()
file = 'DATA/FORCESOURCE'
_writelines(file, lines)
# adjust coordinates
setpar('xs', h.sx[0], file)
setpar('zs', h.sz[0], file)
setpar('ts', h.ts, file)
# adjust wavelet
setpar('f0', PAR['F0'], file)
def write_sources(PAR, h, path='.'):
""" Writes source information to text file
"""
file = findpath('seistools') + '/' + 'specfem3d/FORCESOLUTION'
with open(file, 'r') as f:
lines = f.readlines()
file = 'DATA/FORCESOURCE'
_writelines(file, lines)
# adjust coordinates
setpar('xs', h.sx[0], file)
setpar('zs', h.sz[0], file)
setpar('ts', h.ts, file)
# adjust wavelet
setpar('f0', PAR['F0'], file)
def submit(self, workflow):
""" Submits workflow
"""
unix.mkdir(PATH.OUTPUT)
unix.cd(PATH.OUTPUT)
unix.mkdir(PATH.SUBMIT+'/'+'output.slurm')
self.checkpoint()
# prepare sbatch arguments
unix.run('sbatch '
+ '--job-name=%s ' % PAR.SUBTITLE
+ '--output %s ' % (PATH.SUBMIT+'/'+'output.log')
+ '--ntasks-per-node=%d ' % PAR.NODESIZE
+ '--nodes=%d ' % 1
+ '--time=%d ' % PAR.WALLTIME
+ findpath('system') +'/'+ 'slurm/wrapper_sbatch '
+ PATH.OUTPUT)
def write_sources(par, hdr, path=".", suffix=""):
""" Writes source information to text file
"""
file = findpath("seistools") + "/" + "specfem2d/SOURCE"
with open(file, "r") as f:
lines = f.readlines()
file = path + "/" + "DATA/SOURCE" + suffix
_writelines(file, lines)
# adjust source coordinates
setpar("xs", hdr.sx[0], file)
setpar("zs", hdr.sy[0], file)
setpar("ts", hdr.ts, file)
# adjust source amplitude
try:
fs = float(getpar("factor", file))
setpar("factor", str(fs * hdr.fs), file)
except:
pass
# adjust source wavelet
if 1:
# Ricker wavelet
setpar("time_function_type", 1, file)
elif 0:
# first derivative of Gaussian
setpar("time_function_type", 2, file)
elif 0:
# Gaussian
setpar("time_function_type", 3, file)
elif 0:
# Dirac
setpar("time_function_type", 4, file)
elif 0:
# Heaviside
setpar("time_function_type", 5, file)
setpar("f0", par["F0"], file)
def write_sources(par, hdr, path='.', suffix=''):
""" Writes source information to text file
"""
file = findpath('seistools') + '/' + 'specfem2d/SOURCE'
with open(file, 'r') as f:
lines = f.readlines()
file = path + '/' + 'DATA/SOURCE' + suffix
_writelines(file, lines)
# adjust source coordinates
setpar('xs', hdr.sx[0], file)
setpar('zs', hdr.sy[0], file)
setpar('ts', hdr.ts, file)
# adjust source amplitude
try:
fs = float(getpar('factor', file))
setpar('factor', str(fs * hdr.fs), file)
except:
pass
# adjust source wavelet
if 1:
# Ricker wavelet
setpar('time_function_type', 1, file)
elif 0:
# first derivative of Gaussian
setpar('time_function_type', 2, file)
elif 0:
# Gaussian
setpar('time_function_type', 3, file)
elif 0:
# Dirac
setpar('time_function_type', 4, file)
elif 0:
# Heaviside
setpar('time_function_type', 5, file)
setpar('f0', par['F0'], file)
def submit(self, workflow):
""" Submits workflow
"""
unix.mkdir(PATH.OUTPUT)
unix.cd(PATH.OUTPUT)
unix.mkdir(PATH.SUBMIT+'/'+'output.pbs')
self.checkpoint()
# construct resource list
nodes = PAR.NTASK/PAR.NODESIZE
cores = PAR.NTASK%PAR.NODESIZE
hours = PAR.WALLTIME/60
minutes = PAR.WALLTIME%60
resources = 'walltime=%02d:%02d:00 '%(hours, minutes)
#if nodes == 0:
# resources += ',nodes=1:ppn=%d'%cores
#lif cores == 0:
# resources += ',nodes=%d:ppn=16'%nodes
#else:
# resources += ',nodes=%d:ppn=16+1:ppn=%d'%(nodes, cores)
# prepare qsub arguments
print('qsub -l select=1:ncpus=32:mpiprocs=2 -A ERDCH38424KSC -q debug -N %s -o %s -l %s -j %s')%(PAR.SUBTITLE, 'output.log', resources, 'oe'+findpath('system')+'/'+'wrapper/submit ' + PATH.OUTPUT)
args = ('qsub '
+ '-l select=1:ncpus=32:mpiprocs=32 '
+ '-l %s '%resources
+ '-q standard '
+ '-A ERDCH38424KSC '
+ '-N %s ' % PAR.SUBTITLE
+ '-j %s '%'oe'
+ '-o %s ' % (PATH.SUBMIT+'/'+'output.log')
+ '-v PYTHONPATH '
+ ' -- ' + findpath('system') +'/'+ 'wrappers/submit '
+ PATH.OUTPUT)
print 'args:', args
subprocess.call(args, shell=True)
def submit(self, workflow):
""" Submits workflow
"""
unix.mkdir(PATH.OUTPUT)
unix.cd(PATH.OUTPUT)
# save current state
save_objects('SeisflowsObjects')
save_parameters('SeisflowsParameters.json')
save_paths('SeisflowsPaths.json')
# submit workflow
args = ('sbatch '
+ '--job-name=%s '%PAR.TITLE
+ '--output=%s '%(PATH.SUBMIT + '/' + 'output.log')
+ '--cpus-per-task=%d '%PAR.NPROC
+ '--ntasks=%d '%PAR.NTASK
+ '--time=%d '%PAR.WALLTIME
+ findpath('system') + '/' + 'slurm/wrapper_sbatch '
+ PATH.OUTPUT)
subprocess.call(args, shell=1)
def _launch(self, classname, funcname, hosts='all'):
unix.mkdir(PATH.SYSTEM)
# prepare sbatch arguments
if hosts == 'all':
args = ('--array=%d-%d ' % (0,PAR.NTASK-1)
+'--output %s ' % (PATH.SUBMIT+'/'+'output.slurm/'+'%A_%a'))
elif hosts == 'head':
args = ('--array=%d-%d ' % (0,0)
+'--output=%s ' % (PATH.SUBMIT+'/'+'output.slurm/'+'%j'))
#+('--export=SEISFLOWS_TASK_ID=%s ' % 0
# submit job
with open(PATH.SYSTEM+'/'+'job_id', 'w') as f:
subprocess.call('sbatch '
+ '--job-name=%s ' % PAR.TITLE
+ '--nodes=%d ' % math.ceil(PAR.NPROC/float(PAR.NODESIZE))
+ '--ntasks-per-node=%d ' % PAR.NODESIZE
+ '--time=%d ' % PAR.STEPTIME
+ args
+ findpath('system') +'/'+ 'slurm/wrapper_srun '
+ PATH.OUTPUT + ' '
+ classname + ' '
+ funcname + ' ',
shell=1,
stdout=f)
# retrieve job ids
with open(PATH.SYSTEM+'/'+'job_id', 'r') as f:
line = f.readline()
job = line.split()[-1].strip()
if hosts == 'all' and PAR.NTASK > 1:
nn = range(PAR.NTASK)
return [job+'_'+str(ii) for ii in nn]
else:
return [job]
def test_find_std_module_path(self):
mod = __import__('math')
self.assertTrue(os.path.exists(tools.findpath(mod)))
def test_find_file_path_by_name(self):
tmp_file = NamedTemporaryFile(mode='wb', delete=True)
self.assertEqual(gettempdir(), tools.findpath(tmp_file.name))
def test_find_seisflows_module_path_by_names(self):
names = 'seisflows.tools.code'
self.assertIsNotNone(tools.findpath(names))