def submit(self, *args, **kwargs):
""" Submits job
"""
# create scratch directories
if not exists(PATH.SCRATCH):
path = '/scratch/gpfs' + '/' + getuser(
) + '/' + 'seisflows' + '/' + str(uuid4())
unix.mkdir(path)
unix.ln(path, PATH.SCRATCH)
unix.mkdir(PATH.SYSTEM)
# create output directories
unix.mkdir(PATH.OUTPUT)
unix.mkdir(PATH.WORKDIR + '/' + 'output.slurm')
self.checkpoint()
if not exists(PATH.SCRATCH):
path = '/scratch/gpfs' + '/' + getuser(
) + '/' + 'seisflows' + '/' + str(uuid4())
unix.mkdir(path)
unix.ln(path, PATH.SCRATCH)
call('sbatch ' + '%s ' % PAR.SLURMARGS + '--job-name=%s ' % PAR.TITLE +
'--output=%s ' % (PATH.WORKDIR + '/' + 'output.log') +
'--ntasks-per-node=%d ' % 28 + '--gres=gpu:%d ' % 4 +
'--nodes=%d ' % 1 + '--time=%d ' % PAR.WALLTIME +
pkgpath('seisflows') + '/' + 'system/wrappers/submit ' +
PATH.OUTPUT)
def submit(self, *args, **kwargs):
""" Submits job
"""
if not exists(PATH.SUBMIT + '/' + 'scratch'):
unix.ln(PATH.SCRATCH, PATH.SUBMIT + '/' + 'scratch')
super(tiger_md, self).submit(*args, **kwargs)
def combine(self, path=''):
""" combines SPECFEM3D kernels
"""
unix.cd(self.getpath)
# create temporary files and directories
dirs = unix.ls(path)
with open('kernels_list.txt', 'w') as file:
file.write('\n'.join(dirs) + '\n')
unix.mkdir('INPUT_KERNELS')
unix.mkdir('OUTPUT_SUM')
for dir in dirs:
src = path + '/' + dir
dst = 'INPUT_KERNELS' + '/' + dir
unix.ln(src, dst)
# sum kernels
self.mpirun(PATH.SOLVER_BINARIES + '/' + 'xsum_kernels')
unix.mv('OUTPUT_SUM', path + '/' + 'sum')
# remove temporary files and directories
unix.rm('INPUT_KERNELS')
unix.rm('kernels_list.txt')
unix.cd(path)
def combine_vol_data(self, output_path='', quantity=''):
"""
This does not work
Call Specfems executable combine_vol_data_vtk on kernels or model files
"""
if not exists(output_path):
unix.mkdir(output_path)
# This should probably be moved to its own function
# def import_kernels()
unix.cd(self.cwd)
src = glob(join(PATH.GRAD, self.source_name, "*{}*".format(quantity)))
dst = join(self.cwd, "kernels")
unix.mkdir(dst)
unix.ln(src=src, dst=dst)
solver_call = " ".join([
PATH.SPECFEM_BIN + '/' + 'xcombine_vol_data_vtk',
0, # NPROC_START
PAR.NPROC, # NPROC_END
quantity, # QUANTITY
dst, # DIR_IN
dst, # DIR_OUT, we will rename the files first
0 # GPU ACCEL
])
call_solver(system_mpiexec(), solver_call)
unix.rm(dst)
print ''
def submit(self, workflow):
""" Submits workflow
"""
# create scratch directories
if not exists(PATH.SCRATCH):
path = '/scratch/gpfs'+'/'+getuser()+'/'+'seisflows'+'/'+str(uuid4())
unix.mkdir(path)
unix.ln(path, PATH.SCRATCH)
unix.mkdir(PATH.SYSTEM)
# create output directories
unix.mkdir(PATH.OUTPUT)
unix.mkdir(PATH.WORKDIR+'/'+'output.slurm')
self.checkpoint()
# prepare sbatch arguments
call('sbatch '
+ '%s ' % PAR.SLURMARGS
+ '--job-name=%s ' % PAR.TITLE
+ '--output %s ' % (PATH.WORKDIR+'/'+'output.log')
+ '--ntasks-per-node=28 '
+ '--ntasks=28 '
+ '--gres=gpu:4 '
+ '--nodes=%d ' % 1
+ '--time=%d ' % PAR.WALLTIME
+ findpath('seisflows.system') +'/'+ 'wrappers/submit '
+ PATH.OUTPUT)
def submit(self, *args, **kwargs):
""" Submits job
"""
if not exists(PATH.SUBMIT + '/' + 'scratch'):
unix.ln(PATH.SCRATCH, PATH.SUBMIT + '/' + 'scratch')
super(tiger_md, self).submit(*args, **kwargs)
def adjoint(self):
""" Calls SPECFEM3D_GLOBE adjoint solver
"""
solvertools.setpar('SIMULATION_TYPE', '3')
solvertools.setpar('SAVE_FORWARD', '.false.')
unix.rm('SEM')
unix.ln('traces/adj', 'SEM')
call_solver(system.mpiexec(), 'bin/xspecfem3D')
def adjoint(self):
""" Calls SPECFEM3D_GLOBE adjoint solver
"""
solvertools.setpar('SIMULATION_TYPE', '3')
solvertools.setpar('SAVE_FORWARD', '.false.')
unix.rm('SEM')
unix.ln('traces/adj', 'SEM')
call_solver(system.mpiexec(), 'bin/xspecfem3D')
def adjoint(self):
""" Calls SPECFEM3D adjoint solver
"""
setpar('SIMULATION_TYPE', '3')
setpar('SAVE_FORWARD', '.false.')
unix.rm('SEM')
unix.ln('traces/adj', 'SEM')
self.mpirun('bin/xspecfem3D')
def adjoint(self):
""" Calls SPECFEM3D adjoint solver
"""
setpar('SIMULATION_TYPE', '3')
setpar('SAVE_FORWARD', '.false.')
unix.rm('SEM')
unix.ln('traces/adj', 'SEM')
self.call('bin/xspecfem3D')
def submit(self, *args, **kwargs):
""" Submits job
"""
if not exists(PATH.SCRATCH):
path = '/scratch/gpfs'+'/'+getuser()+'/'+'seisflows'+'/'+str(uuid4())
unix.mkdir(path)
unix.ln(path, PATH.SCRATCH)
super(tiger_lg, self).submit(*args, **kwargs)
def submit(self, *args, **kwargs):
""" Submits job
"""
if not exists(PATH.SCRATCH):
path = '/scratch/gpfs' + '/' + getuser(
) + '/' + 'seisflows' + '/' + str(uuid4())
unix.mkdir(path)
unix.ln(path, PATH.SCRATCH)
super(tiger_sm, self).submit(*args, **kwargs)
def adjoint(self):
""" Calls SPECFEM3D adjoint solver
"""
setpar('SIMULATION_TYPE', '3')
setpar('SAVE_FORWARD', '.false.')
unix.rm('SEM')
unix.ln('traces/adj', 'SEM')
call_solver(system.mpiexec(), 'bin/xspecfem3D')
# work around SPECFEM3D conflicting name conventions
self.rename_data()
def submit(self, workflow):
""" Submits workflow to maui_ancil cluster
This needs to be run on maui_ancil because maui does not have the
ability to run the command "sacct"
"""
# create scratch directories
unix.mkdir(PATH.SCRATCH)
unix.mkdir(PATH.SYSTEM)
# create output directories
unix.mkdir(PATH.OUTPUT)
unix.mkdir(PATH.WORKDIR + '/' + 'output.slurm')
if not exists('./scratch'):
unix.ln(PATH.SCRATCH, PATH.WORKDIR + '/' + 'scratch')
# if resuming, rename the old log files so they don't get overwritten
output_log = os.path.join(PATH.WORKDIR, 'output.log')
error_log = os.path.join(PATH.WORKDIR, 'error.log')
for log in [output_log, error_log]:
log_prior = log + '_prior'
log_temp = log + '_temp'
if os.path.exists(log):
# If a prior log exists, move to temp file and then rewrite
# with new log file
if os.path.exists(log_prior):
os.rename(log_prior, log_temp)
with open(log_prior, 'w') as f_out:
for fid in [log_temp, log]:
with open(fid) as f_in:
f_out.write(f_in.read())
unix.rm(log_temp)
else:
os.rename(log, log_prior)
workflow.checkpoint()
# Submit to maui_ancil
call(" ".join([
'sbatch',
'%s' % PAR.SLURMARGS,
'--account=%s' % PAR.ACCOUNT,
'--clusters=%s' % PAR.ANCIL_CLUSTER,
'--partition=%s' % PAR.ANCIL_PARTITION,
'--job-name=%s' % 'M_' + PAR.TITLE,
'--output=%s' % output_log,
'--error=%s' % error_log,
'--ntasks=%d' % 1,
'--cpus-per-task=%d' % 1,
'--time=%d' % PAR.WALLTIME,
findpath('seisflows.system') + '/' + 'wrappers/submit ',
PATH.OUTPUT
]))
def adjoint_att(self):
""" Calls SPECFEM2D adjoint solver
"""
unix.rm('SEM')
unix.ln('traces/adj_att', 'SEM')
# hack to deal with different SPECFEM2D name conventions for
# regular traces and 'adjoint' traces
if PAR.FORMAT in ['SU', 'su']:
files = glob('traces/adj_att/*.su')
unix.rename('.su', '.su.adj', files)
call_solver(system.mpiexec(), 'bin/xspecfem2D')
def adjoint(self):
""" Calls SPECFEM2D adjoint solver
"""
setpar('SIMULATION_TYPE', '3')
setpar('SAVE_FORWARD', '.false.')
unix.rm('SEM')
unix.ln('traces/adj', 'SEM')
# hack to deal with different SPECFEM2D name conventions for
# regular traces and 'adjoint' traces
if PAR.FORMAT in ['SU', 'su']:
files = glob('traces/adj/*.su')
unix.rename('.su', '.su.adj', files)
call_solver(system.mpiexec(), 'bin/xspecfem2D')
def adjoint(self):
""" Calls SPECFEM2D adjoint solver
"""
setpar('SIMULATION_TYPE', '3')
setpar('SAVE_FORWARD', '.false.')
unix.rm('SEM')
unix.ln('traces/adj', 'SEM')
# hack to deal with different SPECFEM2D name conventions for
# regular traces and 'adjoint' traces
if PAR.FORMAT in ['SU', 'su']:
files = glob('traces/adj/*.su')
unix.rename('.su', '.su.adj', files)
call_solver(system.mpiexec(), 'bin/xmeshfem2D')
call_solver(system.mpiexec(), 'bin/xspecfem2D')
def submit(self, workflow):
""" Submits workflow
"""
unix.cd(PATH.WORKDIR)
if not exists('./scratch'):
unix.ln(PATH.SCRATCH, PATH.WORKDIR + '/' + 'scratch')
unix.mkdir(PATH.OUTPUT)
workflow.checkpoint()
# prepare sbatch arguments
call('sbatch ' + '%s ' % PAR.SLURMARGS +
'--partition=%s ' % PAR.PARTITION + '--job-name=%s ' % PAR.TITLE +
'--output %s ' % (PATH.WORKDIR + '/' + 'output.log') +
'--cpus-per-task=%d ' % PAR.NPROC + '--ntasks=%d ' % PAR.NTASK +
'--time=%d ' % PAR.WALLTIME + findpath('seisflows.system') + '/' +
'wrappers/submit ' + PATH.OUTPUT)
def submit(self, workflow):
""" Submits workflow
"""
unix.mkdir(PATH.OUTPUT)
unix.cd(PATH.OUTPUT)
self.checkpoint()
if not exists(PATH.SUBMIT + '/' + 'scratch'):
unix.ln(PATH.SCRATCH, PATH.SUBMIT + '/' + 'scratch')
call('sbatch '
+ '--job-name=%s ' % PAR.SUBTITLE
+ '--output=%s ' % (PATH.SUBMIT +'/'+ 'output.log')
+ '--nodes 1 '
+ '--ntasks=% ' % PAR.NGPU
+ '--ntasks-per-socket=%d ' % PAR.NGPU
+ '--gres=gpu:%d ' % PAR.NGPU
+ '--time=%d ' % PAR.WALLTIME
+ findpath('seisflows.system') +'/'+ 'wrappers/submit '
+ PATH.OUTPUT)
def combine(self, path=''):
""" combines SPECFEM3D_GLOBE kernels
"""
dirs = unix.ls(path)
# initialize kernels
unix.cd(path)
for key in self.model_parameters:
if key not in self.inversion_parameters:
for i in range(PAR.NPROC):
proc = '%06d' % i
name = self.kernel_map[key]
src = PATH.GLOBAL + '/' + 'mesh' + '/' + key + '/' + proc
dst = path + '/' + 'sum' + '/' + 'proc' + proc + '_' + name + '.bin'
savebin(np.load(src), dst)
# create temporary files and directories
unix.cd(self.getpath)
with open('kernels_list.txt', 'w') as file:
file.write('\n'.join(dirs) + '\n')
unix.mkdir('INPUT_KERNELS')
unix.mkdir('OUTPUT_SUM')
for dir in dirs:
src = path + '/' + dir
dst = 'INPUT_KERNELS' + '/' + dir
unix.ln(src, dst)
# sum kernels
self.mpirun(PATH.SOLVER_BINARIES + '/' + 'xsum_kernels')
unix.mv('OUTPUT_SUM', path + '/' + 'sum')
# remove temporary files and directories
unix.rm('INPUT_KERNELS')
unix.rm('kernels_list.txt')
unix.cd(path)
def submit(self, workflow):
""" Submits workflow
"""
# create scratch directories
unix.mkdir(PATH.SCRATCH)
unix.mkdir(PATH.SYSTEM)
# create output directories
unix.mkdir(PATH.OUTPUT)
unix.mkdir(PATH.WORKDIR + '/' + 'output.slurm')
if not exists('./scratch'):
unix.ln(PATH.SCRATCH, PATH.WORKDIR + '/' + 'scratch')
workflow.checkpoint()
# prepare sbatch arguments
call('sbatch ' + '%s ' % PAR.SLURMARGS +
'--partition=%s ' % 't1small' # overloads SLURMARGS
+ '--job-name=%s ' % PAR.TITLE + '--output %s ' %
(PATH.WORKDIR + '/' + 'output.log') +
'--ntasks-per-node=%d ' % PAR.NODESIZE + '--nodes=%d ' % 1 +
'--time=%d ' % PAR.WALLTIME + findpath('seisflows.system') + '/' +
'wrappers/submit ' + PATH.OUTPUT)
def prepare_model(self):
model = PATH.OUTPUT +'/'+ 'model_init'
assert exists(model)
unix.ln(model, PATH.SCRATCH +'/'+ 'model')
def prepare_model(self):
model = PATH.OUTPUT + '/' + 'model_init'
assert exists(model)
unix.ln(model, PATH.SCRATCH + '/' + 'model')
def submit(self, *args, **kwargs):
"""Submits job
"""
unix.ln(PATH.GLOBAL, PATH.SUBMIT + '/' + 'scratch')
super(tiger_lg, self).submit(*args, **kwargs)
def submit(self, *args, **kwargs):
""" Submits job
"""
if not exists(PATH.WORKDIR + '/' + 'scratch'):
unix.ln(PATH.SCRATCH, PATH.WORKDIR + '/' + 'scratch')
super(tigercpu_lg, self).submit(*args, **kwargs)
def submit(self, *args, **kwargs):
"""Submits job
"""
unix.ln(PATH.GLOBAL, PATH.SUBMIT + '/' + 'scratch')
super(tiger_lg, self).submit(*args, **kwargs)
