def generate_mesh(self, model_path=None, model_name=None, model_type='gll'):
""" Performs meshing and database generation
"""
assert(model_name)
assert(model_type)
self.initialize_solver_directories()
unix.cd(self.getpath)
if model_type in ['gll']:
par = getpar('MODEL').strip()
if par != 'gll':
if self.getnode == 0:
print 'WARNING: Unexpected Par_file setting:'
print 'MODEL =', par
assert(exists(model_path))
self.check_mesh_properties(model_path)
src = glob(model_path +'/'+ '*')
dst = self.model_databases
unix.cp(src, dst)
call_solver(system.mpiexec(), 'bin/xmeshfem3D')
call_solver(system.mpiexec(), 'bin/xgenerate_databases')
self.export_model(PATH.OUTPUT +'/'+ model_name)
else:
raise NotImplementedError
def forward(self, path='traces/syn'):
""" Calls SPECFEM3D forward solver
"""
setpar('SIMULATION_TYPE', '1')
setpar('SAVE_FORWARD', '.true.')
call_solver(system.mpiexec(), 'bin/xgenerate_databases')
call_solver(system.mpiexec(), 'bin/xspecfem3D')
if PAR.FORMAT in ['SU', 'su']:
src = glob('OUTPUT_FILES/*_d?_SU')
dst = path
unix.mv(src, dst)
def call(self, executable, output='/dev/null'):
""" Calls solver through subprocess
"""
# a less complicated version, without error catching, would simply be
# subprocess.call(system.mpiexec() + executable, shell=True)
try:
f = open(output, 'w')
subprocess.check_call(system.mpiexec() + executable,
shell=True,
stdout=f)
except subprocess.CalledProcessError, err:
print msg.SolverError % (system.mpiexec() + executable)
sys.exit(-1)
def smooth(self, path='', parameters=[], span=0.):
""" Smooths kernels by convolving them with a Gaussian. Wrapper over
xsmooth_sem utility.
"""
assert exists(path)
assert len(parameters) > 0
# apply smoothing operator
unix.cd(self.getpath)
for name in parameters or self.parameters:
print ' smoothing', name
call_solver(system.mpiexec(),
PATH.SPECFEM_BIN + '/' + 'xsmooth_sem ' + str(span) +
' ' + str(span) + ' ' + name + '_kernel' + ' ' + path +
'/ ' + path + '/ ',
output=self.getpath + '/' +
'OUTPUT_FILES/output_smooth_sem.txt')
print ''
# move input files
src = path
dst = path + '_nosmooth'
unix.mkdir(dst)
for name in self.parameters:
unix.mv(glob(src + '/*' + name + '.bin'), dst)
# rename output files
unix.rename('_smooth', '', glob(src + '/*'))
def forward(self, path='traces/syn'):
""" Calls SPECFEM2D forward solver
"""
setpar('SIMULATION_TYPE', '1')
setpar('SAVE_FORWARD', '.true.')
if PAR.WITH_MPI:
call_solver(system.mpiexec(), 'bin/xmeshfem2D')
call_solver(system.mpiexec(), 'bin/xspecfem2D')
else:
call_solver_nompi('bin/xmeshfem2D')
call_solver_nompi('bin/xspecfem2D')
if PAR.FORMAT in ['SU', 'su']:
filenames = glob('OUTPUT_FILES/*.su')
unix.mv(filenames, path)
def generate_mesh(self,
model_path=None,
model_name=None,
model_type='gll'):
""" Performs meshing and database generation
"""
assert (model_name)
assert (model_type)
self.initialize_solver_directories()
unix.cd(self.cwd)
if model_type == 'gll':
assert (exists(model_path))
self.check_mesh_properties(model_path)
unix.cp(glob(model_path + '/' + '*'), self.model_databases)
call_solver(system.mpiexec(), 'bin/xmeshfem3D')
if self.taskid == 0:
self.export_model(PATH.OUTPUT + '/' + model_name)
else:
raise NotImplementedError
def adjoint(self):
""" Calls SPECFEM3D_GLOBE adjoint solver
"""
solvertools.setpar('SIMULATION_TYPE', '3')
solvertools.setpar('SAVE_FORWARD', '.false.')
unix.rm('SEM')
unix.ln('traces/adj', 'SEM')
call_solver(system.mpiexec(), 'bin/xspecfem3D')
def mpirun(self, script, output='/dev/null'):
""" Wrapper for mpirun
"""
with open(output,'w') as f:
subprocess.call(
system.mpiexec() + script,
shell=True,
stdout=f)
def adjoint(self):
""" Calls SPECFEM3D_GLOBE adjoint solver
"""
solvertools.setpar('SIMULATION_TYPE', '3')
solvertools.setpar('SAVE_FORWARD', '.false.')
unix.rm('SEM')
unix.ln('traces/adj', 'SEM')
call_solver(system.mpiexec(), 'bin/xspecfem3D')
def adjoint(self):
""" Calls SPECFEM3D adjoint solver
"""
setpar('SIMULATION_TYPE', '3')
setpar('SAVE_FORWARD', '.false.')
unix.rm('SEM')
unix.ln('traces/adj', 'SEM')
call_solver(system.mpiexec(), 'bin/xspecfem3D')
# work around SPECFEM3D conflicting name conventions
self.rename_data()
def generate_data(self, **model_kwargs):
""" Generates data
"""
self.generate_mesh(**model_kwargs)
unix.cd(self.getpath)
setpar('SIMULATION_TYPE', '1')
setpar('SAVE_FORWARD', '.false.')
if PAR.WITH_MPI:
call_solver(system.mpiexec(), 'bin/xmeshfem2D')
call_solver(system.mpiexec(), 'bin/xspecfem2D')
else:
call_solver_nompi('bin/xmeshfem2D')
call_solver_nompi('bin/xspecfem2D')
if PAR.FORMAT in ['SU', 'su']:
src = glob('OUTPUT_FILES/*.su')
dst = 'traces/obs'
unix.mv(src, dst)
def forward(self, path='traces/syn'):
""" Calls SPECFEM3D_GLOBE forward solver
"""
solvertools.setpar('SIMULATION_TYPE', '1')
solvertools.setpar('SAVE_FORWARD', '.true.')
call_solver(system.mpiexec(), 'bin/xspecfem3D')
if PAR.FORMAT in ['ASCII', 'ascii']:
src = glob('OUTPUT_FILES/*.sem.ascii')
dst = path
unix.mv(src, dst)
def forward(self, path='traces/syn'):
""" Calls SPECFEM3D_GLOBE forward solver
"""
solvertools.setpar('SIMULATION_TYPE', '1')
solvertools.setpar('SAVE_FORWARD', '.true.')
call_solver(system.mpiexec(), 'bin/xspecfem3D')
if PAR.FORMAT in ['ASCII', 'ascii']:
src = glob('OUTPUT_FILES/*.sem.ascii')
dst = path
unix.mv(src, dst)
def adjoint(self):
""" Calls SPECFEM2D adjoint solver
"""
setpar('SIMULATION_TYPE', '3')
setpar('SAVE_FORWARD', '.false.')
unix.rm('SEM')
unix.ln('traces/adj', 'SEM')
# hack to deal with SPECFEM2D's use of different name conventions for
# regular traces and 'adjoint' traces
if PAR.FORMAT in ['SU', 'su']:
files = glob('traces/adj/*.su')
unix.rename('.su', '.su.adj', files)
if PAR.WITH_MPI:
call_solver(system.mpiexec(), 'bin/xmeshfem2D')
call_solver(system.mpiexec(), 'bin/xspecfem2D')
else:
call_solver_nompi('bin/xmeshfem2D')
call_solver_nompi('bin/xspecfem2D')
def generate_data(self, **model_kwargs):
""" Generates data
"""
self.generate_mesh(**model_kwargs)
unix.cd(self.getpath)
setpar('SIMULATION_TYPE', '1')
setpar('SAVE_FORWARD', '.true.')
call_solver(system.mpiexec(), 'bin/xspecfem3D')
if PAR.FORMAT in ['ASCII', 'ascii']:
src = glob('OUTPUT_FILES/*.sem.ascii')
dst = 'traces/obs'
unix.mv(src, dst)
def combine(self, path='', parameters=[]):
""" Sums individual source contributions. Wrapper over xcombine_sem
utility.
"""
unix.cd(self.getpath)
names = self.check_source_names()
with open('kernel_paths', 'w') as f:
f.writelines([join(path, dir) + '\n' for dir in names])
unix.mkdir(path + '/' + 'sum')
for name in parameters or self.parameters:
call_solver(
system.mpiexec(),
PATH.SPECFEM_BIN + '/' + 'xcombine_sem ' + name + '_kernel' +
' ' + 'kernel_paths' + ' ' + path + '/' + 'sum')
def generate_data(self, **model_kwargs):
""" Generates data
"""
self.generate_mesh(**model_kwargs)
unix.cd(self.cwd)
setpar('SIMULATION_TYPE', '1')
setpar('SAVE_FORWARD', '.true.')
call_solver(system.mpiexec(), 'bin/xspecfem3D')
if PAR.FORMAT in ['ASCII', 'ascii']:
src = glob('OUTPUT_FILES/*.sem.ascii')
dst = 'traces/obs'
unix.mv(src, dst)
if PAR.SAVETRACES:
self.export_traces(PATH.OUTPUT+'/'+'traces/obs')
def generate_mesh(self, model_path=None, model_name=None, model_type='gll'):
""" Performs meshing and database generation
"""
assert(model_name)
assert(model_type)
self.initialize_solver_directories()
unix.cd(self.cwd)
if model_type == 'gll':
assert (exists(model_path))
self.check_mesh_properties(model_path)
unix.cp(glob(model_path +'/'+ '*'), self.model_databases)
call_solver(system.mpiexec(), 'bin/xmeshfem3D')
if self.taskid == 0:
self.export_model(PATH.OUTPUT +'/'+ model_name)
else:
raise NotImplementedError
class base(object):
""" Base class for SPECFEM2D, SPECFEM3D and SPECFEM3D_GLOBE
eval_func, eval_grad, apply_hess
These methods deal with evaluation of the misfit function or its
derivatives. Together, they provide the primary interface through which
SeisFlows interacts with SPECFEM.
forward, adjoint
These methods allow direct access to low-level SPECFEM components,
providing another interface through which to interact with the solver.
generate_data, generate_model
One time operations performed at beginning of an inversion or
migration.
initialize_solver_directories, initialize_adjoint_traces
SPECFEM requires a particular directory structure in which to run and
particular file formats for models, data, and parameter files. These
methods help put in place all these prerequisites.
load, save
For reading and writing SPECFEM models and kernels. On the disk,
models and kernels are stored as binary files, and in memory, as
dictionaries with different keys corresponding to different material
parameters.
split, merge
Within the solver routines, it is natural to store models as
dictionaries. Within the optimization routines, it is natural to store
models as vectors. Two methods, 'split' and 'merge', are used to convert
back and forth between these two representations.
combine, smooth
Utilities for combining and smoothing kernels.
"""
assert 'MATERIALS' in PAR
assert 'DENSITY' in PAR
parameters = []
if PAR.MATERIALS == 'Elastic':
parameters += ['vp']
parameters += ['vs']
elif PAR.MATERIALS == 'Acoustic':
parameters += ['vp']
if PAR.DENSITY == 'Variable':
density_scaling = None
parameters += ['rho']
elif PAR.DENSITY == 'Constant':
density_scaling = None
def check(self):
""" Checks parameters and paths
"""
if 'NPROC' not in PAR:
raise ParameterError(PAR, 'NPROC')
# check scratch paths
if 'SCRATCH' not in PATH:
raise ParameterError(PATH, 'SCRATCH')
if 'LOCAL' not in PATH:
setattr(PATH, 'LOCAL', None)
if 'SOLVER' not in PATH:
if PATH.LOCAL:
setattr(PATH, 'SOLVER', join(PATH.LOCAL, 'solver'))
else:
setattr(PATH, 'SOLVER', join(PATH.SCRATCH, 'solver'))
# check solver input paths
if 'SPECFEM_BIN' not in PATH:
raise ParameterError(PATH, 'SPECFEM_BIN')
if 'SPECFEM_DATA' not in PATH:
raise ParameterError(PATH, 'SPECFEM_DATA')
# assertions
assert self.parameters != []
def setup(self):
""" Prepares solver for inversion or migration
"""
# clean up for new inversion
unix.rm(self.getpath)
# As input for an inversion or migration, users can choose between
# providing data, or providing a target model from which data are
# generated on the fly. In the former case, a value for PATH.DATA must
# be supplied; in the latter case, a value for PATH.MODEL_TRUE must be
# provided
if PATH.DATA:
# copy user supplied data
self.initialize_solver_directories()
src = glob(PATH.DATA + '/' + basename(self.getpath) + '/' + '*')
dst = 'traces/obs/'
unix.cp(src, dst)
else:
# generate data on the fly
self.generate_data(model_path=PATH.MODEL_TRUE,
model_name='model_true',
model_type='gll')
# prepare initial model
self.generate_mesh(model_path=PATH.MODEL_INIT,
model_name='model_init',
model_type='gll')
self.initialize_adjoint_traces()
def clean(self):
pass
#unix.rm('OUTPUT_FILES')
#unix.mkdir('OUTPUT_FILES')
def generate_data(self, *args, **kwargs):
""" Generates data
"""
# must be implemented by subclass
raise NotImplementedError
def generate_mesh(self, *args, **kwargs):
""" Performs meshing and database generation
"""
# must be implemented by subclass
raise NotImplementedError
### high-level solver interface
def eval_func(self, path='', export_traces=False):
""" Evaluates misfit function by carrying out forward simulation and
comparing observations and synthetics.
"""
unix.cd(self.getpath)
self.import_model(path)
self.forward()
unix.mv(self.data_wildcard, 'traces/syn')
preprocess.prepare_eval_grad(self.getpath)
self.export_residuals(path)
if export_traces:
self.export_traces(path, prefix='traces/syn')
def eval_grad(self, path='', export_traces=False):
""" Evaluates gradient by carrying out adjoint simulation. Adjoint traces
must be in place beforehand.
"""
unix.cd(self.getpath)
self.adjoint()
self.export_kernels(path)
if export_traces:
self.export_traces(path, prefix='traces/adj')
def apply_hess(self, path=''):
""" Computes action of Hessian on a given model vector.
"""
# a gradient evaluation must have already been carried out
unix.cd(self.getpath)
unix.mkdir('traces/lcg')
self.import_model(path)
self.forward()
unix.mv(self.data_wildcard, 'traces/lcg')
preprocess.prepare_apply_hess(self.getpath)
self.adjoint()
self.export_kernels(path)
### low-level solver interface
def forward(self):
""" Calls forward solver
"""
# must be implemented by subclass
raise NotImplementedError
def adjoint(self):
""" Calls adjoint solver
"""
# must be implemented by subclass
raise NotImplementedError
### model input/output
def load(self, path, prefix='', suffix='', verbose=False):
""" reads SPECFEM model or kernel
Models are stored in Fortran binary format and separated into multiple
files according to material parameter and processor rank.
"""
minmax = Minmax(self.parameters)
model = Model(self.parameters)
for iproc in range(self.mesh.nproc):
# read database files
keys, vals = loadbypar(path, self.parameters, iproc, prefix,
suffix)
for key, val in zip(keys, vals):
model[key] += [val]
minmax.update(keys, vals)
if verbose:
minmax.write(path, logpath=PATH.SUBMIT)
return model
def save(self,
path,
model,
prefix='',
suffix='',
solver_parameters=['vp', 'vs']):
""" writes SPECFEM model
The following method writes acoustic and elastic models. For an
example of how to write transversely isotropic models, see
solver.specfem3d_globe.
There is a tradeoff here between being simple and being
flexible. In this case we opt for a simple hardwired approach. For
a more flexible, more complex approach see SEISFLOWS-RESEARCH.
"""
unix.mkdir(path)
if 'kernel' in suffix:
for iproc in range(self.mesh.nproc):
for key in solver_parameters:
if key in self.parameters:
savebin(model[key][iproc], path, iproc,
prefix + key + suffix)
if 'rho' in self.parameters:
savebin(model['rho'][iproc], path, iproc,
prefix + 'rho' + suffix)
else:
for iproc in range(self.mesh.nproc):
for key in solver_parameters:
if key in self.parameters:
savebin(model[key][iproc], path, iproc,
prefix + key + suffix)
else:
src = PATH.OUTPUT + '/' + 'model_init'
dst = path
copybin(src, dst, iproc, prefix + key + suffix)
if 'rho' in self.parameters:
savebin(model['rho'][iproc], path, iproc,
prefix + 'rho' + suffix)
else:
src = PATH.OUTPUT + '/' + 'model_init'
dst = path
copybin(src, dst, iproc, 'rho')
def merge(self, model):
""" Converts model from dictionary to vector representation
"""
v = np.array([])
for key in self.parameters:
for iproc in range(self.mesh.nproc):
v = np.append(v, model[key][iproc])
return v
def split(self, v):
""" Converts model from vector to dictionary representation
"""
model = {}
for idim, key in enumerate(self.parameters):
model[key] = splitvec(v, self.mesh.nproc, self.mesh.ngll, idim)
return model
### postprocessing utilities
def combine(self, path='', parameters=[]):
""" Sums individual source contributions. Wrapper over xcombine_sem
utility.
"""
unix.cd(self.getpath)
names = self.check_source_names()
with open('kernel_paths', 'w') as f:
f.writelines([join(path, dir) + '\n' for dir in names])
unix.mkdir(path + '/' + 'sum')
for name in parameters:
self.call(PATH.SPECFEM_BIN + '/' + 'xcombine_sem ' + name +
'_kernel' + ' ' + 'kernel_paths' + ' ' + path + '/' +
'sum')
def smooth(self, path='', parameters=[], span=0.):
""" Smooths kernels by convolving them with a Gaussian. Wrapper over
xsmooth_sem utility.
"""
assert exists(path)
assert len(parameters) > 0
# apply smoothing operator
unix.cd(self.getpath)
for name in parameters:
print ' smoothing', name
self.call(PATH.SPECFEM_BIN + '/' + 'xsmooth_sem ' + str(span) +
' ' + str(span) + ' ' + name + '_kernel' + ' ' + path +
'/ ' + path + '/ ',
output=self.getpath + '/' +
'OUTPUT_FILES/output_smooth_sem.txt')
print ''
# move input files
src = path
dst = path + '_nosmooth'
unix.mkdir(dst)
for name in self.parameters:
unix.mv(glob(src + '/*' + name + '.bin'), dst)
# rename output files
unix.rename('_smooth', '', glob(src + '/*'))
def clip(self, path='', parameters=[], minval=-np.inf, maxval=np.inf):
""" Clips kernels by convolving them with a Gaussian. Wrapper over
xclip_sem utility.
"""
assert exists(path)
assert len(parameters) > 0
unix.cd(self.getpath)
for name in self.parameters:
self.call(PATH.SPECFEM_BIN + '/' + 'xclip_sem ' + str(minval) +
' ' + str(maxval) + ' ' + name + '_kernel' + ' ' + path +
'/ ' + path + '/ ')
# move input files
src = path
dst = path + '_noclip'
unix.mkdir(dst)
for name in self.parameters:
unix.mv(glob(src + '/*' + name + '.bin'), dst)
# rename output files
unix.rename('_clip', '', glob(src + '/*'))
### file transfer utilities
def import_model(self, path):
src = join(path, 'model')
dst = self.model_databases
if self.getnode == 0:
self.save(dst, self.load(src, verbose=True))
else:
self.save(dst, self.load(src))
def import_traces(self, path):
src = glob(join(path, 'traces', basename(self.getpath), '*'))
dst = join(self.getpath, 'traces/obs')
unix.cp(src, dst)
def export_model(self, path, solver_parameters=['rho', 'vp', 'vs']):
if self.getnode == 0:
unix.mkdir(path)
for key in solver_parameters:
files = glob(join(self.model_databases, '*' + key + '.bin'))
unix.cp(files, path)
def export_kernels(self, path):
unix.cd(self.kernel_databases)
# work around conflicting name conventions
files = []
files += glob('*proc??????_alpha_kernel.bin')
files += glob('*proc??????_alpha[hv]_kernel.bin')
files += glob('*proc??????_reg1_alpha_kernel.bin')
files += glob('*proc??????_reg1_alpha[hv]_kernel.bin')
unix.rename('alpha', 'vp', files)
files = []
files += glob('*proc??????_beta_kernel.bin')
files += glob('*proc??????_beta[hv]_kernel.bin')
files += glob('*proc??????_reg1_beta_kernel.bin')
files += glob('*proc??????_reg1_beta[hv]_kernel.bin')
unix.rename('beta', 'vs', files)
# hack to deal with problems on parallel filesystem
unix.mkdir(join(path, 'kernels'), noexit=True)
unix.mkdir(join(path, 'kernels', basename(self.getpath)))
src = join(glob('*_kernel.bin'))
dst = join(path, 'kernels', basename(self.getpath))
unix.mv(src, dst)
def export_residuals(self, path):
# hack deals with problems on parallel filesystem
unix.mkdir(join(path, 'residuals'), noexit=True)
src = join(self.getpath, 'residuals')
dst = join(path, 'residuals', basename(self.getpath))
unix.mv(src, dst)
def export_traces(self, path, prefix='traces/obs'):
# hack deals with problems on parallel filesystem
unix.mkdir(join(path, 'traces'), noexit=True)
src = join(self.getpath, prefix)
dst = join(path, 'traces', basename(self.getpath))
unix.cp(src, dst)
### setup utilities
def initialize_solver_directories(self):
""" Creates directory structure expected by SPECFEM3D, copies
executables, and prepares input files. Executables must be supplied
by user as there is currently no mechanism for automatically
compiling from source.
"""
unix.mkdir(self.getpath)
unix.cd(self.getpath)
# create directory structure
unix.mkdir('bin')
unix.mkdir('DATA')
unix.mkdir('OUTPUT_FILES')
unix.mkdir('traces/obs')
unix.mkdir('traces/syn')
unix.mkdir('traces/adj')
unix.mkdir(self.model_databases)
unix.mkdir(self.kernel_databases)
# copy exectuables
src = glob(PATH.SPECFEM_BIN + '/' + '*')
dst = 'bin/'
unix.cp(src, dst)
# copy input files
src = glob(PATH.SPECFEM_DATA + '/' + '*')
dst = 'DATA/'
unix.cp(src, dst)
src = 'DATA/' + self.source_prefix + '_' + basename(self.getpath)
dst = 'DATA/' + self.source_prefix
unix.cp(src, dst)
self.check_solver_parameter_files()
def initialize_adjoint_traces(self):
""" Adjoint traces are initialized by writing zeros for all components.
Components actually in use during an inversion or migration will be
overwritten with nonzero values later on.
"""
path = self.getpath + '/' + 'traces/adj/'
for channel in ['x', 'y', 'z']:
preprocess.write_zero_traces(path, channel)
def check_mesh_properties(self, path=None, parameters=None):
if not hasattr(self, '_mesh_properties'):
if not path:
path = PATH.MODEL_INIT
if not parameters:
parameters = self.parameters
nproc = 0
ngll = []
while True:
dummy = loadbin(path, nproc, parameters[0])
ngll += [len(dummy)]
nproc += 1
if not exists('%s/proc%06d_%s.bin' %
(path, nproc, parameters[0])):
break
self._mesh_properties = Struct([['nproc', nproc], ['ngll', ngll]])
return self._mesh_properties
def check_source_names(self):
""" Checks names of sources
"""
if not hasattr(self, '_source_names'):
path = PATH.SPECFEM_DATA
wildcard = self.source_prefix + '_*'
globstar = sorted(glob(path + '/' + wildcard))
if not globstar:
print msg.SourceError_SPECFEM % (path, wildcard)
sys.exit(-1)
names = []
for path in globstar:
names += [basename(path).split('_')[-1]]
self._source_names = names[:PAR.NTASK]
return self._source_names
def check_solver_parameter_files(self):
# must be implemented by subclass
pass
### miscellaneous
def call(self, executable, output='/dev/null'):
""" Calls solver through subprocess
"""
# a less complicated version, without error catching, would simply be
# subprocess.call(system.mpiexec() + executable, shell=True)
try:
f = open(output, 'w')
subprocess.check_call(system.mpiexec() + executable,
shell=True,
stdout=f)
except subprocess.CalledProcessError, err:
print msg.SolverError % (system.mpiexec() + executable)
sys.exit(-1)
except OSError:
print msg.SolverError % (system.mpiexec() + executable)
sys.exit(-1)
