def test_exists(self):
existent = os.listdir(os.curdir)
non_existent = [str(uuid.uuid4().get_hex().upper()[0:6])
for i in range(10)]
not_names = [None, 0, 1, False, True]
for name in existent:
self.assertTrue(tools.exists(name))
for name in non_existent:
self.assertFalse(tools.exists(name))
for name in not_names:
self.assertFalse(tools.exists(name))
def check(self):
""" Checks parameters and paths
"""
# check paths
if 'SCRATCH' not in PATH:
raise ParameterError(PATH, 'SCRATCH')
if 'LOCAL' not in PATH:
setattr(PATH, 'LOCAL', None)
if 'OUTPUT' not in PATH:
raise ParameterError(PATH, 'OUTPUT')
if 'SOLVER' not in PATH:
raise ParameterError(PATH, 'SOLVER')
# check input
if 'DATA' not in PATH:
setattr(PATH, 'DATA', None)
if not exists(PATH.DATA):
assert 'MODEL_TRUE' in PATH
if 'MODEL_INIT' not in PATH:
raise ParameterError(PATH, 'MODEL_INIT')
# assertions
if PAR.NSRC != 1:
raise ParameterError(PAR, 'NSRC')
def check(self):
""" Checks parameters and paths
"""
# check parameters
if 'CLIP' not in PAR:
setattr(PAR, 'CLIP', 0.)
if 'SMOOTH' not in PAR:
setattr(PAR, 'SMOOTH', 0.)
if 'LOGARITHMIC' not in PAR:
setattr(PAR, 'LOGARITHMIC', True)
if 'PRECOND' not in PAR:
setattr(PAR, 'PRECOND', False)
# check paths
if 'MASK' not in PATH:
setattr(PATH, 'MASK', None)
if 'PRECOND' not in PATH:
setattr(PATH, 'PRECOND', None)
if PATH.MASK:
assert exists(PATH.MASK)
def smooth(self, path='', parameters=[], span=0.):
""" Smooths kernels by convolving them with a Gaussian. Wrapper over
xsmooth_sem utility.
"""
assert exists(path)
assert len(parameters) > 0
# apply smoothing operator
unix.cd(self.getpath)
for name in parameters:
print ' smoothing', name
self.mpirun(
PATH.SPECFEM_BIN +'/'+ 'xsmooth_sem '
+ str(span) + ' '
+ str(span) + ' '
+ name + '_kernel' + ' '
+ path + '/ '
+ path + '/ ',
output=self.getpath+'/'+'OUTPUT_FILES/output_smooth_sem.txt')
print ''
# move input files
src = path
dst = path + '_nosmooth'
unix.mkdir(dst)
for name in self.parameters:
unix.mv(glob(src+'/*'+name+'.bin'), dst)
# rename output files
unix.rename('_smooth', '', glob(src+'/*'))
def clip(self, path='', parameters=[], minval=-np.inf, maxval=np.inf):
""" Clips kernels by convolving them with a Gaussian. Wrapper over
xclip_sem utility.
"""
assert exists(path)
assert len(parameters) > 0
unix.cd(self.getpath)
for name in self.parameters:
self.mpirun(
PATH.SPECFEM_BIN +'/'+ 'xclip_sem '
+ str(minval) + ' '
+ str(maxval) + ' '
+ name + '_kernel' + ' '
+ path + '/ '
+ path + '/ ')
# move input files
src = path
dst = path + '_noclip'
unix.mkdir(dst)
for name in self.parameters:
unix.mv(glob(src+'/*'+name+'.bin'), dst)
# rename output files
unix.rename('_clip', '', glob(src+'/*'))
def check(self):
""" Checks parameters and paths
"""
# check paths
if 'GLOBAL' not in PATH:
raise ParameterError(PATH, 'GLOBAL')
if 'LOCAL' not in PATH:
setattr(PATH, 'LOCAL', None)
if 'OUTPUT' not in PATH:
raise ParameterError(PATH, 'OUTPUT')
# check input
if 'DATA' not in PATH:
setattr(PATH, 'DATA', None)
if not exists(PATH.DATA):
assert 'MODEL_TRUE' in PATH
if 'MODEL_INIT' not in PATH:
raise ParameterError(PATH, 'MODEL_INIT')
# check output
if 'SAVEGRADIENT' not in PAR:
setattr(PAR, 'SAVEGRADIENT', 1)
if 'SAVEKERNELS' not in PAR:
setattr(PAR, 'SAVEKERNELS', 0)
if 'SAVETRACES' not in PAR:
setattr(PAR, 'SAVETRACES', 0)
def setup(self):
""" Sets up nonlinear optimization machinery
"""
unix.mkdir(PATH.OPTIMIZE)
# prepare output writers
self.writer = Writer(path=PATH.OUTPUT)
self.stepwriter = StepWriter(path=PATH.SUBMIT)
# prepare algorithm machinery
if PAR.SCHEME in ["NLCG"]:
self.NLCG = NLCG(path=PATH.OPTIMIZE, maxiter=PAR.NLCGMAX, thresh=PAR.NLCGTHRESH, precond=self.precond)
elif PAR.SCHEME in ["LBFGS"]:
self.LBFGS = LBFGS(
path=PATH.OPTIMIZE,
memory=PAR.LBFGSMEM,
maxiter=PAR.LBFGSMAX,
thresh=PAR.LBFGSTHRESH,
precond=self.precond,
)
# write initial model
if exists(PATH.MODEL_INIT):
src = PATH.MODEL_INIT
dst = join(PATH.OPTIMIZE, "m_new")
savenpy(dst, solver.merge(solver.load(src)))
def submit(self, *args, **kwargs):
"""Submits job
"""
if not exists(PATH.SUBMIT + '/' + 'scratch'):
unix.ln(PATH.GLOBAL, PATH.SUBMIT + '/' + 'scratch')
super(tiger_sm, self).submit(*args, **kwargs)
def generate_mesh(self, model_path=None, model_name=None, model_type='gll'):
""" Performs meshing and database generation
"""
assert(model_name)
assert(model_type)
self.initialize_solver_directories()
unix.cd(self.getpath)
if model_type in ['gll']:
par = getpar('MODEL').strip()
if par != 'gll':
if self.getnode == 0:
print 'WARNING: Unexpected Par_file setting:'
print 'MODEL =', par
assert(exists(model_path))
self.check_mesh_properties(model_path)
src = glob(model_path +'/'+ '*')
dst = self.model_databases
unix.cp(src, dst)
self.call('bin/xmeshfem3D')
self.call('bin/xgenerate_databases')
self.export_model(PATH.OUTPUT +'/'+ model_name)
else:
raise NotImplementedError
def write_gradient(self, path):
""" Reads kernels and writes gradient of objective function
"""
if not exists(path):
unix.mkdir(path)
self.combine_kernels(path, solver.parameters)
self.process_kernels(path, solver.parameters)
def clean_directory(self, path):
""" If dir exists clean otherwise make
"""
if not exists(path):
unix.mkdir(path)
else:
unix.rm(path)
unix.mkdir(path)
def process_kernels(self, path, parameters):
""" Processes kernels in accordance with parameter settings
"""
fullpath = path +'/'+ 'kernels'
assert exists(path)
if exists(fullpath +'/'+ 'sum'):
unix.mv(fullpath +'/'+ 'sum', fullpath +'/'+ 'sum_nofix')
# mask sources and receivers
system.run('postprocess', 'fix_near_field',
hosts='all',
path=fullpath)
system.run('solver', 'combine',
hosts='head',
path=fullpath,
parameters=parameters)
def generate_precond(self, process_traces=None, model_path=None, model_name=None, model_type='gll'):
assert(model_name)
assert(model_type)
assert (exists(model_path))
self.initialize_solver_directories()
unix.cd(self.getpath)
assert(exists(model_path))
self.check_mesh_properties(model_path)
src = model_path
dst = join(self.getpath, 'DATA/proc000000_rho_vp_vs.dat')
unix.cp(src, dst)
self.export_model(PATH.OUTPUT +'/'+ model_name)
self.forward()
unix.mv(self.data_wildcard, 'traces/syn')
self.initialize_adjoint_traces('traces/syn')
process_traces(self.getpath)
self.adjoint()
self.export_kernels(PATH.GLOBAL)
def __init__(self, path='.', load=loadnpy, save=savenpy, memory=5, thresh=0., maxiter=np.inf, precond=None):
assert exists(path)
unix.cd(path)
unix.mkdir('LBFGS')
self.path = path
self.load = load
self.save = save
self.thresh = thresh
self.maxiter = maxiter
self.precond = precond
self.memory = memory
self.iter = 0
self.memory_used = 0
def write_residuals(self, path, s, d):
""" Computes residuals from observations and synthetics
"""
nt, dt, _ = self.get_time_scheme(s)
n, _ = self.get_network_size(s)
filename = path +'/'+ 'residuals'
if exists(filename):
r = np.loadtxt(filename)
else:
r = []
for i in range(n):
r.append(self.misfit(s[i].data, d[i].data, nt, dt))
np.savetxt(filename, r)
def regularize(self, path):
assert (exists(path))
g = solver.load(path +'/'+ 'gradient', suffix='_kernel')
if not PAR.LAMBDA:
return solver.merge(g)
m = solver.load(path +'/'+ 'model')
mesh = self.getmesh()
for key in solver.parameters:
for iproc in range(PAR.NPROC):
g[key][iproc] += PAR.LAMBDA *\
self.nabla(mesh, m[key][iproc], g[key][iproc])
#self.nabla(m[key][iproc], g[key][iproc] , mesh, h)
return solver.merge(g)
def generate_mesh(self, model_path=None, model_name=None, model_type='gll'):
""" Performs meshing and database generation
"""
assert(model_name)
assert(model_type)
self.initialize_solver_directories()
unix.cd(self.getpath)
assert(exists(model_path))
self.check_mesh_properties(model_path)
src = model_path
dst = join(self.getpath, 'DATA/proc000000_rho_vp_vs.dat')
unix.cp(src, dst)
self.export_model(PATH.OUTPUT +'/'+ model_name)
def generate_mesh(self, model_path=None, model_name=None, model_type='gll'):
""" Performs meshing and database generation
"""
assert(model_name)
assert(model_type)
self.initialize_solver_directories()
unix.cd(self.getpath)
if model_type == 'gll':
assert (exists(model_path))
self.check_mesh_properties(model_path)
unix.cp(glob(model_path +'/'+ '*'), self.model_databases)
self.mpirun('bin/xmeshfem3D')
self.export_model(PATH.OUTPUT +'/'+ model_name)
else:
raise NotImplementedError
def check_mesh_properties(self, path=None, parameters=None):
if not hasattr(self, '_mesh_properties'):
if not path:
path = PATH.MODEL_INIT
if not parameters:
parameters = self.parameters
nproc = 0
ngll = []
while True:
dummy = loadbin(path, nproc, 'reg1_'+parameters[0])
ngll += [len(dummy)]
nproc += 1
if not exists('%s/proc%06d_reg1_%s.bin' % (path, nproc, parameters[0])):
break
self._mesh_properties = Struct([
['nproc', nproc],
['ngll', ngll]])
return self._mesh_properties
def clip(self, path='', parameters=[], minval=-np.inf, maxval=np.inf):
""" Clips kernels by convolving them with a Gaussian. Wrapper over
xclip_sem utility.
"""
assert exists(path)
assert len(parameters) > 0
unix.cd(self.getpath)
for name in self.parameters:
self.call(PATH.SPECFEM_BIN + '/' + 'xclip_sem ' + str(minval) +
' ' + str(maxval) + ' ' + name + '_kernel' + ' ' + path +
'/ ' + path + '/ ')
# move input files
src = path
dst = path + '_noclip'
unix.mkdir(dst)
for name in self.parameters:
unix.mv(glob(src + '/*' + name + '.bin'), dst)
# rename output files
unix.rename('_clip', '', glob(src + '/*'))
def write_gradient(self, path):
""" Reads kernels and writes gradient of objective function
"""
if not exists(path):
raise Exception()
self.combine_kernels(path, solver.parameters)
self.process_kernels(path, solver.parameters)
g = solver.merge(solver.load(
path +'/'+ 'kernels/sum',
suffix='_kernel',
verbose=True))
if PAR.LOGARITHMIC:
# convert from logarithmic to absolute perturbations
g *= solver.merge(solver.load(path +'/'+ 'model'))
self.save(path, g)
if PATH.MASK:
# apply mask
g *= solver.merge(solver.load(PATH.MASK))
self.save(path, g, backup='nomask')
def generate_mesh(self,
model_path=None,
model_name=None,
model_type='gll'):
""" Performs meshing and database generation
"""
assert (model_name)
assert (model_type)
self.initialize_solver_directories()
unix.cd(self.getpath)
if model_type == 'gll':
assert (exists(model_path))
self.check_mesh_properties(model_path)
unix.cp(glob(model_path + '/' + '*'), self.model_databases)
call_solver(system.mpiexec(), 'bin/xmeshfem3D')
self.export_model(PATH.OUTPUT + '/' + model_name)
else:
raise NotImplementedError
def check(self):
""" Checks parameters and paths
"""
# check parameters
if 'BEGIN' not in PAR:
raise ParameterError(PAR, 'BEGIN')
if 'END' not in PAR:
raise ParameterError(PAR, 'END')
if 'VERBOSE' not in PAR:
setattr(PAR, 'VERBOSE', 1)
# check paths
if 'GLOBAL' not in PATH:
raise ParameterError(PATH, 'GLOBAL')
if 'LOCAL' not in PATH:
setattr(PATH, 'LOCAL', None)
if 'FUNC' not in PATH:
setattr(PATH, 'FUNC', join(PATH.GLOBAL, 'evalfunc'))
if 'GRAD' not in PATH:
setattr(PATH, 'GRAD', join(PATH.GLOBAL, 'evalgrad'))
if 'HESS' not in PATH:
setattr(PATH, 'HESS', join(PATH.GLOBAL, 'evalhess'))
if 'OPTIMIZE' not in PATH:
setattr(PATH, 'OPTIMIZE', join(PATH.GLOBAL, 'optimize'))
# input settings
if 'DATA' not in PATH:
setattr(PATH, 'DATA', None)
if not exists(PATH.DATA):
assert 'MODEL_TRUE' in PATH
if 'MODEL_INIT' not in PATH:
raise ParameterError(PATH, 'MODEL_INIT')
# output settings
if 'OUTPUT' not in PATH:
raise ParameterError(PATH, 'OUTPUT')
if 'SAVEMODEL' not in PAR:
setattr(PAR, 'SAVEMODEL', 1)
if 'SAVEGRADIENT' not in PAR:
setattr(PAR, 'SAVEGRADIENT', 0)
if 'SAVEKERNELS' not in PAR:
setattr(PAR, 'SAVEKERNELS', 0)
if 'SAVETRACES' not in PAR:
setattr(PAR, 'SAVETRACES', 0)
if 'SAVERESIDUALS' not in PAR:
setattr(PAR, 'SAVERESIDUALS', 0)
# assertions
assert 1 <= PAR.BEGIN <= PAR.END
def prepare_model(self):
model = PATH.OUTPUT + '/' + 'model_init'
assert exists(model)
unix.cp(model, PATH.SCRATCH + '/' + 'model')
def prepare_model(self):
model = PATH.OUTPUT +'/'+ 'model_init'
assert exists(model)
unix.cp(model, PATH.GLOBAL +'/'+ 'model')
def check(self):
""" Checks parameters, paths, and dependencies
"""
# check parameters
if 'BEGIN' not in PAR:
raise Exception
if 'END' not in PAR:
raise Exception
if 'VERBOSE' not in PAR:
setattr(PAR, 'VERBOSE', 1)
# check paths
if 'GLOBAL' not in PATH:
raise Exception
if 'LOCAL' not in PATH:
setattr(PATH, 'LOCAL', None)
if 'FUNC' not in PATH:
setattr(PATH, 'FUNC', join(PATH.GLOBAL, 'evalfunc'))
if 'GRAD' not in PATH:
setattr(PATH, 'GRAD', join(PATH.GLOBAL, 'evalgrad'))
if 'HESS' not in PATH:
setattr(PATH, 'HESS', join(PATH.GLOBAL, 'evalhess'))
if 'OPTIMIZE' not in PATH:
setattr(PATH, 'OPTIMIZE', join(PATH.GLOBAL, 'optimize'))
# input settings
if 'DATA' not in PATH:
setattr(PATH, 'DATA', None)
if not exists(PATH.DATA):
assert 'MODEL_TRUE' in PATH
if 'MODEL_INIT' not in PATH:
raise Exception
# output settings
if 'OUTPUT' not in PATH:
raise Exception
if 'SAVEMODEL' not in PAR:
setattr(PAR, 'SAVEMODEL', 1)
if 'SAVEGRADIENT' not in PAR:
setattr(PAR, 'SAVEGRADIENT', 0)
if 'SAVEKERNELS' not in PAR:
setattr(PAR, 'SAVEKERNELS', 0)
if 'SAVETRACES' not in PAR:
setattr(PAR, 'SAVETRACES', 0)
if 'SAVERESIDUALS' not in PAR:
setattr(PAR, 'SAVERESIDUALS', 0)
def check(self):
""" Checks parameters and paths
"""
# check parameters
if 'BEGIN' not in PAR:
raise ParameterError(PAR, 'BEGIN')
if 'END' not in PAR:
raise ParameterError(PAR, 'END')
if 'VERBOSE' not in PAR:
setattr(PAR, 'VERBOSE', 1)
# scratch paths
if 'SCRATCH' not in PATH:
raise ParameterError(PATH, 'SCRATCH')
if 'LOCAL' not in PATH:
setattr(PATH, 'LOCAL', None)
if 'FUNC' not in PATH:
setattr(PATH, 'FUNC', join(PATH.SCRATCH, 'evalfunc'))
if 'GRAD' not in PATH:
setattr(PATH, 'GRAD', join(PATH.SCRATCH, 'evalgrad'))
if 'HESS' not in PATH:
setattr(PATH, 'HESS', join(PATH.SCRATCH, 'evalhess'))
if 'OPTIMIZE' not in PATH:
setattr(PATH, 'OPTIMIZE', join(PATH.SCRATCH, 'optimize'))
# input paths
if 'DATA' not in PATH:
setattr(PATH, 'DATA', None)
if 'MODEL_INIT' not in PATH:
raise ParameterError(PATH, 'MODEL_INIT')
# output paths
if 'OUTPUT' not in PATH:
raise ParameterError(PATH, 'OUTPUT')
if 'SAVEMODEL' not in PAR:
setattr(PAR, 'SAVEMODEL', 1)
if 'SAVEGRADIENT' not in PAR:
setattr(PAR, 'SAVEGRADIENT', 0)
if 'SAVEKERNELS' not in PAR:
setattr(PAR, 'SAVEKERNELS', 0)
if 'SAVETRACES' not in PAR:
setattr(PAR, 'SAVETRACES', 0)
if 'SAVERESIDUALS' not in PAR:
setattr(PAR, 'SAVERESIDUALS', 0)
# parameter assertions
assert 1 <= PAR.BEGIN <= PAR.END
# path assertions
if not exists(PATH.DATA):
assert 'MODEL_TRUE' in PATH
assert exists(PATH.MODEL_TRUE)
if not exists(PATH.MODEL_INIT):
raise Exception()
def prepare_model(self):
model = PATH.OUTPUT + '/' + 'model_init'
assert exists(model)
unix.ln(model, PATH.GLOBAL + '/' + 'model')
def prepare_model(self):
model = PATH.OUTPUT +'/'+ 'model_init'
assert exists(model)
unix.ln(model, PATH.SCRATCH +'/'+ 'model')
