def test_exists(self): existent = os.listdir(os.curdir) non_existent = [str(uuid.uuid4().get_hex().upper()[0:6]) for i in range(10)] not_names = [None, 0, 1, False, True] for name in existent: self.assertTrue(tools.exists(name)) for name in non_existent: self.assertFalse(tools.exists(name)) for name in not_names: self.assertFalse(tools.exists(name))
def check(self): """ Checks parameters and paths """ # check paths if 'SCRATCH' not in PATH: raise ParameterError(PATH, 'SCRATCH') if 'LOCAL' not in PATH: setattr(PATH, 'LOCAL', None) if 'OUTPUT' not in PATH: raise ParameterError(PATH, 'OUTPUT') if 'SOLVER' not in PATH: raise ParameterError(PATH, 'SOLVER') # check input if 'DATA' not in PATH: setattr(PATH, 'DATA', None) if not exists(PATH.DATA): assert 'MODEL_TRUE' in PATH if 'MODEL_INIT' not in PATH: raise ParameterError(PATH, 'MODEL_INIT') # assertions if PAR.NSRC != 1: raise ParameterError(PAR, 'NSRC')
def check(self): """ Checks parameters and paths """ # check parameters if 'CLIP' not in PAR: setattr(PAR, 'CLIP', 0.) if 'SMOOTH' not in PAR: setattr(PAR, 'SMOOTH', 0.) if 'LOGARITHMIC' not in PAR: setattr(PAR, 'LOGARITHMIC', True) if 'PRECOND' not in PAR: setattr(PAR, 'PRECOND', False) # check paths if 'MASK' not in PATH: setattr(PATH, 'MASK', None) if 'PRECOND' not in PATH: setattr(PATH, 'PRECOND', None) if PATH.MASK: assert exists(PATH.MASK)
def smooth(self, path='', parameters=[], span=0.): """ Smooths kernels by convolving them with a Gaussian. Wrapper over xsmooth_sem utility. """ assert exists(path) assert len(parameters) > 0 # apply smoothing operator unix.cd(self.getpath) for name in parameters: print ' smoothing', name self.mpirun( PATH.SPECFEM_BIN +'/'+ 'xsmooth_sem ' + str(span) + ' ' + str(span) + ' ' + name + '_kernel' + ' ' + path + '/ ' + path + '/ ', output=self.getpath+'/'+'OUTPUT_FILES/output_smooth_sem.txt') print '' # move input files src = path dst = path + '_nosmooth' unix.mkdir(dst) for name in self.parameters: unix.mv(glob(src+'/*'+name+'.bin'), dst) # rename output files unix.rename('_smooth', '', glob(src+'/*'))
def clip(self, path='', parameters=[], minval=-np.inf, maxval=np.inf): """ Clips kernels by convolving them with a Gaussian. Wrapper over xclip_sem utility. """ assert exists(path) assert len(parameters) > 0 unix.cd(self.getpath) for name in self.parameters: self.mpirun( PATH.SPECFEM_BIN +'/'+ 'xclip_sem ' + str(minval) + ' ' + str(maxval) + ' ' + name + '_kernel' + ' ' + path + '/ ' + path + '/ ') # move input files src = path dst = path + '_noclip' unix.mkdir(dst) for name in self.parameters: unix.mv(glob(src+'/*'+name+'.bin'), dst) # rename output files unix.rename('_clip', '', glob(src+'/*'))
def check(self): """ Checks parameters and paths """ # check paths if 'GLOBAL' not in PATH: raise ParameterError(PATH, 'GLOBAL') if 'LOCAL' not in PATH: setattr(PATH, 'LOCAL', None) if 'OUTPUT' not in PATH: raise ParameterError(PATH, 'OUTPUT') # check input if 'DATA' not in PATH: setattr(PATH, 'DATA', None) if not exists(PATH.DATA): assert 'MODEL_TRUE' in PATH if 'MODEL_INIT' not in PATH: raise ParameterError(PATH, 'MODEL_INIT') # check output if 'SAVEGRADIENT' not in PAR: setattr(PAR, 'SAVEGRADIENT', 1) if 'SAVEKERNELS' not in PAR: setattr(PAR, 'SAVEKERNELS', 0) if 'SAVETRACES' not in PAR: setattr(PAR, 'SAVETRACES', 0)
def setup(self): """ Sets up nonlinear optimization machinery """ unix.mkdir(PATH.OPTIMIZE) # prepare output writers self.writer = Writer(path=PATH.OUTPUT) self.stepwriter = StepWriter(path=PATH.SUBMIT) # prepare algorithm machinery if PAR.SCHEME in ["NLCG"]: self.NLCG = NLCG(path=PATH.OPTIMIZE, maxiter=PAR.NLCGMAX, thresh=PAR.NLCGTHRESH, precond=self.precond) elif PAR.SCHEME in ["LBFGS"]: self.LBFGS = LBFGS( path=PATH.OPTIMIZE, memory=PAR.LBFGSMEM, maxiter=PAR.LBFGSMAX, thresh=PAR.LBFGSTHRESH, precond=self.precond, ) # write initial model if exists(PATH.MODEL_INIT): src = PATH.MODEL_INIT dst = join(PATH.OPTIMIZE, "m_new") savenpy(dst, solver.merge(solver.load(src)))
def submit(self, *args, **kwargs): """Submits job """ if not exists(PATH.SUBMIT + '/' + 'scratch'): unix.ln(PATH.GLOBAL, PATH.SUBMIT + '/' + 'scratch') super(tiger_sm, self).submit(*args, **kwargs)
def generate_mesh(self, model_path=None, model_name=None, model_type='gll'): """ Performs meshing and database generation """ assert(model_name) assert(model_type) self.initialize_solver_directories() unix.cd(self.getpath) if model_type in ['gll']: par = getpar('MODEL').strip() if par != 'gll': if self.getnode == 0: print 'WARNING: Unexpected Par_file setting:' print 'MODEL =', par assert(exists(model_path)) self.check_mesh_properties(model_path) src = glob(model_path +'/'+ '*') dst = self.model_databases unix.cp(src, dst) self.call('bin/xmeshfem3D') self.call('bin/xgenerate_databases') self.export_model(PATH.OUTPUT +'/'+ model_name) else: raise NotImplementedError
def write_gradient(self, path): """ Reads kernels and writes gradient of objective function """ if not exists(path): unix.mkdir(path) self.combine_kernels(path, solver.parameters) self.process_kernels(path, solver.parameters)
def clean_directory(self, path): """ If dir exists clean otherwise make """ if not exists(path): unix.mkdir(path) else: unix.rm(path) unix.mkdir(path)
def process_kernels(self, path, parameters): """ Processes kernels in accordance with parameter settings """ fullpath = path +'/'+ 'kernels' assert exists(path) if exists(fullpath +'/'+ 'sum'): unix.mv(fullpath +'/'+ 'sum', fullpath +'/'+ 'sum_nofix') # mask sources and receivers system.run('postprocess', 'fix_near_field', hosts='all', path=fullpath) system.run('solver', 'combine', hosts='head', path=fullpath, parameters=parameters)
def generate_precond(self, process_traces=None, model_path=None, model_name=None, model_type='gll'): assert(model_name) assert(model_type) assert (exists(model_path)) self.initialize_solver_directories() unix.cd(self.getpath) assert(exists(model_path)) self.check_mesh_properties(model_path) src = model_path dst = join(self.getpath, 'DATA/proc000000_rho_vp_vs.dat') unix.cp(src, dst) self.export_model(PATH.OUTPUT +'/'+ model_name) self.forward() unix.mv(self.data_wildcard, 'traces/syn') self.initialize_adjoint_traces('traces/syn') process_traces(self.getpath) self.adjoint() self.export_kernels(PATH.GLOBAL)
def __init__(self, path='.', load=loadnpy, save=savenpy, memory=5, thresh=0., maxiter=np.inf, precond=None): assert exists(path) unix.cd(path) unix.mkdir('LBFGS') self.path = path self.load = load self.save = save self.thresh = thresh self.maxiter = maxiter self.precond = precond self.memory = memory self.iter = 0 self.memory_used = 0
def write_residuals(self, path, s, d): """ Computes residuals from observations and synthetics """ nt, dt, _ = self.get_time_scheme(s) n, _ = self.get_network_size(s) filename = path +'/'+ 'residuals' if exists(filename): r = np.loadtxt(filename) else: r = [] for i in range(n): r.append(self.misfit(s[i].data, d[i].data, nt, dt)) np.savetxt(filename, r)
def regularize(self, path): assert (exists(path)) g = solver.load(path +'/'+ 'gradient', suffix='_kernel') if not PAR.LAMBDA: return solver.merge(g) m = solver.load(path +'/'+ 'model') mesh = self.getmesh() for key in solver.parameters: for iproc in range(PAR.NPROC): g[key][iproc] += PAR.LAMBDA *\ self.nabla(mesh, m[key][iproc], g[key][iproc]) #self.nabla(m[key][iproc], g[key][iproc] , mesh, h) return solver.merge(g)
def generate_mesh(self, model_path=None, model_name=None, model_type='gll'): """ Performs meshing and database generation """ assert(model_name) assert(model_type) self.initialize_solver_directories() unix.cd(self.getpath) assert(exists(model_path)) self.check_mesh_properties(model_path) src = model_path dst = join(self.getpath, 'DATA/proc000000_rho_vp_vs.dat') unix.cp(src, dst) self.export_model(PATH.OUTPUT +'/'+ model_name)
def generate_mesh(self, model_path=None, model_name=None, model_type='gll'): """ Performs meshing and database generation """ assert(model_name) assert(model_type) self.initialize_solver_directories() unix.cd(self.getpath) if model_type == 'gll': assert (exists(model_path)) self.check_mesh_properties(model_path) unix.cp(glob(model_path +'/'+ '*'), self.model_databases) self.mpirun('bin/xmeshfem3D') self.export_model(PATH.OUTPUT +'/'+ model_name) else: raise NotImplementedError
def check_mesh_properties(self, path=None, parameters=None): if not hasattr(self, '_mesh_properties'): if not path: path = PATH.MODEL_INIT if not parameters: parameters = self.parameters nproc = 0 ngll = [] while True: dummy = loadbin(path, nproc, 'reg1_'+parameters[0]) ngll += [len(dummy)] nproc += 1 if not exists('%s/proc%06d_reg1_%s.bin' % (path, nproc, parameters[0])): break self._mesh_properties = Struct([ ['nproc', nproc], ['ngll', ngll]]) return self._mesh_properties
def clip(self, path='', parameters=[], minval=-np.inf, maxval=np.inf): """ Clips kernels by convolving them with a Gaussian. Wrapper over xclip_sem utility. """ assert exists(path) assert len(parameters) > 0 unix.cd(self.getpath) for name in self.parameters: self.call(PATH.SPECFEM_BIN + '/' + 'xclip_sem ' + str(minval) + ' ' + str(maxval) + ' ' + name + '_kernel' + ' ' + path + '/ ' + path + '/ ') # move input files src = path dst = path + '_noclip' unix.mkdir(dst) for name in self.parameters: unix.mv(glob(src + '/*' + name + '.bin'), dst) # rename output files unix.rename('_clip', '', glob(src + '/*'))
def write_gradient(self, path): """ Reads kernels and writes gradient of objective function """ if not exists(path): raise Exception() self.combine_kernels(path, solver.parameters) self.process_kernels(path, solver.parameters) g = solver.merge(solver.load( path +'/'+ 'kernels/sum', suffix='_kernel', verbose=True)) if PAR.LOGARITHMIC: # convert from logarithmic to absolute perturbations g *= solver.merge(solver.load(path +'/'+ 'model')) self.save(path, g) if PATH.MASK: # apply mask g *= solver.merge(solver.load(PATH.MASK)) self.save(path, g, backup='nomask')
def generate_mesh(self, model_path=None, model_name=None, model_type='gll'): """ Performs meshing and database generation """ assert (model_name) assert (model_type) self.initialize_solver_directories() unix.cd(self.getpath) if model_type == 'gll': assert (exists(model_path)) self.check_mesh_properties(model_path) unix.cp(glob(model_path + '/' + '*'), self.model_databases) call_solver(system.mpiexec(), 'bin/xmeshfem3D') self.export_model(PATH.OUTPUT + '/' + model_name) else: raise NotImplementedError
def check(self): """ Checks parameters and paths """ # check parameters if 'BEGIN' not in PAR: raise ParameterError(PAR, 'BEGIN') if 'END' not in PAR: raise ParameterError(PAR, 'END') if 'VERBOSE' not in PAR: setattr(PAR, 'VERBOSE', 1) # check paths if 'GLOBAL' not in PATH: raise ParameterError(PATH, 'GLOBAL') if 'LOCAL' not in PATH: setattr(PATH, 'LOCAL', None) if 'FUNC' not in PATH: setattr(PATH, 'FUNC', join(PATH.GLOBAL, 'evalfunc')) if 'GRAD' not in PATH: setattr(PATH, 'GRAD', join(PATH.GLOBAL, 'evalgrad')) if 'HESS' not in PATH: setattr(PATH, 'HESS', join(PATH.GLOBAL, 'evalhess')) if 'OPTIMIZE' not in PATH: setattr(PATH, 'OPTIMIZE', join(PATH.GLOBAL, 'optimize')) # input settings if 'DATA' not in PATH: setattr(PATH, 'DATA', None) if not exists(PATH.DATA): assert 'MODEL_TRUE' in PATH if 'MODEL_INIT' not in PATH: raise ParameterError(PATH, 'MODEL_INIT') # output settings if 'OUTPUT' not in PATH: raise ParameterError(PATH, 'OUTPUT') if 'SAVEMODEL' not in PAR: setattr(PAR, 'SAVEMODEL', 1) if 'SAVEGRADIENT' not in PAR: setattr(PAR, 'SAVEGRADIENT', 0) if 'SAVEKERNELS' not in PAR: setattr(PAR, 'SAVEKERNELS', 0) if 'SAVETRACES' not in PAR: setattr(PAR, 'SAVETRACES', 0) if 'SAVERESIDUALS' not in PAR: setattr(PAR, 'SAVERESIDUALS', 0) # assertions assert 1 <= PAR.BEGIN <= PAR.END
def prepare_model(self): model = PATH.OUTPUT + '/' + 'model_init' assert exists(model) unix.cp(model, PATH.SCRATCH + '/' + 'model')
def prepare_model(self): model = PATH.OUTPUT +'/'+ 'model_init' assert exists(model) unix.cp(model, PATH.GLOBAL +'/'+ 'model')
def check(self): """ Checks parameters, paths, and dependencies """ # check parameters if 'BEGIN' not in PAR: raise Exception if 'END' not in PAR: raise Exception if 'VERBOSE' not in PAR: setattr(PAR, 'VERBOSE', 1) # check paths if 'GLOBAL' not in PATH: raise Exception if 'LOCAL' not in PATH: setattr(PATH, 'LOCAL', None) if 'FUNC' not in PATH: setattr(PATH, 'FUNC', join(PATH.GLOBAL, 'evalfunc')) if 'GRAD' not in PATH: setattr(PATH, 'GRAD', join(PATH.GLOBAL, 'evalgrad')) if 'HESS' not in PATH: setattr(PATH, 'HESS', join(PATH.GLOBAL, 'evalhess')) if 'OPTIMIZE' not in PATH: setattr(PATH, 'OPTIMIZE', join(PATH.GLOBAL, 'optimize')) # input settings if 'DATA' not in PATH: setattr(PATH, 'DATA', None) if not exists(PATH.DATA): assert 'MODEL_TRUE' in PATH if 'MODEL_INIT' not in PATH: raise Exception # output settings if 'OUTPUT' not in PATH: raise Exception if 'SAVEMODEL' not in PAR: setattr(PAR, 'SAVEMODEL', 1) if 'SAVEGRADIENT' not in PAR: setattr(PAR, 'SAVEGRADIENT', 0) if 'SAVEKERNELS' not in PAR: setattr(PAR, 'SAVEKERNELS', 0) if 'SAVETRACES' not in PAR: setattr(PAR, 'SAVETRACES', 0) if 'SAVERESIDUALS' not in PAR: setattr(PAR, 'SAVERESIDUALS', 0)
def check(self): """ Checks parameters and paths """ # check parameters if 'BEGIN' not in PAR: raise ParameterError(PAR, 'BEGIN') if 'END' not in PAR: raise ParameterError(PAR, 'END') if 'VERBOSE' not in PAR: setattr(PAR, 'VERBOSE', 1) # scratch paths if 'SCRATCH' not in PATH: raise ParameterError(PATH, 'SCRATCH') if 'LOCAL' not in PATH: setattr(PATH, 'LOCAL', None) if 'FUNC' not in PATH: setattr(PATH, 'FUNC', join(PATH.SCRATCH, 'evalfunc')) if 'GRAD' not in PATH: setattr(PATH, 'GRAD', join(PATH.SCRATCH, 'evalgrad')) if 'HESS' not in PATH: setattr(PATH, 'HESS', join(PATH.SCRATCH, 'evalhess')) if 'OPTIMIZE' not in PATH: setattr(PATH, 'OPTIMIZE', join(PATH.SCRATCH, 'optimize')) # input paths if 'DATA' not in PATH: setattr(PATH, 'DATA', None) if 'MODEL_INIT' not in PATH: raise ParameterError(PATH, 'MODEL_INIT') # output paths if 'OUTPUT' not in PATH: raise ParameterError(PATH, 'OUTPUT') if 'SAVEMODEL' not in PAR: setattr(PAR, 'SAVEMODEL', 1) if 'SAVEGRADIENT' not in PAR: setattr(PAR, 'SAVEGRADIENT', 0) if 'SAVEKERNELS' not in PAR: setattr(PAR, 'SAVEKERNELS', 0) if 'SAVETRACES' not in PAR: setattr(PAR, 'SAVETRACES', 0) if 'SAVERESIDUALS' not in PAR: setattr(PAR, 'SAVERESIDUALS', 0) # parameter assertions assert 1 <= PAR.BEGIN <= PAR.END # path assertions if not exists(PATH.DATA): assert 'MODEL_TRUE' in PATH assert exists(PATH.MODEL_TRUE) if not exists(PATH.MODEL_INIT): raise Exception()
def prepare_model(self): model = PATH.OUTPUT + '/' + 'model_init' assert exists(model) unix.ln(model, PATH.GLOBAL + '/' + 'model')
def prepare_model(self): model = PATH.OUTPUT +'/'+ 'model_init' assert exists(model) unix.ln(model, PATH.SCRATCH +'/'+ 'model')