class ChenTromp_tti(custom_import('solver', 'ChenTromp_base')): # model parameters included in inversion parameters = [] parameters += ['A'] parameters += ['C'] parameters += ['L'] parameters += ['N'] parameters += ['F'] parameters += ['Jc'] parameters += ['Js'] parameters += ['Kc'] parameters += ['Ks'] parameters += ['Mc'] parameters += ['Ms'] parameters += ['Gc'] parameters += ['Gs'] parameters += ['Bc'] parameters += ['Bs'] parameters += ['Hc'] parameters += ['Hs'] parameters += ['Dc'] parameters += ['Ds'] parameters += ['Ec'] parameters += ['Es']
class tiger_sm(custom_import('system', 'slurm_sm')): """ Specially designed system interface for tiger.princeton.edu See parent class SLURM_SM for more information """ def check(self): """ Checks parameters and paths """ # where job was submitted if 'WORKDIR' not in PATH: setattr(PATH, 'WORKDIR', abspath('.')) # where temporary files are written if 'SCRATCH' not in PATH: setattr(PATH, 'SCRATCH', PATH.WORKDIR + '/' + 'scratch') super(tiger_sm, self).check() def submit(self, *args, **kwargs): """ Submits job """ if not exists(PATH.SCRATCH): path = '/scratch/gpfs' + '/' + getuser( ) + '/' + 'seisflows' + '/' + str(uuid4()) unix.mkdir(path) unix.ln(path, PATH.SCRATCH) super(tiger_sm, self).submit(*args, **kwargs)
class tikhonov1(custom_import('postprocess', 'regularize')): """ Adds regularization options to base class Available options include 0-, 1-, and 2- order Tikhonov and total variation regularization. While the underlying theory is classical, application to unstructured numerical grids via the "seisflows.tools.math.nabla" operator is somewhat complicated. So far, can only be used for 2D inversion, because the required spatial derivative operator "nabla" is not yet available for 3D grids. """ def check(self): """ Checks parameters and paths """ super(tikhonov1, self).check() if 'CREEPING' not in PAR: setattr(PAR, 'CREEPING', False) if not PAR.LAMBDA: raise ValueError def nabla(self, mesh, m, g): if PAR.CREEPING: G, grid = mesh2grid(g, mesh) DG = nabla(G, order=1) dg = grid2mesh(DG, grid, mesh) return -dg / np.mean(m) else: M, grid = mesh2grid(m, mesh) DM = nabla(M, order=1) dm = grid2mesh(DM, grid, mesh) return dm / np.mean(m)
class ChenTromp_base(custom_import('solver', 'specfem3d_legacy')): # model parameters expected by solver solver_parameters = [] solver_parameters += ['A'] solver_parameters += ['C'] solver_parameters += ['N'] solver_parameters += ['L'] solver_parameters += ['F'] solver_parameters += ['Jc'] solver_parameters += ['Js'] solver_parameters += ['Kc'] solver_parameters += ['Ks'] solver_parameters += ['Mc'] solver_parameters += ['Ms'] solver_parameters += ['Gc'] solver_parameters += ['Gs'] solver_parameters += ['Bc'] solver_parameters += ['Bs'] solver_parameters += ['Hc'] solver_parameters += ['Hs'] solver_parameters += ['Dc'] solver_parameters += ['Ds'] solver_parameters += ['Ec'] solver_parameters += ['Es'] def save(self, path, model, prefix='', suffix=''): super(ChenTromp_base, self).save( path, model, prefix, suffix, self.solver_parameters) def export_model(self, path): super(ChenTromp_base, self).export_model( path, self.solver_parameters+['rho'])
class tiger_lg(custom_import('system', 'slurm_lg')): """ Specially designed system interface for tiger.princeton.edu See parent class SLURM_LG for more information """ def check(self): """ Checks parameters and paths """ if 'UUID' not in PAR: setattr(PAR, 'UUID', str(uuid4())) if 'SCRATCH' not in PATH: setattr(PATH, 'SCRATCH', join('/scratch/gpfs', getuser(), 'seisflows', PAR.UUID)) if 'LOCAL' not in PATH: setattr(PATH, 'LOCAL', '') if 'NODESIZE' not in PAR: setattr(PAR, 'NODESIZE', 16) super(tiger_lg, self).check() def submit(self, *args, **kwargs): """ Submits job """ if not exists(PATH.WORKDIR + '/' + 'scratch'): unix.ln(PATH.SCRATCH, PATH.WORKDIR + '/' + 'scratch') super(tiger_lg, self).submit(*args, **kwargs)
class steepest_descent(custom_import('optimize', 'base')): """ Steepest descent method """ restarted = False def check(self): """ Checks parameters, paths, and dependencies """ # line search algorithm if 'LINESEARCH' not in PAR: setattr(PAR, 'LINESEARCH', 'Bracket') super(steepest_descent, self).check() def setup(self): super(steepest_descent, self).setup() def compute_direction(self): g_new = self.load('g_new') if self.precond: p_new = -self.precond(g_new) else: p_new = -g_new self.save('p_new', p_new) savetxt('s_new', self.dot(g_new, p_new)) return p_new def restart(self): pass
class stochastic_saga_2d(custom_import('solver', 'stochastic_saga'), custom_import('solver', 'specfem2d')): """ Adds stochastic optimization (SAGA) machinery to SPECFEM2D """ def get_source_positions(self): """ Read in source positions. Order coincides with self._source_names. """ positions = np.zeros((len(self._source_names), 2)) for isrc, source_name in enumerate(self._source_names): source_file = join(PATH.SPECFEM_DATA, self.source_prefix + '_' + source_name) xs = getpar('xs', source_file, cast=float) zs = getpar('zs', source_file, cast=float) positions[isrc, :] = [xs, zs] return positions
class default(custom_import('postprocess', 'base')): """ Default postprocesing option Provides default image processing and regularization functions for models or gradients """ # currently identical to base class pass
class default(custom_import('preprocess', 'base')): """ Default preprocesing class Provides data processing functions for seismic traces, with options for data misfit, filtering, normalization and muting """ # currently identical to base class pass
class ChenTromp_vti(custom_import('solver', 'ChenTromp_base')): # model parameters included in inversion parameters = [] parameters += ['A'] parameters += ['C'] parameters += ['L'] parameters += ['N'] parameters += ['F']
class Thomsen_vti(custom_import('solver', 'Thomsen_base')): # model parameters included in inversion parameters = [] parameters += ['vp'] parameters += ['vs'] parameters += ['epsilon'] parameters += ['delta'] parameters += ['gamma']
class gauss_newton(custom_import('optimize', 'newton')): """ Implements Gauss-Newton-CG algorithm """ def check(cls): """ Checks parameters and paths """ super(gauss_newton, cls).check() def hessian_product(cls, h): return cls.load('g_lcg') / h
class thrifty_inversion(custom_import('workflow', 'inversion')): """ Thrifty inversion subclass Provides savings over conventional inversion by carrying over forward simulations from line search The results of 'inversion' and 'thrifty_inversion' should be exactly the same """ status=0 def initialize(self): if self.status==0: super(thrifty_inversion, self).initialize() def clean(self): # can forward simulations from line search be carried over? self.update_status() if self.status==1: unix.rm(PATH.GRAD) unix.mv(PATH.FUNC, PATH.GRAD) unix.mkdir(PATH.FUNC) else: super(thrifty_inversion, self).clean() def update_status(self): if PAR.LINESEARCH != 'Backtrack': # only works for backtracking line search self.status=0 elif optimize.iter==PAR.BEGIN or \ optimize.restarted: # even if backtracking line search is chosen, may not work on # first iteration or following a restart self.status=0 elif optimize.iter==PAR.END: # may not work after resuming saved workflow self.status=0 elif PATH.LOCAL: # may not work if using local filesystems self.status=0 else: self.status=1
class saga(custom_import('preprocess', 'base')): """ SAGA data processing class Adds finite sum data misfit functions to base class """ def sum_residuals(self, files, n=1): """ Sums squares of residuals (normalized by 1/N) INPUT FILES - files containing residuals """ total_misfit = 0. for file in files: total_misfit += np.sum(np.loadtxt(file)**2.) return total_misfit / n
class anisotropic(custom_import('solver', 'elastic')): """ Adds elastic inversion machinery """ model_parameters = [] model_parameters += ['c11'] model_parameters += ['c13'] model_parameters += ['c15'] model_parameters += ['c33'] model_parameters += ['c35'] model_parameters += ['c55'] if PAR.MATERIALS == 'ChenTromp2d': from seisflows.plugins.maps import voigt_chentromp_2d as map_forward from seisflows.plugins.maps import chentromp_voigt_2d as map_inverse kernel_parameters = [] kernel_parameters += ['A'] kernel_parameters += ['C'] kernel_parameters += ['N'] kernel_parameters += ['L'] kernel_parameters += ['F'] elif PAR.MATERIALS == 'Voigt2d': from seisflows.plugins.maps import voigt_voigt_2d as map_forward from seisflows.plugins.maps import voigt_voigt_2d as map_inverse kernel_parameters = [] kernel_parameters += ['c11'] kernel_parameters += ['c13'] kernel_parameters += ['c15'] kernel_parameters += ['c33'] kernel_parameters += ['c35'] kernel_parameters += ['c55'] elif PAR.MATERIALS == 'Thomsen2d': from seisflows.plugins.maps import voigt_thomsen_2d as map_forward from seisflows.plugins.maps import thomsen_voigt_2d as map_inverse kernel_parameters = [] kernel_parameters += ['vp'] kernel_parameters += ['vs'] kernel_parameters += ['epsilon'] kernel_parameters += ['delta'] kernel_parameters += ['gamma'] kernel_parameters += ['theta'] else: raise ParameterError(PAR, 'MATERIALS')
class ChenTromp(custom_import('postprocess', 'base')): """ Postprocessing class """ def write_gradient(self, path): """ Writes gradient of objective function """ # check parameters if 'OPTIMIZE' not in PATH: raise ParameterError(PATH, 'OPTIMIZE') # check input arguments if not exists(path): raise Exception() self.combine_kernels(path) self.process_kernels(path) g = solver.merge(solver.load( path +'/'+ 'kernels/sum', suffix='_kernel', verbose=True)) # apply scaling if float(PAR.SCALE) == 1.: pass elif not PAR.SCALE: pass else: g *= PAR.SCALE # write gradient solver.save(PATH.GRAD +'/'+ 'gradient', solver.split(g), suffix='_kernel') savenpy(PATH.OPTIMIZE +'/'+ 'g_new', g) try: for iproc in range(PAR.NPROC): y = g['Gs'][iproc] x = - g['Gc'][iproc] t = 0.5*np.arctan2(y, x) filename = 'proc%06d_%s.bin' % (iproc, 'azimuth') savebin(t, PATH.GRAD +'/'+ filename) except: pass
class NLCG(custom_import('optimize', 'base')): """ Nonlinear conjugate gradient method """ def check(self): """ Checks parameters, paths, and dependencies """ # line search algorithm if 'LINESEARCH' not in PAR: setattr(PAR, 'LINESEARCH', 'Bracket') # NLCG memory if 'NLCGMEM' not in PAR: setattr(PAR, 'NLCGMEM', 3) # NLCG periodic restart interval if 'NLCGMAX' not in PAR: setattr(PAR, 'NLCGMAX', np.inf) # NLCG conjugacy restart threshold if 'NLCGTHRESH' not in PAR: setattr(PAR, 'NLCGTHRESH', np.inf) super(NLCG, self).check() def setup(self): super(NLCG, self).setup() self.NLCG = getattr(optimize, 'NLCG')( path=PATH.OPTIMIZE, maxiter=PAR.NLCGMAX, thresh=PAR.NLCGTHRESH, precond=self.precond) def compute_direction(self): g_new = self.load('g_new') p_new, self.restarted = self.NLCG() self.save('p_new', p_new) def restart(self): super(NLCG, self).restart() self.NLCG.restart()
class LBFGS(custom_import('optimize', 'base')): """ Limited memory BFGS algorithm """ def check(self): """ Checks parameters, paths, and dependencies """ # line search algorithm if 'LINESEARCH' not in PAR: setattr(PAR, 'LINESEARCH', 'Backtrack') # LBFGS memory if 'LBFGSMEM' not in PAR: setattr(PAR, 'LBFGSMEM', 3) # LBFGS periodic restart interval if 'LBFGSMAX' not in PAR: setattr(PAR, 'LBFGSMAX', np.inf) # LBFGS angle restart threshold if 'LBFGSTHRESH' not in PAR: setattr(PAR, 'LBFGSTHRESH', 0.) super(LBFGS, self).check() def setup(self): super(LBFGS, self).setup() self.LBFGS = lib(path=PATH.OPTIMIZE, memory=PAR.LBFGSMEM, maxiter=PAR.LBFGSMAX, thresh=PAR.LBFGSTHRESH, precond=self.precond()) def compute_direction(self): g_new = self.load('g_new') p_new, self.restarted = self.LBFGS() self.save('p_new', p_new) self.savetxt('s_new', self.dot(g_new, p_new)) return p_new def restart(self): super(LBFGS, self).restart() self.LBFGS.restart()
class tikhonov0(custom_import('postprocess', 'regularize')): """ Adds regularization options to base class Available options include 0-, 1-, and 2- order Tikhonov and total variation regularization. While the underlying theory is classical, application to unstructured numerical grids via the "seisflows.tools.math.nabla" operator is somewhat complicated. So far, can only be used for 2D inversion, because the required spatial derivative operator "nabla" is not yet available for 3D grids. """ def check(self): """ Checks parameters and paths """ super(tikhonov0, self).check() if not PAR.LAMBDA: raise ValueError def nabla(self, mesh, m, g): return m / np.mean(m)
class total_variation(custom_import('postprocess', 'regularize')): """ Adds regularization options to base class So far, can only be used for 2D inversion, because the required spatial derivative operator "nabla" is not yet available for 3D grids. """ def check(self): """ Checks parameters and paths """ super(total_variation, self).check() if not PAR.LAMBDA: raise ValueError if not hasattr(PAR, 'EPSILON'): setattr(PAR, 'EPSILON', 0.) def nabla(self, mesh, m, g): M, grid = mesh2grid(g, mesh) DM = tv(M, epsilon=PAR.EPSILON) dm = grid2mesh(DM, grid, mesh) return dm / np.mean(m)
class Thomsen_base(custom_import('solver', 'specfem3d_legacy')): #raise NotImplementedError("Need to fix xsum_kernels utility".) # parameters expected by solver solver_parameters = [] solver_parameters += ['vp'] solver_parameters += ['vs'] solver_parameters += ['epsilon'] solver_parameters += ['delta'] solver_parameters += ['gamma'] solver_parameters += ['theta'] solver_parameters += ['azimuth'] def save(self, path, model, prefix='', suffix=''): super(Thomsen_base, self).save( path, model, prefix, suffix, self.solver_parameters) def export_model(self, path): super(Thomsen_base, self).export_model( path, self.solver_parameters+['rho'])
class multithreaded(custom_import('system', 'multicore')): """ An interface through which to submit workflows, run tasks in serial or parallel, and perform other system functions. By hiding environment details behind a python interface layer, these classes provide a consistent command set across different computing environments. For important additional information, please see http://seisflows.readthedocs.org/en/latest/manual/manual.html#system-configuration """ def check(self): """ Checks parameters and paths """ print(""" DEPRECATION WARNING SYSTEM.MULTITHREADED has been renamed SYSTEM.MULTICORE Please update your parameter file. """) super(multithreaded, self).check()
class steepest_descent(custom_import('optimize', 'base')): """ Steepest descent method """ restarted = False def check(self): """ Checks parameters, paths, and dependencies """ if 'LINESEARCH' not in PAR: setattr(PAR, 'LINESEARCH', 'Bracket') super(steepest_descent, self).check() def setup(self): super(steepest_descent, self).setup() def compute_direction(self): super(steepest_descent, self).compute_direction() def restart(self): # steepest descent never requires restarts pass
class source_encoding_2d(custom_import('solver', 'specfem2d')): def check(self): """ Checks parameters, paths, and dependencies """ super(source_encoding_2d, self).check() def initialize_solver_directories(self): """ Sets up directory in which to run solver """ if 'NT_PADDED' not in PAR: raise Exception super(source_encoding_2d, self).initialize_solver_directories() solvertools.setpar('NSOURCES', PAR.NSRC) solvertools.setpar('nt', PAR.NT_PADDED) def write_receivers(self, coords): """ Writes receivers file """ solvertools.write_receivers(coords, self.cwd) def write_sources(self, coords, stats=[], mapping=lambda i: [i]): """ Writes sources file """ unix.cd(self.cwd) nodes = mapping(system.taskid()) lines = [] for i in nodes: solvertools.write_sources( [coords[0][i], coords[1][i], coords[2][i]], self.cwd, stats['ws'][i]) with open('DATA/SOURCE', 'r') as f: lines.extend(f.readlines()) with open('DATA/SOURCE', 'w') as f: f.writelines(lines)
class lambda_mu_2d(custom_import('solver', 'elastic2d')): """ Adds Lame parameter machinery to SPECFEM2D """ assert PAR.MATERIALS == 'lambda_mu' def export_kernels(self, path): assert self.mesh_properties.nproc == 1 iproc = 0 path = join(path, 'kernels') model_parameters = ['rho', 'vp', 'vs'] kernel_parameters = ['rho', 'kappa', 'mu'] model = getstruct(self.cwd + '/' + 'DATA/', model_parameters, iproc) kernels = getstruct(self.cwd + '/' + 'OUTPUT_FILES/', kernel_parameters, iproc, suffix='_kernel') unix.mkdir(join(path, basename(self.cwd))) self.save(join(path, basename(self.cwd)), map(model, kernels), suffix='_kernel')
class pbs_lg(custom_import('system', 'base')): """ An interface through which to submit workflows, run tasks in serial or parallel, and perform other system functions. By hiding environment details behind a python interface layer, these classes provide a consistent command set across different computing environments. Intermediate files are written to a global scratch path PATH.SCRATCH, which must be accessible to all compute nodes. Optionally, users can provide a local scratch path PATH.LOCAL if each compute node has its own local filesystem. For important additional information, please see http://seisflows.readthedocs.org/en/latest/manual/manual.html#system-configuration """ def check(self): """ Checks parameters and paths """ print msg.Warning_pbs_lg # name of job if 'TITLE' not in PAR: setattr(PAR, 'TITLE', basename(abspath('.'))) # time allocated for entire workflow if 'WALLTIME' not in PAR: setattr(PAR, 'WALLTIME', 30.) # time allocated for each individual task if 'STEPTIME' not in PAR: setattr(PAR, 'STEPTIME', 15.) # number of tasks if 'NTASK' not in PAR: raise ParameterError(PAR, 'NTASK') # number of cores per task if 'NPROC' not in PAR: raise ParameterError(PAR, 'NPROC') # number of cores per node if 'NODESIZE' not in PAR: raise ParameterError(PAR, 'NODESIZE') # optional additional PBS arguments if 'PBSARGS' not in PAR: setattr(PAR, 'PBSARGS', '') # optional environment variable list VAR1=val1,VAR2=val2,... if 'ENVIRONS' not in PAR: setattr(PAR, 'ENVIRONS', '') # level of detail in output messages if 'VERBOSE' not in PAR: setattr(PAR, 'VERBOSE', 1) # where job was submitted if 'WORKDIR' not in PATH: setattr(PATH, 'WORKDIR', abspath('.')) # where output files are written if 'OUTPUT' not in PATH: setattr(PATH, 'OUTPUT', PATH.WORKDIR+'/'+'output') # where temporary files are written if 'SCRATCH' not in PATH: setattr(PATH, 'SCRATCH', PATH.WORKDIR+'/'+'scratch') # where system files are written if 'SYSTEM' not in PATH: setattr(PATH, 'SYSTEM', PATH.SCRATCH+'/'+'system') # optional local scratch path if 'LOCAL' not in PATH: setattr(PATH, 'LOCAL', None) def submit(self, workflow): """ Submits workflow """ # create scratch directories unix.mkdir(PATH.SCRATCH) unix.mkdir(PATH.SYSTEM) # create output directories unix.mkdir(PATH.OUTPUT) unix.mkdir(PATH.WORKDIR+'/'+'output.pbs') self.checkpoint() hours = PAR.WALLTIME/60 minutes = PAR.WALLTIME%60 walltime = 'walltime=%02d:%02d:00 ' % (hours, minutes) ncpus = PAR.NODESIZE mpiprocs = PAR.NODESIZE # prepare qsub arguments call( 'qsub ' + '%s ' % PAR.PBSARGS + '-l select=1:ncpus=%d:mpiprocs=%d ' % (ncpus, mpiprocs) + '-l %s ' % walltime + '-N %s ' % PAR.TITLE + '-j %s '%'oe' + '-o %s ' % (PATH.WORKDIR+'/'+'output.log') + '-V ' + ' -- ' + findpath('seisflows.system') +'/'+ 'wrappers/submit ' + PATH.OUTPUT) def run(self, classname, funcname, hosts='all', **kwargs): """ Runs tasks in serial or parallel on specified hosts. """ self.checkpoint() self.save_kwargs(classname, funcname, kwargs) jobs = self.submit_job_array(classname, funcname, hosts) while True: # wait a few seconds before checking again time.sleep(5) self._timestamp() isdone, jobs = self.job_array_status(classname, funcname, jobs) if isdone: return def mpiexec(self): """ Specifies MPI exectuable; used to invoke solver """ return 'mpiexec ' def getnode(self): """ Gets number of running task """ try: return os.getenv('PBS_ARRAY_INDEX') except: raise Exception("PBS_ARRAY_INDEX environment variable not defined.") ### private methods def submit_job_array(self, classname, funcname, hosts='all'): with open(PATH.SYSTEM+'/'+'job_id', 'w') as f: call(self.job_array_cmd(classname, funcname, hosts), stdout=f) # retrieve job ids with open(PATH.SYSTEM+'/'+'job_id', 'r') as f: line = f.readline() job = line.split()[-1].strip() if hosts == 'all' and PAR.NTASK > 1: nn = range(PAR.NTASK) job0 = job.strip('[].sdb') return [job0+'['+str(ii)+'].sdb' for ii in nn] else: return [job] def job_array_cmd(self, classname, funcname, hosts): nodes = math.ceil(PAR.NTASK/float(PAR.NODESIZE)) ncpus = PAR.NPROC mpiprocs = PAR.NPROC hours = PAR.STEPTIME/60 minutes = PAR.STEPTIME%60 walltime = 'walltime=%02d:%02d:00 '%(hours, minutes) return ('qsub ' + '%s ' % PAR.PBSARGS + '-l select=%d:ncpus=%d:mpiprocs=%d ' (nodes, ncpus, mpiprocs) + '-l %s ' % walltime + '-J 0-%s ' % (PAR.NTASK-1) + '-N %s ' % PAR.TITLE + '-o %s ' % (PATH.WORKDIR+'/'+'output.pbs/' + '$PBS_ARRAYID') + '-r y ' + '-j oe ' + '-V ' + self.job_array_args(hosts) + PATH.OUTPUT + ' ' + classname + ' ' + funcname + ' ' + 'PYTHONPATH='+findpath('seisflows.system'),+',' + PAR.ENVIRONS) def job_array_args(self, hosts): if hosts == 'all': args = ('-J 0-%s ' % (PAR.NTASK-1) +'-o %s ' % (PATH.WORKDIR+'/'+'output.pbs/' + '$PBS_ARRAYID') + ' -- ' + findpath('seisflows.system') +'/'+ 'wrappers/run ') elif hosts == 'head': args = ('-J 0-0 ' +'-o %s ' % (PATH.WORKDIR+'/'+'output.pbs/' + '$PBS_JOBID') + ' -- ' + findpath('seisflows.system') +'/'+ 'wrappers/run ') return args def job_array_status(self, classname, funcname, jobs): """ Determines completion status of one or more jobs """ for job in jobs: state = self._query(job) states = [] if state in ['C']: states += [1] else: states += [0] if state in ['F']: print msg.TaskError_PBS % (classname, funcname, job) sys.exit(-1) isdone = all(states) return isdone, jobs def _query(self, jobid): """ Queries job state from PBS database """ # TODO: replace shell utilities with native Python with open(PATH.SYSTEM+'/'+'job_status', 'w') as f: call('qstat -x -tJ ' + jobid + ' | ' + 'tail -n 1 ' + ' | ' + 'awk \'{print $5}\'', stdout=f) with open(PATH.SYSTEM+'/'+'job_status', 'r') as f: line = f.readline() state = line.strip() return state ### utility function def _timestamp(self): with open(PATH.SYSTEM+'/'+'timestamps', 'a') as f: line = time.strftime('%H:%M:%S')+'\n' f.write(line) def save_kwargs(self, classname, funcname, kwargs): kwargspath = join(PATH.OUTPUT, 'kwargs') kwargsfile = join(kwargspath, classname+'_'+funcname+'.p') unix.mkdir(kwargspath) saveobj(kwargsfile, kwargs)
class specfem3d_globe(custom_import('solver', 'base')): """ Python interface for SPECFEM3D_GLOBE See base class for method descriptions """ if PAR.MATERIALS in ['Isotropic']: parameters = [] parameters += ['vp'] parameters += ['vs'] else: parameters = [] parameters += ['vpv'] parameters += ['vph'] parameters += ['vsv'] parameters += ['vsh'] parameters += ['eta'] def check(self): """ Checks parameters and paths """ super(specfem3d_globe, self).check() if 'CHANNELS' not in PAR: setattr(PAR, 'CHANNELS', 'ENZ') # check data format if 'FORMAT' not in PAR: raise Exception() def generate_data(self, **model_kwargs): """ Generates data """ self.generate_mesh(**model_kwargs) unix.cd(self.cwd) setpar('SIMULATION_TYPE', '1') setpar('SAVE_FORWARD', '.true.') call_solver(system.mpiexec(), 'bin/xspecfem3D') if PAR.FORMAT in ['ASCII', 'ascii']: src = glob('OUTPUT_FILES/*.sem.ascii') dst = 'traces/obs' unix.mv(src, dst) if PAR.SAVETRACES: self.export_traces(PATH.OUTPUT + '/' + 'traces/obs') def generate_mesh(self, model_path=None, model_name=None, model_type='gll'): """ Performs meshing and database generation """ assert (model_name) assert (model_type) self.initialize_solver_directories() unix.cd(self.cwd) if model_type == 'gll': assert (exists(model_path)) self.check_mesh_properties(model_path) unix.cp(glob(model_path + '/' + '*'), self.model_databases) call_solver(system.mpiexec(), 'bin/xmeshfem3D') if self.taskid == 0: self.export_model(PATH.OUTPUT + '/' + model_name) else: raise NotImplementedError ### model input/output def load(self, path, prefix='reg1_', suffix='', verbose=False): """ reads SPECFEM model or kernel Models are stored in Fortran binary format and separated into multiple files according to material parameter and processor rank. """ model = Model(self.parameters) minmax = Minmax(self.parameters) for iproc in range(self.mesh_properties.nproc): # read database files keys, vals = loadbypar(path, self.parameters, iproc, prefix, suffix) for key, val in zip(keys, vals): model[key] += [val] minmax.update(keys, vals) if verbose: minmax.write(path, logpath=PATH.SUBMIT) return model def save(self, path, model, prefix='reg1_', suffix=''): """ writes SPECFEM3D_GLOBE transerverly isotropic model """ unix.mkdir(path) for iproc in range(self.mesh_properties.nproc): for key in ['vpv', 'vph', 'vsv', 'vsh', 'eta']: if key in self.parameters: savebin(model[key][iproc], path, iproc, prefix + key + suffix) elif 'kernel' in suffix: pass else: src = PATH.OUTPUT + '/' + 'model_init' dst = path copybin(src, dst, iproc, prefix + key + suffix) if 'rho' in self.parameters: savebin(model['rho'][iproc], path, iproc, prefix + 'rho' + suffix) elif 'kernel' in suffix: pass else: src = PATH.OUTPUT + '/' + 'model_init' dst = path copybin(src, dst, iproc, prefix + 'rho' + suffix) ### low-level solver interface def forward(self, path='traces/syn'): """ Calls SPECFEM3D_GLOBE forward solver """ solvertools.setpar('SIMULATION_TYPE', '1') solvertools.setpar('SAVE_FORWARD', '.true.') call_solver(system.mpiexec(), 'bin/xspecfem3D') if PAR.FORMAT in ['ASCII', 'ascii']: src = glob('OUTPUT_FILES/*.sem.ascii') dst = path unix.mv(src, dst) def adjoint(self): """ Calls SPECFEM3D_GLOBE adjoint solver """ solvertools.setpar('SIMULATION_TYPE', '3') solvertools.setpar('SAVE_FORWARD', '.false.') unix.rm('SEM') unix.ln('traces/adj', 'SEM') call_solver(system.mpiexec(), 'bin/xspecfem3D') def check_mesh_properties(self, path=None, parameters=None): if not hasattr(self, '_mesh_properties'): if not path: path = PATH.MODEL_INIT if not parameters: parameters = self.parameters nproc = 0 ngll = [] while True: dummy = loadbin(path, nproc, 'reg1_' + parameters[0]) ngll += [len(dummy)] nproc += 1 if not exists('%s/proc%06d_reg1_%s.bin' % (path, nproc, parameters[0])): break self._mesh_properties = Struct([['nproc', nproc], ['ngll', ngll]]) return self._mesh_properties def rename_data(self): """ Works around conflicting data filename conventions """ files = glob(self.cwd + '/' + 'traces/adj/*sem.ascii') unix.rename('sem.ascii', 'sem.ascii.adj', files) def initialize_adjoint_traces(self): super(specfem3d_globe, self).initialize_adjoint_traces() # workaround for SPECFEM2D's use of different name conventions for # regular traces and 'adjoint' traces if PAR.FORMAT in ['ASCII', 'ascii']: files = glob(self.cwd + '/' + 'traces/adj/*sem.ascii') unix.rename('sem.ascii', 'adj', files) @property def data_filenames(self): unix.cd(self.cwd) unix.cd('traces/obs') print 'made it here' if PAR.FORMAT in ['ASCII', 'ascii']: filenames = [] for channel in PAR.CHANNELS: filenames += glob('*.??%s.sem.ascii' % channel) return [filenames] @property def kernel_databases(self): return join(self.cwd, 'OUTPUT_FILES/DATABASES_MPI') @property def model_databases(self): return join(self.cwd, 'OUTPUT_FILES/DATABASES_MPI') @property def source_prefix(self): return 'CMTSOLUTION'
class specfem3d_nz(custom_import('solver', 'base')): """ Python interface for SPECFEM3D See base class for method descriptions """ def check(self): """ Checks parameters and paths """ super(specfem3d_nz, self).check() # check time stepping parameters if 'NT' not in PAR: raise Exception() if 'DT' not in PAR: raise Exception() # check data format if 'FORMAT' not in PAR: raise Exception() # make sure data format is accetapble if PAR.FORMAT not in ['su', 'ascii']: raise Exception() def setup(self): """ Overload of solver.base.setup, removes the need to move data around as Pyatoa takes care of data fetching within eval_func Prepares solver for inversion or migration Sets up directory structure expected by SPECFEM and copies or generates seismic data to be inverted or migrated """ # clean up for new inversion unix.rm(self.cwd) self.initialize_solver_directories() # if synthetic case, create synthetic observations if PAR.CASE == "Synthetic": self.generate_data(model_path=PATH.MODEL_TRUE, model_name='model_true', model_type='gll') # prepare initial model self.generate_mesh(model_path=PATH.MODEL_INIT, model_name='model_init', model_type='gll') def generate_data(self, **model_kwargs): """ Generates data in the synthetic-synthetic comparison case. Not for use in the real-data problem. Differs from specfem3d.nz in that it automatically calls generate mesh for the true model, rather than passing them in as kwargs. """ # Prepare for the forward simulation self.generate_mesh(**model_kwargs) print 'specfem3d_nz.generate data' unix.cd(self.cwd) setpar('SIMULATION_TYPE', '1') setpar('SAVE_FORWARD', '.true.') setpar('ATTENUATION ', '.true.') call_solver(system.mpiexec(), 'bin/xspecfem3D') # seismic unix format if PAR.FORMAT in ['SU', 'su']: src = glob('OUTPUT_FILES/*_d?_SU') dst = 'traces/obs' unix.mv(src, dst) # ascii sem output format elif PAR.FORMAT == "ascii": src = glob('OUTPUT_FILES/*sem?') dst = 'traces/obs' unix.mv(src, dst) if PAR.SAVETRACES: self.export_traces(PATH.OUTPUT + '/' + 'traces/obs') def generate_mesh(self, model_path=None, model_name=None, model_type='gll'): """ Performs meshing and database generation """ print 'specfem3d_nz.generate mesh' assert (model_name) assert (model_type) unix.cd(self.cwd) if model_type in ['gll']: par = getpar('MODEL').strip() if par != 'gll': if self.taskid == 0: print 'WARNING: Unexpected Par_file setting:' print 'MODEL =', par assert (exists(model_path)) self.check_mesh_properties(model_path) src = glob(model_path + '/' + '*') dst = self.model_databases unix.cp(src, dst) call_solver(system.mpiexec(), 'bin/xgenerate_databases') if self.taskid == 0: self.export_model(PATH.OUTPUT + '/' + model_name) else: raise NotImplementedError def eval_fwd(self, path=''): """ Performs forward simulations for misfit function evaluation. Same as solver.base.eval_func without the residual writing. For use in specfem3d_nz where eval_func is taken by Pyatoa. """ print 'specfem3d_nz.eval_fwd' unix.cd(self.cwd) self.import_model(path) self.forward() def eval_func(self, iter='', step=0, suffix=None, *args, **kwargs): """ evaluate the misfit functional using the external package Pyatoa. Pyatoa is written in Python3 so it needs to be called with subprocess :param args: :param kwargs: :return: """ print 'specfem3d_nz.eval_func' load_conda = "module load Anaconda2/5.2.0-GCC-7.1.0;" load_hdf5 = "module load HDF5/1.10.1-GCC-7.1.0;" arguments = " ".join([ "--mode process", "--working_dir {}".format(PATH.WORKDIR), "--current_dir {}".format(self.cwd), "--model_number {}".format("m{:0>2}".format(int(iter) - 1)), "--event_id {}".format(self.source_name), "--step_count {}".format("s{:0>2}".format(step)), "--suffix {}".format(suffix) ]) call_pyatoa = " ".join( [load_conda, load_hdf5, PATH.PYTHON3, PATH.PYATOA_RUN, arguments]) print(call_pyatoa) try: tstart = time.time() stdout = subprocess.check_output(call_pyatoa, shell=True) print '{:.2f}m elapsed'.format((time.time() - tstart) / 60) except subprocess.CalledProcessError as e: print("Pyatoa failed with {}".format(e)) sys.exit(-1) # low-level solver interface def forward(self, path='traces/syn'): """ Calls SPECFEM3D forward solver and then moves files into path """ setpar('SIMULATION_TYPE', '1') setpar('SAVE_FORWARD', '.true.') setpar('ATTENUATION ', '.true.') call_solver(system.mpiexec(), 'bin/xgenerate_databases') call_solver(system.mpiexec(), 'bin/xspecfem3D') # seismic unix output format if PAR.FORMAT in ['SU', 'su']: src = glob('OUTPUT_FILES/*_d?_SU') dst = path unix.mv(src, dst) # ascii sem output format elif PAR.FORMAT == "ascii": src = glob('OUTPUT_FILES/*sem?') dst = path unix.mv(src, dst) def adjoint(self): """ Calls SPECFEM3D adjoint solver """ setpar('SIMULATION_TYPE', '3') setpar('SAVE_FORWARD', '.false.') setpar('ATTENUATION ', '.false.') unix.rm('SEM') unix.ln('traces/adj', 'SEM') call_solver(system.mpiexec(), 'bin/xspecfem3D') # input file writers def check_solver_parameter_files(self): """ Checks solver parameters """ nt = getpar('NSTEP', cast=int) dt = getpar('DT', cast=float) if nt != PAR.NT: if self.taskid == 0: print "WARNING: nt != PAR.NT" setpar('NSTEP', PAR.NT) if dt != PAR.DT: if self.taskid == 0: print "WARNING: dt != PAR.DT" setpar('DT', PAR.DT) if self.mesh_properties.nproc != PAR.NPROC: if self.taskid == 0: print 'Warning: mesh_properties.nproc != PAR.NPROC' if 'MULTIPLES' in PAR: raise NotImplementedError def initialize_adjoint_traces(self): super(specfem3d_nz, self).initialize_adjoint_traces() # workaround for SPECFEM2D's use of different name conventions for # regular traces and 'adjoint' traces if PAR.FORMAT in ['SU', 'su']: files = glob(self.cwd + '/' + 'traces/adj/*SU') unix.rename('_SU', '_SU.adj', files) # workaround for SPECFEM3D's requirement that all components exist, # even ones not in use unix.cd(self.cwd + '/' + 'traces/adj') for iproc in range(PAR.NPROC): for channel in ['x', 'y', 'z']: src = '%d_d%s_SU.adj' % (iproc, PAR.CHANNELS[0]) dst = '%d_d%s_SU.adj' % (iproc, channel) if not exists(dst): unix.cp(src, dst) def rename_data(self): """ Works around conflicting data filename conventions """ if PAR.FORMAT in ['SU', 'su']: files = glob(self.cwd + '/' + 'traces/adj/*SU') unix.rename('_SU', '_SU.adj', files) def write_parameters(self): unix.cd(self.cwd) solvertools.write_parameters(vars(PAR)) def write_receivers(self): unix.cd(self.cwd) key = 'use_existing_STATIONS' val = '.true.' setpar(key, val) _, h = preprocess.load('traces/obs') solvertools.write_receivers(h.nr, h.rx, h.rz) def write_sources(self): unix.cd(self.cwd) _, h = preprocess.load(dir='traces/obs') solvertools.write_sources(vars(PAR), h) # postprocessing wrapper overload def smooth(self, input_path='', output_path='', parameters=[], span_h=0., span_v=0.): """ Smooths kernels by convolving them with a Gaussian. Wrapper over xsmooth_sem utility. smooth() in base.py has the incorrect command line call, specfem requires that NPROC be specified """ if not exists(input_path): raise Exception if not exists(output_path): unix.mkdir(output_path) # apply smoothing operator unix.cd(self.cwd) print 'smoothing parameters ', self.parameters for name in parameters or self.parameters: print 'smoothing', name solver_call = " ".join([ PATH.SPECFEM_BIN + '/' + 'xsmooth_sem', # ./bin/xsmooth_sem str(span_h), # SIGMA_H str(span_v), # SIGMA_V name + '_kernel', # KERNEL_NAME input_path + '/', # INPUT_DIR output_path + '/', # OUTPUT_DIR '.false' # USE_GPU ]) call_solver(system.mpiexec(), solver_call) print '' # rename output files files = glob(output_path + '/*') unix.rename('_smooth', '', files) def combine_vol_data(self, output_path='', quantity=''): """ This does not work Call Specfems executable combine_vol_data_vtk on kernels or model files """ if not exists(output_path): unix.mkdir(output_path) # This should probably be moved to its own function # def import_kernels() unix.cd(self.cwd) src = glob(join(PATH.GRAD, self.source_name, "*{}*".format(quantity))) dst = join(self.cwd, "kernels") unix.mkdir(dst) unix.ln(src=src, dst=dst) solver_call = " ".join([ PATH.SPECFEM_BIN + '/' + 'xcombine_vol_data_vtk', 0, # NPROC_START PAR.NPROC, # NPROC_END quantity, # QUANTITY dst, # DIR_IN dst, # DIR_OUT, we will rename the files first 0 # GPU ACCEL ]) call_solver(system_mpiexec(), solver_call) unix.rm(dst) print '' # miscellaneous @property def data_wildcard(self): channels = PAR.CHANNELS return '*_d[%s]_SU' % channels.lower() @property def data_filenames(self): unix.cd(self.cwd + '/' + 'traces/obs') if PAR.FORMAT in ['SU', 'su']: if not PAR.CHANNELS: return sorted(glob('*_d?_SU')) filenames = [] for channel in PAR.CHANNELS: filenames += sorted(glob('*_d' + channel + '_SU')) return filenames else: raise NotImplementedError @property def kernel_databases(self): return join(self.cwd, 'OUTPUT_FILES/DATABASES_MPI') @property def model_databases(self): return join(self.cwd, 'OUTPUT_FILES/DATABASES_MPI') @property def source_prefix(self): return 'CMTSOLUTION'
class isotropic2d(custom_import('solver', 'specfem2d')): """ Adds isotropic elastic inversion machinery Must supply a model in vp,vs,rho """ def check(self): super(isotropic2d, self).check() if not hasattr(forward, PAR.MATERIALS): raise Exception if not hasattr(reverse, PAR.MATERIALS): raise Exception assert PAR.MATERIALS in [ 'rho_phi_beta', 'rho_kappa_mu' 'rho_lambda_mu', 'rho_alpha_beta' 'phi_beta_gardner', 'kappa_mu_gardner' 'lambda_mu_gardner', 'alpha_beta_gardner', 'phi_beta', 'kappa_mu' 'lambda_mu', 'alpha_beta' ] def __init__(self): # variable density if PAR.MATERIALS == 'rho_phi_beta': self.parameters = [] self.parameters += ['bulk_c'] self.parameters += ['bulk_beta'] self.parameters += ['rho'] if PAR.MATERIALS == 'rho_kappa_mu': self.parameters = [] self.parameters += ['kappa'] self.parameters += ['mu'] self.parameters += ['rho'] if PAR.MATERIALS == 'rho_lambda_mu': self.parameters = [] self.parameters += ['lame1'] self.parameters += ['lame2'] self.parameters += ['rho'] if PAR.MATERIALS == 'rho_alpha_beta': self.parameters = [] self.parameters += ['vp'] self.parameters += ['vs'] self.parameters += ['rho'] # constant density if PAR.MATERIALS == 'phi_beta': self.parameters = [] self.parameters += ['bulk_c'] self.parameters += ['bulk_beta'] if PAR.MATERIALS == 'kappa_mu': self.parameters = [] self.parameters += ['kappa'] self.parameters += ['mu'] if PAR.MATERIALS == 'lambda_mu': self.parameters = [] self.parameters += ['lame1'] self.parameters += ['lame2'] if PAR.MATERIALS == 'alpha_beta': self.parameters = [] self.parameters += ['vp'] self.parameters += ['vs'] # gardner's law density if PAR.MATERIALS == 'phi_beta_gardner': self.parameters = [] self.parameters += ['bulk_c'] self.parameters += ['bulk_beta'] if PAR.MATERIALS == 'kappa_mu_gardner': self.parameters = [] self.parameters += ['kappa'] self.parameters += ['mu'] if PAR.MATERIALS == 'lambda_mu_gardner': self.parameters = [] self.parameters += ['lame1'] self.parameters += ['lame2'] if PAR.MATERIALS == 'alpha_beta_gardner': self.parameters = [] self.parameters += ['vp'] self.parameters += ['vs'] def load(self, path, parameters=['vp', 'vs', 'rho'], prefix='', suffix=''): """ Reads isotropic elastic model """ dict = Container() nproc = self.mesh_properties.nproc for iproc in range(nproc): for key in parameters: key = prefix + key + suffix dict[key] = self.io.read_slice(path, key, iproc) if parameters == ['vp', 'vs', 'rho']: dict.map(self.forward, nproc) return dict def save(self, dict, path, parameters=['vp', 'vs', 'rho'], prefix='', suffix=''): """ Writes isotropic elastic """ nproc = self.mesh_properties.nproc if ['rho'] not in parameters: for iproc in range(nproc): dict['rho'] = self.io.read_slice(PATH.MODEL_INIT, 'rho', iproc) if parameters != ['vp', 'vs', 'rho']: dict.map(self.reverse, nproc) nproc = self.mesh_properties.nproc for key, val in dict.items(): key = prefix + key + suffix self.io.write_slice(val, path, key, iproc) @staticmethod def forward(*args): return getattr(forward, PAR.MATERIALS)(*args) @staticmethod def reverse(*args): return getattr(reverse, PAR.MATERIALS)(*args) def check_mesh_properties(self, path=None, parameters=['vp', 'vs', 'rho']): super(isotropic2d, self).check_mesh_properties(path, parameters)
class slurm_sm(custom_import('system', 'base')): """ An interface through which to submit workflows, run tasks in serial or parallel, and perform other system functions. By hiding environment details behind a python interface layer, these classes provide a consistent command set across different computing environments. Intermediate files are written to a global scratch path PATH.SCRATCH, which must be accessible to all compute nodes. Optionally, users can provide a local scratch path PATH.LOCAL if each compute node has its own local filesystem. For important additional information, please see http://seisflows.readthedocs.org/en/latest/manual/manual.html#system-configuration """ def check(self): """ Checks parameters and paths """ # name of job if 'TITLE' not in PAR: setattr(PAR, 'TITLE', basename(abspath('.'))) # time allocated for entire workflow if 'WALLTIME' not in PAR: setattr(PAR, 'WALLTIME', 30.) # number of tasks if 'NTASK' not in PAR: raise ParameterError(PAR, 'NTASK') # number of cores per task if 'NPROC' not in PAR: raise ParameterError(PAR, 'NPROC') # optional additional SLURM arguments if 'SLURMARGS' not in PAR: setattr(PAR, 'SLURMARGS', '') # optional environment variable list VAR1=val1,VAR2=val2,... if 'ENVIRONS' not in PAR: setattr(PAR, 'ENVIRONS', '') # level of detail in output messages if 'VERBOSE' not in PAR: setattr(PAR, 'VERBOSE', 1) # where job was submitted if 'WORKDIR' not in PATH: setattr(PATH, 'WORKDIR', abspath('.')) # where output files are written if 'OUTPUT' not in PATH: setattr(PATH, 'OUTPUT', PATH.WORKDIR+'/'+'output') # where temporary files are written if 'SCRATCH' not in PATH: setattr(PATH, 'SCRATCH', PATH.WORKDIR+'/'+'scratch') # where system files are written if 'SYSTEM' not in PATH: setattr(PATH, 'SYSTEM', PATH.SCRATCH+'/'+'system') # optional local scratch path if 'LOCAL' not in PATH: setattr(PATH, 'LOCAL', None) def submit(self, workflow): """ Submits workflow """ # create scratch directories unix.mkdir(PATH.SCRATCH) unix.mkdir(PATH.SYSTEM) # create output directories unix.mkdir(PATH.OUTPUT) self.checkpoint() # submit workflow call('sbatch ' + '%s ' % PAR.SLURMARGS + '--job-name=%s '%PAR.TITLE + '--output=%s '%(PATH.WORKDIR +'/'+ 'output.log') + '--cpus-per-task=%d '%PAR.NPROC + '--ntasks=%d '%PAR.NTASK + '--time=%d '%PAR.WALLTIME + findpath('seisflows.system') +'/'+ 'wrappers/submit ' + PATH.OUTPUT) def run(self, classname, funcname, hosts='all', **kwargs): """ Runs tasks in serial or parallel on specified hosts """ self.checkpoint() self.save_kwargs(classname, funcname, kwargs) if hosts == 'all': # run on all available nodes call('srun ' + '--wait=0 ' + join(findpath('seisflows.system'), 'wrappers/run ') + PATH.OUTPUT + ' ' + classname + ' ' + funcname + ' ' + PAR.ENVIRONS) elif hosts == 'head': # run on head node call('srun ' + '--wait=0 ' + '--ntasks=1 ' + '--nodes=1 ' + join(findpath('seisflows.system'), 'wrappers/run ') + PATH.OUTPUT + ' ' + classname + ' ' + funcname + ' ' + PAR.ENVIRONS) else: raise KeyError('Bad keyword argument: system.run: hosts') def hostlist(self): with open(PATH.SYSTEM+'/'+'hostlist', 'w') as f: call('scontrol show hostname $SLURM_JOB_NODEFILE', stdout=f) with open(PATH.SYSTEM+'/'+'hostlist', 'r') as f: return [line.strip() for line in f.readlines()] def getnode(self): """ Gets number of running task """ gid = os.getenv('SLURM_GTIDS').split(',') lid = int(os.getenv('SLURM_LOCALID')) return int(gid[lid]) def mpiexec(self): """ Specifies MPI exectuable; used to invoke solver """ return '' #return 'mpirun -np %d '%PAR.NPROC def save_kwargs(self, classname, funcname, kwargs): kwargspath = join(PATH.OUTPUT, 'kwargs') kwargsfile = join(kwargspath, classname+'_'+funcname+'.p') unix.mkdir(kwargspath) saveobj(kwargsfile, kwargs)
import sys import numpy as np from seisflows.tools import unix from seisflows.tools.tools import exists from seisflows.config import custom_import, ParameterError from seisflows.workflow.base import base PAR = sys.modules['seisflows_parameters'] PATH = sys.modules['seisflows_paths'] solver = sys.modules['seisflows_solver'] postprocess = sys.modules['seisflows_postprocess'] migration = custom_import('workflow','migration')() class test_postprocess(base): """ Postprocessing class """ def check(self): """ Checks parameters and paths """ migration.check() if 'INPUT' not in PATH: setattr(PATH, 'INPUT', None)