def submit(self, *args, **kwargs): """ Submits job """ # create scratch directories if not exists(PATH.SCRATCH): path = '/scratch/gpfs' + '/' + getuser( ) + '/' + 'seisflows' + '/' + str(uuid4()) unix.mkdir(path) unix.ln(path, PATH.SCRATCH) unix.mkdir(PATH.SYSTEM) # create output directories unix.mkdir(PATH.OUTPUT) unix.mkdir(PATH.WORKDIR + '/' + 'output.slurm') self.checkpoint() if not exists(PATH.SCRATCH): path = '/scratch/gpfs' + '/' + getuser( ) + '/' + 'seisflows' + '/' + str(uuid4()) unix.mkdir(path) unix.ln(path, PATH.SCRATCH) call('sbatch ' + '%s ' % PAR.SLURMARGS + '--job-name=%s ' % PAR.TITLE + '--output=%s ' % (PATH.WORKDIR + '/' + 'output.log') + '--ntasks-per-node=%d ' % 28 + '--gres=gpu:%d ' % 4 + '--nodes=%d ' % 1 + '--time=%d ' % PAR.WALLTIME + pkgpath('seisflows') + '/' + 'system/wrappers/submit ' + PATH.OUTPUT)
def submit(self, *args, **kwargs): """ Submits job """ if not exists(PATH.SUBMIT + '/' + 'scratch'): unix.ln(PATH.SCRATCH, PATH.SUBMIT + '/' + 'scratch') super(tiger_md, self).submit(*args, **kwargs)
def combine(self, path=''): """ combines SPECFEM3D kernels """ unix.cd(self.getpath) # create temporary files and directories dirs = unix.ls(path) with open('kernels_list.txt', 'w') as file: file.write('\n'.join(dirs) + '\n') unix.mkdir('INPUT_KERNELS') unix.mkdir('OUTPUT_SUM') for dir in dirs: src = path + '/' + dir dst = 'INPUT_KERNELS' + '/' + dir unix.ln(src, dst) # sum kernels self.mpirun(PATH.SOLVER_BINARIES + '/' + 'xsum_kernels') unix.mv('OUTPUT_SUM', path + '/' + 'sum') # remove temporary files and directories unix.rm('INPUT_KERNELS') unix.rm('kernels_list.txt') unix.cd(path)
def combine_vol_data(self, output_path='', quantity=''): """ This does not work Call Specfems executable combine_vol_data_vtk on kernels or model files """ if not exists(output_path): unix.mkdir(output_path) # This should probably be moved to its own function # def import_kernels() unix.cd(self.cwd) src = glob(join(PATH.GRAD, self.source_name, "*{}*".format(quantity))) dst = join(self.cwd, "kernels") unix.mkdir(dst) unix.ln(src=src, dst=dst) solver_call = " ".join([ PATH.SPECFEM_BIN + '/' + 'xcombine_vol_data_vtk', 0, # NPROC_START PAR.NPROC, # NPROC_END quantity, # QUANTITY dst, # DIR_IN dst, # DIR_OUT, we will rename the files first 0 # GPU ACCEL ]) call_solver(system_mpiexec(), solver_call) unix.rm(dst) print ''
def submit(self, workflow): """ Submits workflow """ # create scratch directories if not exists(PATH.SCRATCH): path = '/scratch/gpfs'+'/'+getuser()+'/'+'seisflows'+'/'+str(uuid4()) unix.mkdir(path) unix.ln(path, PATH.SCRATCH) unix.mkdir(PATH.SYSTEM) # create output directories unix.mkdir(PATH.OUTPUT) unix.mkdir(PATH.WORKDIR+'/'+'output.slurm') self.checkpoint() # prepare sbatch arguments call('sbatch ' + '%s ' % PAR.SLURMARGS + '--job-name=%s ' % PAR.TITLE + '--output %s ' % (PATH.WORKDIR+'/'+'output.log') + '--ntasks-per-node=28 ' + '--ntasks=28 ' + '--gres=gpu:4 ' + '--nodes=%d ' % 1 + '--time=%d ' % PAR.WALLTIME + findpath('seisflows.system') +'/'+ 'wrappers/submit ' + PATH.OUTPUT)
def adjoint(self): """ Calls SPECFEM3D_GLOBE adjoint solver """ solvertools.setpar('SIMULATION_TYPE', '3') solvertools.setpar('SAVE_FORWARD', '.false.') unix.rm('SEM') unix.ln('traces/adj', 'SEM') call_solver(system.mpiexec(), 'bin/xspecfem3D')
def adjoint(self): """ Calls SPECFEM3D adjoint solver """ setpar('SIMULATION_TYPE', '3') setpar('SAVE_FORWARD', '.false.') unix.rm('SEM') unix.ln('traces/adj', 'SEM') self.mpirun('bin/xspecfem3D')
def adjoint(self): """ Calls SPECFEM3D adjoint solver """ setpar('SIMULATION_TYPE', '3') setpar('SAVE_FORWARD', '.false.') unix.rm('SEM') unix.ln('traces/adj', 'SEM') self.call('bin/xspecfem3D')
def submit(self, *args, **kwargs): """ Submits job """ if not exists(PATH.SCRATCH): path = '/scratch/gpfs'+'/'+getuser()+'/'+'seisflows'+'/'+str(uuid4()) unix.mkdir(path) unix.ln(path, PATH.SCRATCH) super(tiger_lg, self).submit(*args, **kwargs)
def submit(self, *args, **kwargs): """ Submits job """ if not exists(PATH.SCRATCH): path = '/scratch/gpfs' + '/' + getuser( ) + '/' + 'seisflows' + '/' + str(uuid4()) unix.mkdir(path) unix.ln(path, PATH.SCRATCH) super(tiger_sm, self).submit(*args, **kwargs)
def adjoint(self): """ Calls SPECFEM3D adjoint solver """ setpar('SIMULATION_TYPE', '3') setpar('SAVE_FORWARD', '.false.') unix.rm('SEM') unix.ln('traces/adj', 'SEM') call_solver(system.mpiexec(), 'bin/xspecfem3D') # work around SPECFEM3D conflicting name conventions self.rename_data()
def submit(self, workflow): """ Submits workflow to maui_ancil cluster This needs to be run on maui_ancil because maui does not have the ability to run the command "sacct" """ # create scratch directories unix.mkdir(PATH.SCRATCH) unix.mkdir(PATH.SYSTEM) # create output directories unix.mkdir(PATH.OUTPUT) unix.mkdir(PATH.WORKDIR + '/' + 'output.slurm') if not exists('./scratch'): unix.ln(PATH.SCRATCH, PATH.WORKDIR + '/' + 'scratch') # if resuming, rename the old log files so they don't get overwritten output_log = os.path.join(PATH.WORKDIR, 'output.log') error_log = os.path.join(PATH.WORKDIR, 'error.log') for log in [output_log, error_log]: log_prior = log + '_prior' log_temp = log + '_temp' if os.path.exists(log): # If a prior log exists, move to temp file and then rewrite # with new log file if os.path.exists(log_prior): os.rename(log_prior, log_temp) with open(log_prior, 'w') as f_out: for fid in [log_temp, log]: with open(fid) as f_in: f_out.write(f_in.read()) unix.rm(log_temp) else: os.rename(log, log_prior) workflow.checkpoint() # Submit to maui_ancil call(" ".join([ 'sbatch', '%s' % PAR.SLURMARGS, '--account=%s' % PAR.ACCOUNT, '--clusters=%s' % PAR.ANCIL_CLUSTER, '--partition=%s' % PAR.ANCIL_PARTITION, '--job-name=%s' % 'M_' + PAR.TITLE, '--output=%s' % output_log, '--error=%s' % error_log, '--ntasks=%d' % 1, '--cpus-per-task=%d' % 1, '--time=%d' % PAR.WALLTIME, findpath('seisflows.system') + '/' + 'wrappers/submit ', PATH.OUTPUT ]))
def adjoint_att(self): """ Calls SPECFEM2D adjoint solver """ unix.rm('SEM') unix.ln('traces/adj_att', 'SEM') # hack to deal with different SPECFEM2D name conventions for # regular traces and 'adjoint' traces if PAR.FORMAT in ['SU', 'su']: files = glob('traces/adj_att/*.su') unix.rename('.su', '.su.adj', files) call_solver(system.mpiexec(), 'bin/xspecfem2D')
def adjoint(self): """ Calls SPECFEM2D adjoint solver """ setpar('SIMULATION_TYPE', '3') setpar('SAVE_FORWARD', '.false.') unix.rm('SEM') unix.ln('traces/adj', 'SEM') # hack to deal with different SPECFEM2D name conventions for # regular traces and 'adjoint' traces if PAR.FORMAT in ['SU', 'su']: files = glob('traces/adj/*.su') unix.rename('.su', '.su.adj', files) call_solver(system.mpiexec(), 'bin/xspecfem2D')
def adjoint(self): """ Calls SPECFEM2D adjoint solver """ setpar('SIMULATION_TYPE', '3') setpar('SAVE_FORWARD', '.false.') unix.rm('SEM') unix.ln('traces/adj', 'SEM') # hack to deal with different SPECFEM2D name conventions for # regular traces and 'adjoint' traces if PAR.FORMAT in ['SU', 'su']: files = glob('traces/adj/*.su') unix.rename('.su', '.su.adj', files) call_solver(system.mpiexec(), 'bin/xmeshfem2D') call_solver(system.mpiexec(), 'bin/xspecfem2D')
def submit(self, workflow): """ Submits workflow """ unix.cd(PATH.WORKDIR) if not exists('./scratch'): unix.ln(PATH.SCRATCH, PATH.WORKDIR + '/' + 'scratch') unix.mkdir(PATH.OUTPUT) workflow.checkpoint() # prepare sbatch arguments call('sbatch ' + '%s ' % PAR.SLURMARGS + '--partition=%s ' % PAR.PARTITION + '--job-name=%s ' % PAR.TITLE + '--output %s ' % (PATH.WORKDIR + '/' + 'output.log') + '--cpus-per-task=%d ' % PAR.NPROC + '--ntasks=%d ' % PAR.NTASK + '--time=%d ' % PAR.WALLTIME + findpath('seisflows.system') + '/' + 'wrappers/submit ' + PATH.OUTPUT)
def submit(self, workflow): """ Submits workflow """ unix.mkdir(PATH.OUTPUT) unix.cd(PATH.OUTPUT) self.checkpoint() if not exists(PATH.SUBMIT + '/' + 'scratch'): unix.ln(PATH.SCRATCH, PATH.SUBMIT + '/' + 'scratch') call('sbatch ' + '--job-name=%s ' % PAR.SUBTITLE + '--output=%s ' % (PATH.SUBMIT +'/'+ 'output.log') + '--nodes 1 ' + '--ntasks=% ' % PAR.NGPU + '--ntasks-per-socket=%d ' % PAR.NGPU + '--gres=gpu:%d ' % PAR.NGPU + '--time=%d ' % PAR.WALLTIME + findpath('seisflows.system') +'/'+ 'wrappers/submit ' + PATH.OUTPUT)
def combine(self, path=''): """ combines SPECFEM3D_GLOBE kernels """ dirs = unix.ls(path) # initialize kernels unix.cd(path) for key in self.model_parameters: if key not in self.inversion_parameters: for i in range(PAR.NPROC): proc = '%06d' % i name = self.kernel_map[key] src = PATH.GLOBAL + '/' + 'mesh' + '/' + key + '/' + proc dst = path + '/' + 'sum' + '/' + 'proc' + proc + '_' + name + '.bin' savebin(np.load(src), dst) # create temporary files and directories unix.cd(self.getpath) with open('kernels_list.txt', 'w') as file: file.write('\n'.join(dirs) + '\n') unix.mkdir('INPUT_KERNELS') unix.mkdir('OUTPUT_SUM') for dir in dirs: src = path + '/' + dir dst = 'INPUT_KERNELS' + '/' + dir unix.ln(src, dst) # sum kernels self.mpirun(PATH.SOLVER_BINARIES + '/' + 'xsum_kernels') unix.mv('OUTPUT_SUM', path + '/' + 'sum') # remove temporary files and directories unix.rm('INPUT_KERNELS') unix.rm('kernels_list.txt') unix.cd(path)
def submit(self, workflow): """ Submits workflow """ # create scratch directories unix.mkdir(PATH.SCRATCH) unix.mkdir(PATH.SYSTEM) # create output directories unix.mkdir(PATH.OUTPUT) unix.mkdir(PATH.WORKDIR + '/' + 'output.slurm') if not exists('./scratch'): unix.ln(PATH.SCRATCH, PATH.WORKDIR + '/' + 'scratch') workflow.checkpoint() # prepare sbatch arguments call('sbatch ' + '%s ' % PAR.SLURMARGS + '--partition=%s ' % 't1small' # overloads SLURMARGS + '--job-name=%s ' % PAR.TITLE + '--output %s ' % (PATH.WORKDIR + '/' + 'output.log') + '--ntasks-per-node=%d ' % PAR.NODESIZE + '--nodes=%d ' % 1 + '--time=%d ' % PAR.WALLTIME + findpath('seisflows.system') + '/' + 'wrappers/submit ' + PATH.OUTPUT)
def prepare_model(self): model = PATH.OUTPUT +'/'+ 'model_init' assert exists(model) unix.ln(model, PATH.SCRATCH +'/'+ 'model')
def prepare_model(self): model = PATH.OUTPUT + '/' + 'model_init' assert exists(model) unix.ln(model, PATH.SCRATCH + '/' + 'model')
def submit(self, *args, **kwargs): """Submits job """ unix.ln(PATH.GLOBAL, PATH.SUBMIT + '/' + 'scratch') super(tiger_lg, self).submit(*args, **kwargs)
def submit(self, *args, **kwargs): """ Submits job """ if not exists(PATH.WORKDIR + '/' + 'scratch'): unix.ln(PATH.SCRATCH, PATH.WORKDIR + '/' + 'scratch') super(tigercpu_lg, self).submit(*args, **kwargs)