def updatebc_run(self, domain): # set domain specific workdir wrfda_workdir = os.path.join(self.wrfda_workdir, "d0" + str(domain)) # run da_update_bc.exe j_id = None if len(self.config['options_slurm']['slurm_updatebc.exe']): if j_id: mid = "--dependency=afterok:%d" % j_id updatebc_command = [ 'sbatch', mid, self.config['options_slurm']['slurm_updatebc.exe'] ] else: updatebc_command = [ 'sbatch', self.config['options_slurm']['slurm_updatebc.exe'] ] try: res = subprocess.check_output(updatebc_command, cwd=wrfda_workdir, stderr=utils.devnull()) j_id = int(res.split()[-1]) # slurm job-id except subprocess.CalledProcessError: #logger.error('Updatebc failed %s:' % updatebc_command) raise # re-raise exception utils.waitJobToFinish(j_id) else: # run locally subprocess.check_call(os.path.join(wrfda_workdir, 'da_update_bc.exe'), cwd=wrfda_workdir, stdout=utils.devnull(), stderr=utils.devnull())
def _run_metgrid(self, j_id=None): ''' run metgrid.exe (locally or using slurm script defined in config.json) ''' if len(self.config['options_slurm']['slurm_metgrid.exe']): if j_id: mid = "--dependency=afterok:%d" %j_id metgrid_command = ['sbatch', mid, self.config['options_slurm']['slurm_metgrid.exe']] else: metgrid_command = ['sbatch', self.config['options_slurm']['slurm_metgrid.exe']] utils.check_file_exists(metgrid_command[-1]) utils.silentremove(os.path.join(self.wps_workdir, 'metgrid', 'metgrid.exe')) os.symlink(os.path.join(self.config['filesystem']['wps_dir'],'metgrid','metgrid.exe'), os.path.join(self.wps_workdir, 'metgrid', 'metgrid.exe')) try: res = subprocess.check_output(metgrid_command, cwd=self.wps_workdir, stderr=utils.devnull()) j_id = int(res.split()[-1]) # slurm job-id except subprocess.CalledProcessError: #logger.error('Metgrid failed %s:' %metgrid_command) raise # re-raise exception return j_id # return slurm job-id else: metgrid_command = os.path.join(self.config['filesystem']['wps_dir'], 'metgrid', 'metgrid.exe') utils.check_file_exists(metgrid_command) try: subprocess.check_call(metgrid_command, cwd=self.wps_workdir, stdout=utils.devnull(), stderr=utils.devnull()) except subprocess.CalledProcessError: #logger.error('Metgrid failed %s:' %metgrid_command) raise # re-raise exception
def run_wrf(self): ''' run wrf ''' j_id = None if len(self.config['options_slurm']['slurm_wrf.exe']): # run using slurm if j_id: mid = "--dependency=afterok:%d" %j_id wrf_command = ['sbatch', mid, self.config['options_slurm']['slurm_wrf.exe']] else: wrf_command = ['sbatch', self.config['options_slurm']['slurm_wrf.exe']] utils.check_file_exists(wrf_command[-1]) try: res = subprocess.check_output(wrf_command, cwd=self.wrf_run_dir, stderr=utils.devnull()) j_id = int(res.split()[-1]) # slurm job-id except subprocess.CalledProcessError: #logger.error('WRF failed %s:' %wrf_command) raise # re-raise exception utils.waitJobToFinish(j_id) else: # run locally subprocess.check_call(os.path.join(self.wrf_run_dir, 'wrf.exe'), cwd=self.wrf_run_dir, stdout=utils.devnull(), stderr=utils.devnull())
def _run_unipost_step(self, wrfout, current_time, thours): ''' Input variables for the function are: - full path to a wrfout file (regular wrfout naming) - time to run unipost for in format YYYY-MM-DD_HH - thours: TODO add description The following functionality is provided by the function: - validate input parameters - write itag file - run unipost.exe command - rename output - archive output - cleanup output ''' # see if current_time is in wrfout AND validate time format utils.validate_time_wrfout(wrfout, current_time) # extract domain information from wrfout filename domain = int(wrfout[-22:-20]) # write itag file self._write_itag(wrfout, current_time) # run unipost.exe subprocess.check_call(os.path.join( self.config['filesystem']['upp_dir'], 'bin', 'unipost.exe'), cwd=self.post_dir, stdout=utils.devnull(), stderr=utils.devnull()) # rename and archive output self._archive_output(current_time, thours, domain) # cleanup output files self._cleanup_output_files()
def _run_geogrid(self, j_id=None): ''' run geogrid.exe (locally or using slurm script defined in config.json) ''' # get number of domains from wps namelist wps_nml = f90nml.read(self.config['options_wps']['namelist.wps']) ndoms = wps_nml['share']['max_dom'] # check if geo_em files already exist for all domains try: for dom in range(1, ndoms + 1): fname = "geo_em.d{}.nc".format(str(dom).zfill(2)) ncfile = Dataset(os.path.join(self.wps_workdir, fname)) ncfile.close() except IOError: # create geo_em nc files if len(self.config['options_slurm']['slurm_geogrid.exe']): # run using slurm if j_id: mid = "--dependency=afterok:%d" % j_id geogrid_command = [ 'sbatch', mid, self.config['options_slurm']['slurm_geogrid.exe'] ] else: geogrid_command = [ 'sbatch', self.config['options_slurm']['slurm_geogrid.exe'] ] utils.check_file_exists(geogrid_command[1]) utils.silentremove( os.path.join(self.wps_workdir, 'geogrid', 'geogrid.exe')) os.symlink( os.path.join(self.config['filesystem']['wps_dir'], 'geogrid', 'geogrid.exe'), os.path.join(self.wps_workdir, 'geogrid', 'geogrid.exe')) try: res = subprocess.check_output(geogrid_command, cwd=self.wps_workdir, stderr=utils.devnull()) j_id = int(res.split()[-1]) # slurm job-id except subprocess.CalledProcessError: #logger.error('Metgrid failed %s:' %geogrid_command) raise # re-raise exception utils.waitJobToFinish(j_id) else: geogrid_command = os.path.join( self.config['filesystem']['wps_dir'], 'geogrid', 'geogrid.exe') utils.check_file_exists(geogrid_command) try: subprocess.check_call(geogrid_command, cwd=self.wps_workdir, stdout=utils.devnull(), stderr=utils.devnull()) except subprocess.CalledProcessError: #logger.error('Geogrid failed %s:' %geogrid_command) raise # re-raise exception
def _run_ungrib(self, j_id=None): ''' run ungrib.exe (locally or using slurm script defined in config.json) ''' if len(self.config['options_slurm']['slurm_ungrib.exe']): # run using slurm if j_id: mid = "--dependency=afterok:%d" % j_id ungrib_command = [ 'sbatch', mid, self.config['options_slurm']['slurm_ungrib.exe'] ] else: ungrib_command = [ 'sbatch', self.config['options_slurm']['slurm_ungrib.exe'] ] utils.check_file_exists(ungrib_command[-1]) utils.silentremove( os.path.join(self.wps_workdir, 'ungrib', 'ungrib.exe')) if not os.path.isdir(os.path.join(self.wps_workdir, 'ungrib')): utils._create_directory( os.path.join(self.wps_workdir, 'ungrib')) os.symlink( os.path.join(self.config['filesystem']['wps_dir'], 'ungrib', 'ungrib.exe'), os.path.join(self.wps_workdir, 'ungrib', 'ungrib.exe')) try: res = subprocess.check_output(ungrib_command, cwd=self.wps_workdir, stderr=utils.devnull()) j_id = int(res.split()[-1]) # slurm job-id except subprocess.CalledProcessError: #logger.error('Ungrib failed %s:' %ungrib_command) raise # re-raise exception utils.waitJobToFinish(j_id) else: ungrib_command = os.path.join(self.config['filesystem']['wps_dir'], 'ungrib', 'ungrib.exe') utils.check_file_exists(ungrib_command) try: subprocess.check_call(ungrib_command, cwd=self.wps_workdir, stdout=utils.devnull(), stderr=utils.devnull()) except subprocess.CalledProcessError: #logger.error('Ungrib failed %s:' %ungrib_command) raise # re-raise exception
def run_wrf(self, j_id=None): ''' run wrf.exe ''' # check if slurm_wrf.exe is defined if len(self.config['options_slurm']['slurm_wrf.exe']): if j_id: mid = "--dependency=afterok:%d" % j_id wrf_command = [ 'sbatch', mid, self.config['options_slurm']['slurm_wrf.exe'] ] else: wrf_command = [ 'sbatch', self.config['options_slurm']['slurm_wrf.exe'] ] utils.check_file_exists(wrf_command[-1]) utils.silentremove(os.path.join(self.wrf_rundir, 'wrf.exe')) os.symlink( os.path.join(self.config['filesystem']['wrf_dir'], 'main', 'wrf.exe'), os.path.join(self.wrf_rundir, 'wrf.exe')) try: res = subprocess.check_output(wrf_command, cwd=self.wrf_rundir, stderr=utils.devnull()) j_id = int(res.split()[-1]) # slurm job-id except subprocess.CalledProcessError: logger.error('Wrf failed %s:' % wrf_command) raise # re-raise exception utils.waitJobToFinish(j_id) else: # run locally wrf_command = os.path.join(self.config['filesystem']['wrf_dir'], 'main', 'wrf.exe') utils.check_file_exists(wrf_command) try: subprocess.check_call(wrf_command, cwd=self.wrf_rundir, stdout=utils.devnull(), stderr=utils.devnull()) except subprocess.CalledProcessError: logger.error('wrf.exe failed %s:' % wrf_command) raise # re-raise exception
def wrfvar_run(self, domain): ''' run da_wrfvar.exe ''' # set domain specific workdir wrfda_workdir = os.path.join(self.wrfda_workdir, "d0" + str(domain)) logfile = os.path.join(wrfda_workdir, 'log.wrfda_d' + str(domain)) j_id = None if len(self.config['options_slurm']['slurm_wrfvar.exe']): if j_id: mid = "--dependency=afterok:%d" % j_id wrfvar_command = [ 'sbatch', mid, self.config['options_slurm']['slurm_wrfvar.exe'] ] else: wrfvar_command = [ 'sbatch', self.config['options_slurm']['slurm_wrfvar.exe'] ] utils.check_file_exists(wrfvar_command[-1]) try: res = subprocess.check_output(wrfvar_command, cwd=wrfda_workdir, stderr=utils.devnull()) j_id = int(res.split()[-1]) # slurm job-id except subprocess.CalledProcessError: logger.error('Wrfvar failed %s:' % wrfvar_command) raise # re-raise exception while True: time.sleep(1) if not utils.testjob(j_id): break else: # run locally subprocess.check_call( [os.path.join(wrfda_workdir, 'da_wrfvar.exe'), '>&!', logfile], cwd=wrfda_workdir, stdout=utils.devnull(), stderr=utils.devnull())
def obsproc_run(self): ''' run obsproc.exe ''' obslist = list(set(self.obs.values())) obsproc_dir = obslist[0][0] # TODO: check if output is file is created and no errors have occurred j_id = None if len(self.config['options_slurm']['slurm_obsproc.exe']): # run using slurm if j_id: mid = "--dependency=afterok:%d" % j_id obsproc_command = [ 'sbatch', mid, self.config['options_slurm']['slurm_obsproc.exe'] ] else: obsproc_command = [ 'sbatch', self.config['options_slurm']['slurm_obsproc.exe'] ] utils.check_file_exists(obsproc_command[-1]) try: res = subprocess.check_output(obsproc_command, cwd=obsproc_dir, stderr=utils.devnull()) j_id = int(res.split()[-1]) # slurm job-id except subprocess.CalledProcessError: #logger.error('Obsproc failed %s:' % obsproc_command) raise # re-raise exception utils.waitJobToFinish(j_id) else: # run locally subprocess.check_call(os.path.join(obsproc_dir, 'obsproc.exe'), cwd=obsproc_dir, stdout=utils.devnull(), stderr=utils.devnull()) return None