def _run_metgrid(self, j_id=None): ''' run metgrid.exe (locally or using slurm script defined in config.json) ''' if len(self.config['options_slurm']['slurm_metgrid.exe']): if j_id: mid = "--dependency=afterok:%d" %j_id metgrid_command = ['sbatch', mid, self.config['options_slurm']['slurm_metgrid.exe']] else: metgrid_command = ['sbatch', self.config['options_slurm']['slurm_metgrid.exe']] utils.check_file_exists(metgrid_command[-1]) utils.silentremove(os.path.join(self.wps_workdir, 'metgrid', 'metgrid.exe')) os.symlink(os.path.join(self.config['filesystem']['wps_dir'],'metgrid','metgrid.exe'), os.path.join(self.wps_workdir, 'metgrid', 'metgrid.exe')) try: res = subprocess.check_output(metgrid_command, cwd=self.wps_workdir, stderr=utils.devnull()) j_id = int(res.split()[-1]) # slurm job-id except subprocess.CalledProcessError: logger.error('Metgrid failed %s:' %metgrid_command) raise # re-raise exception return j_id # return slurm job-id else: metgrid_command = os.path.join(self.config['filesystem']['wps_dir'], 'metgrid', 'metgrid.exe') utils.check_file_exists(metgrid_command) try: subprocess.check_call(metgrid_command, cwd=self.wps_workdir, stdout=utils.devnull(), stderr=utils.devnull()) except subprocess.CalledProcessError: logger.error('Metgrid failed %s:' %metgrid_command) raise # re-raise exception
def _prepare_namelist(self, datestart, dateend): ''' prepare wps namelist ''' # read WPS namelist in WPS work_dir wps_nml = f90nml.read(self.config['options_wps']['namelist.wps']) # get numer of domains ndoms = wps_nml['share']['max_dom'] # check if ndoms is an integer and >0 if not (isinstance(ndoms, int) and ndoms>0): raise ValueError("'domains_max_dom' namelist variable should be an " \ "integer>0") # check if both datestart and dateend are a datetime instance if not all([ isinstance(dt, datetime) for dt in [datestart, dateend] ]): raise TypeError("datestart and dateend must be an instance of datetime") # set new datestart and dateend wps_nml['share']['start_date'] = [datetime.strftime(datestart, '%Y-%m-%d_%H:%M:%S')] * ndoms wps_nml['share']['end_date'] = [datetime.strftime(dateend, '%Y-%m-%d_%H:%M:%S')] * ndoms # write namelist in wps work_dir utils.silentremove(os.path.join( self.config['filesystem']['work_dir'], 'wps', 'namelist.wps')) wps_nml.write(os.path.join( self.config['filesystem']['work_dir'], 'wps', 'namelist.wps'))
def _write_itag(self, wrfout, current_time): ''' Create input file for unipost --------content itag file --------------------------------------- First line is location of wrfout data Second line is required format Third line is the modeltime to process Fourth line is the model identifier (WRF, NMM) ----------------------------------------------------------------- ''' logger.debug('Enter write_itag') logger.debug('Time in itag file is: %s' %current_time) # set itag filename and cleanup filename = os.path.join(config['post_dir'], 'itag') utils.silentremove(filename) # template of itag file template = """{wrfout} netcdf {current_time}:00:00 NCAR """ # context variables in template context = { "wrfout":wrfout, "current_time":current_time } # create the itag file and write content to it based on the template try: with open(filename, 'w') as itag: itag.write(template.format(**context)) except IOError as e: logger.error('Unable to write itag file: %s' %filename) raise # re-raise exception logger.debug('Leave write_itag')
def _initialize(self): ''' Check if archive dir exists, create if not. The archive dir is used to ... ''' utils._create_directory(config['upp_archive_dir']) # create archive dir # create post_dir (remove old one if needed) utils.silentremove(config['post_dir']) utils._create_directory(config['post_dir'])
def _link_vtable(self): ''' link the required Vtable ''' utils.silentremove(os.path.join(self.wps_workdir, 'Vtable')) # TODO: make vtable depend on the boundary source vtable = 'Vtable.GFS' vtable_path = os.path.join(self.config['filesystem']['wps_dir'], 'ungrib', 'Variable_Tables', vtable) os.symlink(vtable_path, os.path.join(self.wps_workdir, 'Vtable'))
def prepare_wrfda_namelist(self, domain): # set domain specific workdir wrfda_workdir = os.path.join(self.wrfda_workdir, "d0" + str(domain)) # read WRFDA namelist wrfda_namelist = os.path.join(self.config['filesystem']['wrfda_dir'], 'var/test/tutorial/namelist.input') wrfda_nml = f90nml.read(wrfda_namelist) # read WRF namelist in WRF work_dir wrf_nml = f90nml.read(os.path.join(self.config['filesystem']['wrf_run_dir'], 'namelist.input')) ## silent remove file if exists ##utils.silentremove(os.path.join(wrfda_workdir, 'fg')) ## create symlink of wrfinput_d0${domain} ##os.symlink(os.path.join(self.rundir, 'wrfinput_d0' + str(domain)), ## os.path.join(wrfda_workdir, 'fg')) # set domain specific information in namelist for var in ['e_we', 'e_sn', 'e_vert', 'dx', 'dy']: # get variable from ${RUNDIR}/namelist.input var_value = wrf_nml['domains'][var] # set domain specific variable in WRDFA_WORKDIR/namelist.input wrfda_nml['domains'][var] = var_value[domain - 1] for var in ['mp_physics', 'ra_lw_physics', 'ra_sw_physics', 'radt', 'sf_sfclay_physics', 'sf_surface_physics', 'bl_pbl_physics', 'cu_physics', 'cudt', 'num_soil_layers']: # get variable from ${RUNDIR}/namelist.input var_value = wrf_nml['physics'][var] # set domain specific variable in WRDFA_WORKDIR/namelist.input try: wrfda_nml['physics'][var] = var_value[domain - 1] except TypeError: wrfda_nml['physics'][var] = var_value obsproc_nml = f90nml.read(os.path.join(self.obs[domain][0], 'namelist.obsproc')) # sync wrfda namelist with obsproc namelist wrfda_nml['wrfvar18']['analysis_date'] = obsproc_nml['record2']['time_analysis'] wrfda_nml['wrfvar21']['time_window_min'] = obsproc_nml['record2']['time_window_min'] wrfda_nml['wrfvar22']['time_window_max'] = obsproc_nml['record2']['time_window_max'] if check_cv5(): wrfda_nml['wrfvar7']['cv_options'] = 5 wrfda_nml['wrfvar6']['max_ext_its'] = 2 wrfda_nml['wrfvar5']['check_max_iv'] = True else: wrfda_nml['wrfvar7']['cv_options'] = 3 tana = utils.return_validate(obsproc_nml['record2']['time_analysis'][:-6]) wrfda_nml['time_control']['start_year'] = tana.year wrfda_nml['time_control']['start_month'] = tana.month wrfda_nml['time_control']['start_day'] = tana.day wrfda_nml['time_control']['start_hour'] = tana.hour wrfda_nml['time_control']['end_year'] = tana.year wrfda_nml['time_control']['end_month'] = tana.month wrfda_nml['time_control']['end_day'] = tana.day wrfda_nml['time_control']['end_hour'] = tana.hour # save changes to wrfda_nml utils.silentremove(os.path.join(wrfda_workdir, 'namelist.input')) wrfda_nml.write(os.path.join(wrfda_workdir, 'namelist.input'))
def __init__(self): config.__init__(self) rundir = self.config['filesystem']['wrf_run_dir'] wpsdir = os.path.join(self.config['filesystem']['work_dir'], 'wps') ## wrf run dir # cleanup old met_em files # create list of files to remove #files = [glob.glob(os.path.join(rundir, ext)) # for ext in ['met_em*']] # flatten list #files_flat = [item for sublist in files for item in sublist] # remove files silently #[ utils.silentremove(filename) for filename in files_flat ] # copy new met_em files # create list of files to copy files = [glob.glob(os.path.join(wpsdir, ext)) for ext in ['met_em*']] # flatten list files_flat = [item for sublist in files for item in sublist] [ shutil.copyfile(filename, os.path.join(rundir, os.path.basename(filename))) for filename in files_flat ] ## wps workdir # create list of files to remove files = [glob.glob(os.path.join(wpsdir, ext)) for ext in ['met_em*', 'FILE*', 'PFILE*', 'GRIBFILE*']] # flatten list files_flat = [item for sublist in files for item in sublist] # remove files silently [ utils.silentremove(filename) for filename in files_flat ]
def create_obsproc_dir(self, workdir): ''' symlink all files required to run obsproc.exe into obsproc workdir ''' # cleanup utils.silentremove(workdir) # create work directory utils._create_directory(workdir) # symlink error files files = ['DIR.txt', 'HEIGHT.txt', 'PRES.txt', 'RH.txt', 'TEMP.txt', 'UV.txt', 'obserr.txt'] for fl in files: os.symlink(os.path.join(self.obsproc_dir, fl), os.path.join(workdir, fl)) # symlink obsproc.exe os.symlink(os.path.join(self.obsproc_dir, 'src', 'obsproc.exe'), os.path.join(workdir, 'obsproc.exe'))
def wrfda_post(self): ''' Move files into WRF run dir after all data assimilation steps have completed ''' # prepare a WRFDA workdirectory for each domain for domain in range(1, self.max_dom+1): # set domain specific workdir wrfda_workdir = os.path.join(self.wrfda_workdir, "d0" + str(domain)) if domain==1: # copy over updated lateral boundary conditions to RUNDIR # only for outer domain utils.silentremove(os.path.join(self.rundir, 'wrfbdy_d01')) shutil.copyfile(os.path.join(wrfda_workdir, 'wrfbdy_d01'), os.path.join(self.rundir, 'wrfbdy_d01')) # copy wrfvar_output_d0${domain} to ${RUNDIR}/wrfinput_d0${domain} utils.silentremove(os.path.join(self.rundir ,'wrfinput_d0' + str(domain))) shutil.copyfile(os.path.join(wrfda_workdir, 'wrfvar_output'), os.path.join(self.rundir, 'wrfinput_d0' + str(domain)))
def obsproc_init(self, datestart): ''' Sync obsproc namelist with WRF namelist.input ''' from shutil import copyfile from datetime import timedelta from datetime import datetime # convert to unique list obslist = list(set(self.obs.values())) # read WRF namelist in WRF work_dir wrf_nml = f90nml.read(os.path.join(self.config['filesystem']['wrf_run_dir'], 'namelist.input')) for obs in obslist: # read obsproc namelist obsproc_nml = f90nml.read(os.path.join(self.obsproc_dir, 'namelist.obsproc.3dvar.wrfvar-tut')) # create obsproc workdir self.create_obsproc_dir(obs[0]) # copy observation in LITTLE_R format to obsproc_dir shutil.copyfile(os.path.join( self.config['filesystem']['obs_dir'], obs[1]), os.path.join(obs[0], obs[1])) # sync obsproc namelist variables with wrf namelist.input obsproc_nml['record1']['obs_gts_filename'] = obs[1] obsproc_nml['record8']['nesti'] = wrf_nml['domains']['i_parent_start'] obsproc_nml['record8']['nestj'] = wrf_nml['domains']['j_parent_start'] obsproc_nml['record8']['nestix'] = wrf_nml['domains']['e_we'] obsproc_nml['record8']['nestjx'] = wrf_nml['domains']['e_sn'] obsproc_nml['record8']['numc'] = wrf_nml['domains']['parent_id'] obsproc_nml['record8']['dis'] = wrf_nml['domains']['dx'] obsproc_nml['record8']['maxnes'] = wrf_nml['domains']['max_dom'] # set time_analysis, time_window_min, time_window_max # check if both datestart and dateend are a datetime instance if not isinstance(datestart, datetime): raise TypeError("datestart must be an instance of datetime") obsproc_nml['record2']['time_analysis'] = datetime.strftime(datestart, '%Y-%m-%d_%H:%M:%S') obsproc_nml['record2']['time_window_min'] = datetime.strftime( datestart - timedelta(minutes=15), '%Y-%m-%d_%H:%M:%S') obsproc_nml['record2']['time_window_max'] = datetime.strftime( datestart + timedelta(minutes=15), '%Y-%m-%d_%H:%M:%S') # save obsproc_nml utils.silentremove(os.path.join(obs[0], 'namelist.obsproc')) obsproc_nml.write(os.path.join(obs[0], 'namelist.obsproc'))
def _clean_boundaries_wps(self): ''' clean old leftover boundary files in WPS directory ''' # create list of files to remove files = [glob.glob(os.path.join(self.wps_workdir, ext)) for ext in ['GRIBFILE.*', 'FILE:', 'PFILE:', 'PRES:']] # flatten list files_flat = [item for sublist in files for item in sublist] # remove files silently [ utils.silentremove(filename) for filename in files_flat ]
def _cleanup_output_files(self): ''' Clean up old output files in post_dir ''' logger.debug('Enter cleanup_output_files') file_ext = [ '*.out', '*.tm00', 'fort.110', 'itag'] files_found = [ f for files in [ glob.glob(os.path.join(config['post_dir'], ext)) for ext in file_ext ] for f in files] # try to remove files, raise exception if needed [ utils.silentremove(fl) for fl in files_found ] logger.debug('Leave cleanup_output_files')
def create_parame(self, parame_type, domain): # set domain specific workdir wrfda_workdir = os.path.join(self.wrfda_workdir, "d0" + str(domain)) filename = os.path.join(wrfda_workdir, 'parame.in') utils.silentremove(filename) # add configuration to parame.in file parame = open(filename, 'w') # open file if parame_type == 'lower': ## start config file lower boundary conditions parame.write("""&control_param da_file = './fg' wrf_input = './wrfinput_d01' wrf_input = '/home/WUR/haren009/sources/WRFV3/run/wrfinput_d01' domain_id = 1 cycling = .true. debug = .true. low_bdy_only = .true. update_lsm = .false. var4d_lbc = .false. iswater = 16 / """) ## end config file lower boundary conditions else: ## start config file lateral boundary conditions parame.write("""&control_param da_file = '/home/haren/model/WRFV3/run2/wrfinput_d01' wrf_bdy_file = './wrfbdy_d01' domain_id = 1 cycling = .true. debug = .true. update_low_bdy = .false. update_lateral_bdy = .true. update_lsm = .false. var4d_lbc = .false. / """) ## end config file lateral boundary conditions parame.close() # close file
def prepare_updatebc_type(self, boundary_type, datestart, domain): # set domain specific workdir wrfda_workdir = os.path.join(self.wrfda_workdir, "d0" + str(domain)) if boundary_type == 'lower' : # define parame.in file self.create_parame(boundary_type, domain) # copy first guess (wrfout in wrfinput format) for WRFDA first_guess = os.path.join(self.rundir, 'wrfvar_input_d0' + str(domain) + '_' + datetime.strftime(datestart, '%Y-%m-%d_%H:%M:%S')) try: shutil.copyfile(first_guess, os.path.join(wrfda_workdir, 'fg')) except Exception: shutil.copyfile(os.path.join(self.rundir, 'wrfinput_d0' + str(domain)), os.path.join(wrfda_workdir, 'fg')) # read parame.in file parame = f90nml.read(os.path.join(wrfda_workdir, 'parame.in')) # set domain in parame.in parame['control_param']['domain_id'] = domain # set wrf_input (IC from WPS and WRF real) parame['control_param']['wrf_input'] = str(os.path.join( self.rundir, 'wrfinput_d0' + str(domain))) # save changes to parame.in file utils.silentremove(os.path.join(wrfda_workdir, 'parame.in')) parame.write(os.path.join(wrfda_workdir, 'parame.in')) elif boundary_type == 'lateral' : # define parame.in file self.create_parame(boundary_type, domain) # read parame.in file parame = f90nml.read(os.path.join(wrfda_workdir, 'parame.in')) # set output from WRFDA parame['control_param']['da_file'] = os.path.join(wrfda_workdir, 'wrfvar_output') # save changes to parame.in file utils.silentremove(os.path.join(wrfda_workdir, 'parame.in')) parame.write(os.path.join(wrfda_workdir, 'parame.in')) else: raise Exception('unknown boundary type')
def _save_namelists(self): ''' write coarse and fine WRF namelist.input to the respective run directories as namelist.forecast ''' # define namelist directories coarse_namelist_dir = os.path.join(self.config['filesystem']['work_dir'], 'wrf_coarse') fine_namelist_dir = os.path.join(self.config['filesystem']['work_dir'], 'wrf_fine') # create directories [utils._create_directory(directory) for directory in [coarse_namelist_dir, fine_namelist_dir]] # remove old files if needed [utils.silentremove(filename) for filename in [ os.path.join(dn, 'namelist.forecast') for dn in [coarse_namelist_dir, fine_namelist_dir]]] # write namelists self.nml_coarse.write(os.path.join(coarse_namelist_dir, 'namelist.forecast')) self.nml_fine.write(os.path.join(fine_namelist_dir, 'namelist.forecast'))
def cleanup_previous_wrf_run(self): from utils import silentremove ''' cleanup initial/boundary conditions and namelist from previous WRF run ''' # remove initial conditions (wrfinput files) for filename in glob.glob( os.path.join(self.config['filesystem']['wrf_run_dir'], 'wrfinput_d*')): silentremove(filename) # remove lateral boundary conditions (wrfbdy_d01) silentremove( os.path.join(self.config['filesystem']['wrf_run_dir'], 'wrfbdy_d01')) silentremove( os.path.join(self.config['filesystem']['wrf_run_dir'], 'namelist.input'))
def __run_job(self, job): success = False try: obj = self.driver.get_object(job["bucket"], job["name"]) except ObjectDoesNotExistError: self.error_message("Could not get file object, skipping: %s/%s" % (job["bucket"], job["name"])) return CONTINUE if job["size"] < 1024 * 10: try: self.debug_message( 110, "%3d: Put complete file %s into queue" % (self.worker_id, job["bucket"] + job["name"]), ) stream = obj.as_stream() content = b"".join(list(stream)) size_of_fetched_object = len(content) if size_of_fetched_object != job["size"]: self.error_message( "prefetched file %s: got %s bytes, not the real size (%s bytes)" % (job["name"], size_of_fetched_object, job["size"]), ) return CONTINUE job["data"] = io.BytesIO(content) job["type"] = TASK_TYPE.DOWNLOADED success = True except LibcloudError: self.error_message("Libcloud error, could not download file") return CONTINUE except Exception: self.error_message("Could not download file") return CONTINUE elif job["size"] < self.options["prefetch_size"]: try: self.debug_message( 110, "%3d: Prefetch file %s" % (self.worker_id, job["bucket"] + job["name"]), ) tmpfilename = self.tmp_dir_path + "/" + str(uuid.uuid4()) obj.download(tmpfilename) job["data"] = None job["tmpfile"] = tmpfilename job["type"] = TASK_TYPE.TEMP_FILE success = True except OSError as e: self.error_message("Could not open temporary file %s" % e.filename) self.abort_message() return FINISH except ObjectDoesNotExistError as e: silentremove(tmpfilename) self.error_message("Could not open object, skipping: %s" % e.object_name) return CONTINUE except LibcloudError: silentremove(tmpfilename) self.error_message( "Error downloading object, skipping: %s/%s" % (job["bucket"], job["name"])) return CONTINUE except Exception: silentremove(tmpfilename) self.error_message( "Error using temporary file for, skipping: %s/%s" % (job["bucket"], job["name"])) return CONTINUE else: try: self.debug_message( 110, "%3d: Prepare file as stream for download %s" % (self.worker_id, job["bucket"] + job["name"]), ) job["data"] = obj job["type"] = TASK_TYPE.STREAM success = True except LibcloudError: self.error_message( "Libcloud error preparing stream object, skipping: %s/%s" % (job["bucket"], job["name"])) return CONTINUE except Exception: self.error_message( "Error preparing stream object, skipping: %s/%s" % (job["bucket"], job["name"])) return CONTINUE if success == True: self.queue_try_put(self.output_queue, job) return CONTINUE