def make_namelist_and_inflate(datein, prevdate): # Determine if we need to copy in the previous inflation # values. If so, do it and unzip them. If not, just # write the namelist. # Use inflate_start to find the times we don't need # Make a list of all assim times #start = datetime.strptime(date_start,'%Y%m%d%H') #assim_dt = timedelta(minutes=int(fct_len)) start = 0 assim_dt = int(cycle_len) # Start on the first actual assimilation time curdate = start + (assim_dt * (assim_start)) no_adaptive_inf_dates = [] step = 1 # WRF dart param tells us which assimilation step to start using inflation while step < inflate_start: no_adaptive_inf_dates.append(curdate) curdate = curdate + assim_dt step = step + 1 print(no_adaptive_inf_dates) # Now check to see if we are in the list of no inflation dates if datein not in no_adaptive_inf_dates: print("Using adaptive inflation values from previous time") nmld['filter_nml']['inf_initial_from_restart'] = [True, True] nmld['filter_nml']['inf_sd_initial_from_restart'] = [True, True] prior_inf_mean = os.path.join(dir_longsave, '{:06d}_inf_ic_mean.nc'.format(prevdate)) prior_inf_sd = os.path.join(dir_longsave, '{:06d}_inf_ic_sd.nc'.format(prevdate)) if os.path.exists(prior_inf_mean): os.system('cp {:s} {:s}/prior_inf_ic_old_mean.nc'.format(prior_inf_mean, dir_assim)) os.system('cp {:s} {:s}/prior_inf_ic_old_sd.nc'.format(prior_inf_sd, dir_assim)) else: error_handler('Could not find {:s}'.format(prior_inf_file),'submit_filter') else: nmld['filter_nml']['inf_initial_from_restart'] = [False, False] nmld['filter_nml']['inf_sd_initial_from_restart'] = [False, False] print("Using initial values for inflation mean and std") # now figure out the "date" we are at based on the CM1 namelist cm1date = cm1nmld['param11'] start_date = datetime(cm1date['year'], cm1date['month'], cm1date['day'], cm1date['hour'],\ cm1date['minute'], cm1date['second']) # Make sure num members is right nmld['filter_nml']['ens_size'] = Ne # Check to see if we need to copy in the sampling error correction if nmld['assim_tools_nml']['sampling_error_correction']: os.system('cp -f {:s}/../../../system_simulation/final_full_precomputed_tables/final_full.{:d} {:s}/final_full.{:d}'.format(dir_src_dart, Ne, dir_assim, Ne)) # Write the namelist write_dart_namelist(nmld, date=start_date + timedelta(seconds=datein)) os.system('cp input.nml {:s}/input.nml'.format(dir_assim))
def make_namelist_and_inflate(datein, prevdate): # Determine if we need to copy in the previous inflation # values. If so, do it and unzip them. If not, just # write the namelist. # Use inflate_start to find the times we don't need # Make a list of all assim times #start = datetime.strptime(date_start,'%Y%m%d%H') #assim_dt = timedelta(minutes=int(fct_len)) start = 0 assim_dt = int(cycle_len) # Start on the first actual assimilation time curdate = start + (assim_dt * (assim_start)) no_adaptive_inf_dates = [] step = 1 # WRF dart param tells us which assimilation step to start using inflation while step < inflate_start: no_adaptive_inf_dates.append(curdate) curdate = curdate + assim_dt step = step + 1 print(no_adaptive_inf_dates) # Now check to see if we are in the list of no inflation dates if datein not in no_adaptive_inf_dates: print("Using adaptive inflation values from previous time") nmld['filter_nml']['inf_initial_from_restart'] = [True, True] nmld['filter_nml']['inf_sd_initial_from_restart'] = [True, True] prior_inf_mean = os.path.join(dir_longsave, '{:06d}_inf_ic_mean.nc'.format(prevdate)) prior_inf_sd = os.path.join(dir_longsave, '{:06d}_inf_ic_sd.nc'.format(prevdate)) if os.path.exists(prior_inf_mean): os.system('cp {:s} {:s}/prior_inf_ic_old_mean.nc'.format( prior_inf_mean, dir_assim)) os.system('cp {:s} {:s}/prior_inf_ic_old_sd.nc'.format( prior_inf_sd, dir_assim)) else: error_handler('Could not find {:s}'.format(prior_inf_file), 'submit_filter') else: nmld['filter_nml']['inf_initial_from_restart'] = [False, False] nmld['filter_nml']['inf_sd_initial_from_restart'] = [False, False] print("Using initial values for inflation mean and std") # now figure out the "date" we are at based on the CM1 namelist cm1date = cm1nmld['param11'] start_date = datetime(cm1date['year'], cm1date['month'], cm1date['day'], cm1date['hour'],\ cm1date['minute'], cm1date['second']) # Make sure num members is write nmld['filter_nml']['ens_size'] = Ne write_dart_namelist(nmld, date=start_date + timedelta(seconds=datein)) os.system('cp input.nml {:s}/input.nml'.format(dir_assim))
def run_dart_to_cm1(mem, start): """ If starting post-DART, run dart_to_wrf to populate the wrfinput file. """ # Refresh with new WRF_dart_param variables #from WRF_dart_param import * print "Running dart_to_wrf" # We know the member number, so mv the new dart state vector # for that member number into the working directory if os.path.exists('%s/wrfdart/filter_ic_new.%04d' % (dir_wrf_dom, mem)): os.system('cp %s/wrfdart/filter_ic_new.%04d dart_wrf_vector' % (dir_wrf_dom, mem)) else: error_handler('Could not find update ic file wrfdart/filter_ic_new.%04d' % mem, \ 'run_dart_to_wrf') # Write a new dart namelist for this time nml_good = write_dart_namelist(mem) if not nml_good: error_handler('Trouble writing input.nml for member %d' % mem, 'run_dart_to_wrf') # Loop through each domain to be sure that wrfinput files are present for dn in range(max_dom + 1)[1:]: os.system('cp %s/archive_bdy/%s_wrfinput_d01 wrfinput_d01' % (dir_wrf_dom, start.strftime('%Y%m%d%H'))) if not os.path.exists('wrfinput_d%02d' % dn): error_handler('Could not find wrfinput_d%02d for mem %d' % (dn,mem), \ 'run_dart_to_wrf') # Link in the dart_to_wrf executable os.system('ln -sf %s/dart_to_wrf .' % dir_src_dart) # Run the executable for the current member if os.path.exists('dtw.out'): os.system('rm dtw.out') os.system('./dart_to_wrf >> dtw.out') # Check for errors in the dart_to_wrf_sequence darterror = False logfile = open('dtw.out', 'r') for line in logfile: if re.search('error', str(line)): darterror = True # Error out if errors found if darterror: error_handler('Error creating wrfinput files from filter_ic_new.%04d' % (mem), \ 'run_dart_to_wrf') else: # If no errors, just clean up the directory os.system('rm -f dart_wrf_vector') os.system('rm -f dart_to_wrf') os.system('rm input.nml')
def run_dart_to_cm1(mem,start): """ If starting post-DART, run dart_to_wrf to populate the wrfinput file. """ # Refresh with new WRF_dart_param variables #from WRF_dart_param import * print "Running dart_to_wrf" # We know the member number, so mv the new dart state vector # for that member number into the working directory if os.path.exists('%s/wrfdart/filter_ic_new.%04d' % (dir_wrf_dom, mem)): os.system('cp %s/wrfdart/filter_ic_new.%04d dart_wrf_vector' % (dir_wrf_dom, mem)) else: error_handler('Could not find update ic file wrfdart/filter_ic_new.%04d' % mem, \ 'run_dart_to_wrf') # Write a new dart namelist for this time nml_good = write_dart_namelist(mem) if not nml_good: error_handler('Trouble writing input.nml for member %d' % mem, 'run_dart_to_wrf') # Loop through each domain to be sure that wrfinput files are present for dn in range(max_dom+1)[1:]: os.system('cp %s/archive_bdy/%s_wrfinput_d01 wrfinput_d01' % (dir_wrf_dom,start.strftime('%Y%m%d%H'))) if not os.path.exists('wrfinput_d%02d' % dn): error_handler('Could not find wrfinput_d%02d for mem %d' % (dn,mem), \ 'run_dart_to_wrf') # Link in the dart_to_wrf executable os.system('ln -sf %s/dart_to_wrf .' % dir_src_dart) # Run the executable for the current member if os.path.exists('dtw.out'): os.system('rm dtw.out') os.system('./dart_to_wrf >> dtw.out') # Check for errors in the dart_to_wrf_sequence darterror = False logfile = open('dtw.out','r') for line in logfile: if re.search('error',str(line)): darterror = True # Error out if errors found if darterror: error_handler('Error creating wrfinput files from filter_ic_new.%04d' % (mem), \ 'run_dart_to_wrf') else: # If no errors, just clean up the directory os.system('rm -f dart_wrf_vector') os.system('rm -f dart_to_wrf') os.system('rm input.nml')
def run_cm1_to_dart(mem, end): """ Run wrf_to_dart to prepare for assimilation""" print "Beginning wrf_to_dart." # Check to be sure that the date in wrfinput_d01 matches the end time for dn in range(max_dom + 1)[1:]: # Use ncdump to get the time from the wrfinput_d?? file ncout = os.popen('ncdump -v Times wrfinput_d%02d' % dn) timeline = ncout.readlines()[-2] wrfin_timestring = re.search('"(\d{4}-\d{2}-\d{2}_\d{2}:\d{2}:\d{2})"', timeline).groups()[0] wrfin_time = datetime.strptime(wrfin_timestring, '%Y-%m-%d_%H:%M:%S') # Make sure the wrfinput file is at the right time if wrfin_time != end: error_handler('wrfinput_d%02d time (%s) does not match assim time (%s) for mem %d' \ % (dn,wrfin_time.strftime('%Y%m%d%H'), end.strftime('%Y%m%d%H'), mem), \ 'run_wrf_to_dart') # Write a new dart namelist for this time nml_good = write_dart_namelist(mem) if not nml_good: error_handler('Trouble writing input.nml for member %d' % mem, 'run_wrf_to_dart') # Link in the dart_to_wrf executable os.system('ln -sf %s/wrf_to_dart .' % dir_src_dart) # Run the executable for the current member if os.path.exists('wtd.out'): os.system('rm wtd.out') os.system('./wrf_to_dart >> wtd.out') # Check for errors in the wrf_to_dart sequence darterror = False logfile = open('wtd.out', 'r') for line in logfile: if re.search('error', str(line)): darterror = True # Error out if errors found if darterror or not os.path.exists('dart_wrf_vector'): error_handler('Error creating dart_wrf_vector from wrfinput on mem %d' % (mem), \ 'run_wrf_to_dart') else: # If no errors, just clean up the directory os.system('mv -f dart_wrf_vector %s/wrfdart/filter_ic_old.%04d' % (dir_wrf_dom, mem)) os.system('rm -f wrf_to_dart') os.system('rm input.nml')
def run_cm1_to_dart(mem,end): """ Run wrf_to_dart to prepare for assimilation""" print "Beginning wrf_to_dart." # Check to be sure that the date in wrfinput_d01 matches the end time for dn in range(max_dom+1)[1:]: # Use ncdump to get the time from the wrfinput_d?? file ncout = os.popen('ncdump -v Times wrfinput_d%02d' % dn) timeline = ncout.readlines()[-2] wrfin_timestring = re.search('"(\d{4}-\d{2}-\d{2}_\d{2}:\d{2}:\d{2})"',timeline).groups()[0] wrfin_time = datetime.strptime(wrfin_timestring,'%Y-%m-%d_%H:%M:%S') # Make sure the wrfinput file is at the right time if wrfin_time != end: error_handler('wrfinput_d%02d time (%s) does not match assim time (%s) for mem %d' \ % (dn,wrfin_time.strftime('%Y%m%d%H'), end.strftime('%Y%m%d%H'), mem), \ 'run_wrf_to_dart') # Write a new dart namelist for this time nml_good = write_dart_namelist(mem) if not nml_good: error_handler('Trouble writing input.nml for member %d' % mem, 'run_wrf_to_dart') # Link in the dart_to_wrf executable os.system('ln -sf %s/wrf_to_dart .' % dir_src_dart) # Run the executable for the current member if os.path.exists('wtd.out'): os.system('rm wtd.out') os.system('./wrf_to_dart >> wtd.out') # Check for errors in the wrf_to_dart sequence darterror = False logfile = open('wtd.out','r') for line in logfile: if re.search('error',str(line)): darterror = True # Error out if errors found if darterror or not os.path.exists('dart_wrf_vector'): error_handler('Error creating dart_wrf_vector from wrfinput on mem %d' % (mem), \ 'run_wrf_to_dart') else: # If no errors, just clean up the directory os.system('mv -f dart_wrf_vector %s/wrfdart/filter_ic_old.%04d' % (dir_wrf_dom,mem)) os.system('rm -f wrf_to_dart') os.system('rm input.nml')
def post_model_cleanup(mem,start,end,fcst_end): """ Function to move output files to appropriate places, handle calculating tendency if desired and process auxilliary outputs """ print "Beginning post-model cleanup" # First, verify that CM1 finished correctly if start != 0: # Find the second restart name rst_files = [f for f in os.listdir('.') if\ f.startswith('cm1out_rst') and f.endswith('.nc')] rst_files.sort() # It's the seocnd file restart_name = rst_files[1] if not os.path.exists(restart_name): error_handler('Restart file {:s} not found for member {:d}'.format(restart_name, mem),'post_model_cleanup') # Check to be sure this is at the right time with Dataset(restart_name,'r') as rstfile: rst_time = rstfile.variables['time'][0] if rst_time != end: error_handler('Restart file {:s} not at correct time {:d} min. for member {:d}'.format(restart_name, int(end), mem),'post_model_cleanup') # Also check for out file if not os.path.exists('cm1out.nc'): error_handler('Output file cm1out.nc not found for member {:d}'.format(mem), 'post_model_cleanup') print "CM1 restart file found. Success!" # Check the rsl.error files to be doubly sure # INSERT CODE HERE # Now run through a litany of possible post-processing things to be done # Check if we're computing tendency if flag_compute_tendency: print "Computing altimeter tendency" if not os.path.exists('wrf_tendency'): os.system('ln -sf %s/wrf_tendency .' % dir_utils) # Write a new dart namelist for this time nml_good = write_dart_namelist(mem) if not nml_good: error_handler('Trouble writing input.nml for member %d' % mem, 'post_model_cleanup') for dn in range(int(max_dom)+1)[1:]: # Don't overwrite the wrfinput_d01 file if dn > 1: os.system('cp wrfinput_d01 wrfinput_d01_orig') os.system('rm wrfinput_d01') os.system('ln -sf wrfinput_d%02d wrfinput_d01' % dn) os.system('ln -sf wrfout_d%02d_%s wrfout_d01' % (dn, end.strftime('%Y-%m-%d_%H:%M:%S'))) os.system('./wrf_tendency') os.system('rm wrfout_d01 wrf_tendency') if dn > 1: os.system('rm wrfinput_d01') os.system('mv wrfinput_d01_orig wrfinput_d01') # Actual file turnaround goes here print "Copying over files for restart." # Save the previous file in case something goes wrong if start != 0: os.system('mv cm1out_rst_000001.nc prev_cm1out_rst_000001.nc') # THIS IS WHERE THE ACTUAL SWITCHOVER GOES print("Copying over restart:", restart_name) os.system('cp {:s} cm1out_rst_000001.nc'.format(restart_name)) # Now remove all other restart files rst_file = [f for f in os.listdir('.') if 'cm1out_rst' in f and not f.endswith('000001.nc')] for f in rst_file: os.system('rm -f {:s}'.format(f)) # Check to see if we keep the regular out files if not flag_keep_outs: print "Removing old out files." os.system('rm -f cm1out.nc') # Archive the forecast if fcst_len != 0 if fcst_len: os.system('mv cm1out.nc cm1out_m{:d}_{:06d}.nc'.format(mem,start))
def build_obs_structure(intime, rst_file, gridspace=16): """ Function to specify what variables we want and how dense they should be """ #from make_namelist_dart import set_namelist_sectors, write_namelist #from write_cm1_namelist import set_namelist_defaults from namelist_utils import read_namelist, write_dart_namelist # Generate the DART namelist structure so we # can query (and modify) it dartnml = read_namelist('input.nml') cm1nml = read_namelist('namelist.input') # Build the epoch from the cm1 namelist param11 = cm1nml['param11'] startdate = datetime(param11['year'], param11['month'],\ param11['day'], param11['hour'],\ param11['minute'], param11['second']) # Parse out the observations to assimilate use_obs = dartnml['obs_kind_nml']['assimilate_these_obs_types'] # I'm just testing this for now #use_obs = use_obs[0:2] use_obs = ['LAND_SFC_PRESSURE'] print(use_obs) # Put time as datetime intime = startdate + timedelta(seconds=intime) epoch = datetime(1601,1,1,0) print("Indate:", intime) # Figure out the grid structure dx = cm1nml['param1']['dx'] dy = cm1nml['param1']['dy'] nx = cm1nml['param0']['nx'] ny = cm1nml['param0']['ny'] # Now figure out how many obs we'll need gridspace *= 1000 obx = range(int(dx/2.),int(nx*dx),gridspace) oby = range(int(dy/2.),int(ny*dx),gridspace) #obx = [16000] #oby = [16000] total_obs = len(obx)*len(oby)*len(use_obs) print("Total number of obs:", total_obs) # Write a new dart namelist for the current time delt = intime - epoch #dartnml['perfect_model_obs_nml']['first_obs_days'] = str(delt.days) #dartnml['perfect_model_obs_nml']['first_obs_seconds'] = str(delt.seconds) #dartnml['perfect_model_obs_nml']['last_obs_days'] = str(delt.days) #dartnml['perfect_model_obs_nml']['last_obs_seconds'] = str(delt.seconds) #dartnml['perfect_model_obs_nml']['init_time_days'] = str(delt.days) #dartnml['perfect_model_obs_nml']['init_time_seconds'] = str(delt.seconds) dartnml['perfect_model_obs_nml']['obs_seq_in_file_name'] = 'obs_seq.in' dartnml['perfect_model_obs_nml']['restart_in_file_name'] = rst_file # Make sure we have the right io pattern dartnml['io_filenames_nml']['restart_in_stub'] = 'notinuse' dartnml['io_filenames_nml']['overwrite_input'] = False dartnml['io_filenames_nml']['rpointer'] = True dartnml['io_filenames_nml']['rpointer_file'] = 'input_filelist.txt' # Write the pointer file with open('input_filelist.txt', 'w') as pointerfile: pointerfile.write(rst_file) os.system('mv input_filelist.txt {:s}/'.format(dir_obs)) # Write the modified namelist write_dart_namelist(dartnml) # Set up the input file with open('obs_seq_input.txt','w') as infile: infile.write(str(total_obs)+'\n') # Total num obs infile.write('0\n') # 0 copies infile.write('0\n') # 0 QC values for obtype in use_obs: for x in obx: for y in oby: infile.write('0\n') infile.write(obtype+'\n') # Identify ob type infile.write('0\n') # Specify location infile.write(str(x)+'\n') # X coordinate infile.write(str(y)+'\n') # Y coordinate infile.write('0\n') # Z coordinate infile.write('{:d} {:d} {:d} {:d} {:d} {:d}\n'.format(\ intime.year, intime.month,\ intime.day, intime.hour,\ intime.minute, intime.second)) infile.write(str(error_var[obtype])+'\n') # Error variance infile.write('obs_seq.in\n') # Move these files to the obs dir if os.path.exists(os.path.join(dir_obs, 'input.nml')): os.system('rm -f {:s}'.format(os.path.join(dir_obs, 'input.nml'))) if os.path.exists(os.path.join(dir_obs, 'obs_seq_input.txt')): os.system('rm -f {:s}'.format(os.path.join(dir_obs, 'obs_seq_input.txt'))) os.system('mv obs_seq_input.txt {:s}'.format(dir_obs)) os.system('cp input.nml {:s}'.format(dir_obs)) return None
def build_obs_structure(intime, rst_file, gridspace=4): """ Function to specify what variables we want and how dense they should be """ #from make_namelist_dart import set_namelist_sectors, write_namelist #from write_cm1_namelist import set_namelist_defaults from namelist_utils import read_namelist, write_dart_namelist # Generate the DART namelist structure so we # can query (and modify) it dartnml = read_namelist('input.nml') cm1nml = read_namelist('namelist.input') # Build the epoch from the cm1 namelist param11 = cm1nml['param11'] startdate = datetime(param11['year'], param11['month'],\ param11['day'], param11['hour'],\ param11['minute'], param11['second']) # Parse out the observations to assimilate use_obs = dartnml['obs_kind_nml']['assimilate_these_obs_types'] # I'm just testing this for now #use_obs = use_obs[0:2] #use_obs = ['LAND_SFC_U_WIND_COMPONENT', 'LAND_SFC_V_WIND_COMPONENT', 'LAND_SFC_TEMPERATURE', # 'LAND_SFC_SPECIFIC_HUMIDITY'] use_obs = [ 'TEMPERATURE_2M', 'SURFACE_PRESSURE', 'SPECIFIC_HUMIDITY_2M', 'V_WIND_10M', 'U_WIND_10M' ] print(use_obs) # Put time as datetime intime = startdate + timedelta(seconds=intime) epoch = datetime(1601, 1, 1, 0) print("Indate:", intime) # Figure out the grid structure dx = cm1nml['param1']['dx'] dy = cm1nml['param1']['dy'] nx = cm1nml['param0']['nx'] ny = cm1nml['param0']['ny'] # Now figure out how many obs we'll need gridspace *= 1000 obx = range(int(dx / 2.), int(nx * dx), gridspace) oby = range(int(dy / 2.), int(ny * dx), gridspace) #obx = [16000] #oby = [16000] total_obs = len(obx) * len(oby) * len(use_obs) print("Total number of obs:", total_obs) # Write a new dart namelist for the current time delt = intime - epoch #dartnml['perfect_model_obs_nml']['first_obs_days'] = str(delt.days) #dartnml['perfect_model_obs_nml']['first_obs_seconds'] = str(delt.seconds) #dartnml['perfect_model_obs_nml']['last_obs_days'] = str(delt.days) #dartnml['perfect_model_obs_nml']['last_obs_seconds'] = str(delt.seconds) #dartnml['perfect_model_obs_nml']['init_time_days'] = str(delt.days) #dartnml['perfect_model_obs_nml']['init_time_seconds'] = str(delt.seconds) dartnml['perfect_model_obs_nml']['obs_seq_in_file_name'] = 'obs_seq.in' dartnml['perfect_model_obs_nml']['restart_in_file_name'] = rst_file # Make sure we have the right io pattern dartnml['io_filenames_nml']['restart_in_stub'] = 'notinuse' dartnml['io_filenames_nml']['overwrite_input'] = False dartnml['io_filenames_nml']['rpointer'] = True dartnml['io_filenames_nml']['rpointer_file'] = 'input_filelist.txt' # Write the pointer file with open('input_filelist.txt', 'w') as pointerfile: pointerfile.write(rst_file) os.system('mv input_filelist.txt {:s}/'.format(dir_obs)) # Write the modified namelist write_dart_namelist(dartnml) # Set up the input file with open('obs_seq_input.txt', 'w') as infile: infile.write(str(total_obs) + '\n') # Total num obs infile.write('0\n') # 0 copies infile.write('0\n') # 0 QC values for obtype in use_obs: for x in obx: for y in oby: infile.write('0\n') infile.write(obtype + '\n') # Identify ob type infile.write('0\n') # Specify location infile.write(str(x) + '\n') # X coordinate infile.write(str(y) + '\n') # Y coordinate infile.write(str(heights[obtype]) + '\n') # Z coordinate infile.write('{:d} {:d} {:d} {:d} {:d} {:d}\n'.format(\ intime.year, intime.month,\ intime.day, intime.hour,\ intime.minute, intime.second)) infile.write(str(error_var[obtype]) + '\n') # Error variance infile.write('obs_seq.in\n') # Move these files to the obs dir if os.path.exists(os.path.join(dir_obs, 'input.nml')): os.system('rm -f {:s}'.format(os.path.join(dir_obs, 'input.nml'))) if os.path.exists(os.path.join(dir_obs, 'obs_seq_input.txt')): os.system('rm -f {:s}'.format( os.path.join(dir_obs, 'obs_seq_input.txt'))) os.system('mv obs_seq_input.txt {:s}'.format(dir_obs)) os.system('cp input.nml {:s}'.format(dir_obs)) return None