def genSplitCommands(cm_args, batch, file_dir, file_base): work_path, input_path, debug_path = \ getPaths(cm_args.base_path, cm_args.job_name, work=True, input=True, debug=True) nx, ny = cm_args.mpi_config input_file_name = "%s/%s.hdfsubdomain.input" % (input_path, file_base) debug_file_name = "%s/%s.hdfsubdomain.debug" % (debug_path, file_base) kwargs = { 'hisfile(1)':"%s/%s.hdf%06d" % (work_path, file_base, t_ens) } editNamelistFile("%s/hdfsubdomain.input" % base_path, input_file_name, hdmpinopt=2, nproc_x=1, nproc_y=1, runname=file_base, dirname="%s/%s" % (work_path, file_dir), hdmpfheader="%s/%s/%s" % (work_path, file_dir, file_base), grdbasfn="%s/%s.hdfgrdbas" % (work_path, file_base), nprocx_in=1, nprocy_in=1, nprocx_out=nx, nprocy_out=ny, **kwargs ) commands = [] cmd = batch.genMultiMPICommand("$base/hdfsubdomain", cm_args.mpi_config, stdin=input_file_name, stdout=debug_file_name) commands.append(cmd) return commands
def prepareJoinNamelist(cm_args, file_dir, file_base): work_path, input_path, debug_path = \ getPaths(cm_args.base_path, cm_args.job_name, work=True, input=True, debug=True) start_time = cm_args.t_ens_start for t_ens in xrange(cm_args.t_ens_start, cm_args.t_ens_end + cm_args.dt_ens_step, cm_args.dt_ens_step): if os.path.exists("%s/%s.hdf%06d" % (work_path, file_base, t_ens)): start_time = t_ens + cm_args.dt_ens_step if start_time > cm_args.t_ens_end: print "Ensemble member %s is done, moving on ..." % file_base return False txt_file_base = "%s.%d-%d.hdfsubdomain" % (file_base, cm_args.t_ens_start, cm_args.t_ens_end) input_file_name = "%s/%s.input" % (input_path, txt_file_base) nx, ny = cm_args.mpi_config editNamelistFile("%s/hdfsubdomain.input" % cm_args.base_path, input_file_name, hdmpinopt=1, nproc_x=nx, nproc_y=ny, runname=file_base, dirname="%s" % work_path, tbgn_dmpin=start_time, tend_dmpin=cm_args.t_ens_end, tintv_dmpin=cm_args.dt_ens_step, hdmpfheader="%s/%s/%s" % (work_path, file_dir, file_base), nprocx_in=nx, nprocy_in=ny, nprocx_out=1, nprocy_out=1, ) return True
def generateEnsemblePerturbations(cm_args, batch, start_time): work_path, input_path, debug_path = \ getPaths(cm_args.base_path, cm_args.job_name, work=True, input=True, debug=True) for n_ens in cm_args.members: ens_member_name = "ena%03d" % (n_ens + 1) arpsenkfic_input_file_name = "%s/%s.arpsenkfic.input" % (input_path, ens_member_name) editNamelistFile("%s/%s" % (cm_args.base_path, cm_args.arpsenkfic_template), arpsenkfic_input_file_name, seeds=-n_ens, dirnamp="%s/" % work_path, outdumpdir="%s/" % work_path, outname=ens_member_name, tfgs=start_time) arps_input_file_name = "%s/arps.input" % input_path arpsenkfic_input_file_name = "%s/%s.arpsenkfic.input" % (input_path, 'ena%(ens)03d') arpsenkfic_debug_file_name = "%s/%s.arpsenkfic.debug" % (debug_path, 'ena%(ens)03d') command = "%s -n 1 $base/arpsenkfic %s < %s > %s" % \ (batch.getMPIprogram(), arps_input_file_name, arpsenkfic_input_file_name, arpsenkfic_debug_file_name) command_lines = doForEnsemble(command, cm_args.members) return command_lines
def main(): radar_location_KCYS = (41.15194, -104.80611) radar_location_KFTG = (39.78667, -104.54583) radar_location_KRIW = (43.06611, -108.47722) radar_location_05XP = (41.56150, -104.298996) domain_center_1km = (41.61795, -104.34843) domain_center_3km = (40.61795, -107.344) domain_std_lon = -107.344 # Set these! ALL OF THESE! radar_id = "KCYS" path_to_data = "raw" radar_location = radar_location_KCYS domain_center = domain_center_3km domain_grid_spacing = 3000 domain_grid_size = 411 manual_qc_flag = 'manual' tags = findTags(path_to_data, radar_id) initial_time = toSeconds("180000") max_pids = 8 pids = [] if radar_id == "KRIW": apparent_lat = domain_center[0] apparent_lon = domain_center[1] apparent_std_lon = domain_std_lon if os.path.exists("88d2arps"): os.unlink("88d2arps") os.link("/data6/tsupinie/88d2arps_KRIW", "88d2arps") else: # apparent_lat = domain_center[0] # apparent_lon = -domain_center[1] + 2 * (domain_center[1] - radar_location[1]) # apparent_std_lon = -domain_std_lon + 2 * (domain_std_lon - radar_location[1]) apparent_lat = domain_center[0] apparent_lon = domain_center[1] apparent_std_lon = domain_std_lon if manual_qc_flag == "manual": good_flag = "good" else: good_flag = "bad" if os.path.exists("solo2arps"): os.unlink("solo2arps") os.link("/data6/tsupinie/solo2arps_%s" % (radar_id[-4:], ), "solo2arps") for tag in tags: print "Extracting tag %s ..." % tag files = makeSweepFile(path_to_data, radar_id, tag) if radar_id != "KRIW": time_lbound = files[0].find("swp") + 11 time_ubound = time_lbound + 6 time = files[0][time_lbound:time_ubound] if initial_time == -1: initial_time = toSeconds(time) offset_seconds = toSeconds(time) - initial_time else: offset_seconds = toSeconds(tag) - initial_time kwargs = {} if radar_id == "KRIW": kwargs['radfname'] = files[0] kwargs['refvarname'] = "DBZ" kwargs['velvarname'] = "VEL" src_remap_file = "radremap_88D.input" elif radar_id == "KCYS": if manual_qc_flag == "manual": kwargs['refvarname'] = "DZ" kwargs['velvarname'] = "DV" else: kwargs['refvarname'] = "DBZ" kwargs['velvarname'] = "VEL" src_remap_file = "radremap_88D.input" elif radar_id == "KFTG": if manual_qc_flag == "manual": kwargs['refvarname'] = "DZ" kwargs['velvarname'] = "DV" else: kwargs['refvarname'] = "DBZ" kwargs['velvarname'] = "VEL" src_remap_file = "radremap_88D.input" elif radar_id == "MWR05XP": src_remap_file = "radremap_MWR05XP.input" kwargs['refvarname'] = "DZ" kwargs['velvarname'] = "DV" if manual_qc_flag == "automated": kwargs['rngmin'] = 100000.0 editNamelistFile(src_remap_file, "input/radremap.%s.input" % tag, nx=domain_grid_size, ny=domain_grid_size, dx=domain_grid_spacing, dy=domain_grid_spacing, radname=radar_id[-4:], ctrlat=apparent_lat, ctrlon=apparent_lon, trulon=apparent_std_lon, outtime=offset_seconds, **kwargs) if radar_id == "KRIW": pid = subprocess.Popen(["./88d2arps"], stdin=open("input/radremap.%s.input" % tag, 'r'), stdout=open("debug/radremap.%s.debug" % tag, 'w')).pid else: pid = subprocess.Popen( ["./solo2arps", "-fdisk", "sweeps/sweeps.%s.txt" % tag], stdin=open("input/radremap.%s.input" % tag, 'r'), stdout=open("debug/radremap.%s.debug" % tag, 'w')).pid pids.append(pid) if len(pids) == max_pids or tag == tags[-1]: for pid in pids: os.waitpid(pid, 0) pids = [] for file in glob.glob("%s.20090605.*" % radar_id): os.rename( file, "qc/%s/%dkm/." % (manual_qc_flag, domain_grid_spacing / 1000)) for file in glob.glob("goshen.hdfr*"): os.rename(file, "hdf/%s/%dkm/." % (radar_id, domain_grid_spacing / 1000)) return
def main(): radar_location_KCYS = (41.15194, -104.80611) radar_location_KFTG = (39.78667, -104.54583) radar_location_KRIW = (43.06611, -108.47722) radar_location_05XP = (41.56150, -104.298996) domain_center_1km = (41.61795, -104.34843) domain_center_3km = (40.61795, -107.344) domain_std_lon = -107.344 # Set these! ALL OF THESE! radar_id = "KCYS" path_to_data = "raw" radar_location = radar_location_KCYS domain_center = domain_center_3km domain_grid_spacing = 3000 domain_grid_size = 411 manual_qc_flag = 'manual' tags = findTags(path_to_data, radar_id) initial_time = toSeconds("180000") max_pids = 8 pids = [] if radar_id == "KRIW": apparent_lat = domain_center[0] apparent_lon = domain_center[1] apparent_std_lon = domain_std_lon if os.path.exists("88d2arps"): os.unlink("88d2arps") os.link("/data6/tsupinie/88d2arps_KRIW", "88d2arps") else: # apparent_lat = domain_center[0] # apparent_lon = -domain_center[1] + 2 * (domain_center[1] - radar_location[1]) # apparent_std_lon = -domain_std_lon + 2 * (domain_std_lon - radar_location[1]) apparent_lat = domain_center[0] apparent_lon = domain_center[1] apparent_std_lon = domain_std_lon if manual_qc_flag == "manual": good_flag = "good" else: good_flag = "bad" if os.path.exists("solo2arps"): os.unlink("solo2arps") os.link("/data6/tsupinie/solo2arps_%s" % (radar_id[-4:],), "solo2arps") for tag in tags: print "Extracting tag %s ..." % tag files = makeSweepFile(path_to_data, radar_id, tag) if radar_id != "KRIW": time_lbound = files[0].find("swp") + 11 time_ubound = time_lbound + 6 time = files[0][time_lbound:time_ubound] if initial_time == -1: initial_time = toSeconds(time) offset_seconds = toSeconds(time) - initial_time else: offset_seconds = toSeconds(tag) - initial_time kwargs = {} if radar_id == "KRIW": kwargs['radfname'] = files[0] kwargs['refvarname'] = "DBZ" kwargs['velvarname'] = "VEL" src_remap_file = "radremap_88D.input" elif radar_id == "KCYS": if manual_qc_flag == "manual": kwargs['refvarname'] = "DZ" kwargs['velvarname'] = "DV" else: kwargs['refvarname'] = "DBZ" kwargs['velvarname'] = "VEL" src_remap_file = "radremap_88D.input" elif radar_id == "KFTG": if manual_qc_flag == "manual": kwargs['refvarname'] = "DZ" kwargs['velvarname'] = "DV" else: kwargs['refvarname'] = "DBZ" kwargs['velvarname'] = "VEL" src_remap_file = "radremap_88D.input" elif radar_id == "MWR05XP": src_remap_file = "radremap_MWR05XP.input" kwargs['refvarname'] = "DZ" kwargs['velvarname'] = "DV" if manual_qc_flag == "automated": kwargs['rngmin'] = 100000.0 editNamelistFile(src_remap_file, "input/radremap.%s.input" % tag, nx=domain_grid_size, ny=domain_grid_size, dx=domain_grid_spacing, dy=domain_grid_spacing, radname=radar_id[-4:], ctrlat=apparent_lat, ctrlon=apparent_lon, trulon=apparent_std_lon, outtime=offset_seconds, **kwargs ) if radar_id == "KRIW": pid = subprocess.Popen([ "./88d2arps" ], stdin=open("input/radremap.%s.input" % tag, 'r'), stdout=open("debug/radremap.%s.debug" % tag, 'w')).pid else: pid = subprocess.Popen([ "./solo2arps", "-fdisk", "sweeps/sweeps.%s.txt" % tag ], stdin=open("input/radremap.%s.input" % tag, 'r'), stdout=open("debug/radremap.%s.debug" % tag, 'w')).pid pids.append(pid) if len(pids) == max_pids or tag == tags[-1]: for pid in pids: os.waitpid(pid, 0) pids = [] for file in glob.glob("%s.20090605.*" % radar_id): os.rename(file, "qc/%s/%dkm/." % (manual_qc_flag, domain_grid_spacing / 1000)) for file in glob.glob("goshen.hdfr*"): os.rename(file, "hdf/%s/%dkm/." % (radar_id, domain_grid_spacing / 1000)) return
member_paths = ["%s/EN%03d/%s" % (basedir, m, ens_member_name) for m, ens_member_name in zip(ens_member_list, ens_member_names)] for ens_member_name, t0_interp_input_file_name, lbc_interp_input_file_name, member_path in \ zip(ens_member_names, t0_interp_input_file_names, lbc_interp_input_file_names, \ member_paths): # Initial conditions editNamelistFile("%s" % template, t0_interp_input_file_name, runname="%s" % ens_member_name, hdmpfheader=member_path, dirname=outputdir, tbgn_dmpin=start_time, tend_dmpin=start_time, tintv_dmpin=step_time, nproc_x=nproc_x, nproc_y=nproc_y, nproc_x_in=nproc_x_in, nproc_y_in=nproc_y_in, nproc_x_out=nproc_x_out, nproc_y_out=nproc_y_out, inisplited=inisplited, dmp_out_joined=dmp_out_joined, terndta1="%s/%s.trndata" % (trndata, expPrefix)) # Boundary conditions editNamelistFile("%s" % template, lbc_interp_input_file_name, runname="%s" % ens_member_name, hdmpfheader=member_path, dirname=outputdir, tbgn_dmpin=start_time, tend_dmpin=end_time,
def generateEnsembleIntegration(cm_args, batch, start_time, end_time, dump_time, split_files='neither', move_for_assim=True): work_path, input_path, debug_path, bc_path, hx_path = \ getPaths(cm_args.base_path, cm_args.job_name, work=True, input=True, debug=True, boundary=True, hx=True) nproc_x, nproc_y = cm_args.mpi_config extraneous_files = [ "%s/%s.hdf%06d.01" % (work_path, 'ena%(ens)03d', start_time), "%s/%s.hdfgrdbas.01" % (work_path, 'ena%(ens)03d'), "%s/%s.log" % (work_path, 'ena%(ens)03d'), "%s/%s.maxmin" % (work_path, 'ena%(ens)03d'), ] read_split = split_files in ['read', 'both'] dump_split = split_files in ['dump', 'both'] if move_for_assim: if dump_split: epilogue = doForMPIConfig( "mv %s/%s/%s.hdf%06d_%s %s/%s/%s.hdf%06d_%s" % \ (work_path, 'EN%(ens)s', 'ena%(ens)s', end_time, '%(x_proc)03d%(y_proc)03d', work_path, 'ENF%(ens)s', 'enf%(ens)s', end_time, '%(x_proc)03d%(y_proc)03d'), cm_args.mpi_config ) else: epilogue = [ "mv %s/%s.hdf%06d %s/%s.hdf%06d" % (work_path, 'ena%(ens)03d', end_time, work_path, 'enf%(ens)03d', end_time) ] else: epilogue = [] arps_input_file_name = "%s/%s.%d-%d.arps.input" % (input_path, 'ena%(ens)03d', start_time, end_time) arps_debug_file_name = "%s/%s.%d-%d.arps.debug" % (debug_path, 'ena%(ens)03d', start_time, end_time) if cm_args.algorithm == "4densrf": arpsenkf_input_file_name = "%s/%d.arpsenkf.input" % (input_path, end_time) epilogue.extend([ "mv ????????????_%s.lso %s" % ('%(ens)03d', hx_path), "mv ????????????_%s.snd %s" % ('%(ens)03d', hx_path), "mv ????_*_??????_%s %s" % ('%(ens)03d', hx_path), ]) for n_ens in cm_args.members: ens_member_name = "ena%03d" % (n_ens + 1) ens_member_directory = "EN%03d" % (n_ens + 1) ens_arps_file_name = arps_input_file_name % { 'ens':(n_ens + 1) } if read_split: init_file = "%s/%s/%s.hdf%06d" % (work_path, ens_member_directory, ens_member_name, start_time), init_grdbas = "%s/%s/%s.hdfgrdbas" % (work_path, ens_member_directory, ens_member_name), else: init_file = "%s/%s.hdf%06d" % (work_path, ens_member_name, start_time), init_grdbas = "%s/%s.hdfgrdbas" % (work_path, ens_member_name), if dump_split: out_dir="%s/%s/" % (work_path, ens_member_directory) else: out_dir="%s/" % work_path kwargs = {'hxopt':0, 'tstop':end_time} if cm_args.algorithm == '4densrf': kwargs['memid'] = n_ens + 1 kwargs['hxopt'] = 1 kwargs['tstop'] += 150 editNamelistFile("%s/%s" % (cm_args.base_path, cm_args.arps_template), ens_arps_file_name, nproc_x=nproc_x, nproc_y=nproc_y, runname=ens_member_name, initopt=3, inifile=init_file, inigbf=init_grdbas, exbcname="%s/%s" % (bc_path, ens_member_name), tstart=start_time, tstrtdmp=start_time + dump_time, thisdmp=dump_time, dmp_out_joined=int(not dump_split), inisplited=int(read_split), dirname=out_dir, **kwargs ) if cm_args.algorithm == 'ensrf': command = [ "%s %s $base/arps %s > %s " % (batch.getMPIprogram(), batch.getMPIargs()%(nproc_x*nproc_y), arps_input_file_name, arps_debug_file_name), "rm %s" % ( " ".join(extraneous_files) ) ] command.extend(epilogue) elif cm_args.algorithm == '4densrf': command = [ "%s %s $base/arps %s %s > %s " % (batch.getMPIprogram(), batch.getMPIargs()%(nproc_x*nproc_y), arps_input_file_name, arpsenkf_input_file_name, arps_debug_file_name), "rm %s" % ( " ".join(extraneous_files) ) ] command.extend(epilogue) command_lines = doForEnsemble(command, cm_args.members) return command_lines
def main(): ap = argparse.ArgumentParser() ap.add_argument('--n-ens', dest='n_ens_members', default=4, type=int) ap.add_argument('--members', dest='members', nargs='+', default=[], type=int) ap.add_argument('--base-path', dest='base_path', default=os.getcwd()) ap.add_argument('--job-name', dest='job_name', default="run_osse_test") ap.add_argument('--mpi-config', dest='mpi_config', nargs=2, default=(3, 4), type=int) ap.add_argument('--algorithm', dest='algorithm', choices=['ensrf', '4densrf'], default='ensrf') ap.add_argument('--ens-start', dest='t_ens_start', default=1200, type=int) ap.add_argument('--ens-end', dest='t_ens_end', default=1500, type=int) ap.add_argument('--assim-step', dest='dt_assim_step', default=300, type=int) ap.add_argument('--ens-step', dest='dt_ens_step', default=300, type=int) ap.add_argument('--initial-conditions', dest='init_cond', default='') ap.add_argument('--boundary-conditions', dest='bound_cond', default='') ap.add_argument('--subset-ic', dest='subset', action='store_true') ap.add_argument('--free-forecast', dest='free_forecast', action='store_true') ap.add_argument('--covariance-inflation', dest='cov_infl', nargs='+', default=["mult=1.1"]) ap.add_argument('--arps-template', dest='arps_template', default='arps.input') ap.add_argument('--arpsenkf-template', dest='arpsenkf_template', default='arpsenkf.input') ap.add_argument('--arpsenkfic-template', dest='arpsenkfic_template', default='arpsenkfic.input') ap.add_argument('--assim-radar', dest='radflags') ap.add_argument('--assim-sndg', dest='sndgflags', default='yes') ap.add_argument('--assim-prof', dest='profflags', default='yes') ap.add_argument('--assim-surf', dest='surfflags', default='yes') ap.add_argument('--init-fcst-req', dest='init_fcst_req', default='0:40') ap.add_argument('--fcst-req', dest='fcst_req', default='0:20') ap.add_argument('--init-free-fcst-req', dest='init_free_fcst_req', default='1:45') ap.add_argument('--free-fcst-req', dest='free_fcst_req', default='1:30') ap.add_argument('--assim-on-req', dest='assim_on_req', default='1:00') ap.add_argument('--assim-off-req', dest='assim_off_req', default='0:45') ap.add_argument('--chunk-size', dest='chunk_size', default=1200, type=int) ap.add_argument('--join-files', dest='split_files', action='store_false') ap.add_argument('--split-init', dest='split_init', choices=['auto', 'yes', 'no'], default='auto') # Whether or not the initialization is split. ap.add_argument('--no-submit', dest='submit', action='store_false') ap.add_argument('--restart', dest='restart', action='store_true') ap.add_argument('--debug', dest='debug', action='store_true') args = ap.parse_args() batch = Batch('rice') # stampede # work_path = "%s/%s" % (args.base_path, args.job_name) # input_path = "%s/input" % args.base_path # boundary_path = "%s/boundary/" % args.base_path work_path, input_path, boundary_path, hx_path =\ getPaths(args.base_path, args.job_name, work=True, input=True, boundary=True, hx=True) if not os.path.exists(work_path): os.mkdir(work_path, 0755) if not os.path.exists(hx_path): os.mkdir(hx_path, 0755) member_list = [ m - 1 for m in args.members ] if member_list == []: member_list = range(args.n_ens_members) args.members = member_list nproc_x, nproc_y = args.mpi_config prologue = [ "echo \"\" > %s/debug/ena%s-%s.output" % (args.base_path, '%(ens)03d', args.job_name), "", ] command_lines = doForEnsemble(prologue, member_list) command = [ "set base=%s" % args.base_path, "cd $base", "" ] appendCommands(command_lines, doForEnsemble(command, member_list) ) joined = 0 if args.split_files else 1 editNamelistFile("%s/%s" % (args.base_path, args.arps_template), "%s/arps.input" % input_path, nproc_x=nproc_x, nproc_y=nproc_y, dmp_out_joined=joined, inisplited=3 * (1 - joined), sfcdat=3, dirname=work_path ) do_first_integration = True ens_chunk_start = args.t_ens_start if not args.free_forecast and args.chunk_size > args.dt_assim_step: args.chunk_size = args.dt_assim_step if args.dt_ens_step > args.chunk_size: args.dt_ens_step = args.chunk_size print "Warning: resetting dt_ens_step to be chunk_size (%d)" % args.dt_ens_step if not args.free_forecast: exec open("%s/%s" % (args.base_path, args.radflags), 'r') in locals() # Get the radar assimilation flags from the file # args.radar_data_flags = radar_data_flag exp_start = args.t_ens_start # Copy the configuration information to the working directory, so we'll **ALWAYS HAVE IT IF WE NEED TO GO BACK AND LOOK AT IT** config_files = [ "%s/%s" % (args.base_path, f) for f in [ args.arps_template, args.arpsenkf_template, args.arpsenkfic_template ] ] config_files.extend(['run_real_data_case.py', 'run_real_data_case.csh']) for file in config_files: subprocess.Popen(['cp', file, "%s/." % work_path]) if args.restart: for t_ens in xrange(args.t_ens_start, args.t_ens_end + args.dt_assim_step, args.dt_assim_step): if args.split_files and args.split_init != 'no': ena_exist = [ os.path.exists("%s/EN%03d/ena%03d.hdf%06d_001001" % (work_path, n_ens + 1, n_ens + 1, t_ens)) for n_ens in member_list ] enf_exist = [ os.path.exists("%s/ENF%03d/enf%03d.hdf%06d_001001" % (work_path, n_ens + 1, n_ens + 1, t_ens)) for n_ens in member_list ] else: ena_exist = [ os.path.exists("%s/ena%03d.hdf%06d" % (work_path, n_ens + 1, t_ens)) for n_ens in member_list ] enf_exist = [ os.path.exists("%s/enf%03d.hdf%06d" % (work_path, n_ens + 1, t_ens)) for n_ens in member_list ] all_ena_exist = all(ena_exist) all_enf_exist = all(enf_exist) if all_ena_exist: args.t_ens_start = t_ens ens_chunk_start = t_ens elif all_enf_exist and not all_ena_exist: args.t_ens_start = t_ens - args.dt_assim_step do_first_integration = False for t_chunk in xrange(t_ens, t_ens + args.dt_assim_step, args.chunk_size): if args.split_files and args.split_init != 'no': ena_exist = [ os.path.exists("%s/EN%03d/ena%03d.hdf%06d_001001" % (work_path, n_ens + 1, n_ens + 1, t_chunk)) for n_ens in member_list ] else: ena_exist = [ os.path.exists("%s/ena%03d.hdf%06d" % (work_path, n_ens + 1, t_chunk)) for n_ens in member_list ] if all(ena_exist): ens_chunk_start = t_chunk if do_first_integration: print "Restarting from time %d (with integration) ..." % (args.t_ens_start) else: print "Restarting from time %d (no integration) ..." % (args.t_ens_start + args.dt_assim_step) else: print "New experiment ..." if args.split_files: command = "mkdir %s/%s ; mkdir %s/%s" % (work_path, 'EN%(ens)03d', work_path, 'ENF%(ens)03d') appendCommands(command_lines, doForEnsemble(command, member_list) ) if args.init_cond == "": print "Generate random initial conditions ..." appendCommands(command_lines, generateEnsemblePerturbations(args, batch, args.t_ens_start) ) command = "cp %s/%s.hdfgrdbas %s/%s.hdfgrdbas" % (work_path, 'ena%(ens)03d', work_path, 'enf%(ens)03d') appendCommands(command_lines, doForEnsemble(command, member_list) ) else: print "Use supplied initial conditions ..." if args.subset: print "Subset and perturb the domain ..." appendCommands(command_lines, generateDomainSubset(args, batch, args.init_cond, args.t_ens_start, args.t_ens_end, args.dt_ens_step, perturb_ic=True) ) command = "cp %s/%s.hdfgrdbas %s/%s.hdfgrdbas" % (boundary_path, 'ena%(ens)03d', work_path, 'enf%(ens)03d') appendCommands(command_lines, doForEnsemble(command, member_list) ) else: print "No domain subset ..." command = [ "cp %s/%s.hdf%06d %s" % (boundary_path, 'ena%(ens)03d', args.t_ens_start, work_path), "cp %s/%s.hdfgrdbas %s" % (boundary_path, 'ena%(ens)03d', work_path), "cp %s/%s.hdfgrdbas %s/%s.hdfgrdbas" % (boundary_path, 'ena%(ens)03d', work_path, 'enf%(ens)03d') ] appendCommands(command_lines, doForEnsemble(command, member_list) ) if args.free_forecast: if args.subset and args.t_ens_start == exp_start: print "Subset the boundary conditions ..." appendCommands(command_lines, generateDomainSubset(args, batch, args.bound_cond, args.t_ens_start, args.t_ens_end, args.dt_ens_step, perturb_ic=False, copy_ic=False) ) command = "cp %s/%s.hdfgrdbas %s/%s.hdfgrdbas" % (boundary_path, 'ena%(ens)03d', work_path, 'enf%(ens)03d') appendCommands(command_lines, doForEnsemble(command, member_list) ) n_chunks = int(ceil(float(args.t_ens_end - args.t_ens_start) / args.chunk_size)) n_chunk_start = 0 n_chunk_start = (ens_chunk_start - args.t_ens_start) / args.chunk_size for n_chunk, t_chunk in enumerate(range(ens_chunk_start, args.t_ens_end, args.chunk_size)): print "Generating free forecast from %d to %d (chunk %d of %d) ..." % (args.t_ens_start, args.t_ens_end, n_chunk + n_chunk_start + 1, n_chunks) chunk_start = t_chunk chunk_end = t_chunk + args.chunk_size which_split = 'neither' if args.split_files and (args.split_init == 'auto' or args.split_init == 'yes'): which_split = 'both' elif args.split_files and args.split_init == 'no': which_split = 'dump' if chunk_end > args.t_ens_end: chunk_end = args.t_ens_end appendCommands(command_lines, generateEnsembleIntegration(args, batch, chunk_start, chunk_end, args.dt_ens_step, split_files=which_split, move_for_assim=False) ) req_time = args.free_fcst_req if args.subset and t_chunk == exp_start: req_time = args.init_free_fcst_req submit(args, batch, command_lines, req_time, nproc_x * nproc_y, hybrid=False) command_lines.clear() command = [ "set base=%s" % args.base_path, "cd $base", "" ] appendCommands(command_lines, doForEnsemble(command, member_list) ) else: for t_ens in xrange(args.t_ens_start, args.t_ens_end, args.dt_assim_step): print "Generating timestep %d ..." % t_ens start_time = t_ens end_time = t_ens + args.dt_assim_step if do_first_integration or t_ens > args.t_ens_start: n_chunks = int(ceil(float(end_time - start_time) / args.chunk_size)) n_chunk_start = 0 if start_time == args.t_ens_start: n_chunk_start = (ens_chunk_start - start_time) / args.chunk_size start_time = ens_chunk_start for n_chunk, t_chunk in enumerate(range(start_time, end_time, args.chunk_size)): print "Submitting ensemble integration for timestep %d (chunk %d of %d) ..." % (t_ens, n_chunk + n_chunk_start + 1, n_chunks) chunk_start = t_chunk chunk_end = t_chunk + args.chunk_size if chunk_end > end_time: chunk_end = end_time which_split = 'neither' if args.split_files: if chunk_start == exp_start: # For the first chunk if args.restart: # We're restarting if args.split_init == 'auto' or args.split_init == 'yes': which_split = 'both' elif args.split_init == 'no': which_split = 'dump' else: # No restart if args.split_init == 'auto' or args.split_init == 'no': which_split = 'dump' elif args.split_init == 'yes': which_split = 'both' else: # Everything after the first chunk which_split = 'both' if args.algorithm == '4densrf': # ARPS for the 4DEnSRF wants an EnKF input file, too. generateEnKFAssimilation(args, batch, chunk_end, radar_data_flag[end_time]) appendCommands(command_lines, generateEnsembleIntegration(args, batch, chunk_start, chunk_end, args.dt_ens_step, split_files=which_split, move_for_assim=(chunk_end == end_time)) ) if args.split_files and t_chunk == exp_start: command = doForMPIConfig("cp %s/%s/%s/%s.hdfgrdbas_%s %s/%s/%s/%s.hdfgrdbas_%s" % (args.base_path, args.job_name, 'EN%(ens)s', 'ena%(ens)s', '%(x_proc)03d%(y_proc)03d', args.base_path, args.job_name, 'ENF%(ens)s', 'enf%(ens)s', '%(x_proc)03d%(y_proc)03d'), args.mpi_config) appendCommands(command_lines, doForEnsemble(command, member_list) ) req_time = args.fcst_req if (args.subset or not args.restart) and t_chunk == exp_start: req_time = args.init_fcst_req submit(args, batch, command_lines, req_time, nproc_x * nproc_y, hybrid=False, squash_jobs=False) command_lines.clear() command = [ "set base=%s" % args.base_path, "cd $base", "" ] appendCommands(command_lines, doForEnsemble(command, member_list) ) req_time = args.assim_off_req if isDivisible(end_time, 3600): req_time = args.assim_on_req # Ask Tim about rationale for using hybrid with so many total cores... assimilation_lines = [ "set base=%s" % args.base_path, "cd $base", "", "setenv OMP_NUM_THREADS %d" % batch.getNCoresPerNode(), "" ] #assimilation_lines = [ "set base=%s" % args.base_path, "cd $base", "" ] assimilation_lines.extend( generateEnKFAssimilation(args, batch, end_time, radar_data_flag[end_time]) ) print "Submitting assimilation for timestep %d ..." % t_ens submit(args, batch, assimilation_lines, req_time, nproc_x * nproc_y * batch.getNCoresPerNode(), hybrid=True) # Will have to ask Tim about this... #submit(args, batch, assimilation_lines, req_time, nproc_x * nproc_y, hybrid=False) print "Experiment complete!" return
def generateDomainSubset(cm_args, batch, src_path, start_time, end_time, step_time, perturb_ic=True, copy_ic=False): work_path, input_path, debug_path, bc_path = \ getPaths(cm_args.base_path, cm_args.job_name, work=True, input=True, debug=True, boundary=True) for n_ens in cm_args.members: ens_member_name = "ena%03d" % (n_ens + 1) interp_input_file_name = "%s/%s.arpsintrp.input" % (input_path, ens_member_name) arpsenkfic_input_file_name = "%s/%s.arpsenkfic.input" % (input_path, ens_member_name) arps_input_file_name = "%s/%s.arps.input" % (input_path, ens_member_name) editNamelistFile("%s/%s" % (cm_args.base_path, cm_args.arpsintrp_template), interp_input_file_name, runname="%s" % ens_member_name, hdmpfheader="%s/%s" % (src_path, ens_member_name), dirname=bc_path, tbgn_dmpin=start_time, tend_dmpin=end_time, tintv_dmpin=step_time, ) editNamelistFile("%s/%s" % (cm_args.base_path, cm_args.arps_template), arps_input_file_name, runname="%s" % ens_member_name, initopt=3, inifile="%s/%s.hdf%06d" % (bc_path, ens_member_name, start_time), inigbf="%s/%s.hdfgrdbas" % (bc_path, ens_member_name), tstart=start_time, tstop=end_time, dirname="%s/" % work_path ) if perturb_ic: editNamelistFile("%s/%s" % (cm_args.base_path, cm_args.arpsenkfic_template), arpsenkfic_input_file_name, seeds=-n_ens, dirnamp="%s/" % work_path, outdumpdir="%s/" % work_path, outname=ens_member_name, tfgs=start_time ) interp_input_file_name = "%s/ena%s.arpsintrp.input" % (input_path, '%(ens)03d') interp_debug_file_name = "%s/ena%s.arpsintrp.debug" % (debug_path, '%(ens)03d') arps_input_file_name = "%s/ena%s.arps.input" % (input_path, '%(ens)03d') arpsenkfic_input_file_name = "%s/ena%s.arpsenkfic.input" % (input_path, '%(ens)03d') arpsenkfic_debug_file_name = "%s/ena%s.arpsenkfic.debug" % (debug_path, '%(ens)03d') commands = [ "rm %s/%sicbc.*" % (bc_path, "ena%(ens)03d"), "" ] if perturb_ic: commands.append("%s -n 1 $base/arpsintrp %s/arps.input < %s > %s ; %s -n 1 $base/arpsenkfic %s < %s > %s" % (batch.getMPIprogram(), input_path, interp_input_file_name, interp_debug_file_name, batch.getMPIprogram(), arps_input_file_name, arpsenkfic_input_file_name, arpsenkfic_debug_file_name)) else: commands.append("%s -n 1 $base/arpsintrp %s/arps.input < %s > %s" % (batch.getMPIprogram(), input_path, interp_input_file_name, interp_debug_file_name)) if copy_ic: commands.extend([ "cp %s/%s.hdf%06d %s" % (bc_path, 'ena%(ens)03d', start_time, work_path), "cp %s/%s.hdfgrdbas %s" % (bc_path, 'ena%(ens)03d', work_path), "cp %s/%s.hdfgrdbas %s/%s.hdfgrdbas" % (bc_path, 'ena%(ens)03d', work_path, 'enf%(ens)03d') ]) command_lines = doForEnsemble(commands, cm_args.members) return command_lines
def generateEnKFAssimilation(cm_args, batch, assim_time, radar_data_flag): work_path, input_path, debug_path, batch_path = \ getPaths(cm_args.base_path, cm_args.job_name, work=True, input=True, debug=True, batch=True) nproc_x, nproc_y = cm_args.mpi_config arps_input_file_name = "%s/%d.arps.input" % (input_path, assim_time) enkf_input_file_name = "%s/%d.arpsenkf.input" % (input_path, assim_time) enkf_debug_file_name = "%s/%d.arpsenkf.debug" % (debug_path, assim_time) batch_file_name = "%s/%d.csh" % (batch_path, assim_time) kwargs = {} # Figure out what conventional data we're assimilating (combine this with the next section!) cvn_data_flags = dict( (k, getattr(cm_args, k)) for k in ['sndgflags', 'profflags', 'surfflags' ] ) assim_all = False if isDivisible(assim_time, 3600): print "Assimilate all data ..." else: cvn_data_flags['sndgflags'] = 'no' cvn_data_flags['profflags'] = 'no' # Conventional DA flags for cvn_flag, kw_flag in [ ('sndgflags', 'sndassim'), ('profflags', 'proassim'), ('surfflags', 'sfcassim') ]: if cvn_flag in cvn_data_flags and cvn_data_flags[cvn_flag].lower() == 'yes': kwargs[kw_flag] = 1 else: kwargs[kw_flag] = 0 if cm_args.split_files: kwargs['nproc_x_in'] = nproc_x kwargs['nproc_y_in'] = nproc_y kwargs['inidirname'] = "%s/ENF%s" % (work_path, "%3N") else: kwargs['inidirname'] = work_path # Figure out what our covariance inflation will be (combine this with the next section!) for cov_infl in cm_args.cov_infl: if ':' in cov_infl: time, factors = cov_infl.split(':') if assim_time >= int(time): covariance_inflation = factors else: covariance_inflation = cov_infl print "Covariance inflation for this timestep is", covariance_inflation # Covariance inflation flags kwargs['mult_inflat'] = 0 kwargs['adapt_inflat'] = 0 kwargs['relax_inflat'] = 0 try: covariance_inflation = covariance_inflation.split(',') except ValueError: covariance_inflation = [ covariance_inflation ] for cov_infl in covariance_inflation: if '=' in cov_infl: inflation_method, inflation_factor = cov_infl.split('=') inflation_factor = float(inflation_factor) if inflation_method == "mults": # Multiplicative inflation in the storm region only kwargs['mult_inflat'] = 1 kwargs['cinf'] = inflation_factor elif inflation_method == "multd": # Multiplicative inflation over the entire domain kwargs['mult_inflat'] = 2 kwargs['cinf'] = inflation_factor elif inflation_method == "adapt": # Relaxation to Prior Spread ("Adaptive") inflation kwargs['adapt_inflat'] = 1 kwargs['rlxf'] = inflation_factor elif inflation_method == "relax": # Relaxation to Prior Perturbation ("Relaxation") inflation kwargs['relax_inflat'] = 1 kwargs['rlxf'] = inflation_factor if cm_args.algorithm == 'ensrf': kwargs['anaopt'] = 2 elif cm_args.algorithm == '4densrf': kwargs['anaopt'] = 5 n_radars = len(radar_data_flag[True]) if True in radar_data_flag else 0 radardaopt = 1 if n_radars > 0 else 0 editNamelistFile("%s/%s" % (cm_args.base_path, cm_args.arpsenkf_template), enkf_input_file_name, nen=cm_args.n_ens_members, casenam=cm_args.job_name, enkfdtadir="%s/" % work_path, cvndatadir="%s/obs/" % cm_args.base_path, assim_time=assim_time, radardaopt=radardaopt, nrdrused=n_radars, rmsfcst=2, hdmpfheader=cm_args.job_name, **kwargs ) if cm_args.algorithm == '4densrf': #Hard-code the 4densrf assimilation window for now, change me later kwargs = {'hdmptim(1)':assim_time, 'tstop':assim_time + 150} else: kwargs = {} joined = 0 if cm_args.split_files else 1 editNamelistFile("%s/%s" % (cm_args.base_path, cm_args.arps_template), arps_input_file_name, nproc_x=nproc_x, nproc_y=nproc_y, dmp_out_joined=joined, inisplited=3 * (1 - joined), sfcdat=3, dirname=work_path, **kwargs ) command_lines = [ "cd %s" % work_path, "%s %s $base/arpsenkf %s < %s > %s" % (batch.getMPIprogram(), batch.getMPIargs()%(nproc_x*nproc_y), arps_input_file_name, enkf_input_file_name, enkf_debug_file_name), "cd -", "", ] return command_lines