def genSplitCommands(cm_args, batch, file_dir, file_base):
work_path, input_path, debug_path = \
getPaths(cm_args.base_path, cm_args.job_name, work=True, input=True, debug=True)
nx, ny = cm_args.mpi_config
input_file_name = "%s/%s.hdfsubdomain.input" % (input_path, file_base)
debug_file_name = "%s/%s.hdfsubdomain.debug" % (debug_path, file_base)
kwargs = { 'hisfile(1)':"%s/%s.hdf%06d" % (work_path, file_base, t_ens) }
editNamelistFile("%s/hdfsubdomain.input" % base_path, input_file_name,
hdmpinopt=2,
nproc_x=1, nproc_y=1,
runname=file_base,
dirname="%s/%s" % (work_path, file_dir),
hdmpfheader="%s/%s/%s" % (work_path, file_dir, file_base),
grdbasfn="%s/%s.hdfgrdbas" % (work_path, file_base),
nprocx_in=1, nprocy_in=1,
nprocx_out=nx, nprocy_out=ny,
**kwargs
)
commands = []
cmd = batch.genMultiMPICommand("$base/hdfsubdomain", cm_args.mpi_config, stdin=input_file_name, stdout=debug_file_name)
commands.append(cmd)
return commands
def prepareJoinNamelist(cm_args, file_dir, file_base):
work_path, input_path, debug_path = \
getPaths(cm_args.base_path, cm_args.job_name, work=True, input=True, debug=True)
start_time = cm_args.t_ens_start
for t_ens in xrange(cm_args.t_ens_start, cm_args.t_ens_end + cm_args.dt_ens_step, cm_args.dt_ens_step):
if os.path.exists("%s/%s.hdf%06d" % (work_path, file_base, t_ens)):
start_time = t_ens + cm_args.dt_ens_step
if start_time > cm_args.t_ens_end:
print "Ensemble member %s is done, moving on ..." % file_base
return False
txt_file_base = "%s.%d-%d.hdfsubdomain" % (file_base, cm_args.t_ens_start, cm_args.t_ens_end)
input_file_name = "%s/%s.input" % (input_path, txt_file_base)
nx, ny = cm_args.mpi_config
editNamelistFile("%s/hdfsubdomain.input" % cm_args.base_path, input_file_name,
hdmpinopt=1,
nproc_x=nx, nproc_y=ny,
runname=file_base,
dirname="%s" % work_path,
tbgn_dmpin=start_time,
tend_dmpin=cm_args.t_ens_end,
tintv_dmpin=cm_args.dt_ens_step,
hdmpfheader="%s/%s/%s" % (work_path, file_dir, file_base),
nprocx_in=nx, nprocy_in=ny,
nprocx_out=1, nprocy_out=1,
)
return True
def generateEnsemblePerturbations(cm_args, batch, start_time):
work_path, input_path, debug_path = \
getPaths(cm_args.base_path, cm_args.job_name, work=True, input=True, debug=True)
for n_ens in cm_args.members:
ens_member_name = "ena%03d" % (n_ens + 1)
arpsenkfic_input_file_name = "%s/%s.arpsenkfic.input" % (input_path, ens_member_name)
editNamelistFile("%s/%s" % (cm_args.base_path, cm_args.arpsenkfic_template), arpsenkfic_input_file_name,
seeds=-n_ens,
dirnamp="%s/" % work_path,
outdumpdir="%s/" % work_path,
outname=ens_member_name,
tfgs=start_time)
arps_input_file_name = "%s/arps.input" % input_path
arpsenkfic_input_file_name = "%s/%s.arpsenkfic.input" % (input_path, 'ena%(ens)03d')
arpsenkfic_debug_file_name = "%s/%s.arpsenkfic.debug" % (debug_path, 'ena%(ens)03d')
command = "%s -n 1 $base/arpsenkfic %s < %s > %s" % \
(batch.getMPIprogram(), arps_input_file_name, arpsenkfic_input_file_name, arpsenkfic_debug_file_name)
command_lines = doForEnsemble(command, cm_args.members)
return command_lines
def main():
radar_location_KCYS = (41.15194, -104.80611)
radar_location_KFTG = (39.78667, -104.54583)
radar_location_KRIW = (43.06611, -108.47722)
radar_location_05XP = (41.56150, -104.298996)
domain_center_1km = (41.61795, -104.34843)
domain_center_3km = (40.61795, -107.344)
domain_std_lon = -107.344
# Set these! ALL OF THESE!
radar_id = "KCYS"
path_to_data = "raw"
radar_location = radar_location_KCYS
domain_center = domain_center_3km
domain_grid_spacing = 3000
domain_grid_size = 411
manual_qc_flag = 'manual'
tags = findTags(path_to_data, radar_id)
initial_time = toSeconds("180000")
max_pids = 8
pids = []
if radar_id == "KRIW":
apparent_lat = domain_center[0]
apparent_lon = domain_center[1]
apparent_std_lon = domain_std_lon
if os.path.exists("88d2arps"):
os.unlink("88d2arps")
os.link("/data6/tsupinie/88d2arps_KRIW", "88d2arps")
else:
# apparent_lat = domain_center[0]
# apparent_lon = -domain_center[1] + 2 * (domain_center[1] - radar_location[1])
# apparent_std_lon = -domain_std_lon + 2 * (domain_std_lon - radar_location[1])
apparent_lat = domain_center[0]
apparent_lon = domain_center[1]
apparent_std_lon = domain_std_lon
if manual_qc_flag == "manual":
good_flag = "good"
else:
good_flag = "bad"
if os.path.exists("solo2arps"):
os.unlink("solo2arps")
os.link("/data6/tsupinie/solo2arps_%s" % (radar_id[-4:], ),
"solo2arps")
for tag in tags:
print "Extracting tag %s ..." % tag
files = makeSweepFile(path_to_data, radar_id, tag)
if radar_id != "KRIW":
time_lbound = files[0].find("swp") + 11
time_ubound = time_lbound + 6
time = files[0][time_lbound:time_ubound]
if initial_time == -1:
initial_time = toSeconds(time)
offset_seconds = toSeconds(time) - initial_time
else:
offset_seconds = toSeconds(tag) - initial_time
kwargs = {}
if radar_id == "KRIW":
kwargs['radfname'] = files[0]
kwargs['refvarname'] = "DBZ"
kwargs['velvarname'] = "VEL"
src_remap_file = "radremap_88D.input"
elif radar_id == "KCYS":
if manual_qc_flag == "manual":
kwargs['refvarname'] = "DZ"
kwargs['velvarname'] = "DV"
else:
kwargs['refvarname'] = "DBZ"
kwargs['velvarname'] = "VEL"
src_remap_file = "radremap_88D.input"
elif radar_id == "KFTG":
if manual_qc_flag == "manual":
kwargs['refvarname'] = "DZ"
kwargs['velvarname'] = "DV"
else:
kwargs['refvarname'] = "DBZ"
kwargs['velvarname'] = "VEL"
src_remap_file = "radremap_88D.input"
elif radar_id == "MWR05XP":
src_remap_file = "radremap_MWR05XP.input"
kwargs['refvarname'] = "DZ"
kwargs['velvarname'] = "DV"
if manual_qc_flag == "automated":
kwargs['rngmin'] = 100000.0
editNamelistFile(src_remap_file,
"input/radremap.%s.input" % tag,
nx=domain_grid_size,
ny=domain_grid_size,
dx=domain_grid_spacing,
dy=domain_grid_spacing,
radname=radar_id[-4:],
ctrlat=apparent_lat,
ctrlon=apparent_lon,
trulon=apparent_std_lon,
outtime=offset_seconds,
**kwargs)
if radar_id == "KRIW":
pid = subprocess.Popen(["./88d2arps"],
stdin=open("input/radremap.%s.input" % tag,
'r'),
stdout=open("debug/radremap.%s.debug" % tag,
'w')).pid
else:
pid = subprocess.Popen(
["./solo2arps", "-fdisk",
"sweeps/sweeps.%s.txt" % tag],
stdin=open("input/radremap.%s.input" % tag, 'r'),
stdout=open("debug/radremap.%s.debug" % tag, 'w')).pid
pids.append(pid)
if len(pids) == max_pids or tag == tags[-1]:
for pid in pids:
os.waitpid(pid, 0)
pids = []
for file in glob.glob("%s.20090605.*" % radar_id):
os.rename(
file,
"qc/%s/%dkm/." % (manual_qc_flag, domain_grid_spacing / 1000))
for file in glob.glob("goshen.hdfr*"):
os.rename(file,
"hdf/%s/%dkm/." % (radar_id, domain_grid_spacing / 1000))
return
def main():
radar_location_KCYS = (41.15194, -104.80611)
radar_location_KFTG = (39.78667, -104.54583)
radar_location_KRIW = (43.06611, -108.47722)
radar_location_05XP = (41.56150, -104.298996)
domain_center_1km = (41.61795, -104.34843)
domain_center_3km = (40.61795, -107.344)
domain_std_lon = -107.344
# Set these! ALL OF THESE!
radar_id = "KCYS"
path_to_data = "raw"
radar_location = radar_location_KCYS
domain_center = domain_center_3km
domain_grid_spacing = 3000
domain_grid_size = 411
manual_qc_flag = 'manual'
tags = findTags(path_to_data, radar_id)
initial_time = toSeconds("180000")
max_pids = 8
pids = []
if radar_id == "KRIW":
apparent_lat = domain_center[0]
apparent_lon = domain_center[1]
apparent_std_lon = domain_std_lon
if os.path.exists("88d2arps"):
os.unlink("88d2arps")
os.link("/data6/tsupinie/88d2arps_KRIW", "88d2arps")
else:
# apparent_lat = domain_center[0]
# apparent_lon = -domain_center[1] + 2 * (domain_center[1] - radar_location[1])
# apparent_std_lon = -domain_std_lon + 2 * (domain_std_lon - radar_location[1])
apparent_lat = domain_center[0]
apparent_lon = domain_center[1]
apparent_std_lon = domain_std_lon
if manual_qc_flag == "manual":
good_flag = "good"
else:
good_flag = "bad"
if os.path.exists("solo2arps"):
os.unlink("solo2arps")
os.link("/data6/tsupinie/solo2arps_%s" % (radar_id[-4:],), "solo2arps")
for tag in tags:
print "Extracting tag %s ..." % tag
files = makeSweepFile(path_to_data, radar_id, tag)
if radar_id != "KRIW":
time_lbound = files[0].find("swp") + 11
time_ubound = time_lbound + 6
time = files[0][time_lbound:time_ubound]
if initial_time == -1:
initial_time = toSeconds(time)
offset_seconds = toSeconds(time) - initial_time
else:
offset_seconds = toSeconds(tag) - initial_time
kwargs = {}
if radar_id == "KRIW":
kwargs['radfname'] = files[0]
kwargs['refvarname'] = "DBZ"
kwargs['velvarname'] = "VEL"
src_remap_file = "radremap_88D.input"
elif radar_id == "KCYS":
if manual_qc_flag == "manual":
kwargs['refvarname'] = "DZ"
kwargs['velvarname'] = "DV"
else:
kwargs['refvarname'] = "DBZ"
kwargs['velvarname'] = "VEL"
src_remap_file = "radremap_88D.input"
elif radar_id == "KFTG":
if manual_qc_flag == "manual":
kwargs['refvarname'] = "DZ"
kwargs['velvarname'] = "DV"
else:
kwargs['refvarname'] = "DBZ"
kwargs['velvarname'] = "VEL"
src_remap_file = "radremap_88D.input"
elif radar_id == "MWR05XP":
src_remap_file = "radremap_MWR05XP.input"
kwargs['refvarname'] = "DZ"
kwargs['velvarname'] = "DV"
if manual_qc_flag == "automated":
kwargs['rngmin'] = 100000.0
editNamelistFile(src_remap_file, "input/radremap.%s.input" % tag,
nx=domain_grid_size, ny=domain_grid_size,
dx=domain_grid_spacing, dy=domain_grid_spacing,
radname=radar_id[-4:],
ctrlat=apparent_lat,
ctrlon=apparent_lon,
trulon=apparent_std_lon,
outtime=offset_seconds,
**kwargs
)
if radar_id == "KRIW":
pid = subprocess.Popen([ "./88d2arps" ], stdin=open("input/radremap.%s.input" % tag, 'r'), stdout=open("debug/radremap.%s.debug" % tag, 'w')).pid
else:
pid = subprocess.Popen([ "./solo2arps", "-fdisk", "sweeps/sweeps.%s.txt" % tag ], stdin=open("input/radremap.%s.input" % tag, 'r'), stdout=open("debug/radremap.%s.debug" % tag, 'w')).pid
pids.append(pid)
if len(pids) == max_pids or tag == tags[-1]:
for pid in pids:
os.waitpid(pid, 0)
pids = []
for file in glob.glob("%s.20090605.*" % radar_id):
os.rename(file, "qc/%s/%dkm/." % (manual_qc_flag, domain_grid_spacing / 1000))
for file in glob.glob("goshen.hdfr*"):
os.rename(file, "hdf/%s/%dkm/." % (radar_id, domain_grid_spacing / 1000))
return
member_paths = ["%s/EN%03d/%s" % (basedir, m, ens_member_name) for m, ens_member_name in
zip(ens_member_list, ens_member_names)]
for ens_member_name, t0_interp_input_file_name, lbc_interp_input_file_name, member_path in \
zip(ens_member_names, t0_interp_input_file_names, lbc_interp_input_file_names, \
member_paths):
# Initial conditions
editNamelistFile("%s" % template, t0_interp_input_file_name,
runname="%s" % ens_member_name,
hdmpfheader=member_path,
dirname=outputdir,
tbgn_dmpin=start_time,
tend_dmpin=start_time,
tintv_dmpin=step_time,
nproc_x=nproc_x,
nproc_y=nproc_y,
nproc_x_in=nproc_x_in,
nproc_y_in=nproc_y_in,
nproc_x_out=nproc_x_out,
nproc_y_out=nproc_y_out,
inisplited=inisplited,
dmp_out_joined=dmp_out_joined,
terndta1="%s/%s.trndata" % (trndata, expPrefix))
# Boundary conditions
editNamelistFile("%s" % template, lbc_interp_input_file_name,
runname="%s" % ens_member_name,
hdmpfheader=member_path,
dirname=outputdir,
tbgn_dmpin=start_time,
tend_dmpin=end_time,
def generateEnsembleIntegration(cm_args, batch, start_time, end_time, dump_time, split_files='neither', move_for_assim=True):
work_path, input_path, debug_path, bc_path, hx_path = \
getPaths(cm_args.base_path, cm_args.job_name, work=True, input=True, debug=True, boundary=True, hx=True)
nproc_x, nproc_y = cm_args.mpi_config
extraneous_files = [
"%s/%s.hdf%06d.01" % (work_path, 'ena%(ens)03d', start_time),
"%s/%s.hdfgrdbas.01" % (work_path, 'ena%(ens)03d'),
"%s/%s.log" % (work_path, 'ena%(ens)03d'),
"%s/%s.maxmin" % (work_path, 'ena%(ens)03d'),
]
read_split = split_files in ['read', 'both']
dump_split = split_files in ['dump', 'both']
if move_for_assim:
if dump_split:
epilogue = doForMPIConfig(
"mv %s/%s/%s.hdf%06d_%s %s/%s/%s.hdf%06d_%s" % \
(work_path, 'EN%(ens)s', 'ena%(ens)s', end_time, '%(x_proc)03d%(y_proc)03d', work_path, 'ENF%(ens)s', 'enf%(ens)s', end_time, '%(x_proc)03d%(y_proc)03d'), cm_args.mpi_config
)
else:
epilogue = [
"mv %s/%s.hdf%06d %s/%s.hdf%06d" % (work_path, 'ena%(ens)03d', end_time, work_path, 'enf%(ens)03d', end_time)
]
else:
epilogue = []
arps_input_file_name = "%s/%s.%d-%d.arps.input" % (input_path, 'ena%(ens)03d', start_time, end_time)
arps_debug_file_name = "%s/%s.%d-%d.arps.debug" % (debug_path, 'ena%(ens)03d', start_time, end_time)
if cm_args.algorithm == "4densrf":
arpsenkf_input_file_name = "%s/%d.arpsenkf.input" % (input_path, end_time)
epilogue.extend([
"mv ????????????_%s.lso %s" % ('%(ens)03d', hx_path),
"mv ????????????_%s.snd %s" % ('%(ens)03d', hx_path),
"mv ????_*_??????_%s %s" % ('%(ens)03d', hx_path),
])
for n_ens in cm_args.members:
ens_member_name = "ena%03d" % (n_ens + 1)
ens_member_directory = "EN%03d" % (n_ens + 1)
ens_arps_file_name = arps_input_file_name % { 'ens':(n_ens + 1) }
if read_split:
init_file = "%s/%s/%s.hdf%06d" % (work_path, ens_member_directory, ens_member_name, start_time),
init_grdbas = "%s/%s/%s.hdfgrdbas" % (work_path, ens_member_directory, ens_member_name),
else:
init_file = "%s/%s.hdf%06d" % (work_path, ens_member_name, start_time),
init_grdbas = "%s/%s.hdfgrdbas" % (work_path, ens_member_name),
if dump_split:
out_dir="%s/%s/" % (work_path, ens_member_directory)
else:
out_dir="%s/" % work_path
kwargs = {'hxopt':0, 'tstop':end_time}
if cm_args.algorithm == '4densrf':
kwargs['memid'] = n_ens + 1
kwargs['hxopt'] = 1
kwargs['tstop'] += 150
editNamelistFile("%s/%s" % (cm_args.base_path, cm_args.arps_template), ens_arps_file_name,
nproc_x=nproc_x, nproc_y=nproc_y,
runname=ens_member_name,
initopt=3,
inifile=init_file,
inigbf=init_grdbas,
exbcname="%s/%s" % (bc_path, ens_member_name),
tstart=start_time,
tstrtdmp=start_time + dump_time,
thisdmp=dump_time,
dmp_out_joined=int(not dump_split),
inisplited=int(read_split),
dirname=out_dir,
**kwargs
)
if cm_args.algorithm == 'ensrf':
command = [
"%s %s $base/arps %s > %s " % (batch.getMPIprogram(), batch.getMPIargs()%(nproc_x*nproc_y), arps_input_file_name, arps_debug_file_name),
"rm %s" % ( " ".join(extraneous_files) )
]
command.extend(epilogue)
elif cm_args.algorithm == '4densrf':
command = [
"%s %s $base/arps %s %s > %s " % (batch.getMPIprogram(), batch.getMPIargs()%(nproc_x*nproc_y), arps_input_file_name, arpsenkf_input_file_name, arps_debug_file_name),
"rm %s" % ( " ".join(extraneous_files) )
]
command.extend(epilogue)
command_lines = doForEnsemble(command, cm_args.members)
return command_lines
def main():
ap = argparse.ArgumentParser()
ap.add_argument('--n-ens', dest='n_ens_members', default=4, type=int)
ap.add_argument('--members', dest='members', nargs='+', default=[], type=int)
ap.add_argument('--base-path', dest='base_path', default=os.getcwd())
ap.add_argument('--job-name', dest='job_name', default="run_osse_test")
ap.add_argument('--mpi-config', dest='mpi_config', nargs=2, default=(3, 4), type=int)
ap.add_argument('--algorithm', dest='algorithm', choices=['ensrf', '4densrf'], default='ensrf')
ap.add_argument('--ens-start', dest='t_ens_start', default=1200, type=int)
ap.add_argument('--ens-end', dest='t_ens_end', default=1500, type=int)
ap.add_argument('--assim-step', dest='dt_assim_step', default=300, type=int)
ap.add_argument('--ens-step', dest='dt_ens_step', default=300, type=int)
ap.add_argument('--initial-conditions', dest='init_cond', default='')
ap.add_argument('--boundary-conditions', dest='bound_cond', default='')
ap.add_argument('--subset-ic', dest='subset', action='store_true')
ap.add_argument('--free-forecast', dest='free_forecast', action='store_true')
ap.add_argument('--covariance-inflation', dest='cov_infl', nargs='+', default=["mult=1.1"])
ap.add_argument('--arps-template', dest='arps_template', default='arps.input')
ap.add_argument('--arpsenkf-template', dest='arpsenkf_template', default='arpsenkf.input')
ap.add_argument('--arpsenkfic-template', dest='arpsenkfic_template', default='arpsenkfic.input')
ap.add_argument('--assim-radar', dest='radflags')
ap.add_argument('--assim-sndg', dest='sndgflags', default='yes')
ap.add_argument('--assim-prof', dest='profflags', default='yes')
ap.add_argument('--assim-surf', dest='surfflags', default='yes')
ap.add_argument('--init-fcst-req', dest='init_fcst_req', default='0:40')
ap.add_argument('--fcst-req', dest='fcst_req', default='0:20')
ap.add_argument('--init-free-fcst-req', dest='init_free_fcst_req', default='1:45')
ap.add_argument('--free-fcst-req', dest='free_fcst_req', default='1:30')
ap.add_argument('--assim-on-req', dest='assim_on_req', default='1:00')
ap.add_argument('--assim-off-req', dest='assim_off_req', default='0:45')
ap.add_argument('--chunk-size', dest='chunk_size', default=1200, type=int)
ap.add_argument('--join-files', dest='split_files', action='store_false')
ap.add_argument('--split-init', dest='split_init', choices=['auto', 'yes', 'no'], default='auto') # Whether or not the initialization is split.
ap.add_argument('--no-submit', dest='submit', action='store_false')
ap.add_argument('--restart', dest='restart', action='store_true')
ap.add_argument('--debug', dest='debug', action='store_true')
args = ap.parse_args()
batch = Batch('rice') # stampede
# work_path = "%s/%s" % (args.base_path, args.job_name)
# input_path = "%s/input" % args.base_path
# boundary_path = "%s/boundary/" % args.base_path
work_path, input_path, boundary_path, hx_path =\
getPaths(args.base_path, args.job_name, work=True, input=True, boundary=True, hx=True)
if not os.path.exists(work_path):
os.mkdir(work_path, 0755)
if not os.path.exists(hx_path):
os.mkdir(hx_path, 0755)
member_list = [ m - 1 for m in args.members ]
if member_list == []: member_list = range(args.n_ens_members)
args.members = member_list
nproc_x, nproc_y = args.mpi_config
prologue = [
"echo \"\" > %s/debug/ena%s-%s.output" % (args.base_path, '%(ens)03d', args.job_name),
"",
]
command_lines = doForEnsemble(prologue, member_list)
command = [ "set base=%s" % args.base_path, "cd $base", "" ]
appendCommands(command_lines,
doForEnsemble(command, member_list)
)
joined = 0 if args.split_files else 1
editNamelistFile("%s/%s" % (args.base_path, args.arps_template), "%s/arps.input" % input_path,
nproc_x=nproc_x, nproc_y=nproc_y,
dmp_out_joined=joined,
inisplited=3 * (1 - joined),
sfcdat=3,
dirname=work_path
)
do_first_integration = True
ens_chunk_start = args.t_ens_start
if not args.free_forecast and args.chunk_size > args.dt_assim_step:
args.chunk_size = args.dt_assim_step
if args.dt_ens_step > args.chunk_size:
args.dt_ens_step = args.chunk_size
print "Warning: resetting dt_ens_step to be chunk_size (%d)" % args.dt_ens_step
if not args.free_forecast:
exec open("%s/%s" % (args.base_path, args.radflags), 'r') in locals() # Get the radar assimilation flags from the file
# args.radar_data_flags = radar_data_flag
exp_start = args.t_ens_start
# Copy the configuration information to the working directory, so we'll **ALWAYS HAVE IT IF WE NEED TO GO BACK AND LOOK AT IT**
config_files = [ "%s/%s" % (args.base_path, f) for f in [ args.arps_template, args.arpsenkf_template, args.arpsenkfic_template ] ]
config_files.extend(['run_real_data_case.py', 'run_real_data_case.csh'])
for file in config_files:
subprocess.Popen(['cp', file, "%s/." % work_path])
if args.restart:
for t_ens in xrange(args.t_ens_start, args.t_ens_end + args.dt_assim_step, args.dt_assim_step):
if args.split_files and args.split_init != 'no':
ena_exist = [ os.path.exists("%s/EN%03d/ena%03d.hdf%06d_001001" % (work_path, n_ens + 1, n_ens + 1, t_ens)) for n_ens in member_list ]
enf_exist = [ os.path.exists("%s/ENF%03d/enf%03d.hdf%06d_001001" % (work_path, n_ens + 1, n_ens + 1, t_ens)) for n_ens in member_list ]
else:
ena_exist = [ os.path.exists("%s/ena%03d.hdf%06d" % (work_path, n_ens + 1, t_ens)) for n_ens in member_list ]
enf_exist = [ os.path.exists("%s/enf%03d.hdf%06d" % (work_path, n_ens + 1, t_ens)) for n_ens in member_list ]
all_ena_exist = all(ena_exist)
all_enf_exist = all(enf_exist)
if all_ena_exist:
args.t_ens_start = t_ens
ens_chunk_start = t_ens
elif all_enf_exist and not all_ena_exist:
args.t_ens_start = t_ens - args.dt_assim_step
do_first_integration = False
for t_chunk in xrange(t_ens, t_ens + args.dt_assim_step, args.chunk_size):
if args.split_files and args.split_init != 'no':
ena_exist = [ os.path.exists("%s/EN%03d/ena%03d.hdf%06d_001001" % (work_path, n_ens + 1, n_ens + 1, t_chunk)) for n_ens in member_list ]
else:
ena_exist = [ os.path.exists("%s/ena%03d.hdf%06d" % (work_path, n_ens + 1, t_chunk)) for n_ens in member_list ]
if all(ena_exist):
ens_chunk_start = t_chunk
if do_first_integration:
print "Restarting from time %d (with integration) ..." % (args.t_ens_start)
else:
print "Restarting from time %d (no integration) ..." % (args.t_ens_start + args.dt_assim_step)
else:
print "New experiment ..."
if args.split_files:
command = "mkdir %s/%s ; mkdir %s/%s" % (work_path, 'EN%(ens)03d', work_path, 'ENF%(ens)03d')
appendCommands(command_lines,
doForEnsemble(command, member_list)
)
if args.init_cond == "":
print "Generate random initial conditions ..."
appendCommands(command_lines,
generateEnsemblePerturbations(args, batch, args.t_ens_start)
)
command = "cp %s/%s.hdfgrdbas %s/%s.hdfgrdbas" % (work_path, 'ena%(ens)03d', work_path, 'enf%(ens)03d')
appendCommands(command_lines,
doForEnsemble(command, member_list)
)
else:
print "Use supplied initial conditions ..."
if args.subset:
print "Subset and perturb the domain ..."
appendCommands(command_lines,
generateDomainSubset(args, batch, args.init_cond, args.t_ens_start, args.t_ens_end, args.dt_ens_step, perturb_ic=True)
)
command = "cp %s/%s.hdfgrdbas %s/%s.hdfgrdbas" % (boundary_path, 'ena%(ens)03d', work_path, 'enf%(ens)03d')
appendCommands(command_lines,
doForEnsemble(command, member_list)
)
else:
print "No domain subset ..."
command = [
"cp %s/%s.hdf%06d %s" % (boundary_path, 'ena%(ens)03d', args.t_ens_start, work_path),
"cp %s/%s.hdfgrdbas %s" % (boundary_path, 'ena%(ens)03d', work_path),
"cp %s/%s.hdfgrdbas %s/%s.hdfgrdbas" % (boundary_path, 'ena%(ens)03d', work_path, 'enf%(ens)03d')
]
appendCommands(command_lines,
doForEnsemble(command, member_list)
)
if args.free_forecast:
if args.subset and args.t_ens_start == exp_start:
print "Subset the boundary conditions ..."
appendCommands(command_lines,
generateDomainSubset(args, batch, args.bound_cond, args.t_ens_start, args.t_ens_end, args.dt_ens_step, perturb_ic=False, copy_ic=False)
)
command = "cp %s/%s.hdfgrdbas %s/%s.hdfgrdbas" % (boundary_path, 'ena%(ens)03d', work_path, 'enf%(ens)03d')
appendCommands(command_lines,
doForEnsemble(command, member_list)
)
n_chunks = int(ceil(float(args.t_ens_end - args.t_ens_start) / args.chunk_size))
n_chunk_start = 0
n_chunk_start = (ens_chunk_start - args.t_ens_start) / args.chunk_size
for n_chunk, t_chunk in enumerate(range(ens_chunk_start, args.t_ens_end, args.chunk_size)):
print "Generating free forecast from %d to %d (chunk %d of %d) ..." % (args.t_ens_start, args.t_ens_end, n_chunk + n_chunk_start + 1, n_chunks)
chunk_start = t_chunk
chunk_end = t_chunk + args.chunk_size
which_split = 'neither'
if args.split_files and (args.split_init == 'auto' or args.split_init == 'yes'):
which_split = 'both'
elif args.split_files and args.split_init == 'no':
which_split = 'dump'
if chunk_end > args.t_ens_end:
chunk_end = args.t_ens_end
appendCommands(command_lines,
generateEnsembleIntegration(args, batch, chunk_start, chunk_end, args.dt_ens_step, split_files=which_split, move_for_assim=False)
)
req_time = args.free_fcst_req
if args.subset and t_chunk == exp_start:
req_time = args.init_free_fcst_req
submit(args, batch, command_lines, req_time, nproc_x * nproc_y, hybrid=False)
command_lines.clear()
command = [ "set base=%s" % args.base_path, "cd $base", "" ]
appendCommands(command_lines,
doForEnsemble(command, member_list)
)
else:
for t_ens in xrange(args.t_ens_start, args.t_ens_end, args.dt_assim_step):
print "Generating timestep %d ..." % t_ens
start_time = t_ens
end_time = t_ens + args.dt_assim_step
if do_first_integration or t_ens > args.t_ens_start:
n_chunks = int(ceil(float(end_time - start_time) / args.chunk_size))
n_chunk_start = 0
if start_time == args.t_ens_start:
n_chunk_start = (ens_chunk_start - start_time) / args.chunk_size
start_time = ens_chunk_start
for n_chunk, t_chunk in enumerate(range(start_time, end_time, args.chunk_size)):
print "Submitting ensemble integration for timestep %d (chunk %d of %d) ..." % (t_ens, n_chunk + n_chunk_start + 1, n_chunks)
chunk_start = t_chunk
chunk_end = t_chunk + args.chunk_size
if chunk_end > end_time:
chunk_end = end_time
which_split = 'neither'
if args.split_files:
if chunk_start == exp_start:
# For the first chunk
if args.restart:
# We're restarting
if args.split_init == 'auto' or args.split_init == 'yes':
which_split = 'both'
elif args.split_init == 'no':
which_split = 'dump'
else:
# No restart
if args.split_init == 'auto' or args.split_init == 'no':
which_split = 'dump'
elif args.split_init == 'yes':
which_split = 'both'
else:
# Everything after the first chunk
which_split = 'both'
if args.algorithm == '4densrf':
# ARPS for the 4DEnSRF wants an EnKF input file, too.
generateEnKFAssimilation(args, batch, chunk_end, radar_data_flag[end_time])
appendCommands(command_lines,
generateEnsembleIntegration(args, batch, chunk_start, chunk_end, args.dt_ens_step, split_files=which_split, move_for_assim=(chunk_end == end_time))
)
if args.split_files and t_chunk == exp_start:
command = doForMPIConfig("cp %s/%s/%s/%s.hdfgrdbas_%s %s/%s/%s/%s.hdfgrdbas_%s" % (args.base_path, args.job_name, 'EN%(ens)s', 'ena%(ens)s', '%(x_proc)03d%(y_proc)03d',
args.base_path, args.job_name, 'ENF%(ens)s', 'enf%(ens)s', '%(x_proc)03d%(y_proc)03d'), args.mpi_config)
appendCommands(command_lines,
doForEnsemble(command, member_list)
)
req_time = args.fcst_req
if (args.subset or not args.restart) and t_chunk == exp_start:
req_time = args.init_fcst_req
submit(args, batch, command_lines, req_time, nproc_x * nproc_y, hybrid=False, squash_jobs=False)
command_lines.clear()
command = [ "set base=%s" % args.base_path, "cd $base", "" ]
appendCommands(command_lines,
doForEnsemble(command, member_list)
)
req_time = args.assim_off_req
if isDivisible(end_time, 3600):
req_time = args.assim_on_req
# Ask Tim about rationale for using hybrid with so many total cores...
assimilation_lines = [ "set base=%s" % args.base_path, "cd $base", "", "setenv OMP_NUM_THREADS %d" % batch.getNCoresPerNode(), "" ]
#assimilation_lines = [ "set base=%s" % args.base_path, "cd $base", "" ]
assimilation_lines.extend(
generateEnKFAssimilation(args, batch, end_time, radar_data_flag[end_time])
)
print "Submitting assimilation for timestep %d ..." % t_ens
submit(args, batch, assimilation_lines, req_time, nproc_x * nproc_y * batch.getNCoresPerNode(), hybrid=True) # Will have to ask Tim about this...
#submit(args, batch, assimilation_lines, req_time, nproc_x * nproc_y, hybrid=False)
print "Experiment complete!"
return
def generateDomainSubset(cm_args, batch, src_path, start_time, end_time, step_time, perturb_ic=True, copy_ic=False):
work_path, input_path, debug_path, bc_path = \
getPaths(cm_args.base_path, cm_args.job_name, work=True, input=True, debug=True, boundary=True)
for n_ens in cm_args.members:
ens_member_name = "ena%03d" % (n_ens + 1)
interp_input_file_name = "%s/%s.arpsintrp.input" % (input_path, ens_member_name)
arpsenkfic_input_file_name = "%s/%s.arpsenkfic.input" % (input_path, ens_member_name)
arps_input_file_name = "%s/%s.arps.input" % (input_path, ens_member_name)
editNamelistFile("%s/%s" % (cm_args.base_path, cm_args.arpsintrp_template), interp_input_file_name,
runname="%s" % ens_member_name,
hdmpfheader="%s/%s" % (src_path, ens_member_name),
dirname=bc_path,
tbgn_dmpin=start_time,
tend_dmpin=end_time,
tintv_dmpin=step_time,
)
editNamelistFile("%s/%s" % (cm_args.base_path, cm_args.arps_template), arps_input_file_name,
runname="%s" % ens_member_name,
initopt=3,
inifile="%s/%s.hdf%06d" % (bc_path, ens_member_name, start_time),
inigbf="%s/%s.hdfgrdbas" % (bc_path, ens_member_name),
tstart=start_time,
tstop=end_time,
dirname="%s/" % work_path
)
if perturb_ic:
editNamelistFile("%s/%s" % (cm_args.base_path, cm_args.arpsenkfic_template), arpsenkfic_input_file_name,
seeds=-n_ens,
dirnamp="%s/" % work_path,
outdumpdir="%s/" % work_path,
outname=ens_member_name,
tfgs=start_time
)
interp_input_file_name = "%s/ena%s.arpsintrp.input" % (input_path, '%(ens)03d')
interp_debug_file_name = "%s/ena%s.arpsintrp.debug" % (debug_path, '%(ens)03d')
arps_input_file_name = "%s/ena%s.arps.input" % (input_path, '%(ens)03d')
arpsenkfic_input_file_name = "%s/ena%s.arpsenkfic.input" % (input_path, '%(ens)03d')
arpsenkfic_debug_file_name = "%s/ena%s.arpsenkfic.debug" % (debug_path, '%(ens)03d')
commands = [ "rm %s/%sicbc.*" % (bc_path, "ena%(ens)03d"), "" ]
if perturb_ic:
commands.append("%s -n 1 $base/arpsintrp %s/arps.input < %s > %s ; %s -n 1 $base/arpsenkfic %s < %s > %s" % (batch.getMPIprogram(), input_path, interp_input_file_name,
interp_debug_file_name, batch.getMPIprogram(), arps_input_file_name, arpsenkfic_input_file_name, arpsenkfic_debug_file_name))
else:
commands.append("%s -n 1 $base/arpsintrp %s/arps.input < %s > %s" % (batch.getMPIprogram(), input_path, interp_input_file_name, interp_debug_file_name))
if copy_ic:
commands.extend([
"cp %s/%s.hdf%06d %s" % (bc_path, 'ena%(ens)03d', start_time, work_path),
"cp %s/%s.hdfgrdbas %s" % (bc_path, 'ena%(ens)03d', work_path),
"cp %s/%s.hdfgrdbas %s/%s.hdfgrdbas" % (bc_path, 'ena%(ens)03d', work_path, 'enf%(ens)03d')
])
command_lines = doForEnsemble(commands, cm_args.members)
return command_lines
def generateEnKFAssimilation(cm_args, batch, assim_time, radar_data_flag):
work_path, input_path, debug_path, batch_path = \
getPaths(cm_args.base_path, cm_args.job_name, work=True, input=True, debug=True, batch=True)
nproc_x, nproc_y = cm_args.mpi_config
arps_input_file_name = "%s/%d.arps.input" % (input_path, assim_time)
enkf_input_file_name = "%s/%d.arpsenkf.input" % (input_path, assim_time)
enkf_debug_file_name = "%s/%d.arpsenkf.debug" % (debug_path, assim_time)
batch_file_name = "%s/%d.csh" % (batch_path, assim_time)
kwargs = {}
# Figure out what conventional data we're assimilating (combine this with the next section!)
cvn_data_flags = dict( (k, getattr(cm_args, k)) for k in ['sndgflags', 'profflags', 'surfflags' ] )
assim_all = False
if isDivisible(assim_time, 3600):
print "Assimilate all data ..."
else:
cvn_data_flags['sndgflags'] = 'no'
cvn_data_flags['profflags'] = 'no'
# Conventional DA flags
for cvn_flag, kw_flag in [ ('sndgflags', 'sndassim'), ('profflags', 'proassim'), ('surfflags', 'sfcassim') ]:
if cvn_flag in cvn_data_flags and cvn_data_flags[cvn_flag].lower() == 'yes':
kwargs[kw_flag] = 1
else:
kwargs[kw_flag] = 0
if cm_args.split_files:
kwargs['nproc_x_in'] = nproc_x
kwargs['nproc_y_in'] = nproc_y
kwargs['inidirname'] = "%s/ENF%s" % (work_path, "%3N")
else:
kwargs['inidirname'] = work_path
# Figure out what our covariance inflation will be (combine this with the next section!)
for cov_infl in cm_args.cov_infl:
if ':' in cov_infl:
time, factors = cov_infl.split(':')
if assim_time >= int(time):
covariance_inflation = factors
else:
covariance_inflation = cov_infl
print "Covariance inflation for this timestep is", covariance_inflation
# Covariance inflation flags
kwargs['mult_inflat'] = 0
kwargs['adapt_inflat'] = 0
kwargs['relax_inflat'] = 0
try:
covariance_inflation = covariance_inflation.split(',')
except ValueError:
covariance_inflation = [ covariance_inflation ]
for cov_infl in covariance_inflation:
if '=' in cov_infl:
inflation_method, inflation_factor = cov_infl.split('=')
inflation_factor = float(inflation_factor)
if inflation_method == "mults":
# Multiplicative inflation in the storm region only
kwargs['mult_inflat'] = 1
kwargs['cinf'] = inflation_factor
elif inflation_method == "multd":
# Multiplicative inflation over the entire domain
kwargs['mult_inflat'] = 2
kwargs['cinf'] = inflation_factor
elif inflation_method == "adapt":
# Relaxation to Prior Spread ("Adaptive") inflation
kwargs['adapt_inflat'] = 1
kwargs['rlxf'] = inflation_factor
elif inflation_method == "relax":
# Relaxation to Prior Perturbation ("Relaxation") inflation
kwargs['relax_inflat'] = 1
kwargs['rlxf'] = inflation_factor
if cm_args.algorithm == 'ensrf':
kwargs['anaopt'] = 2
elif cm_args.algorithm == '4densrf':
kwargs['anaopt'] = 5
n_radars = len(radar_data_flag[True]) if True in radar_data_flag else 0
radardaopt = 1 if n_radars > 0 else 0
editNamelistFile("%s/%s" % (cm_args.base_path, cm_args.arpsenkf_template), enkf_input_file_name,
nen=cm_args.n_ens_members,
casenam=cm_args.job_name,
enkfdtadir="%s/" % work_path,
cvndatadir="%s/obs/" % cm_args.base_path,
assim_time=assim_time,
radardaopt=radardaopt,
nrdrused=n_radars,
rmsfcst=2,
hdmpfheader=cm_args.job_name,
**kwargs
)
if cm_args.algorithm == '4densrf':
#Hard-code the 4densrf assimilation window for now, change me later
kwargs = {'hdmptim(1)':assim_time, 'tstop':assim_time + 150}
else:
kwargs = {}
joined = 0 if cm_args.split_files else 1
editNamelistFile("%s/%s" % (cm_args.base_path, cm_args.arps_template), arps_input_file_name,
nproc_x=nproc_x, nproc_y=nproc_y,
dmp_out_joined=joined,
inisplited=3 * (1 - joined),
sfcdat=3,
dirname=work_path,
**kwargs
)
command_lines = [
"cd %s" % work_path,
"%s %s $base/arpsenkf %s < %s > %s" % (batch.getMPIprogram(), batch.getMPIargs()%(nproc_x*nproc_y), arps_input_file_name, enkf_input_file_name, enkf_debug_file_name),
"cd -",
"",
]
return command_lines