def run_wrfplus_ad(work_root, wrfplus_root, config, args):
start_time = config['custom']['start_time']
end_time = config['custom']['end_time']
datetime_fmt = 'YYYY-MM-DD_HH:mm:ss'
start_time_str = start_time.format(datetime_fmt)
max_dom = config['domains']['max_dom']
wrf_work_dir = os.path.abspath(work_root) + '/wrf'
if not os.path.isdir(wrf_work_dir):
cli.error(f'WRF work directory {wrf_work_dir} does not exist!')
wrfplus_work_dir = os.path.abspath(work_root) + '/wrfplus'
if not os.path.isdir(wrfplus_work_dir):
cli.error(f'WRFPLUS has not been configured! Run config_wrfplus.py first.')
os.chdir(wrfplus_work_dir)
if os.path.isfile(f'{wrf_work_dir}/wrfinput_d01_{start_time_str}'):
run(f'ln -sf {wrf_work_dir}/wrfinput_d01 .')
elif os.path.isfile(f'{wrf_work_dir}/wrfout_d01_{start_time_str}'):
run(f'ln -sf {wrf_work_dir}/wrfout_d01_{start_time_str} wrfinput_d01')
run(f'ln -sf {wrf_work_dir}/wrfbdy_d01 .')
if not os.path.isfile('final_sens_d01'):
cli.error('There is no final_sens_d01 file!')
version = wrf_version(wrfplus_root)
cli.stage(f'Run WRFPLUS at {wrfplus_work_dir} ...')
expected_files = ['wrfout_d{:02d}_{}'.format(i + 1, start_time_str) for i in range(max_dom)]
expected_files.append(f'init_sens_d01_{start_time_str}')
if not check_files(expected_files) or args.force:
run('rm -f wrfout_*')
run(f'ln -sf {wrfplus_root}/run/LANDUSE.TBL .')
run(f'ln -sf {wrfplus_root}/run/VEGPARM.TBL .')
run(f'ln -sf {wrfplus_root}/run/SOILPARM.TBL .')
run(f'ln -sf {wrfplus_root}/run/GENPARM.TBL .')
run(f'ln -sf {wrfplus_root}/run/RRTM_DATA_DBL RRTM_DATA')
run(f'ln -sf {wrfplus_root}/run/ETAMPNEW_DATA_DBL ETAMPNEW_DATA')
if version >= Version('4.0'):
cmd = f'{wrfplus_root}/run/wrfplus.exe'
else:
cmd = f'{wrfplus_root}/run/wrf.exe'
retries = 0
while True:
submit_job(cmd, args.np, config, args, wait=True)
if os.path.isfile(f'gradient_wrfplus_d01_{start_time_str}'):
run(f'mv gradient_wrfplus_d01_{start_time_str} init_sens_d01_{start_time_str}')
if not check_files(expected_files):
if retries == 10:
cli.error(f'Failed! Check output {os.path.abspath(wrfplus_work_dir)}/rsl.error.0000.')
retries = retries + 1
cli.warning('Failed to run wrfplus, retry it!')
else:
break
cli.notice('Succeeded.')
else:
cli.notice('File wrfout_* already exist.')
run(f'ls -l {wrfplus_work_dir}/wrfout_*')
def add_to_db(wrap: str, versions: T.List[T.Tuple[str, str]], releases: T.Dict[str, T.Dict[str, T.List[str]]]):
releases.setdefault(wrap, {})
releases[wrap].setdefault('versions', [])
releases[wrap].setdefault('dependency_names', [])
releases[wrap].setdefault('program_names', [])
versions: T.List[Version] = [Version(f'{version}-{revision}') for version, revision in versions]
versions = sorted(versions, reverse=True)
versions: T.List[str] = [v._s for v in versions]
progs, deps = get_provide(wrap)
releases[wrap]['versions'] = versions
releases[wrap]['program_names'] = progs
releases[wrap]['dependency_names'] = deps
def config_wps(work_root, wps_root, geog_root, config, args):
if has_key(config, ('custom', 'start_time')):
start_time = config['custom']['start_time']
start_time_str = start_time.format('YYYY-MM-DD_HH:mm:ss')
if not has_key(config, ('custom', 'end_time')):
cli.error('custom->end_time does not exist in config file!')
end_time = config['custom']['end_time']
end_time_str = end_time.format('YYYY-MM-DD_HH:mm:ss')
if not has_key(config, ('domains', 'max_dom')):
cli.error('domains->max_dom does not exist in config file!')
max_dom = config['domains']['max_dom']
wps_work_dir = work_root + '/wps'
if not os.path.isdir(wps_work_dir): os.makedirs(wps_work_dir)
os.chdir(wps_work_dir)
version = wrf_version(wps_root)
if version < Version('3.9.1'):
cli.error(
f'WPS {version} may not handle GFS data correctly! Please use WPS >= 3.9.1.'
)
cli.notice('Edit namelist.wps for WPS.')
copy(f'{wps_root}/namelist.wps', 'namelist.wps')
namelist_wps = f90nml.read('namelist.wps')
namelist_wps['share']['max_dom'] = max_dom
if has_key(config, ('custom', 'start_time')):
namelist_wps['share']['start_date'] = [
start_time_str for i in range(max_dom)
]
if has_key(config, ('custom', 'end_time')):
namelist_wps['share']['end_date'] = [
end_time_str if i == 0 else start_time_str for i in range(max_dom)
]
if has_key(config, ('custom', 'background')) and has_key(
config, ('custom', 'background', 'interval_seconds')):
namelist_wps['share']['interval_seconds'] = config['custom'][
'background']['interval_seconds']
namelist_wps['geogrid']['geog_data_path'] = geog_root
for key, value in config['geogrid'].items():
namelist_wps['geogrid'][key] = value
namelist_wps['geogrid']['opt_geogrid_tbl_path'] = wps_work_dir
namelist_wps['metgrid']['opt_metgrid_tbl_path'] = wps_work_dir
if 'metgrid' in config:
for key, value in config['metgrid'].items():
namelist_wps['metgrid'][key] = value
namelist_wps.write('./namelist.wps', force=True)
run(f'ncl -Q {script_root}/../plots/plot_domains.ncl')
cli.notice(f'Check {wps_work_dir}/wps_show_dom.pdf for domains.')
cli.notice('Succeeded.')
def test_releases(self):
# Take list of git tags
stdout = subprocess.check_output(['git', 'tag'])
tags = [t.strip() for t in stdout.decode().splitlines()]
with open('releases.json', 'r') as f:
releases = json.load(f)
# All tags must be in the releases file
for t in tags:
name, version = t.rsplit('_', 1)
self.assertIn(name, releases)
self.assertIn(version, releases[name]['versions'])
# Verify keys are sorted
self.assertEqual(sorted(releases.keys()), list(releases.keys()))
# Get the list of wraps that has modified packagefiles
with open(Path.home() / 'files.json', 'r') as f:
changed_files = json.load(f)
self.changed_wraps = set()
for f in changed_files:
if f.startswith('subprojects/packagefiles'):
self.changed_wraps.add(f.split('/')[2])
for name, info in releases.items():
# Make sure we can load wrap file
config = configparser.ConfigParser()
config.read(f'subprojects/{name}.wrap')
self.assertEqual(config.sections()[0], 'wrap-file')
wrap_section = config['wrap-file']
self.check_has_no_path_separators(wrap_section['directory'])
self.check_has_no_path_separators(wrap_section['source_filename'])
# Basic checks
self.assertTrue(re.fullmatch('[a-z][a-z0-9._-]*', name))
patch_directory = wrap_section.get('patch_directory')
if patch_directory:
patch_path = Path('subprojects', 'packagefiles', patch_directory)
self.assertTrue(patch_path.is_dir())
# FIXME: Not all wraps currently complies, only check for wraps we modify.
if name in self.changed_wraps:
self.assertTrue(Path(patch_path, 'LICENSE.build').is_file())
self.check_files(patch_path)
# Make sure it has the same deps/progs provided
progs = []
deps = []
if 'provide' in config.sections():
provide = config['provide']
progs = [i.strip() for i in provide.get('program_names', '').split(',')]
deps = [i.strip() for i in provide.get('dependency_names', '').split(',')]
for k in provide:
if k not in {'dependency_names', 'program_names'}:
deps.append(k.strip())
progs = [i for i in progs if i]
deps = [i for i in deps if i]
self.assertEqual(sorted(progs), sorted(info['program_names']))
self.assertEqual(sorted(deps), sorted(info['dependency_names']))
# Verify versions are sorted
versions = info['versions']
self.assertGreater(len(versions), 0)
versions_obj = [Version(v) for v in versions]
self.assertEqual(sorted(versions_obj, reverse=True), versions_obj)
# The first version could be a new release, all others must have
# a corresponding tag already.
for i, v in enumerate(versions):
t = f'{name}_{v}'
ver, rev = v.rsplit('-', 1)
self.assertTrue(re.fullmatch('[a-z0-9._]+', ver))
self.assertTrue(re.fullmatch('[0-9]+', rev))
if i == 0:
self.check_source_url(name, wrap_section, ver)
if i == 0 and t not in tags:
self.check_new_release(name, info, wrap_section)
else:
self.assertIn(t, tags)
def config_wrfda_sens(work_root, wrfda_root, config, args, wrf_work_dir=None):
if not 'wrfda' in config:
cli.error('There is no "wrfda" in configuration file!')
wrfda_config = config['wrfda']
phys_config = config['physics'] if 'physics' in config else {}
start_time = config['custom']['start_time']
end_time = config['custom']['end_time']
datetime_fmt = 'YYYY-MM-DD_HH:mm:ss'
start_time_str = start_time.format(datetime_fmt)
max_dom = config['domains']['max_dom']
if not wrf_work_dir: wrf_work_dir = work_root + '/wrf'
if not os.path.isdir(wrf_work_dir): cli.error(f'{wrf_work_dir} does not exist!')
wrfda_work_dir = os.path.abspath(work_root) + '/wrfda'
if not os.path.isdir(wrfda_work_dir): os.mkdir(wrfda_work_dir)
os.chdir(wrfda_work_dir)
version = wrf_version(wrfda_root)
wrfinput = Dataset(f'{wrf_work_dir}/wrfinput_d01_{start_time_str}')
num_land_cat = wrfinput.getncattr('NUM_LAND_CAT')
wrfinput.close()
time_window = config['wrfda']['time_window'] if 'time_window' in config['wrfda'] else 360
# Read in namelist template (not exact Fortran namelist format, we need to change it).
template = open(f'{wrfda_root}/var/README.namelist').read()
template = re.sub(r'^[^&]*', '', template, flags=re.DOTALL)
template = re.sub(r';.*', '', template)
template = re.sub(r'\([^\)]*\)', '', template)
namelist_input = f90nml.read(StringIO(template))
# Merge namelist.input in tutorial.
tmp = f90nml.read(f'{wrfda_root}/var/test/tutorial/namelist.input')
for key, value in tmp.items():
if not key in namelist_input:
namelist_input[key] = value
namelist_input['wrfvar1'] ['var4d_lbc'] = False
namelist_input['wrfvar3'] ['ob_format'] = wrfda_config['ob_format']
namelist_input['wrfvar6'] ['orthonorm_gradient'] = True
namelist_input['wrfvar6'] ['use_lanczos'] = True
namelist_input['wrfvar6'] ['read_lanczos'] = True
namelist_input['wrfvar17'] ['adj_sens'] = True
namelist_input['wrfvar17'] ['sensitivity_option'] = 0
namelist_input['wrfvar17'] ['analysis_type'] = 'QC-OBS'
namelist_input['wrfvar18'] ['analysis_date'] = start_time_str
namelist_input['wrfvar21'] ['time_window_min'] = start_time.subtract(minutes=time_window/2).format(datetime_fmt)
namelist_input['wrfvar22'] ['time_window_max'] = start_time.add(minutes=time_window/2).format(datetime_fmt)
# Fix bugs
namelist_input['wrfvar2'] ['qc_rej_both'] = False
namelist_input['wrfvar7'] ['cv_options'] = wrfda_config['cv_options']
namelist_input['wrfvar14'] ['rtminit_satid'] = -1
namelist_input['wrfvar14'] ['rtminit_sensor'] = -1
namelist_input['time_control']['run_hours'] = config['custom']['forecast_hours']
namelist_input['time_control']['start_year'] = [int(start_time.format("Y")) for i in range(max_dom)]
namelist_input['time_control']['start_month'] = [int(start_time.format("M")) for i in range(max_dom)]
namelist_input['time_control']['start_day'] = [int(start_time.format("D")) for i in range(max_dom)]
namelist_input['time_control']['start_hour'] = [int(start_time.format("H")) for i in range(max_dom)]
namelist_input['time_control']['end_year'] = [int(end_time.format("Y")) for i in range(max_dom)]
namelist_input['time_control']['end_month'] = [int(end_time.format("M")) for i in range(max_dom)]
namelist_input['time_control']['end_day'] = [int(end_time.format("D")) for i in range(max_dom)]
namelist_input['time_control']['end_hour'] = [int(end_time.format("H")) for i in range(max_dom)]
namelist_input['time_control']['io_form_auxinput17'] = 2
namelist_input['time_control']['auxinput17_inname'] = './gr01'
namelist_input['time_control']['iofields_filename'] = f'{wrfda_root}/var/run/fso.io_config'
for key, value in config['domains'].items():
namelist_input['domains'][key] = value
# Sync physics parameters.
for key, value in phys_config.items():
namelist_input['physics'][key] = value
namelist_input['physics']['num_land_cat'] = num_land_cat
if version == Version('3.9.1'):
namelist_input['dynamics']['gwd_opt'] = 0
namelist_input.write(f'{wrfda_work_dir}/namelist.input', force=True)
cli.notice('Succeeded.')
