import xarray as xr
import sys
import os
import pandas as pd
from tonic.io import read_configobj
cfg = read_configobj(sys.argv[1])
i = int(sys.argv[2])
def to_netcdf_history_file_compress(ds_hist, out_nc):
''' This function saves a VIC-history-file-format ds to netCDF, with
compression.
Parameters
----------
ds_hist: <xr.Dataset>
History dataset to save
out_nc: <str>
Path of output netCDF file
'''
dict_encode = {}
for var in ds_hist.data_vars:
# skip variables not starting with "OUT_"
if var.split('_')[0] != 'OUT':
continue
# determine chunksizes
chunksizes = []
for i, dim in enumerate(ds_hist[var].dims):
import os
import xarray as xr
import subprocess
import string
import pandas as pd
import warnings
warnings.filterwarnings('ignore')
from optimize_utils import read_RVIC_output, read_USGS_data, kge
from tonic.io import read_configobj
# ======================================================== #
# Parse in parameters from MOCOM
# ======================================================== #
cfg = read_configobj(sys.argv[1])
infilt = float(sys.argv[2])
d1 = float(sys.argv[3])
d2 = float(sys.argv[4])
d3 = float(sys.argv[5])
expt = float(sys.argv[6])
Ksat = float(sys.argv[7])
statsfile = sys.argv[8]
stor_dir = sys.argv[9]
# ======================================================== #
# Parameter setting
# ======================================================== #
# --- VIC --- #
def main(cfg_file, nproc=1):
''' Main function
Parameters
----------
cfg_file: <str>
Input config file
nproc: <int>
Number of processors to use
'''
# ====================================================== #
# Load in config file
# ====================================================== #
cfg = read_configobj(cfg_file)
# ====================================================== #
# Process some cfg variables
# ====================================================== #
start_date = pd.to_datetime(cfg['FORCING']['start_date'])
end_date = pd.to_datetime(cfg['FORCING']['end_date'])
start_year = start_date.year
end_year = end_date.year
ens_list = range(cfg['FORCING']['ens_start'],
cfg['FORCING']['ens_end'] + 1)
# ====================================================== #
# Set up output directories
# ====================================================== #
dirs = setup_output_dirs(cfg['OUTPUT']['out_basedir'],
mkdirs=['forc_orig_nc', 'forc_orig_asc',
'forc_disagg_asc', 'forc_disagg_nc',
'config_files', 'logs_vic'])
# Subdirs for config files for ensemble
subdirs_config = setup_output_dirs(
dirs['config_files'],
mkdirs=['netcdf2vic', 'vic4', 'vic2nc'])
# ====================================================== #
# Load in domain file
# ====================================================== #
ds_domain = xr.open_dataset(cfg['DOMAIN']['domain_nc'])
da_domain = ds_domain[cfg['DOMAIN']['mask_name']]
lat_min = da_domain['lat'].min().values
lat_max = da_domain['lat'].max().values
lon_min = da_domain['lon'].min().values
lon_max = da_domain['lon'].max().values
# ====================================================== #
# Load in and process Newman ensemble forcings (for prec, Tmax and Tmin)
# and orig. Maurer forcing (for wind speed)
# ====================================================== #
# --- Load Maurer forcings --- #
print('Processing Maurer forcings...')
# Loop over each year
list_da_wind = []
for year in range(start_year, end_year+1):
print('Year {}'.format(year))
# --- Load in netCDF file for this year --- #
da_wind = xr.open_dataset(os.path.join(
cfg['FORCING']['maurer_dir'],
'nldas_met_update.obs.daily.wind.{}.nc'.format(year)))['wind']
# --- Mask out the target area --- #
da_wind = da_wind.sel(latitude=slice(lat_min, lat_max),
longitude=slice(lon_min, lon_max))
da_wind = da_wind.where(da_domain.values)
# --- Rename lat and lon --- #
da_wind = da_wind.rename({'latitude': 'lat', 'longitude': 'lon'})
# --- Put in list --- #
list_da_wind.append(da_wind)
# Concat all years together
da_wind_allyears = xr.concat(list_da_wind, dim='time')
# --- Load Newman forcings --- #
print('Processing Newman forcings...')
# If 1 processor, do a regular process
if nproc == 1:
# Loop over each ensemble member
for ens in ens_list:
load_and_process_Newman(ens, cfg, da_domain, lat_min, lat_max,
lon_min, lon_max, start_date, end_date,
dirs, da_wind_allyears)
# If multiple processors, use mp
elif nproc > 1:
# Set up multiprocessing
pool = mp.Pool(processes=nproc)
# Loop over each ensemble member
for ens in ens_list:
pool.apply_async(load_and_process_Newman,
(ens, cfg, da_domain, lat_min, lat_max, lon_min,
lon_max, start_date, end_date, dirs,
da_wind_allyears,))
# Finish multiprocessing
pool.close()
pool.join()
# ====================================================== #
# Convert orig. forcings to ascii format
# ====================================================== #
print('Converting orig. netCDF forcings to VIC ascii...')
# --- Setup subdirs for asc VIC orig. forcings for each ensemble member
# --- #
list_ens = []
for ens in ens_list:
list_ens.append('ens_{}'.format(ens))
subdirs_output = setup_output_dirs(
dirs['forc_orig_asc'],
mkdirs=list_ens)
# --- Prepare netcdf2vic config file --- #
dict_cfg_file = {}
for ens in ens_list:
cfg_file = os.path.join(subdirs_config['netcdf2vic'],
'ens_{}.cfg'.format(ens))
dict_cfg_file[ens] = cfg_file
with open(cfg_file, 'w') as f:
f.write('[options]\n')
f.write('files: forc_orig.{}.nc\n')
f.write('verbose: True\n')
f.write('output_format: ASCII\n')
f.write('out_prefix: forc_orig_\n')
f.write('coord_keys: lon,lat\n')
f.write('var_keys: pr,tasmax,tasmin,wind\n')
f.write('start_year: {}\n'.format(start_year))
f.write('end_year: {}\n'.format(end_year))
f.write('latlon_precision: {}\n'.format(
cfg['OUTPUT']['latlon_precision']))
f.write('\n[paths]\n')
f.write('in_path: {}\n'.format(os.path.join(
dirs['forc_orig_nc'],
'ens_{}'.format(ens))))
f.write('mask_path: {}\n'.format(cfg['DOMAIN']['domain_nc']))
f.write('mask_varname: {}\n'.format(cfg['DOMAIN']['mask_name']))
f.write('ASCIIoutPath: {}\n'.format(
subdirs_output['ens_{}'.format(ens)]))
# --- Run nc_to_vic --- #
# If 1 processor, do a regular process
if nproc == 1:
for ens in ens_list:
nc_to_vic(dict_cfg_file[ens])
# If multiple processors, use mp
elif nproc > 1:
# Set up multiprocessing
pool = mp.Pool(processes=nproc)
# Loop over each ensemble member
for ens in ens_list:
pool.apply_async(nc_to_vic, (dict_cfg_file[ens],))
# Finish multiprocessing
pool.close()
pool.join()
# ====================================================== #
# Run VIC forcing disaggregator
# ====================================================== #
print('Running VIC as a disaggregator...')
# --- Setup subdirs for asc VIC disagg. forcings and VIC log files for
# each ensemble member --- #
list_ens = []
for ens in ens_list:
list_ens.append('ens_{}'.format(ens))
subdirs_output = setup_output_dirs(
dirs['forc_disagg_asc'],
mkdirs=list_ens)
subdirs_logs = setup_output_dirs(
dirs['logs_vic'],
mkdirs=list_ens)
# --- Prepare VIC global file for the disaggregation run --- #
# Load in global file template
with open(cfg['VIC_DISAGG']['global_template'], 'r') as f:
global_param = f.read()
# Create string template
s = string.Template(global_param)
# Loop over each ensemble member
dict_global_file = {}
for ens in ens_list:
# Fill in variables in the template
global_param = s.safe_substitute(
time_step=cfg['VIC_DISAGG']['time_step'],
startyear=start_year,
startmonth=start_date.month,
startday=start_date.day,
endyear=end_year,
endmonth=end_date.month,
endday=end_date.day,
forcing1=os.path.join(dirs['forc_orig_asc'],
'ens_{}'.format(ens),
'forc_orig_'),
grid_decimal=cfg['OUTPUT']['latlon_precision'],
prec='PREC',
tmax='TMAX',
tmin='TMIN',
wind='WIND',
forceyear=start_year,
forcemonth=start_date.month,
forceday=start_date.day,
result_dir=subdirs_output['ens_{}'.format(ens)])
# Write global param file
global_file = os.path.join(subdirs_config['vic4'],
'vic.global.ens_{}.txt'.format(ens))
dict_global_file[ens] = global_file
with open(global_file, mode='w') as f:
for line in global_param:
f.write(line)
# --- Run VIC --- #
# Prepare VIC exe
vic_exe = VIC(cfg['VIC_DISAGG']['vic4_exe'])
# If 1 processor, do a regular process
if nproc == 1:
for ens in ens_list:
vic_exe.run(dict_global_file[ens],
logdir=subdirs_logs['ens_{}'.format(ens)])
# If multiple processors, use mp
elif nproc > 1:
# Set up multiprocessing
pool = mp.Pool(processes=nproc)
# Loop over each ensemble member
for ens in ens_list:
pool.apply_async(run_vic_for_multiprocess,
(vic_exe, dict_global_file[ens],
subdirs_logs['ens_{}'.format(ens)],))
# Finish multiprocessing
pool.close()
pool.join()
# ====================================================== #
# Convert disaggregated forcings to netCDF format
# ====================================================== #
# --- Prepare config file for vic2nc --- #
print('Converting disaggregated forcings to netCDF...')
# --- Setup subdirs for VIC disagg. netCDF forcings for each ensemble
# member --- #
list_ens = []
for ens in ens_list:
list_ens.append('ens_{}'.format(ens))
subdirs_output = setup_output_dirs(
dirs['forc_disagg_nc'],
mkdirs=list_ens)
# --- Prepare netcdf2vic config file --- #
# Extract disaggregated forcing variable names and order
with open(cfg['VIC_DISAGG']['global_template'], 'r') as f:
global_param = f.read()
outvar_list = find_outvar_global_param(global_param)
for i, var in enumerate(outvar_list):
outvar_list[i] = var.strip('OUT_')
# Extract end date and hour
end_date_with_hour = end_date + pd.DateOffset(days=1) -\
pd.DateOffset(hours=cfg['VIC_DISAGG']['time_step'])
# Loop over each ensemble member
dict_cfg_file = {}
for ens in ens_list:
cfg_file = os.path.join(subdirs_config['vic2nc'],
'ens_{}.cfg'.format(ens))
dict_cfg_file[ens] = cfg_file
with open(cfg_file, 'w') as f:
f.write('[OPTIONS]\n')
f.write('input_files: {}\n'.format(
os.path.join(dirs['forc_disagg_asc'],
'ens_{}'.format(ens),
'force_*')))
f.write('input_file_format: ascii\n')
f.write('bin_dt_sec: {}\n'.format(cfg['VIC_DISAGG']['time_step']*3600))
f.write('bin_start_date: {}\n'.format(start_date.strftime("%Y-%m-%d-%H")))
f.write('bin_end_date: {}\n'.format(end_date_with_hour.strftime("%Y-%m-%d-%H")))
f.write('regular_grid: False\n')
f.write('out_directory: {}\n'.format(subdirs_output['ens_{}'.format(ens)]))
f.write('memory_mode: big_memory\n')
f.write('chunksize: 100\n')
f.write('out_file_prefix: force\n')
f.write('out_file_format: NETCDF4\n')
f.write('precision: single\n')
f.write('start_date: {}\n'.format(start_date.strftime("%Y-%m-%d-%H")))
f.write('end_date: {}\n'.format(end_date_with_hour.strftime("%Y-%m-%d-%H")))
f.write('calendar: proleptic_gregorian\n')
f.write('time_segment: year\n')
f.write('snow_bands: False\n')
f.write('veg_tiles: False\n')
f.write('soil_layers: False\n')
f.write('\n[DOMAIN]\n')
f.write('filename: {}\n'.format(cfg['DOMAIN']['domain_nc']))
f.write('longitude_var: {}\n'.format(cfg['DOMAIN']['lon_name']))
f.write('latitude_var: {}\n'.format(cfg['DOMAIN']['lat_name']))
f.write('y_x_dims: {}, {}\n'.format(cfg['DOMAIN']['lat_name'],
cfg['DOMAIN']['lon_name']))
f.write('copy_vars: {}, {}, {}\n'.format(cfg['DOMAIN']['mask_name'],
cfg['DOMAIN']['lat_name'],
cfg['DOMAIN']['lon_name']))
f.write('\n[GLOBAL_ATTRIBUTES]\n')
f.write('title: VIC forcings\n')
f.write('version: VIC4.2\n')
f.write('grid: 1/8\n')
for i, var in enumerate(outvar_list):
if var == 'AIR_TEMP':
f.write('\n[AIR_TEMP]\n')
f.write('column: {}\n'.format(i))
f.write('units: C\n')
f.write('standard_name: air_temperature\n')
f.write('description: air temperature\n')
elif var == 'PREC':
f.write('\n[PREC]\n')
f.write('column: {}\n'.format(i))
f.write('units: mm/step\n')
f.write('standard_name: precipitation\n')
f.write('description: precipitation\n')
elif var == 'PRESSURE':
f.write('\n[PRESSURE]\n')
f.write('column: {}\n'.format(i))
f.write('units: kPa\n')
f.write('standard_name: surface_air_pressure\n')
f.write('description: near-surface atmospheric pressure\n')
elif var == 'SHORTWAVE':
f.write('\n[SHORTWAVE]\n')
f.write('column: {}\n'.format(i))
f.write('units: W m-2\n')
f.write('standard_name: incoming_shortwave_radiation\n')
f.write('description: incoming shortwave radiation\n')
elif var == 'LONGWAVE':
f.write('\n[LONGWAVE]\n')
f.write('column: {}\n'.format(i))
f.write('units: W m-2\n')
f.write('standard_name: incoming_longwave_radiation\n')
f.write('description: incoming longwave radiation\n')
elif var == 'VP':
f.write('\n[VP]\n')
f.write('column: {}\n'.format(i))
f.write('units: kPa\n')
f.write('standard_name: water_vapor_pressure\n')
f.write('description: near surface vapor pressure\n')
elif var == 'WIND':
f.write('\n[WIND]\n')
f.write('column: {}\n'.format(i))
f.write('units: m/s\n')
f.write('standard_name: surface_air_pressure\n')
f.write('description: near-surface wind speed\n')
# --- Run vic2nc --- #
# If 1 processor, do a regular process
if nproc == 1:
for ens in ens_list:
cfg_vic2nc = read_config(dict_cfg_file[ens])
options = cfg_vic2nc.pop('OPTIONS')
global_atts = cfg_vic2nc.pop('GLOBAL_ATTRIBUTES')
if not options['regular_grid']:
domain_dict = cfg_vic2nc.pop('DOMAIN')
else:
domain_dict = None
# Set aside fields dict
fields = cfg_vic2nc
# Run vic2nc
vic2nc(options, global_atts, domain_dict, fields)
# If multiple processors, use mp
elif nproc > 1:
# Set up multiprocessing
pool = mp.Pool(processes=nproc)
# Loop over each ensemble member
for ens in ens_list:
cfg_vic2nc = read_config(dict_cfg_file[ens])
options = cfg_vic2nc.pop('OPTIONS')
global_atts = cfg_vic2nc.pop('GLOBAL_ATTRIBUTES')
if not options['regular_grid']:
domain_dict = cfg_vic2nc.pop('DOMAIN')
else:
domain_dict = None
# set aside fields dict
fields = cfg_vic2nc
pool.apply_async(vic2nc,
(options, global_atts, domain_dict, fields,))
# Finish multiprocessing
pool.close()
pool.join()
def run_system(config_file, vic_exe, test_data_dir, out_dir, driver):
'''Run system tests from config file
Parameters
----------
config_file : str
Configuration file for system tests.
vic_exe : VIC (object)
VIC executable object (see tonic documentation).
test_data_dir : str
Path to test data sets.
out_dir : str
Path to output location
driver : {'classic', 'image'}
Driver to run tests on.
Returns
-------
test_results : dict
Test results for all tests in config_file.
See Also
--------
run_unit_tests
run_examples
run_science
run_release
'''
# Print test set welcome
print('\n-'.ljust(OUTPUT_WIDTH + 1, '-'))
print('Running System Tests')
print('-'.ljust(OUTPUT_WIDTH, '-'))
# Get setup
config = read_configobj(config_file)
# drop invalid driver tests
config = drop_tests(config, driver)
test_results = OrderedDict()
# Run individual system tests
for i, (testname, test_dict) in enumerate(config.items()):
# print out status info
print('Running test {0}/{1}: {2}'.format(i + 1, len(config.items()),
testname))
# Setup directories for test
dirs = setup_test_dirs(testname, out_dir,
mkdirs=['results', 'state', 'logs', 'plots'])
# read template global parameter file
infile = os.path.join(test_dir, 'system',
test_dict['global_parameter_file'])
with open(infile, 'r') as global_file:
global_param = global_file.read()
# If restart test, prepare running periods
# (1) Find STATESEC option (and STATE_FORMAT option for later use)
statesec = find_global_param_value(global_param, 'STATESEC')
if driver == 'classic':
state_format = find_global_param_value(global_param,
'STATE_FORMAT')
# (2) Prepare running periods and initial state file info for restart
# test
if 'exact_restart' in test_dict['check']:
run_periods = prepare_restart_run_periods(
test_dict['restart'],
dirs['state'], statesec)
# create template string
s = string.Template(global_param)
# fill in global parameter options
# --- if restart test, multiple runs --- #
if 'exact_restart' in test_dict['check']:
# Set up subdirectories and fill in global parameter options
# for restart testing
list_global_param =\
setup_subdirs_and_fill_in_global_param_restart_test(
s, run_periods, driver, dirs['results'], dirs['state'],
test_data_dir)
# else, single run
else:
global_param = s.safe_substitute(test_data_dir=test_data_dir,
result_dir=dirs['results'],
state_dir=dirs['state'])
# replace global options from config file
# --- extract global options to be substitute --- #
if 'options' in test_dict:
replacements = test_dict['options']
else:
replacements = OrderedDict()
# --- if STATE_FORMAT is specified, then the specified value (instead
# of the one in the global template file) --- #
if 'STATE_FORMAT' in replacements:
state_format = replacements['STATE_FORMAT']
# --- replace global options --- #
if 'exact_restart' in test_dict['check']:
for j, gp in enumerate(list_global_param):
# save a copy of replacements for the next global file
replacements_cp = replacements.copy()
# replace global options for this global file
list_global_param[j] = replace_global_values(gp, replacements)
replacements = replacements_cp
else:
global_param = replace_global_values(global_param, replacements)
# write global parameter file
if 'exact_restart' in test_dict['check']:
list_test_global_file = []
for j, gp in enumerate(list_global_param):
test_global_file = os.path.join(
dirs['test'],
'{}_globalparam_{}_{}.txt'.format(
testname,
run_periods[j]['start_date'].strftime("%Y%m%d"),
run_periods[j]['end_date'].strftime("%Y%m%d")))
list_test_global_file.append(test_global_file)
with open(test_global_file, mode='w') as f:
for line in gp:
f.write(line)
else:
test_global_file = os.path.join(
dirs['test'],
'{0}_globalparam.txt'.format(testname))
with open(test_global_file, mode='w') as f:
for line in global_param:
f.write(line)
# Get optional kwargs for run executable
run_kwargs = pop_run_kwargs(test_dict)
# run VIC
test_complete = False
test_passed = False
test_comment = ''
error_message = ''
try:
if 'exact_restart' in test_dict['check']:
for j, test_global_file in enumerate(list_test_global_file):
returncode = vic_exe.run(test_global_file,
logdir=dirs['logs'])
# Check return code
check_returncode(vic_exe,
test_dict.pop('expected_retval', 0))
else:
returncode = vic_exe.run(test_global_file, logdir=dirs['logs'],
**run_kwargs)
# Check return code
check_returncode(vic_exe,
test_dict.pop('expected_retval', 0))
test_complete = True
# check output files (different tests depending on driver)
if 'check' in test_dict:
# Check that the simulation completed for all grid cells
if 'complete' in test_dict['check']:
fnames = glob.glob(os.path.join(dirs['results'], '*'))
if driver == 'classic':
test_classic_driver_all_complete(fnames)
# check for nans in all example files
if 'output_file_nans' in test_dict['check']:
fnames = glob.glob(os.path.join(dirs['results'], '*'))
if driver == 'classic':
test_classic_driver_no_output_file_nans(fnames)
elif driver == 'image':
domain_file = os.path.join(test_data_dir,
test_dict['domain_file'])
test_image_driver_no_output_file_nans(fnames,
domain_file)
else:
raise ValueError('unknown driver')
# check for exact restarts
if 'exact_restart' in test_dict['check']:
check_exact_restart_fluxes(dirs['results'], driver,
run_periods)
if driver == 'classic':
check_exact_restart_states(dirs['state'], driver,
run_periods, statesec,
state_format)
elif driver == 'image':
check_exact_restart_states(dirs['state'], driver,
run_periods, statesec)
if 'multistream' in test_dict['check']:
fnames = glob.glob(os.path.join(dirs['results'], '*'))
if driver == 'classic':
check_multistream_classic(fnames)
elif driver == 'image':
warnings.warn('Skipping multistream image driver test')
# TODO: check_multistream_image(fnames)
# if we got this far, the test passed.
test_passed = True
# Handle errors
except Exception as e:
test_comment, error_message = process_error(e, vic_exe)
# record the test results
test_results[testname] = TestResults(testname,
test_complete=test_complete,
passed=test_passed,
comment=test_comment,
error_message=error_message,
returncode=returncode)
print('-'.ljust(OUTPUT_WIDTH, '-'))
print('Finished testing system tests.')
print('-'.ljust(OUTPUT_WIDTH, '-'))
return test_results
#!/usr/local/anaconda/bin/python
''' This scripts prepares a xmask file for Lohmann routine, using consistant method as in the inverse routing method (i.e., using haversine formula) '''
import numpy as np
import argparse
from tonic.io import read_configobj
from da_utils import generate_xmask_for_route
parser = argparse.ArgumentParser()
parser.add_argument("--cfg", type=str, help="config file for this script")
args = parser.parse_args()
cfg = read_configobj(args.cfg)
#========================================================#
# Calculate xmask (i.e., flow distance)
#========================================================#
flow_distance = generate_xmask_for_route(cfg['INPUT']['fdir_path'])
#========================================================#
# Write to file
#========================================================#
f = open(cfg['OUTPUT']['output_xmask_path'], 'w')
#--- Write header lines (copy from flow direction file) ---#
f_fdir = open(cfg['INPUT']['fdir_path'], 'r')
for i in range(6):
line = f_fdir.readline().rstrip("\n") # read from flow direction file
f.write(line + "\n")
f_fdir.close()
#--- Write xmask values ---#
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
from sklearn.neighbors import NearestNeighbors
from tonic.io import read_config, read_configobj
from utils import calc_monthly_PET, find_1deg_grid_gldas, edges_from_centers
# ========================================= #
# Parameter setting
# ========================================= #
# Load both SMAP and GLDAS, v1 and v2 results
dict_cfg = {}
for v in ['v1', 'v2']:
dict_cfg['SMAP_{}'.format(v)] = read_configobj(
'/civil/hydro/ymao/smap_data/scripts/cfg/20181121.SMAP_recom_qual.exclude_arid/'
'X_{}.linear.cfg'.format(v))
dict_cfg['GLDAS_{}'.format(v)] = read_configobj(
'/civil/hydro/ymao/smap_data/scripts/cfg/20181121.GLDAS_VIC.exclude_arid/smap_freq/'
'X_{}.linear.cfg'.format(v))
# Output data dir
output_dir = 'output/20181121.smap_gldas'
# --- Load GLDAS monthly data --- #
ds_monthly = xr.open_dataset(
'/civil/hydro/ymao/smap_data/data/GLDAS/VIC_monthly/aggregated_variables/soil_moisture.197901_201712.monthly.nc'
)
ds_monthly.load()
ds_monthly.close()
def run_system(config_file, dict_drivers, test_data_dir, out_dir):
'''Run system tests from config file
Parameters
----------
config_file : str
Configuration file for system tests.
dict_drivers : dict
Keys: driver names {'classic', 'image'}
Content: corresponding VIC executable object (see tonic documentation)
test_data_dir : str
Path to test data sets.
out_dir : str
Path to output location
Returns
-------
test_results : dict
Test results for all tests in config_file.
See Also
--------
run_unit_tests
run_examples
run_science
run_release
'''
# Print test set welcome
print('\n-'.ljust(OUTPUT_WIDTH + 1, '-'))
print('Running System Tests')
print('-'.ljust(OUTPUT_WIDTH, '-'))
# Get setup
config = read_configobj(config_file)
# Process driver info
if len(dict_drivers) == 1: # if single driver
driver = list(dict_drivers.keys())[0]
vic_exe = dict_drivers[driver]
# Drop invalid driver tests
if len(dict_drivers) == 1: # if single driver
config = drop_tests(config, driver)
else: # if multiple drivers
config = drop_tests(config, list(dict_drivers))
test_results = OrderedDict()
# Run individual system tests
for i, (testname, test_dict) in enumerate(config.items()):
# print out status info
print('Running test {0}/{1}: {2}'.format(i + 1, len(config.items()),
testname))
# Setup directories for test
dirs = setup_test_dirs(testname,
out_dir,
mkdirs=['results', 'state', 'logs', 'plots'])
# read template global parameter file
dict_global_param = {}
# --- if single driver --- #
if len(dict_drivers) == 1:
infile = os.path.join(test_dir, 'system',
test_dict['global_parameter_file'])
with open(infile, 'r') as global_file:
dict_global_param[driver] = global_file.read()
# --- if multiple drivers --- #
else:
for j, dr in enumerate(test_dict['driver']):
infile = os.path.join(test_dir, 'system',
test_dict['global_parameter_file'][j])
with open(infile, 'r') as global_file:
dict_global_param[dr] = global_file.read()
# If restart test, prepare running periods
if 'exact_restart' in test_dict['check']:
if len(dict_drivers) > 1:
raise ValueError('Only support single driver for restart'
'tests!')
global_param = dict_global_param[driver]
# (1) Find STATESEC option (and STATE_FORMAT option for later use)
statesec = find_global_param_value(global_param, 'STATESEC')
if driver == 'classic':
state_format = find_global_param_value(global_param,
'STATE_FORMAT')
# (2) Prepare running periods and initial state file info for
# restart test
run_periods = prepare_restart_run_periods(test_dict['restart'],
dirs['state'], statesec)
# If mpi test, prepare a list of number of processors to be run
elif 'mpi' in test_dict['check']:
if len(dict_drivers) > 1:
raise ValueError('Only support single driver for MPI' 'tests!')
if not isinstance(test_dict['mpi']['n_proc'], list):
raise ValueError('Need at least two values in n_proc to run'
'mpi test!')
list_n_proc = test_dict['mpi']['n_proc']
# create template string
dict_s = {}
for dr, global_param in dict_global_param.items():
dict_s[dr] = string.Template(global_param)
# fill in global parameter options
# --- if restart test, multiple runs --- #
if 'exact_restart' in test_dict['check']:
s = dict_s[driver]
# Set up subdirectories and fill in global parameter options
# for restart testing
list_global_param =\
setup_subdirs_and_fill_in_global_param_restart_test(
s, run_periods, driver, dirs['results'], dirs['state'],
test_data_dir)
# --- if mpi test, multiple runs --- #
elif 'mpi' in test_dict['check']:
s = dict_s[driver]
# Set up subdirectories and output directories in global file for
# multiprocessor testing
list_global_param = \
setup_subdirs_and_fill_in_global_param_mpi_test(
s, list_n_proc, dirs['results'], dirs['state'],
test_data_dir)
# --- if driver-match test, one run for each driver --- #
elif 'driver_match' in test_dict['check']:
# Set up subdirectories and output directories in global file for
# driver-match testing
dict_global_param = \
setup_subdirs_and_fill_in_global_param_driver_match_test(
dict_s, dirs['results'], dirs['state'], test_data_dir)
# --- else, single run --- #
else:
if len(dict_drivers) > 1:
raise RuntimeError('Only support single driver for test'
'{}!'.format(testname))
s = dict_s[driver]
global_param = s.safe_substitute(test_data_dir=test_data_dir,
result_dir=dirs['results'],
state_dir=dirs['state'])
# replace global options from config file
# --- extract global options to be substitute --- #
if 'options' in test_dict:
replacements = test_dict['options']
else:
replacements = OrderedDict()
# --- replace global options --- #
# For the purpose of exact restart, if STATE_FORMAT is specified,
# then record the specified value (instead of the one in the global
# template file)
if 'exact_restart' in test_dict['check']:
if 'STATE_FORMAT' in replacements:
state_format = replacements['STATE_FORMAT']
if 'exact_restart' in test_dict['check'] or\
'mpi' in test_dict['check']: # if multiple runs
for j, gp in enumerate(list_global_param):
# save a copy of replacements for the next global file
replacements_cp = replacements.copy()
# replace global options for this global file
list_global_param[j] = replace_global_values(gp, replacements)
replacements = replacements_cp
elif 'driver_match' in test_dict['check']: # if cross-driver runs
for dr, gp in dict_global_param.items():
# save a copy of replacements for the next global file
replacements_cp = replacements.copy()
# replace global options for this global file
dict_global_param[dr] = replace_global_values(gp, replacements)
replacements = replacements_cp
else: # if single run
global_param = replace_global_values(global_param, replacements)
# write global parameter file
if 'exact_restart' in test_dict['check']:
list_test_global_file = []
for j, gp in enumerate(list_global_param):
test_global_file = os.path.join(
dirs['test'], '{}_globalparam_{}_{}.txt'.format(
testname,
run_periods[j]['start_date'].strftime("%Y%m%d"),
run_periods[j]['end_date'].strftime("%Y%m%d")))
list_test_global_file.append(test_global_file)
with open(test_global_file, mode='w') as f:
for line in gp:
f.write(line)
elif 'mpi' in test_dict['check']:
list_test_global_file = []
for j, gp in enumerate(list_global_param):
test_global_file = os.path.join(
dirs['test'], '{}_globalparam_processors_{}.txt'.format(
testname, list_n_proc[j]))
list_test_global_file.append(test_global_file)
with open(test_global_file, mode='w') as f:
for line in gp:
f.write(line)
elif 'driver_match' in test_dict['check']:
dict_test_global_file = {}
for dr, gp in dict_global_param.items():
test_global_file = os.path.join(
dirs['test'], '{}_globalparam_{}.txt'.format(testname, dr))
dict_test_global_file[dr] = test_global_file
with open(test_global_file, mode='w') as f:
for line in gp:
f.write(line)
else:
test_global_file = os.path.join(
dirs['test'], '{0}_globalparam.txt'.format(testname))
with open(test_global_file, mode='w') as f:
for line in global_param:
f.write(line)
# Get optional kwargs for run executable
run_kwargs = pop_run_kwargs(test_dict)
# run VIC
test_complete = False
test_passed = False
test_comment = ''
error_message = ''
try:
if 'exact_restart' in test_dict['check']:
for j, test_global_file in enumerate(list_test_global_file):
returncode = vic_exe.run(test_global_file,
logdir=dirs['logs'],
**run_kwargs)
# Check return code
check_returncode(vic_exe,
test_dict.pop('expected_retval', 0))
elif 'mpi' in test_dict['check']:
for j, test_global_file in enumerate(list_test_global_file):
# Overwrite mpi_proc in option kwargs
n_proc = list_n_proc[j]
if n_proc == 1:
run_kwargs['mpi_proc'] = None
else:
run_kwargs['mpi_proc'] = list_n_proc[j]
# Run VIC
returncode = vic_exe.run(test_global_file,
logdir=dirs['logs'],
**run_kwargs)
# Check return code
check_returncode(vic_exe,
test_dict.pop('expected_retval', 0))
elif 'driver_match' in test_dict['check']:
for dr in dict_test_global_file.keys():
# Reset mpi_proc in option kwargs to None for classic
# driver run
if dr == 'classic':
run_kwargs_classic = run_kwargs
run_kwargs_classic['mpi_proc'] = None
returncode = dict_drivers[dr].run(
dict_test_global_file[dr],
logdir=dirs['logs'],
**run_kwargs_classic)
else:
returncode = dict_drivers[dr].run(
dict_test_global_file[dr],
logdir=dirs['logs'],
**run_kwargs)
# Check return code
check_returncode(dict_drivers[dr],
test_dict.pop('expected_retval', 0))
else:
returncode = vic_exe.run(test_global_file,
logdir=dirs['logs'],
**run_kwargs)
# Check return code
check_returncode(vic_exe, test_dict.pop('expected_retval', 0))
test_complete = True
# check output files (different tests depending on driver)
if 'check' in test_dict:
# Check that the simulation completed for all grid cells
if 'complete' in test_dict['check']:
if len(dict_drivers) > 1:
raise RuntimeError('Only support single driver for '
'complete check')
fnames = glob.glob(os.path.join(dirs['results'], '*'))
if driver == 'classic':
test_classic_driver_all_complete(fnames)
else:
raise RuntimeError('complete check only supports '
'classic driver')
# check for nans in all example files
if 'output_file_nans' in test_dict['check']:
if len(dict_drivers) > 1:
raise RuntimeError('Only support single driver for '
'output_file_nans check')
fnames = glob.glob(os.path.join(dirs['results'], '*'))
if driver == 'classic':
test_classic_driver_no_output_file_nans(fnames)
elif driver == 'image':
domain_file = os.path.join(test_data_dir,
test_dict['domain_file'])
test_image_driver_no_output_file_nans(
fnames, domain_file)
else:
raise ValueError('unknown driver')
# check for exact restarts
if 'exact_restart' in test_dict['check']:
check_exact_restart_fluxes(dirs['results'], driver,
run_periods)
if driver == 'classic':
check_exact_restart_states(dirs['state'], driver,
run_periods, statesec,
state_format)
elif driver == 'image':
check_exact_restart_states(dirs['state'], driver,
run_periods, statesec)
else:
raise ValueError('unknown driver')
# check for multistream output
if 'multistream' in test_dict['check']:
if len(dict_drivers) > 1:
raise ValueError('Only support single driver for '
'multistream check')
fnames = glob.glob(os.path.join(dirs['results'], '*'))
if driver == 'classic':
check_multistream_classic(fnames)
elif driver == 'image':
warnings.warn('Skipping multistream image driver test')
# TODO: check_multistream_image(fnames)
# check for mpi multiprocessor results
if 'mpi' in test_dict['check']:
check_mpi_fluxes(dirs['results'], list_n_proc)
check_mpi_states(dirs['state'], list_n_proc)
# check that results from different drivers match
if 'driver_match' in test_dict['check']:
check_drivers_match_fluxes(list(dict_drivers.keys()),
dirs['results'])
# if we got this far, the test passed.
test_passed = True
# Handle errors
except Exception as e:
for dr, exe in dict_drivers.items():
test_comment, error_message = process_error(e, exe)
# record the test results
test_results[testname] = TestResults(testname,
test_complete=test_complete,
passed=test_passed,
comment=test_comment,
error_message=error_message,
returncode=returncode)
print('-'.ljust(OUTPUT_WIDTH, '-'))
print('Finished testing system tests.')
print('-'.ljust(OUTPUT_WIDTH, '-'))
return test_results
def run_science(config_file, vic_exe, science_test_data_dir, test_data_dir,
out_dir, driver, nproc):
'''Run science tests from config file
Parameters
----------
config_file : str
Configuration file for science tests.
vic_exe : VIC (object)
VIC executable object (see tonic documentation).
science_test_data_dir: str
Path to science test data sets (archived VIC runs and observations)
test_data_dir : str
Path to test data sets.
out_dir : str
Path to output location
driver : {'classic', 'image'}
Driver to run tests on.
nproc : int
Number of processors to use for science tests
Returns
-------
test_results : dict
Test results for all tests in config_file.
See Also
--------
run_unit_tests
run_examples
run_system
run_release
'''
# Print test set welcome
print('\n-'.ljust(OUTPUT_WIDTH + 1, '-'))
print('Running Science Tests')
print('-'.ljust(OUTPUT_WIDTH, '-'))
# Get setup
config = read_configobj(config_file)
# drop invalid driver tests
config = drop_tests(config, driver)
test_results = OrderedDict()
# Run individual tests
for i, (test_type, test_dict) in enumerate(config.items()):
# print out status info
print('Running test {0}/{1}: {2}'.format(i + 1, len(config.items()),
test_type))
# Setup directories for test
dirs = setup_test_dirs(test_type,
out_dir,
mkdirs=['results', 'state', 'logs', 'plots'])
# read template global parameter file
infile = os.path.join(test_dir, 'science',
test_dict['global_parameter_file'])
with open(infile, 'r') as global_file:
global_param = global_file.read()
# create template string
s = string.Template(global_param)
# fill in global parameter options
global_param = s.safe_substitute(test_data_dir=science_test_data_dir,
test_dir=test_dir,
result_dir=dirs['results'],
state_dir=dirs['state'],
testname=test_type,
test_root=test_dir)
test_global_file = os.path.join(
dirs['test'], '{0}_globalparam.txt'.format(test_type))
# write global parameter file
with open(test_global_file, 'w') as f:
f.write(global_param)
# Get optional kwargs for run executable
run_kwargs = pop_run_kwargs(test_dict)
# run VIC
test_complete = False
test_passed = False
test_comment = ''
error_message = ''
try:
# Run the VIC simulation
returncode = vic_exe.run(test_global_file,
logdir=dirs['logs'],
**run_kwargs)
test_complete = True
# Check return code
check_returncode(vic_exe)
# check output files (different tests depending on driver)
if test_dict['check']:
fnames = glob.glob(os.path.join(dirs['results'], '*'))
# Check that the simulation completed for all grid cells
if 'complete' in test_dict['check'] and driver == 'classic':
test_classic_driver_all_complete(fnames)
# check for nans in all example files
if 'output_file_nans' in test_dict['check']:
if driver == 'classic':
test_classic_driver_no_output_file_nans(fnames)
elif driver == 'image':
domain_file = os.path.join(test_data_dir,
test_dict['domain_file'])
test_image_driver_no_output_file_nans(
fnames, domain_file)
else:
raise ValueError('unknown driver')
# plot science test results
plot_science_tests(test_dict['driver'],
test_type,
science_test_data_dir,
dirs['results'],
dirs['plots'],
test_dict['plots'],
test_dict['compare_data'],
nproc=nproc)
# if we got this far, the test passed.
test_passed = True
# Handle errors
except Exception as e:
test_comment, error_message = process_error(e, vic_exe)
# record the test results
test_results[test_type] = TestResults(test_type,
test_complete=test_complete,
passed=test_passed,
comment=test_comment,
error_message=error_message,
returncode=returncode)
print('-'.ljust(OUTPUT_WIDTH, '-'))
print('Finished testing science tests.')
print('-'.ljust(OUTPUT_WIDTH, '-'))
return test_results
#!/usr/local/anaconda/bin/python
''' This scripts prepares a xmask file for Lohmann routine, using consistant method as in the inverse routing method (i.e., using haversine formula) '''
import numpy as np
import argparse
from tonic.io import read_configobj
from da_utils import generate_xmask_for_route
parser = argparse.ArgumentParser()
parser.add_argument("--cfg", type=str, help="config file for this script")
args = parser.parse_args()
cfg = read_configobj(args.cfg)
#========================================================#
# Calculate xmask (i.e., flow distance)
#========================================================#
flow_distance = generate_xmask_for_route(cfg['INPUT']['fdir_path'])
#========================================================#
# Write to file
#========================================================#
f = open(cfg['OUTPUT']['output_xmask_path'], 'w')
#--- Write header lines (copy from flow direction file) ---#
f_fdir = open(cfg['INPUT']['fdir_path'], 'r')
for i in range(6):
line = f_fdir.readline().rstrip("\n") # read from flow direction file
f.write(line + "\n")
f_fdir.close()
#--- Write xmask values ---#
for i in range(len(flow_distance)):
import subprocess
import string
import pandas as pd
import warnings
warnings.filterwarnings('ignore')
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from optimize_utils import read_RVIC_output, read_USGS_data, kge
from tonic.io import read_configobj
# ======================================================== #
# Parse in parameters from MOCOM
# ======================================================== #
cfg = read_configobj(sys.argv[1]) # optimize cfg
stor_dir = sys.argv[2] # .../run_ident/
soln_num = sys.argv[3] # the 5-digit parameter solution number
out_plot_dir = sys.argv[4]
# ======================================================== #
# Parameter setting
# ======================================================== #
# --- RVIC --- #
rvic_convolve_template = cfg['RVIC']['rvic_convolve_template']
# Time lag between VIC/RVIC forcing and USGS local time [hour]
time_lag = cfg['RVIC']['time_lag']
# Site name in RVIC
site = cfg['RVIC']['site']
# --- USGS streamflow data --- #
#!/bin/env python
import matplotlib.pyplot as plt
import os
import numpy as np
import pandas as pd
import xarray as xr
from tonic.io import read_config, read_configobj
# read configs for filenames and diagnostics to run
config_files = read_config(os.path.join(os.getcwd(), 'filenames.cfg'))
config_diag = read_configobj(os.path.join(os.getcwd(), 'diagnostics.cfg'))
# full list of l2x coupler fields
l2x_vars = [
'l2x_Sl_t', 'l2x_Sl_tref', 'l2x_Sl_qref', 'l2x_Sl_avsdr', 'l2x_Sl_anidr',
'l2x_Sl_avsdf', 'l2x_Sl_anidf', 'l2x_Sl_snowh', 'l2x_Sl_u10', 'l2x_Sl_fv',
'l2x_Sl_ram1', 'l2x_Sl_logz0', 'l2x_Fall_taux', 'l2x_Fall_tauy',
'l2x_Fall_lat', 'l2x_Fall_sen', 'l2x_Fall_lwup', 'l2x_Fall_evap',
'l2x_Fall_swnet', 'l2x_Fall_flxdst1', 'l2x_Fall_flxdst2',
'l2x_Fall_flxdst3', 'l2x_Fall_flxdst4', 'l2x_Flrl_rofliq',
'l2x_Flrl_rofice'
]
l2x_units = [
'K', 'K', 'g/g', 'fraction', 'fraction', 'fraction', 'fraction', 'm',
'm/s', 'm/s', 's/m', 'm', 'N/m2', 'N/m2', 'W/m2', 'W/m2', 'W/m2', 'kg/m2s',
'W/m2', 'm/s', 'm/s', 'm/s', 'm/s', 'kg/m2s', 'kg/m2s'
]
def run_science(config_file, vic_exe, science_test_data_dir,
test_data_dir, out_dir, driver, nproc):
'''Run science tests from config file
Parameters
----------
config_file : str
Configuration file for science tests.
vic_exe : VIC (object)
VIC executable object (see tonic documentation).
science_test_data_dir: str
Path to science test data sets (archived VIC runs and observations)
test_data_dir : str
Path to test data sets.
out_dir : str
Path to output location
driver : {'classic', 'image'}
Driver to run tests on.
nproc : int
Number of processors to use for science tests
Returns
-------
test_results : dict
Test results for all tests in config_file.
See Also
--------
run_unit_tests
run_examples
run_system
run_release
'''
# Print test set welcome
print('\n-'.ljust(OUTPUT_WIDTH + 1, '-'))
print('Running Science Tests')
print('-'.ljust(OUTPUT_WIDTH, '-'))
# Get setup
config = read_configobj(config_file)
# drop invalid driver tests
config = drop_tests(config, driver)
test_results = OrderedDict()
# Run individual tests
for i, (test_type, test_dict) in enumerate(config.items()):
# print out status info
print('Running test {0}/{1}: {2}'.format(i + 1, len(config.items()),
test_type))
# Setup directories for test
dirs = setup_test_dirs(test_type, out_dir,
mkdirs=['results', 'state', 'logs', 'plots'])
# read template global parameter file
infile = os.path.join(test_dir, 'science',
test_dict['global_parameter_file'])
with open(infile, 'r') as global_file:
global_param = global_file.read()
# create template string
s = string.Template(global_param)
# fill in global parameter options
global_param = s.safe_substitute(test_data_dir=science_test_data_dir,
test_dir=test_dir,
result_dir=dirs['results'],
state_dir=dirs['state'],
testname=test_type,
test_root=test_dir)
test_global_file = os.path.join(
dirs['test'], '{0}_globalparam.txt'.format(test_type))
# write global parameter file
with open(test_global_file, 'w') as f:
f.write(global_param)
# Get optional kwargs for run executable
run_kwargs = pop_run_kwargs(test_dict)
# run VIC
test_complete = False
test_passed = False
test_comment = ''
error_message = ''
try:
# Run the VIC simulation
returncode = vic_exe.run(test_global_file, logdir=dirs['logs'],
**run_kwargs)
test_complete = True
# Check return code
check_returncode(vic_exe)
# check output files (different tests depending on driver)
if test_dict['check']:
fnames = glob.glob(os.path.join(dirs['results'], '*'))
# Check that the simulation completed for all grid cells
if 'complete' in test_dict['check'] and driver == 'classic':
test_classic_driver_all_complete(fnames)
# check for nans in all example files
if 'output_file_nans' in test_dict['check']:
if driver == 'classic':
test_classic_driver_no_output_file_nans(fnames)
elif driver == 'image':
domain_file = os.path.join(test_data_dir,
test_dict['domain_file'])
test_image_driver_no_output_file_nans(fnames,
domain_file)
else:
raise ValueError('unknown driver')
# plot science test results
plot_science_tests(test_dict['driver'],
test_type,
science_test_data_dir,
dirs['results'],
dirs['plots'],
test_dict['plots'],
test_dict['compare_data'],
nproc=nproc)
# if we got this far, the test passed.
test_passed = True
# Handle errors
except Exception as e:
test_comment, error_message = process_error(e, vic_exe)
# record the test results
test_results[test_type] = TestResults(test_type,
test_complete=test_complete,
passed=test_passed,
comment=test_comment,
error_message=error_message,
returncode=returncode)
print('-'.ljust(OUTPUT_WIDTH, '-'))
print('Finished testing science tests.')
print('-'.ljust(OUTPUT_WIDTH, '-'))
return test_results
def run_system(config_file, dict_drivers, test_data_dir, out_dir):
'''Run system tests from config file
Parameters
----------
config_file : str
Configuration file for system tests.
dict_drivers : dict
Keys: driver names {'classic', 'image'}
Content: corresponding VIC executable object (see tonic documentation)
test_data_dir : str
Path to test data sets.
out_dir : str
Path to output location
Returns
-------
test_results : dict
Test results for all tests in config_file.
See Also
--------
run_unit_tests
run_examples
run_science
run_release
'''
# Print test set welcome
print('\n-'.ljust(OUTPUT_WIDTH + 1, '-'))
print('Running System Tests')
print('-'.ljust(OUTPUT_WIDTH, '-'))
# Get setup
config = read_configobj(config_file)
# Process driver info
if len(dict_drivers) == 1: # if single driver
driver = list(dict_drivers.keys())[0]
vic_exe = dict_drivers[driver]
# Drop invalid driver tests
if len(dict_drivers) == 1: # if single driver
config = drop_tests(config, driver)
else: # if multiple drivers
config = drop_tests(config, list(dict_drivers))
test_results = OrderedDict()
# Run individual system tests
for i, (testname, test_dict) in enumerate(config.items()):
# print out status info
print('Running test {0}/{1}: {2}'.format(i + 1, len(config.items()),
testname))
# Setup directories for test
dirs = setup_test_dirs(testname, out_dir,
mkdirs=['results', 'state', 'logs', 'plots'])
# read template global parameter file
dict_global_param = {}
# --- if single driver --- #
if len(dict_drivers) == 1:
infile = os.path.join(test_dir, 'system',
test_dict['global_parameter_file'])
with open(infile, 'r') as global_file:
dict_global_param[driver] = global_file.read()
# --- if multiple drivers --- #
else:
for j, dr in enumerate(test_dict['driver']):
infile = os.path.join(test_dir, 'system',
test_dict['global_parameter_file'][j])
with open(infile, 'r') as global_file:
dict_global_param[dr] = global_file.read()
# If restart test, prepare running periods
if 'exact_restart' in test_dict['check']:
if len(dict_drivers) > 1:
raise ValueError('Only support single driver for restart'
'tests!')
global_param = dict_global_param[driver]
# () Find STATE_FORMAT option for later use
if driver == 'classic':
state_format = find_global_param_value(global_param,
'STATE_FORMAT')
# (2) Prepare running periods and initial state file info for
# restart test
run_periods = prepare_restart_run_periods(
test_dict['restart'],
dirs['state'])
# If mpi test, prepare a list of number of processors to be run
elif 'mpi' in test_dict['check']:
if len(dict_drivers) > 1:
raise ValueError('Only support single driver for MPI'
'tests!')
if not isinstance(test_dict['mpi']['n_proc'], list):
raise ValueError('Need at least two values in n_proc to run'
'mpi test!')
list_n_proc = test_dict['mpi']['n_proc']
# create template string
dict_s = {}
for dr, global_param in dict_global_param.items():
dict_s[dr] = string.Template(global_param)
# fill in global parameter options
# --- if restart test, multiple runs --- #
if 'exact_restart' in test_dict['check']:
s = dict_s[driver]
# Set up subdirectories and fill in global parameter options
# for restart testing
list_global_param =\
setup_subdirs_and_fill_in_global_param_restart_test(
s, run_periods, driver, dirs['results'], dirs['state'],
test_data_dir)
# --- if mpi test, multiple runs --- #
elif 'mpi' in test_dict['check']:
s = dict_s[driver]
# Set up subdirectories and output directories in global file for
# multiprocessor testing
list_global_param = \
setup_subdirs_and_fill_in_global_param_mpi_test(
s, list_n_proc, dirs['results'], dirs['state'],
test_data_dir)
# --- if driver-match test, one run for each driver --- #
elif 'driver_match' in test_dict['check']:
# Set up subdirectories and output directories in global file for
# driver-match testing
dict_global_param = \
setup_subdirs_and_fill_in_global_param_driver_match_test(
dict_s, dirs['results'], dirs['state'], test_data_dir)
# --- else, single run --- #
else:
if len(dict_drivers) > 1:
raise RuntimeError('Only support single driver for test'
'{}!'.format(testname))
s = dict_s[driver]
global_param = s.safe_substitute(test_data_dir=test_data_dir,
result_dir=dirs['results'],
state_dir=dirs['state'])
# replace global options from config file
# --- extract global options to be substitute --- #
if 'options' in test_dict:
replacements = test_dict['options']
else:
replacements = OrderedDict()
# --- replace global options --- #
# For the purpose of exact restart, if STATE_FORMAT is specified,
# then record the specified value (instead of the one in the global
# template file)
if 'exact_restart' in test_dict['check']:
if 'STATE_FORMAT' in replacements:
state_format = replacements['STATE_FORMAT']
if 'exact_restart' in test_dict['check'] or\
'mpi' in test_dict['check']: # if multiple runs
for j, gp in enumerate(list_global_param):
# save a copy of replacements for the next global file
replacements_cp = replacements.copy()
# replace global options for this global file
list_global_param[j] = replace_global_values(gp, replacements)
replacements = replacements_cp
elif 'driver_match' in test_dict['check']: # if cross-driver runs
for dr, gp in dict_global_param.items():
# save a copy of replacements for the next global file
replacements_cp = replacements.copy()
# replace global options for this global file
dict_global_param[dr] = replace_global_values(gp,
replacements)
replacements = replacements_cp
else: # if single run
global_param = replace_global_values(global_param, replacements)
# write global parameter file
if 'exact_restart' in test_dict['check']:
list_test_global_file = []
for j, gp in enumerate(list_global_param):
test_global_file = os.path.join(
dirs['test'],
'{}_globalparam_{}_{}.txt'.format(
testname,
run_periods[j]['start_date'].strftime("%Y%m%d"),
run_periods[j]['end_date'].strftime("%Y%m%d")))
list_test_global_file.append(test_global_file)
with open(test_global_file, mode='w') as f:
for line in gp:
f.write(line)
elif 'mpi' in test_dict['check']:
list_test_global_file = []
for j, gp in enumerate(list_global_param):
test_global_file = os.path.join(
dirs['test'],
'{}_globalparam_processors_{}.txt'.format(
testname, list_n_proc[j]))
list_test_global_file.append(test_global_file)
with open(test_global_file, mode='w') as f:
for line in gp:
f.write(line)
elif 'driver_match' in test_dict['check']:
dict_test_global_file = {}
for dr, gp in dict_global_param.items():
test_global_file = os.path.join(
dirs['test'],
'{}_globalparam_{}.txt'.format(
testname, dr))
dict_test_global_file[dr] = test_global_file
with open(test_global_file, mode='w') as f:
for line in gp:
f.write(line)
else:
test_global_file = os.path.join(
dirs['test'],
'{0}_globalparam.txt'.format(testname))
with open(test_global_file, mode='w') as f:
for line in global_param:
f.write(line)
# Get optional kwargs for run executable
run_kwargs = pop_run_kwargs(test_dict)
# run VIC
test_complete = False
test_passed = False
test_comment = ''
error_message = ''
try:
if 'exact_restart' in test_dict['check']:
for j, test_global_file in enumerate(list_test_global_file):
returncode = vic_exe.run(test_global_file,
logdir=dirs['logs'],
**run_kwargs)
# Check return code
check_returncode(vic_exe,
test_dict.pop('expected_retval', 0))
elif 'mpi' in test_dict['check']:
for j, test_global_file in enumerate(list_test_global_file):
# Overwrite mpi_proc in option kwargs
n_proc = list_n_proc[j]
if n_proc == 1:
run_kwargs['mpi_proc'] = None
else:
run_kwargs['mpi_proc'] = list_n_proc[j]
# Run VIC
returncode = vic_exe.run(test_global_file,
logdir=dirs['logs'],
**run_kwargs)
# Check return code
check_returncode(vic_exe,
test_dict.pop('expected_retval', 0))
elif 'driver_match' in test_dict['check']:
for dr in dict_test_global_file.keys():
# Reset mpi_proc in option kwargs to None for classic
# driver run
if dr == 'classic':
run_kwargs_classic = run_kwargs
run_kwargs_classic['mpi_proc'] = None
returncode = dict_drivers[dr].run(
dict_test_global_file[dr],
logdir=dirs['logs'],
**run_kwargs_classic)
else:
returncode = dict_drivers[dr].run(
dict_test_global_file[dr],
logdir=dirs['logs'],
**run_kwargs)
# Check return code
check_returncode(dict_drivers[dr],
test_dict.pop('expected_retval', 0))
else:
returncode = vic_exe.run(test_global_file, logdir=dirs['logs'],
**run_kwargs)
# Check return code
check_returncode(vic_exe,
test_dict.pop('expected_retval', 0))
test_complete = True
# check output files (different tests depending on driver)
if 'check' in test_dict:
# Check that the simulation completed for all grid cells
if 'complete' in test_dict['check']:
if len(dict_drivers) > 1:
raise RuntimeError('Only support single driver for '
'complete check')
fnames = glob.glob(os.path.join(dirs['results'], '*'))
if driver == 'classic':
test_classic_driver_all_complete(fnames)
else:
raise RuntimeError('complete check only supports '
'classic driver')
# check for nans in all example files
if 'output_file_nans' in test_dict['check']:
if len(dict_drivers) > 1:
raise RuntimeError('Only support single driver for '
'output_file_nans check')
fnames = glob.glob(os.path.join(dirs['results'], '*'))
if driver == 'classic':
test_classic_driver_no_output_file_nans(fnames)
elif driver == 'image':
domain_file = os.path.join(
test_data_dir,
test_dict['domain_file'])
test_image_driver_no_output_file_nans(
fnames,
domain_file)
else:
raise ValueError('unknown driver')
# check for exact restarts
if 'exact_restart' in test_dict['check']:
check_exact_restart_fluxes(dirs['results'], driver,
run_periods)
if driver == 'classic':
check_exact_restart_states(dirs['state'], driver,
run_periods,
state_format)
elif driver == 'image':
check_exact_restart_states(dirs['state'], driver,
run_periods)
else:
raise ValueError('unknown driver')
# check for multistream output
if 'multistream' in test_dict['check']:
if len(dict_drivers) > 1:
raise ValueError('Only support single driver for '
'multistream check')
fnames = glob.glob(os.path.join(dirs['results'], '*'))
if driver == 'classic':
check_multistream_classic(fnames)
elif driver == 'image':
warnings.warn('Skipping multistream image driver test')
# TODO: check_multistream_image(fnames)
# check for mpi multiprocessor results
if 'mpi' in test_dict['check']:
check_mpi_fluxes(dirs['results'], list_n_proc)
check_mpi_states(dirs['state'], list_n_proc)
# check that results from different drivers match
if 'driver_match' in test_dict['check']:
check_drivers_match_fluxes(list(dict_drivers.keys()),
dirs['results'])
# if we got this far, the test passed.
test_passed = True
# Handle errors
except Exception as e:
for dr, exe in dict_drivers.items():
test_comment, error_message = process_error(e, exe)
# record the test results
test_results[testname] = TestResults(testname,
test_complete=test_complete,
passed=test_passed,
comment=test_comment,
error_message=error_message,
returncode=returncode)
print('-'.ljust(OUTPUT_WIDTH, '-'))
print('Finished testing system tests.')
print('-'.ljust(OUTPUT_WIDTH, '-'))
return test_results
