def _run(args):
"""Top level driver"""
print('running now...')
if args.create_batch:
# ------------------------------------------------------------ #
# Create batch files and exit
batch(args.config_file, args.create_batch, args.batch_dir)
# ------------------------------------------------------------ #
else:
# ------------------------------------------------------------ #
# Read Configuration files
config_dict = read_config(args.config_file,
default_config=default_config)
options = config_dict.pop('OPTIONS')
global_atts = config_dict.pop('GLOBAL_ATTRIBUTES')
if not options['regular_grid']:
domain_dict = config_dict.pop('DOMAIN')
else:
domain_dict = None
# set aside fields dict
fields = config_dict
vic2nc(options, global_atts, domain_dict, fields)
# ------------------------------------------------------------ #
return
def _run(args):
config = read_config(args.config)
files = config["options"]["files"]
var_keys = config["options"]["var_keys"]
output = config["options"]["output"]
binary_mult = config["options"]["binary_mult"]
binary_type = (config["options"]["binary_type"],)
paths = config["options"]["paths"]
out_prefix = config["options"]["out_prefix"]
verbose = config["options"]["verbose"]
mask = read_netcdf(paths["mask_path"], nc_vars=["mask"])["mask"]
yi, xi = np.nonzero(mask)
print("found {0} points in mask file.".format(len(yi)))
xlist = []
ylist = []
pointlist = []
append = False
for i, fname in enumerate(files):
d = read_netcdf(os.path.join(paths["in_path"], fname), verbose=verbose)
if i == 0:
# find point locations
xs = d["xc"]
ys = d["yc"]
posinds = np.nonzero(xs > 180)
xs[posinds] -= 360
print("adjusted xs lon minimum")
for y, x in pyzip(yi, xi):
active_flag = False
for key in var_keys:
if (d[key][:, y, x].all() is np.ma.masked) or (mask[y, x] == 0):
active_flag = True
if not active_flag:
point = (ys[y, x], xs[y, x])
xlist.append(x)
ylist.append(y)
pointlist.append(point)
else:
append = True
for y, x, point in pyzip(ylist, xlist, pointlist):
data = np.empty((d[var_keys[0]].shape[0], len(var_keys)))
for j, key in enumerate(var_keys):
data[:, j] = d[key][:, y, x]
if output["Binary"]:
write_binary(data * binary_mult, point, binary_type, out_prefix, paths["BinaryoutPath"], append)
if output["ASCII"]:
write_ascii(data, point, out_prefix, paths["ASCIIoutPath"], append)
return
def batch(config_file, create_batch, batch_dir):
"""Create a set of batch configuration files"""
# Read Configuration files
config_dict = read_config(config_file)
options = config_dict.pop('OPTIONS')
global_atts = config_dict.pop('GLOBAL_ATTRIBUTES')
domain_dict = config_dict.pop('DOMAIN', None)
fields = config_dict
config = SafeConfigParser()
config.optionxform = str
# Figure out what to call the new files
nameprefix = os.path.splitext(os.path.split(config_file)[1])[0]
if create_batch == 'variables':
# batch by variables
# options section
config.add_section('OPTIONS')
for option, value in options.items():
if type(value) == list:
try:
value = ", ".join(value)
except TypeError:
value = ", ".join(repr(e) for e in value)
elif type(value) != str:
value = str(value)
config.set('OPTIONS', option, str(value))
# global_atts section
config.add_section('GLOBAL_ATTRIBUTES')
for option, value in global_atts.items():
if type(value) == list:
try:
value = ", ".join(value)
except TypeError:
value = ", ".join(repr(e) for e in value)
elif type(value) != str:
value = str(value)
config.set('GLOBAL_ATTRIBUTES', option, str(value))
# domain dict section
if domain_dict:
config.add_section('DOMAIN')
for option, value in domain_dict.items():
if type(value) == list:
try:
value = ", ".join(value)
except TypeError:
value = ", ".join(repr(e) for e in value)
elif type(value) != str:
value = str(value)
config.set('DOMAIN', option, value.strip("'"))
for var, field in fields.items():
suffix = "_{0}.cfg".format(var)
new_cfg_file = os.path.join(batch_dir, nameprefix + suffix)
# this var
config.add_section(var)
for option, value in field.items():
if type(value) == list:
try:
value = ", ".join(value)
except TypeError:
value = ", ".join(repr(e) for e in value)
elif type(value) != str:
value = str(value)
config.set(var, option, str(value))
# write that config
with open(new_cfg_file, 'wb') as cf:
config.write(cf)
# clear the var section
config.remove_section(var)
else:
# start with existing config
config.read(config_file)
# by time
start_date = datetime.strptime(options['start_date'], TIMESTAMPFORM)
end_date = datetime.strptime(options['end_date'], TIMESTAMPFORM)
t0 = start_date
if create_batch == 'years':
td = relativedelta.relativedelta(years=1)
t1 = datetime(t0.year, 12, 31, end_date.hour)
elif create_batch == 'months':
td = relativedelta.relativedelta(months=1)
elif create_batch == 'days':
# days option is only valid for gregorian calendar
td = relativedelta.relativedelta(days=1)
hour = relativedelta.relativedelta(hours=-1)
i = 0
while t0 < end_date:
i += 1
t1 = t0 + td
if t1 > end_date:
t1 = end_date
else:
t1 += hour
suffix = '_{0}'.format(i)
new_cfg_file = os.path.join(batch_dir, nameprefix + suffix)
# Write config replacing start and end dates
config.set('OPTIONS', 'start_date', t0.strftime(TIMESTAMPFORM))
config.set('OPTIONS', 'end_date', t1.strftime(TIMESTAMPFORM))
with open(new_cfg_file, 'wb') as cf:
config.write(cf)
t0 += td
return
def run_examples(config_file, vic_exe, test_data_dir, out_dir, driver):
'''Run examples tests from config file
Parameters
----------
config_file : str
Configuration file for example tests.
vic_exe : VIC (object)
VIC executable object (see tonic documentation).
test_data_dir : str
Path to test data sets.
out_dir : str
Path to output location
driver : {'classic', 'image'}
Driver to run tests on.
Returns
-------
test_results : dict
Test results for all tests in config_file.
See Also
--------
run_unit_tests
run_system
run_science
run_release
'''
# Print test set welcome
print('\n-'.ljust(OUTPUT_WIDTH + 1, '-'))
print('Running Examples')
print('-'.ljust(OUTPUT_WIDTH, '-'))
# Get setup
config = read_config(config_file)
# drop invalid driver tests
config = drop_tests(config, driver)
test_results = OrderedDict()
# Run individual examples
for i, (testname, test_dict) in enumerate(config.items()):
# print out status info
print('Running test {0}/{1}: {2}'.format(i + 1, len(config.items()),
testname))
# Setup directories for test
dirs = setup_test_dirs(testname, out_dir,
mkdirs=['results', 'state', 'logs', 'plots'])
# read template global parameter file
infile = os.path.join(test_dir, 'examples',
test_dict['global_parameter_file'])
with open(infile, 'r') as global_file:
global_param = global_file.read()
# create template string
s = string.Template(global_param)
# fill in global parameter options
global_param = s.safe_substitute(test_data_dir=test_data_dir,
result_dir=dirs['results'],
state_dir=dirs['state'],
testname=testname,
test_root=test_dir)
test_global_file = os.path.join(dirs['test'],
'{0}_globalparam.txt'.format(testname))
# write global parameter file
with open(test_global_file, 'w') as f:
f.write(global_param)
# Get optional kwargs for run executable
run_kwargs = pop_run_kwargs(test_dict)
# run VIC
test_complete = False
test_passed = False
test_comment = ''
error_message = ''
try:
# Run the VIC simulation
returncode = vic_exe.run(test_global_file, logdir=dirs['logs'],
**run_kwargs)
test_complete = True
# Check return code
check_returncode(vic_exe)
# check output files (different tests depending on driver)
if test_dict['check']:
fnames = glob.glob(os.path.join(dirs['results'], '*'))
# Check that the simulation completed for all grid cells
if 'complete' in test_dict['check'] and driver == 'classic':
test_classic_driver_all_complete(fnames)
# check for nans in all example files
if 'output_file_nans' in test_dict['check']:
if driver == 'classic':
test_classic_driver_no_output_file_nans(fnames)
elif driver == 'image':
domain_file = os.path.join(test_data_dir,
test_dict['domain_file'])
test_image_driver_no_output_file_nans(fnames,
domain_file)
else:
raise ValueError('unknown driver')
# if we got this far, the test passed.
test_passed = True
# Handle errors
except Exception as e:
test_comment, error_message = process_error(e, vic_exe)
# record the test results
test_results[testname] = TestResults(testname,
test_complete=test_complete,
passed=test_passed,
comment=test_comment,
error_message=error_message,
returncode=returncode)
# Print examples footer
print('-'.ljust(OUTPUT_WIDTH, '-'))
print('Finished testing examples.')
print('-'.ljust(OUTPUT_WIDTH, '-'))
return test_results
def nc_to_vic(config_file):
''' This function converts netCDF files to VIC ascii format files.
(This function is adapted from tonic)
Parameters
----------
config_file: <str>
Path of config file for nc_to_vic
Returns
----------
Requires
----------
write_binary
'''
import numpy as np
import struct
import os
from tonic.io import read_netcdf, read_config
from tonic.pycompat import pyzip
config = read_config(config_file)
files = config['options']['files'] # should contain "{}", which will be replaced by YYYY
var_keys = config['options']['var_keys']
output_format = config['options']['output_format'] # Binary or ASCII
out_prefix = config['options']['out_prefix']
verbose = config['options']['verbose']
coord_keys = config['options']['coord_keys'] # varname of lon and lat in netCDF files
lon_name = coord_keys[0]
lat_name = coord_keys[1]
start_year = config['options']['start_year']
end_year = config['options']['end_year']
latlon_precision = config['options']['latlon_precision']
paths = config['paths']
mask_varname = paths['mask_varname']
mask = read_netcdf(paths['mask_path'], variables=['mask'])[0][mask_varname]
yi, xi = np.nonzero(mask)
print('found {0} points in mask file.'.format(len(yi)))
xlist = []
ylist = []
pointlist = []
append = False
for i, year in enumerate(range(start_year, end_year+1)):
print('Year {}'.format(year))
fname = files.format(year)
d = read_netcdf(os.path.join(paths['in_path'], fname),
verbose=verbose)[0]
if i == 0:
# find point locations
xs = d[lon_name]
ys = d[lat_name]
posinds = np.nonzero(xs > 180)
xs[posinds] -= 360
print('adjusted xs lon minimum')
for y, x in pyzip(yi, xi):
active_flag = False
for key in var_keys:
if (d[key][:, y, x].all() is np.ma.masked) \
or (mask[y, x] == 0):
active_flag = True
if not active_flag:
point = (ys[y], xs[x])
xlist.append(x)
ylist.append(y)
pointlist.append(point)
else:
append = True
for y, x, point in pyzip(ylist, xlist, pointlist):
data = np.empty((d[var_keys[0]].shape[0], len(var_keys)))
for j, key in enumerate(var_keys):
data[:, j] = d[key][:, y, x]
if output_format == 'Binary':
write_binary(data * binary_mult, point, binary_type,
out_prefix, paths['BinaryoutPath'], append)
if output_format == 'ASCII':
write_ascii(data, point, out_prefix, paths['ASCIIoutPath'],
latlon_precision, append)
return
def main(cfg_file, nproc=1):
''' Main function
Parameters
----------
cfg_file: <str>
Input config file
nproc: <int>
Number of processors to use
'''
# ====================================================== #
# Load in config file
# ====================================================== #
cfg = read_configobj(cfg_file)
# ====================================================== #
# Process some cfg variables
# ====================================================== #
start_date = pd.to_datetime(cfg['FORCING']['start_date'])
end_date = pd.to_datetime(cfg['FORCING']['end_date'])
start_year = start_date.year
end_year = end_date.year
ens_list = range(cfg['FORCING']['ens_start'],
cfg['FORCING']['ens_end'] + 1)
# ====================================================== #
# Set up output directories
# ====================================================== #
dirs = setup_output_dirs(cfg['OUTPUT']['out_basedir'],
mkdirs=['forc_orig_nc', 'forc_orig_asc',
'forc_disagg_asc', 'forc_disagg_nc',
'config_files', 'logs_vic'])
# Subdirs for config files for ensemble
subdirs_config = setup_output_dirs(
dirs['config_files'],
mkdirs=['netcdf2vic', 'vic4', 'vic2nc'])
# ====================================================== #
# Load in domain file
# ====================================================== #
ds_domain = xr.open_dataset(cfg['DOMAIN']['domain_nc'])
da_domain = ds_domain[cfg['DOMAIN']['mask_name']]
lat_min = da_domain['lat'].min().values
lat_max = da_domain['lat'].max().values
lon_min = da_domain['lon'].min().values
lon_max = da_domain['lon'].max().values
# ====================================================== #
# Load in and process Newman ensemble forcings (for prec, Tmax and Tmin)
# and orig. Maurer forcing (for wind speed)
# ====================================================== #
# --- Load Maurer forcings --- #
print('Processing Maurer forcings...')
# Loop over each year
list_da_wind = []
for year in range(start_year, end_year+1):
print('Year {}'.format(year))
# --- Load in netCDF file for this year --- #
da_wind = xr.open_dataset(os.path.join(
cfg['FORCING']['maurer_dir'],
'nldas_met_update.obs.daily.wind.{}.nc'.format(year)))['wind']
# --- Mask out the target area --- #
da_wind = da_wind.sel(latitude=slice(lat_min, lat_max),
longitude=slice(lon_min, lon_max))
da_wind = da_wind.where(da_domain.values)
# --- Rename lat and lon --- #
da_wind = da_wind.rename({'latitude': 'lat', 'longitude': 'lon'})
# --- Put in list --- #
list_da_wind.append(da_wind)
# Concat all years together
da_wind_allyears = xr.concat(list_da_wind, dim='time')
# --- Load Newman forcings --- #
print('Processing Newman forcings...')
# If 1 processor, do a regular process
if nproc == 1:
# Loop over each ensemble member
for ens in ens_list:
load_and_process_Newman(ens, cfg, da_domain, lat_min, lat_max,
lon_min, lon_max, start_date, end_date,
dirs, da_wind_allyears)
# If multiple processors, use mp
elif nproc > 1:
# Set up multiprocessing
pool = mp.Pool(processes=nproc)
# Loop over each ensemble member
for ens in ens_list:
pool.apply_async(load_and_process_Newman,
(ens, cfg, da_domain, lat_min, lat_max, lon_min,
lon_max, start_date, end_date, dirs,
da_wind_allyears,))
# Finish multiprocessing
pool.close()
pool.join()
# ====================================================== #
# Convert orig. forcings to ascii format
# ====================================================== #
print('Converting orig. netCDF forcings to VIC ascii...')
# --- Setup subdirs for asc VIC orig. forcings for each ensemble member
# --- #
list_ens = []
for ens in ens_list:
list_ens.append('ens_{}'.format(ens))
subdirs_output = setup_output_dirs(
dirs['forc_orig_asc'],
mkdirs=list_ens)
# --- Prepare netcdf2vic config file --- #
dict_cfg_file = {}
for ens in ens_list:
cfg_file = os.path.join(subdirs_config['netcdf2vic'],
'ens_{}.cfg'.format(ens))
dict_cfg_file[ens] = cfg_file
with open(cfg_file, 'w') as f:
f.write('[options]\n')
f.write('files: forc_orig.{}.nc\n')
f.write('verbose: True\n')
f.write('output_format: ASCII\n')
f.write('out_prefix: forc_orig_\n')
f.write('coord_keys: lon,lat\n')
f.write('var_keys: pr,tasmax,tasmin,wind\n')
f.write('start_year: {}\n'.format(start_year))
f.write('end_year: {}\n'.format(end_year))
f.write('latlon_precision: {}\n'.format(
cfg['OUTPUT']['latlon_precision']))
f.write('\n[paths]\n')
f.write('in_path: {}\n'.format(os.path.join(
dirs['forc_orig_nc'],
'ens_{}'.format(ens))))
f.write('mask_path: {}\n'.format(cfg['DOMAIN']['domain_nc']))
f.write('mask_varname: {}\n'.format(cfg['DOMAIN']['mask_name']))
f.write('ASCIIoutPath: {}\n'.format(
subdirs_output['ens_{}'.format(ens)]))
# --- Run nc_to_vic --- #
# If 1 processor, do a regular process
if nproc == 1:
for ens in ens_list:
nc_to_vic(dict_cfg_file[ens])
# If multiple processors, use mp
elif nproc > 1:
# Set up multiprocessing
pool = mp.Pool(processes=nproc)
# Loop over each ensemble member
for ens in ens_list:
pool.apply_async(nc_to_vic, (dict_cfg_file[ens],))
# Finish multiprocessing
pool.close()
pool.join()
# ====================================================== #
# Run VIC forcing disaggregator
# ====================================================== #
print('Running VIC as a disaggregator...')
# --- Setup subdirs for asc VIC disagg. forcings and VIC log files for
# each ensemble member --- #
list_ens = []
for ens in ens_list:
list_ens.append('ens_{}'.format(ens))
subdirs_output = setup_output_dirs(
dirs['forc_disagg_asc'],
mkdirs=list_ens)
subdirs_logs = setup_output_dirs(
dirs['logs_vic'],
mkdirs=list_ens)
# --- Prepare VIC global file for the disaggregation run --- #
# Load in global file template
with open(cfg['VIC_DISAGG']['global_template'], 'r') as f:
global_param = f.read()
# Create string template
s = string.Template(global_param)
# Loop over each ensemble member
dict_global_file = {}
for ens in ens_list:
# Fill in variables in the template
global_param = s.safe_substitute(
time_step=cfg['VIC_DISAGG']['time_step'],
startyear=start_year,
startmonth=start_date.month,
startday=start_date.day,
endyear=end_year,
endmonth=end_date.month,
endday=end_date.day,
forcing1=os.path.join(dirs['forc_orig_asc'],
'ens_{}'.format(ens),
'forc_orig_'),
grid_decimal=cfg['OUTPUT']['latlon_precision'],
prec='PREC',
tmax='TMAX',
tmin='TMIN',
wind='WIND',
forceyear=start_year,
forcemonth=start_date.month,
forceday=start_date.day,
result_dir=subdirs_output['ens_{}'.format(ens)])
# Write global param file
global_file = os.path.join(subdirs_config['vic4'],
'vic.global.ens_{}.txt'.format(ens))
dict_global_file[ens] = global_file
with open(global_file, mode='w') as f:
for line in global_param:
f.write(line)
# --- Run VIC --- #
# Prepare VIC exe
vic_exe = VIC(cfg['VIC_DISAGG']['vic4_exe'])
# If 1 processor, do a regular process
if nproc == 1:
for ens in ens_list:
vic_exe.run(dict_global_file[ens],
logdir=subdirs_logs['ens_{}'.format(ens)])
# If multiple processors, use mp
elif nproc > 1:
# Set up multiprocessing
pool = mp.Pool(processes=nproc)
# Loop over each ensemble member
for ens in ens_list:
pool.apply_async(run_vic_for_multiprocess,
(vic_exe, dict_global_file[ens],
subdirs_logs['ens_{}'.format(ens)],))
# Finish multiprocessing
pool.close()
pool.join()
# ====================================================== #
# Convert disaggregated forcings to netCDF format
# ====================================================== #
# --- Prepare config file for vic2nc --- #
print('Converting disaggregated forcings to netCDF...')
# --- Setup subdirs for VIC disagg. netCDF forcings for each ensemble
# member --- #
list_ens = []
for ens in ens_list:
list_ens.append('ens_{}'.format(ens))
subdirs_output = setup_output_dirs(
dirs['forc_disagg_nc'],
mkdirs=list_ens)
# --- Prepare netcdf2vic config file --- #
# Extract disaggregated forcing variable names and order
with open(cfg['VIC_DISAGG']['global_template'], 'r') as f:
global_param = f.read()
outvar_list = find_outvar_global_param(global_param)
for i, var in enumerate(outvar_list):
outvar_list[i] = var.strip('OUT_')
# Extract end date and hour
end_date_with_hour = end_date + pd.DateOffset(days=1) -\
pd.DateOffset(hours=cfg['VIC_DISAGG']['time_step'])
# Loop over each ensemble member
dict_cfg_file = {}
for ens in ens_list:
cfg_file = os.path.join(subdirs_config['vic2nc'],
'ens_{}.cfg'.format(ens))
dict_cfg_file[ens] = cfg_file
with open(cfg_file, 'w') as f:
f.write('[OPTIONS]\n')
f.write('input_files: {}\n'.format(
os.path.join(dirs['forc_disagg_asc'],
'ens_{}'.format(ens),
'force_*')))
f.write('input_file_format: ascii\n')
f.write('bin_dt_sec: {}\n'.format(cfg['VIC_DISAGG']['time_step']*3600))
f.write('bin_start_date: {}\n'.format(start_date.strftime("%Y-%m-%d-%H")))
f.write('bin_end_date: {}\n'.format(end_date_with_hour.strftime("%Y-%m-%d-%H")))
f.write('regular_grid: False\n')
f.write('out_directory: {}\n'.format(subdirs_output['ens_{}'.format(ens)]))
f.write('memory_mode: big_memory\n')
f.write('chunksize: 100\n')
f.write('out_file_prefix: force\n')
f.write('out_file_format: NETCDF4\n')
f.write('precision: single\n')
f.write('start_date: {}\n'.format(start_date.strftime("%Y-%m-%d-%H")))
f.write('end_date: {}\n'.format(end_date_with_hour.strftime("%Y-%m-%d-%H")))
f.write('calendar: proleptic_gregorian\n')
f.write('time_segment: year\n')
f.write('snow_bands: False\n')
f.write('veg_tiles: False\n')
f.write('soil_layers: False\n')
f.write('\n[DOMAIN]\n')
f.write('filename: {}\n'.format(cfg['DOMAIN']['domain_nc']))
f.write('longitude_var: {}\n'.format(cfg['DOMAIN']['lon_name']))
f.write('latitude_var: {}\n'.format(cfg['DOMAIN']['lat_name']))
f.write('y_x_dims: {}, {}\n'.format(cfg['DOMAIN']['lat_name'],
cfg['DOMAIN']['lon_name']))
f.write('copy_vars: {}, {}, {}\n'.format(cfg['DOMAIN']['mask_name'],
cfg['DOMAIN']['lat_name'],
cfg['DOMAIN']['lon_name']))
f.write('\n[GLOBAL_ATTRIBUTES]\n')
f.write('title: VIC forcings\n')
f.write('version: VIC4.2\n')
f.write('grid: 1/8\n')
for i, var in enumerate(outvar_list):
if var == 'AIR_TEMP':
f.write('\n[AIR_TEMP]\n')
f.write('column: {}\n'.format(i))
f.write('units: C\n')
f.write('standard_name: air_temperature\n')
f.write('description: air temperature\n')
elif var == 'PREC':
f.write('\n[PREC]\n')
f.write('column: {}\n'.format(i))
f.write('units: mm/step\n')
f.write('standard_name: precipitation\n')
f.write('description: precipitation\n')
elif var == 'PRESSURE':
f.write('\n[PRESSURE]\n')
f.write('column: {}\n'.format(i))
f.write('units: kPa\n')
f.write('standard_name: surface_air_pressure\n')
f.write('description: near-surface atmospheric pressure\n')
elif var == 'SHORTWAVE':
f.write('\n[SHORTWAVE]\n')
f.write('column: {}\n'.format(i))
f.write('units: W m-2\n')
f.write('standard_name: incoming_shortwave_radiation\n')
f.write('description: incoming shortwave radiation\n')
elif var == 'LONGWAVE':
f.write('\n[LONGWAVE]\n')
f.write('column: {}\n'.format(i))
f.write('units: W m-2\n')
f.write('standard_name: incoming_longwave_radiation\n')
f.write('description: incoming longwave radiation\n')
elif var == 'VP':
f.write('\n[VP]\n')
f.write('column: {}\n'.format(i))
f.write('units: kPa\n')
f.write('standard_name: water_vapor_pressure\n')
f.write('description: near surface vapor pressure\n')
elif var == 'WIND':
f.write('\n[WIND]\n')
f.write('column: {}\n'.format(i))
f.write('units: m/s\n')
f.write('standard_name: surface_air_pressure\n')
f.write('description: near-surface wind speed\n')
# --- Run vic2nc --- #
# If 1 processor, do a regular process
if nproc == 1:
for ens in ens_list:
cfg_vic2nc = read_config(dict_cfg_file[ens])
options = cfg_vic2nc.pop('OPTIONS')
global_atts = cfg_vic2nc.pop('GLOBAL_ATTRIBUTES')
if not options['regular_grid']:
domain_dict = cfg_vic2nc.pop('DOMAIN')
else:
domain_dict = None
# Set aside fields dict
fields = cfg_vic2nc
# Run vic2nc
vic2nc(options, global_atts, domain_dict, fields)
# If multiple processors, use mp
elif nproc > 1:
# Set up multiprocessing
pool = mp.Pool(processes=nproc)
# Loop over each ensemble member
for ens in ens_list:
cfg_vic2nc = read_config(dict_cfg_file[ens])
options = cfg_vic2nc.pop('OPTIONS')
global_atts = cfg_vic2nc.pop('GLOBAL_ATTRIBUTES')
if not options['regular_grid']:
domain_dict = cfg_vic2nc.pop('DOMAIN')
else:
domain_dict = None
# set aside fields dict
fields = cfg_vic2nc
pool.apply_async(vic2nc,
(options, global_atts, domain_dict, fields,))
# Finish multiprocessing
pool.close()
pool.join()
def _run(args):
config = read_config(args.config)
files = config['options']['files']
var_keys = config['options']['var_keys']
output = config['options']['output']
binary_mult = config['options']['binary_mult']
binary_type = config['options']['binary_type'],
paths = config['options']['paths']
out_prefix = config['options']['out_prefix']
verbose = config['options']['verbose']
mask = read_netcdf(paths['mask_path'], nc_vars=['mask'])['mask']
yi, xi = np.nonzero(mask)
print('found {0} points in mask file.'.format(len(yi)))
xlist = []
ylist = []
pointlist = []
append = False
for i, fname in enumerate(files):
d = read_netcdf(os.path.join(paths['in_path'], fname), verbose=verbose)
if i == 0:
# find point locations
xs = d['xc']
ys = d['yc']
posinds = np.nonzero(xs > 180)
xs[posinds] -= 360
print('adjusted xs lon minimum')
for y, x in pyzip(yi, xi):
active_flag = False
for key in var_keys:
if (d[key][:, y, x].all() is np.ma.masked) \
or (mask[y, x] == 0):
active_flag = True
if not active_flag:
point = (ys[y, x], xs[y, x])
xlist.append(x)
ylist.append(y)
pointlist.append(point)
else:
append = True
for y, x, point in pyzip(ylist, xlist, pointlist):
data = np.empty((d[var_keys[0]].shape[0], len(var_keys)))
for j, key in enumerate(var_keys):
data[:, j] = d[key][:, y, x]
if output['Binary']:
write_binary(data * binary_mult, point, binary_type,
out_prefix, paths['BinaryoutPath'], append)
if output['ASCII']:
write_ascii(data, point, out_prefix, paths['ASCIIoutPath'],
append)
return
def run_examples(config_file, vic_exe, test_data_dir, out_dir, driver):
'''Run examples tests from config file
Parameters
----------
config_file : str
Configuration file for example tests.
vic_exe : VIC (object)
VIC executable object (see tonic documentation).
test_data_dir : str
Path to test data sets.
out_dir : str
Path to output location
driver : {'classic', 'image'}
Driver to run tests on.
Returns
-------
test_results : dict
Test results for all tests in config_file.
See Also
--------
run_unit_tests
run_system
run_science
run_release
'''
# Print test set welcome
print('\n-'.ljust(OUTPUT_WIDTH + 1, '-'))
print('Running Examples')
print('-'.ljust(OUTPUT_WIDTH, '-'))
# Get setup
config = read_config(config_file)
# drop invalid driver tests
config = drop_tests(config, driver)
test_results = OrderedDict()
# Run individual examples
for i, (testname, test_dict) in enumerate(config.items()):
# print out status info
print('Running test {0}/{1}: {2}'.format(i + 1, len(config.items()),
testname))
# Setup directories for test
dirs = setup_test_dirs(testname,
out_dir,
mkdirs=['results', 'state', 'logs', 'plots'])
# read template global parameter file
infile = os.path.join(test_dir, 'examples',
test_dict['global_parameter_file'])
with open(infile, 'r') as global_file:
global_param = global_file.read()
# create template string
s = string.Template(global_param)
# fill in global parameter options
global_param = s.safe_substitute(test_data_dir=test_data_dir,
result_dir=dirs['results'],
state_dir=dirs['state'],
testname=testname,
test_root=test_dir)
test_global_file = os.path.join(dirs['test'],
'{0}_globalparam.txt'.format(testname))
# write global parameter file
with open(test_global_file, 'w') as f:
f.write(global_param)
# Get optional kwargs for run executable
run_kwargs = pop_run_kwargs(test_dict)
# run VIC
test_complete = False
test_passed = False
test_comment = ''
error_message = ''
try:
# Run the VIC simulation
returncode = vic_exe.run(test_global_file,
logdir=dirs['logs'],
**run_kwargs)
test_complete = True
# Check return code
check_returncode(vic_exe)
# check output files (different tests depending on driver)
if test_dict['check']:
fnames = glob.glob(os.path.join(dirs['results'], '*'))
# Check that the simulation completed for all grid cells
if 'complete' in test_dict['check'] and driver == 'classic':
test_classic_driver_all_complete(fnames)
# check for nans in all example files
if 'output_file_nans' in test_dict['check']:
if driver == 'classic':
test_classic_driver_no_output_file_nans(fnames)
elif driver == 'image':
domain_file = os.path.join(test_data_dir,
test_dict['domain_file'])
test_image_driver_no_output_file_nans(
fnames, domain_file)
else:
raise ValueError('unknown driver')
# if we got this far, the test passed.
test_passed = True
# Handle errors
except Exception as e:
test_comment, error_message = process_error(e, vic_exe)
# record the test results
test_results[testname] = TestResults(testname,
test_complete=test_complete,
passed=test_passed,
comment=test_comment,
error_message=error_message,
returncode=returncode)
# Print examples footer
print('-'.ljust(OUTPUT_WIDTH, '-'))
print('Finished testing examples.')
print('-'.ljust(OUTPUT_WIDTH, '-'))
return test_results
import sys
from tonic.io import read_config
from tonic.models.vic.vic2netcdf import vic2nc
# --- Load config file --- #
cfg_file = sys.argv[1]
# --- Run vic2nc --- #
cfg_vic2nc = read_config(cfg_file)
options = cfg_vic2nc.pop('OPTIONS')
global_atts = cfg_vic2nc.pop('GLOBAL_ATTRIBUTES')
if not options['regular_grid']:
domain_dict = cfg_vic2nc.pop('DOMAIN')
else:
domain_dict = None
# set aside fields dict
fields = cfg_vic2nc
vic2nc(options, global_atts, domain_dict, fields)
f.write('standard_name: incoming_longwave_radiation\n')
f.write('description: incoming longwave radiation\n')
elif var == 'VP':
f.write('\n[VP]\n')
f.write('column: {}\n'.format(i))
f.write('units: kPa\n')
f.write('standard_name: water_vapor_pressure\n')
f.write('description: near surface vapor pressure\n')
elif var == 'WIND':
f.write('\n[WIND]\n')
f.write('column: {}\n'.format(i))
f.write('units: m/s\n')
f.write('standard_name: surface_air_pressure\n')
f.write('description: near-surface wind speed\n')
# --- Run vic2nc --- #
cfg_vic2nc = read_config(cfg_file)
options = cfg_vic2nc.pop('OPTIONS')
global_atts = cfg_vic2nc.pop('GLOBAL_ATTRIBUTES')
if not options['regular_grid']:
domain_dict = cfg_vic2nc.pop('DOMAIN')
else:
domain_dict = None
# set aside fields dict
fields = cfg_vic2nc
vic2nc(options, global_atts, domain_dict, fields)
def _run(args):
config = read_config(args.config)
files = [config['Basics']['files']]
var_keys = [config['Basics']['var_keys']]
output = config['Paths']['ASCIIoutPath']
#binary_mult = config['Basics']['binary_mult']
#binary_type = config['Basics']['binary_type'],
paths = config['Paths']
out_prefix = config['Basics']['out_prefix']
verbose = config['Basics']['verbose']
mask = read_netcdf(paths['mask_path'], variables=['mask'])[0]['mask']
yi, xi = np.nonzero(mask)
print(mask)
print('found {0} points in mask fqile.'.format(len(xi)))
#x = read_netcdf(os.path.join(paths['in_path'], 'pr_1979.nc'))
#print(x)
xlist = []
ylist = []
pointlist = []
append = False
for i, fname in enumerate(files):
d = read_netcdf(os.path.join(paths['in_path'], fname),
verbose=verbose)[0]
print(i)
if i == 0:
# find point locations
xs = d['lon']
ys = d['lat']
posinds = np.nonzero(xs > 180)
xs[posinds] -= 360
print('adjusted xs lon minimum')
for y, x in pyzip(yi, xi):
active_flag = False
for key in var_keys:
if (d[key][:, y, x].all() is np.ma.masked) \
or (mask[y, x] == 0):
active_flag = True
if not active_flag:
point = (ys[y], xs[x])
print(point)
xlist.append(x)
ylist.append(y)
pointlist.append(point)
else:
append = True
for y, x, point in pyzip(ylist, xlist, pointlist):
data = np.empty((d[var_keys[0]].shape[0], len(var_keys)))
for j, key in enumerate(var_keys):
data[:, j] = d[key][:, y, x]
#if output['Binary']:
# write_binary(data * binary_mult, point, binary_type,
# out_prefix, paths['BinaryoutPath'], append)
#if output['ASCII']:
write_ascii(data, point, out_prefix, paths['ASCIIoutPath'],
append)
return
#!/bin/env python
import matplotlib.pyplot as plt
import os
import numpy as np
import pandas as pd
import xarray as xr
from tonic.io import read_config, read_configobj
# read configs for filenames and diagnostics to run
config_files = read_config(os.path.join(os.getcwd(), 'filenames.cfg'))
config_diag = read_configobj(os.path.join(os.getcwd(), 'diagnostics.cfg'))
# full list of l2x coupler fields
l2x_vars = [
'l2x_Sl_t', 'l2x_Sl_tref', 'l2x_Sl_qref', 'l2x_Sl_avsdr', 'l2x_Sl_anidr',
'l2x_Sl_avsdf', 'l2x_Sl_anidf', 'l2x_Sl_snowh', 'l2x_Sl_u10', 'l2x_Sl_fv',
'l2x_Sl_ram1', 'l2x_Sl_logz0', 'l2x_Fall_taux', 'l2x_Fall_tauy',
'l2x_Fall_lat', 'l2x_Fall_sen', 'l2x_Fall_lwup', 'l2x_Fall_evap',
'l2x_Fall_swnet', 'l2x_Fall_flxdst1', 'l2x_Fall_flxdst2',
'l2x_Fall_flxdst3', 'l2x_Fall_flxdst4', 'l2x_Flrl_rofliq',
'l2x_Flrl_rofice'
]
l2x_units = [
'K', 'K', 'g/g', 'fraction', 'fraction', 'fraction', 'fraction', 'm',
'm/s', 'm/s', 's/m', 'm', 'N/m2', 'N/m2', 'W/m2', 'W/m2', 'W/m2', 'kg/m2s',
'W/m2', 'm/s', 'm/s', 'm/s', 'm/s', 'kg/m2s', 'kg/m2s'
]
def run(config_file, outfile):
"""
Generate high-resolution meteorologic forcings by downscaling the GCM
and/or RCM using the Generalized Analog Regression Downscaling (GARD) tool.
Inputs:
Configuration file formatted with the following options:
TODO: Add sample config
"""
# Read configuration file into a dictionary
config = read_config(config_file)
log_level = config['Options']['LogLevel']
chunk_years = relativedelta(years=int(config['Options']['ChunkYears']))
# Set up logging for messaging
logger = set_logger(
os.path.splitext(os.path.split(__file__)[-1])[0], log_level)
logger.info('Downscaling Configuration Options:')
pp.pprint(config)
# Define variables from configuration file
# create directories if they don't exist yet
data_dir = config['Options']['DataDir']
filelist_dir = os.path.join(data_dir, 'gard_filelists')
namelist_dir = os.path.join(data_dir, 'gard_namelists')
for d in [data_dir, filelist_dir, namelist_dir]:
os.makedirs(d, exist_ok=True)
# if outfile is default, put in data_dir
if outfile == 'namelist.txt':
outfile = os.path.join(data_dir, outfile)
# GARD namelist template
namelist_template = config['Options']['NamelistTemplate']
prediction_sets = config['Sets']
# Make training file lists
file_lists = {}
file_lists_len = {}
set_dirs = {}
namelists = []
for dataset, dset_config in config['Datasets'].items():
predict_ranges = {}
gcms = list_like(dset_config['GCMs'])
if isinstance(gcms[0], int):
# work around for these being cast as ints
gcms = ['{0:03}'.format(i) for i in gcms]
print(gcms)
train_calendar = dset_config.get('TrainCalendar', None)
obs_calendar = config['ObsDataset'].get('ObsCalendar', None)
for setname, set_config in prediction_sets.items():
logger.info('Creating configuration files for set: %s', setname)
# Make set directory
set_dirs[setname] = os.path.join(data_dir, dataset, setname)
os.makedirs(set_dirs[setname], exist_ok=True)
# Training/prediction/obs variables
obs_vars = list_like(set_config['ObsVars'])
logger.info('Obs Vars: %s', obs_vars)
# For now, we can assume the training / prediction variable names
# are he same
vars_list = []
for var in obs_vars:
vars_list.append(set_config[var])
vars_list = list(set(flatten(vars_list)))
logger.info('Variables: %s', vars_list)
# Get scenario to process
logger.debug(dset_config['PredictPattern'])
scenarios = dset_config['scenario']
train_range = tuple(dset_config['TrainPeriod'])
transform_range = tuple(dset_config['TransformPeriod'])
transform_scen = dset_config['TransformScenario']
logger.info('training range: %s', train_range)
for scen, drange in scenarios.items():
predict_ranges[scen] = tuple(drange)
# Make the filelists
for var in obs_vars:
# Obs filelist
key = filelistkey(dset=dataset,
var=var,
id='obs',
drange=train_range,
scenario='training')
file_lists[key], file_lists_len[key] = make_filelist(
key,
config['ObsDataset']['ObsInputPattern'],
prefix=filelist_dir,
calendar=obs_calendar)
for var in vars_list:
# training filelist
key = filelistkey(dset=dataset,
var=TRAINVARMAP.get(var, var),
id='training',
drange=train_range,
scenario='training')
file_lists[key], file_lists_len[key] = make_filelist(
key,
dset_config['TrainPattern'],
prefix=filelist_dir,
calendar=train_calendar)
# prediction filelists
for gcm, scen in itertools.product(gcms, scenarios):
for drange in get_drange_chunks(
predict_ranges[scen], max_chunk_size=chunk_years):
key = filelistkey(dset=dataset,
var=var,
id=gcm,
drange=drange,
scenario=scen)
file_lists[key], file_lists_len[key] = make_filelist(
key,
dset_config['PredictPattern'],
prefix=filelist_dir,
transform_range=transform_range,
transform_scen=transform_scen,
calendar=config['Calendars'].get(
gcm, config['Calendars'].get('all', None)))
for gcm, var, setname in itertools.product(gcms, obs_vars,
prediction_sets):
for scen in dset_config['scenario']:
# drange_str = _tslice_to_str(predict_ranges[scen])
for drange in get_drange_chunks(predict_ranges[scen],
max_chunk_size=chunk_years):
drange_str = _tslice_to_str(drange)
# Make GARD namelist
namelist = os.path.join(
namelist_dir,
NAMELIST_TEMPLATE.format(setname=setname,
var=var,
id=gcm,
dset=dataset,
scenario=scen,
drange=drange_str))
kwargs = defaults.copy()
kwargs.update(prediction_sets[setname])
# Set the downscaling mode
mode = prediction_sets[setname]['Mode']
for m in [
'pure_regression', 'analog_regression',
'pure_analog', 'pass_through'
]:
kwargs[m] = (m == mode)
# Now, add a bunch of computed variables
# parameters section
out_dir = os.path.join(set_dirs[setname], drange_str)
os.makedirs(out_dir, exist_ok=True)
kwargs['output_file_prefix'] = os.path.join(
out_dir,
OUTPUT_TEMPLATE.format(setname=setname,
drange=drange_str,
dset=dataset,
id=gcm,
scenario=scen))
kwargs['start_date'] = pd.to_datetime(
drange[0]).strftime(GARD_TIMEFORMAT)
kwargs['end_date'] = pd.to_datetime(
drange[1]).strftime(GARD_TIMEFORMAT)
kwargs['start_train'] = pd.to_datetime(
train_range[0]).strftime(GARD_TIMEFORMAT)
kwargs['end_train'] = pd.to_datetime(
train_range[1]).strftime(GARD_TIMEFORMAT)
kwargs['start_transform'] = pd.to_datetime(
transform_range[0]).strftime(GARD_TIMEFORMAT)
kwargs['end_transform'] = pd.to_datetime(
transform_range[1]).strftime(GARD_TIMEFORMAT)
kwargs['logistic_threshold'] = LOGISTIC_THRESH[var]
# training_parameters section
var_list = list_like(prediction_sets[setname][var])
train_var_list = [TRAINVARMAP.get(v, v) for v in var_list]
train_kwargs = dict(
dset=dataset,
drange=train_range,
id='training',
scenario='training',
)
kwargs['train_nfiles'] = file_lists_len[filelistkey(
var=train_var_list[0], **train_kwargs)]
kwargs['train_nvars'] = len(train_var_list)
kwargs['train_vars'] = _make_variables_str(train_var_list)
kwargs['train_transform'] = get_transform_str(
train_var_list, mode)
kwargs['train_filelists'] = _make_filelist_str(
file_lists, train_var_list, **train_kwargs)
kwargs['train_calendar'] = '"{}"'.format(train_calendar)
# prediction_parameters section
predict_kwargs = dict(dset=dataset,
drange=drange,
id=gcm,
scenario=scen)
kwargs['predict_nfiles'] = file_lists_len[filelistkey(
var=var_list[0], **predict_kwargs)]
nvars = len(var_list)
kwargs['predict_nvars'] = nvars
kwargs['predict_vars'] = _make_variables_str(var_list)
transformations = ','.join([str(QUANTILE_TRANSFORM)] *
nvars)
kwargs['transformations'] = transformations
kwargs['predict_transform'] = get_transform_str(
var_list, mode)
kwargs['predict_filelists'] = _make_filelist_str(
file_lists, var_list, **predict_kwargs)
kwargs['predict_calendar'] = '"{}"'.format(
config['Calendars'].get(
gcm, config['Calendars'].get('all', None)))
# obs_parameters section
obs_kwargs = dict(dset=dataset,
drange=train_range,
id='obs',
scenario='training')
kwargs['obs_nvars'] = 1
key = filelistkey(var=var, **obs_kwargs)
kwargs['obs_nfiles'] = file_lists_len[filelistkey(
var=var, **obs_kwargs)]
kwargs['obs_vars'] = _make_variables_str(var)
kwargs['obs_transform'] = TRANSFORM[var]
kwargs['obs_filelists'] = _make_filelist_str(
file_lists, [var], **obs_kwargs)
kwargs['obs_calendar'] = '"{}"'.format(obs_calendar)
# Special cases for "pass_through" option:
if mode == 'pass_through':
kwargs['train_transform'] = NO_TRANSFORM
kwargs['predict_transform'] = NO_TRANSFORM
kwargs['obs_transform'] = NO_TRANSFORM
kwargs['normalization_method'] = NO_NORM
kwargs['transformations'] = NO_TRANSFORM
# Store the namelist filepath for execuation
namelists.append(namelist)
# Create the namelist by filling in the template
replace_var_pythonic_config(namelist_template, namelist,
**kwargs)
# Write file of namelists
logger.info('writing outfile %s', outfile)
with open(outfile, 'w') as f:
f.writelines('\n'.join(namelists))
config_copy = os.path.join(data_dir, os.path.basename(config_file))
logger.info('writing config file %s', config_copy)
shutil.copyfile(config_file, config_copy)
return
