def plot_veg_types(yc, xc, cv, baresoil):
projection_parameters = projection
labels = ['Evergreen Needleleaf', 'Evergreen Broadleaf',
'Deciduous Needleleaf', 'Deciduous Broadleaf', 'Mixed Cover',
'Woodland', 'Wooded Grasslands', 'Closed Shrublands',
'Open Shrublands', 'Grasslands', 'Crop land', 'Bare Soil/Ice']
fig = plt.figure(figsize=(10, 12))
gs1 = gridspec.GridSpec(4, 3)
for loc in pyrange(11):
ax = fig.add_subplot(gs1[loc])
plot_map(ax, yc, xc, cv[loc], projection_parameters, vmin=0,
cmap='Jet')
ax.set_title(labels[loc])
ax = fig.add_subplot(gs1[11])
plot_map(ax, yc, xc, baresoil, projection_parameters, vmin=0, cmap='Jet')
ax.set_title(labels[11])
sm = plt.cm.ScalarMappable(cmap='Jet', norm=plt.normalize(vmin=0, vmax=1))
colorbar_ax = fig.add_axes([0.92, 0.1, 0.03, 0.8])
sm._A = []
plt.colorbar(sm, cax=colorbar_ax)
fig.suptitle('Fraction of Vegetation Type', fontsize=20, fontweight='bold')
fig.text(0.5, 0.93, 'Regional Arctic Climate Model',
ha='center', fontsize=18)
plt.show()
def nc_add_data_to_array(self, point):
for name in self.three_dim_vars:
self.data[name][:, point.y, point.x] = \
point.df[name].values[self.slice]
for name in self.four_dim_vars:
varshape = self.f.variables[name].shape[1]
for i in pyrange(varshape):
subname = name + str(i)
self.data[name][:, i, point.y,
point.x] = point.df[subname].values[self.slice]
def nc_add_data_standard(self, points):
ys = points.get_ys()
xs = points.get_xs()
for p in points:
for name in self.three_dim_vars:
data = points.get_data(name, self.slice)
self.f.variables[name][:, ys, xs] = data
for name in self.four_dim_vars:
varshape = self.f.variables[name].shape[1]
for i in pyrange(varshape):
sn = name + str(i)
self.f.variables[name][:, i, ys,
xs] = p.df[sn].values[self.slice]
def my_plot9(lons, lats, d1, d2, units=None,
vmin=None, vmax=None, amin=None,
amax=None, mask=True, amap=None,
t1='', t2='', cmap=None):
""" 9 pannel plot to compare soil layers for two different datasets"""
if vmin is None:
vmin = d1.min()
if vmax is None:
vmax = d1.max()
anom = d1 - d2
if amin is None:
amin = -1 * np.max(np.abs(anom))
if amax is None:
amax = np.max(np.abs(anom))
if cmap is None:
cmap = cmap_discretize('cm.winter')
if amap is None:
amap = cmap_discretize('cm.RdBu')
f, axarr = plt.subplots(3, 3, figsize=(13.5, 9), dpi=150)
f.tight_layout()
for layer in pyrange(3):
plt.sca(axarr[layer, 0])
sub_plot_pcolor(lons, lats, np.ma.masked_where(mask, d1[layer]),
vmin=vmin, vmax=vmax, units=units,
ncolors=9, title='{} (A)'.format(t1),
cmap=cmap, cbar_location='right',)
plt.ylabel('Layer {}'.format(layer))
plt.sca(axarr[layer, 1])
sub_plot_pcolor(lons, lats, np.ma.masked_where(mask, d2[layer]),
vmin=vmin, vmax=vmax,
ncolors=9, title='{} (B)'.format(t2), units=units,
cbar_location='right', cmap=cmap)
plt.sca(axarr[layer, 2])
sub_plot_pcolor(lons, lats, np.ma.masked_where(mask, anom[layer]),
ncolors=9, cmap=amap,
title='Difference (A-B)', units=units,
cbar_location='right', vmin=amin, vmax=amax)
return f
def plot_veg_types(yc, xc, cv, baresoil):
projection_parameters = projection
labels = [
'Evergreen Needleleaf', 'Evergreen Broadleaf', 'Deciduous Needleleaf',
'Deciduous Broadleaf', 'Mixed Cover', 'Woodland', 'Wooded Grasslands',
'Closed Shrublands', 'Open Shrublands', 'Grasslands', 'Crop land',
'Bare Soil/Ice'
]
fig = plt.figure(figsize=(10, 12))
gs1 = gridspec.GridSpec(4, 3)
for loc in pyrange(11):
ax = fig.add_subplot(gs1[loc])
plot_map(ax,
yc,
xc,
cv[loc],
projection_parameters,
vmin=0,
cmap='Jet')
ax.set_title(labels[loc])
ax = fig.add_subplot(gs1[11])
plot_map(ax, yc, xc, baresoil, projection_parameters, vmin=0, cmap='Jet')
ax.set_title(labels[11])
sm = plt.cm.ScalarMappable(cmap='Jet', norm=plt.normalize(vmin=0, vmax=1))
colorbar_ax = fig.add_axes([0.92, 0.1, 0.03, 0.8])
sm._A = []
plt.colorbar(sm, cax=colorbar_ax)
fig.suptitle('Fraction of Vegetation Type', fontsize=20, fontweight='bold')
fig.text(0.5,
0.93,
'Regional Arctic Climate Model',
ha='center',
fontsize=18)
plt.show()
def vic2nc(options, global_atts, domain_dict, fields):
""" Convert ascii VIC files to netCDF format"""
# determine run mode
if (options['memory_mode'] == 'standard') \
and (options['chunksize'] in ['all', 'All', 'ALL', 0]):
memory_mode = 'big_memory'
else:
memory_mode = options['memory_mode']
print("\n-------------------------------")
print("Configuration File Options")
print("-------------OPTIONS-------------")
for pair in options.items():
print("{0}: {1}".format(*pair))
print('Fields: {0}'.format(", ".join(fields.keys())))
if domain_dict:
print("-------------DOMAIN--------------")
for pair in domain_dict.items():
print("{0}: {1}".format(*pair))
print("--------GLOBAL_ATTRIBUTES--------")
for pair in global_atts.items():
print("{0}: {1}".format(*pair))
print("--------RUN MODE--------")
print('Memory Mode: {0}'.format(memory_mode))
if memory_mode == 'standard':
print('Chunksize={0}'.format(options['chunksize']))
print("---------------------------------\n")
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# Make output directory
if not os.path.exists(options['out_directory']):
os.makedirs(options['out_directory'])
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# Make pairs (i.e. find inds)
files = glob(options['input_files'])
points = get_file_coords(files)
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# Get target grid information
if domain_dict:
domain = read_domain(domain_dict)
target_grid_file = path.split(domain_dict['filename'])[1]
global_atts['target_grid_file'] = target_grid_file
else:
# must be a regular grid, build from file names
domain = calc_grid(points.get_lats(), points.get_lons())
target_grid_file = None
domain_dict = {'y_x_dims': ['lat', 'lon']}
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# Get grid index locations
points = get_grid_inds(domain, points)
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# Get timestamps
if options['input_file_format'].lower() == 'ascii':
if ('bin_start_date' in options
and 'bin_end_date' in options
and 'bin_dt_sec' in options):
vic_datelist, vic_ordtime = make_dates(
options['bin_start_date'],
options['bin_end_date'],
options['bin_dt_sec'],
calendar=options['calendar'])
else:
vic_datelist = get_dates(files[0])
vic_ordtime = date2num(vic_datelist, TIMEUNITS,
calendar=options['calendar'])
elif options['input_file_format'].lower() in ['binary', 'netcdf']:
vic_datelist, vic_ordtime = make_dates(options['bin_start_date'],
options['bin_end_date'],
options['bin_dt_sec'],
calendar=options['calendar'])
else:
raise ValueError('Unknown input file format: {}. Valid options are \
ascii or binary'.format(options['input_file_format']))
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# Determine time segmentation
if options['start_date']:
start_date = datetime.strptime(options['start_date'], TIMESTAMPFORM)
if start_date < vic_datelist[0]:
print("WARNING: Start date in configuration file is before "
"first date in file.")
start_date = vic_datelist[0]
print('WARNING: New start date is {0}'.format(start_date))
else:
start_date = vic_datelist[0]
if options['end_date']:
end_date = datetime.strptime(options['end_date'], TIMESTAMPFORM)
if end_date > vic_datelist[-1]:
print("WARNING: End date in configuration file is after "
"last date in file.")
end_date = vic_datelist[-1]
print('WARNING: New end date is {0}'.format(end_date))
else:
end_date = vic_datelist[-1]
# Ordinal Time
start_ord = date2num(start_date, TIMEUNITS, calendar=options['calendar'])
end_ord = date2num(end_date, TIMEUNITS, calendar=options['calendar'])
print("netCDF Start Date: {0}".format(start_date))
print("netCDF End Date: {0}".format(end_date))
segment_dates = []
if options['time_segment'] == 'day':
# calendar insensitive
num_segments = np.ceil(end_ord - start_ord)
if start_date.hour == 0:
segment_dates = num2date(np.arange(start_ord, end_ord + 1, 1),
TIMEUNITS, calendar=options['calendar'])
else:
# allow start at time other than 0
temp = [start_ord].append(np.arange(np.ceil(start_ord),
end_ord + 1, 1))
segment_dates = num2date(temp, TIMEUNITS,
calendar=options['calendar'])
elif options['time_segment'] == 'month':
num_segments = (end_date.year - start_date.year) * 12 \
+ end_date.month - start_date.month + 1
month = start_date.month
year = start_date.year
for i in pyrange(num_segments + 1):
segment_dates.append(datetime(year, month, 1))
month += 1
if month == 13:
month = 1
year += 1
elif options['time_segment'] == 'year':
num_segments = end_date.year - start_date.year + 1
year = start_date.year
for i in pyrange(num_segments + 1):
segment_dates.append(datetime(year, 1, 1))
year += 1
elif options['time_segment'] == 'decade':
num_segments = (end_date.year - start_date.year) / 10 + 1
year = start_date.year
for i in pyrange(num_segments + 1):
segment_dates.append(datetime(year, 1, 1))
year += 10
elif options['time_segment'] == 'all':
num_segments = 1
segment_dates = [start_date, end_date]
else:
raise ValueError('Unknown timesegment options \
{0}'.format(options['time_segment']))
print("Number of files: {0}".format(len(segment_dates) - 1))
assert len(segment_dates) == num_segments + 1
# Make sure the first and last dates are start/end_date
segment_dates[0] = start_date
segment_dates[-1] = end_date + timedelta(minutes=1)
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# Setup Segments
segments = deque()
for num in pyrange(num_segments):
# Segment time bounds
t0 = segment_dates[num]
t1 = segment_dates[num + 1]
# Get segment inds
i0 = bisect_left(vic_datelist, t0)
i1 = bisect_left(vic_datelist, t1)
# Make segment filename (with path)
if options['time_segment'] == 'day':
filename = "{0}.{1}.nc".format(options['out_file_prefix'],
t0.strftime('%Y-%m-%d'))
elif options['time_segment'] == 'month':
filename = "{0}.{1}.nc".format(options['out_file_prefix'],
t0.strftime('%Y-%m'))
elif options['time_segment'] == 'year':
filename = "{0}.{1}.nc".format(options['out_file_prefix'],
t0.strftime('%Y'))
elif options['time_segment'] == 'all':
filename = "{0}.{1}-{2}.nc".format(options['out_file_prefix'],
t0.strftime('%Y%m%d'),
t1.strftime('%Y%m%d'))
filename = path.join(options['out_directory'], filename)
# Setup segment and initialize netcdf
segment = Segment(num, i0, i1, options['out_file_format'],
filename, memory_mode=memory_mode)
segment.nc_globals(**global_atts)
segment.nc_time(t0, t1, vic_ordtime, options['calendar'])
segment.nc_dimensions(snow_bands=options['snow_bands'],
veg_tiles=options['veg_tiles'],
soil_layers=options['soil_layers'])
segment.nc_domain(domain)
segment.nc_fields(fields,
domain_dict['y_x_dims'], options['precision'])
print(repr(segment))
segments.append(segment)
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# Get column numbers and names (will help speed up reading)
names = []
usecols = []
dtypes = []
bin_dtypes = []
bin_mults = []
if options['precision'] == 'double':
prec = NC_DOUBLE
else:
prec = NC_FLOAT
for name, field in fields.items():
if not np.isscalar(field['column']):
# multiple levels
for i, col in enumerate(field['column']):
names.append(name + str(i))
usecols.append(col)
if 'type' in field:
if type(field['type']) == list:
dtypes.extend(field['type'])
else:
dtypes.extend([field['type']] * len(field['column']))
else:
dtypes.append([prec] * len(field['column']))
if options['input_file_format'].lower() == 'binary':
if 'bin_dtype' in field:
if type(field['bin_dtype']) == list:
bin_dtypes.extend(field['bin_dtype'])
else:
bin_dtypes.extend([field['bin_dtype']] *
len(field['column']))
else:
raise ValueError('bin_dtype not in field: {}'.format(name))
if 'bin_mult' in field:
if type(field['bin_mult']) == list:
bin_mults.extend(field['bin_mult'])
else:
bin_mults.extend([field['bin_mult']] *
len(field['column']))
else:
bin_mults.extend([1.0] * len(field['column']))
else:
# no levels
names.append(name)
usecols.append(field['column'])
if 'type' in field:
dtypes.append(field['type'])
else:
dtypes.append(prec)
if options['input_file_format'].lower() == 'binary':
if 'bin_dtype' in field:
bin_dtypes.append(field['bin_dtype'])
else:
raise ValueError('bin_dtype not in field: {}'.format(name))
if 'bin_mult' in field:
bin_mults.append(field['bin_mult'])
else:
bin_mults.append(1.0)
print('setting point attributes (fileformat, names, usecols, and dtypes)')
# pandas.read_table does not 'honor' the order of the columns in usecols
# it simply uses them in ascending order. So the names need to be sorted
# the same way. For example, if the columns in the VIC file are:
# 3: prcp; 4: evap; 5: runoff; 6; baseflow; 7: sm1; 8: sm2; 9: sm3; 10: swe
# and this is parsed from the configuration file as
# usecols = [3, 4, 5, 6, 10, 7, 8, 9]
# names=['prcp', 'evap', 'runoff', 'baseflow', 'swe', 'sm1', 'sm2', 'sm3']
# then without sorting, the netcdf file will have the wrong variables:
# nc_swe will contain sm1, nc_sm1 will contain sm2, nc_sm2: sm3 and
# nc_swe: sm3
# the following will ensure that the names are sorted in increasing column
# order. Note that sorted(usecols) is not strictly necessary, since
# apparently that is done in read_table, but it keeps the names and columns
# in the same order
names = [x for (y, x) in sorted(pyzip(usecols, names))]
usecols = sorted(usecols)
points.set_names(names)
points.set_usecols(usecols)
points.set_dtypes(dtypes)
# set binary attributes
if options['input_file_format'].lower() == 'binary':
points.set_bin_dtypes(bin_dtypes)
points.set_bin_mults(bin_mults)
points.set_fileformat(options['input_file_format'])
print('done')
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
if memory_mode == 'big_memory':
# ------------------------------------------------------------ #
# run in big memory mode
for i, segment in enumerate(segments):
segments[i].allocate()
while points:
point = points.popleft()
point.open()
point.read()
point.close()
for segment in segments:
segment.nc_add_data_to_array(point)
for segment in segments:
segment.nc_write_data_from_array()
segment.nc_close()
# ------------------------------------------------------------ #
elif memory_mode == 'standard':
# ------------------------------------------------------------ #
# Open VIC files and put data into netcdfs
chunk = Plist()
while points:
point = points.popleft()
point.open()
point.read()
point.close()
chunk.append(point)
if len(chunk) > int(options['chunksize']) or len(points) == 0:
for segment in segments:
segment.nc_add_data_standard(chunk)
chunk = Plist()
del point
# ------------------------------------------------------------ #
# ------------------------------------------------------------ #
# Close the netcdf files
for segment in segments:
segment.nc_close()
# ------------------------------------------------------------ #
elif memory_mode == 'original':
# ------------------------------------------------------------ #
# Run in original memory mode (a.k.a. vic2nc.c mode)
# Open all files
for point in points:
point.open()
while segments:
segment = segments.popleft()
segment.allocate()
count = segment.count
for point in points:
point.read(count)
segment.nc_add_data_to_array(point)
segment.nc_write_data_from_array()
segment.nc_close()
for point in points:
point.close()
# ------------------------------------------------------------ #
return
def add_ys(self, yinds):
for i in pyrange(len(self)):
self[i].y = yinds[i]
return
def add_xs(self, xinds):
for i in pyrange(len(self)):
self[i].x = xinds[i]
return
def grid_params(soil_dict, target_grid, snow_dict, veg_dict, veglib_dict,
version='4.1.2'):
"""
Reads the coordinate information from the soil_dict and target_grid and
maps all input dictionaries to the target grid. Returns a grid_dict with
the mapped input dictionary data.
"""
print('gridding params now...')
yi, xi = latlon2yx(soil_dict['lats'], soil_dict['lons'],
target_grid[YVAR], target_grid[XVAR])
in_dicts = {'soil_dict': soil_dict}
out_dicts = OrderedDict()
if snow_dict:
in_dicts['snow_dict'] = snow_dict
else:
out_dicts['snow_dict'] = False
if veg_dict:
in_dicts['veg_dict'] = veg_dict
else:
out_dicts['veg_dict'] = False
# get "unmasked" mask
mask = target_grid['mask']
ysize, xsize = target_grid['mask'].shape
ymask, xmask = np.nonzero(mask != 1)
print('{0} masked values'.format(len(ymask)))
for name, mydict in in_dicts.items():
out_dict = OrderedDict()
for var in mydict:
if mydict[var].dtype in [np.int, np.int64, np.int32]:
fill_val = FILLVALUE_I
dtype = np.int
else:
fill_val = FILLVALUE_F
dtype = np.float
if mydict[var].ndim == 1:
out_dict[var] = np.ma.zeros((ysize, xsize),
dtype=dtype)
out_dict[var][yi, xi] = mydict[var]
out_dict[var][ymask, xmask] = fill_val
elif mydict[var].ndim == 2:
steps = mydict[var].shape[1]
out_dict[var] = np.ma.zeros((steps, ysize, xsize),
dtype=dtype)
for i in pyrange(steps):
out_dict[var][i, yi, xi] = mydict[var][:, i]
out_dict[var][:, ymask, xmask] = fill_val
elif mydict[var].ndim == 3:
j = mydict[var].shape[1]
k = mydict[var].shape[2]
out_dict[var] = np.ma.zeros((j, k, ysize, xsize),
dtype=dtype)
for jj in pyrange(j):
for kk in pyrange(k):
out_dict[var][jj, kk, yi, xi] = mydict[var][:, jj, kk]
for y, x in pyzip(ymask, xmask):
out_dict[var][:, :, y, x] = fill_val
out_dict[var] = np.ma.masked_values(out_dict[var], fill_val)
out_dicts[name] = out_dict
if veglib_dict and version == '5.0.dev':
# adjust vars for the following conditions
# bare soil tile
# Add bare soil tile
var = 'Cv'
bare = 1 - out_dicts['veg_dict'][var].sum(axis=0)
bare[bare < 0.0] = 0.0
nveg_clases = out_dicts['veg_dict'][var].shape[0] + 1
shape = (nveg_clases, ) + out_dicts['veg_dict'][var].shape[1:]
new = np.zeros(shape)
new[:-1, :, :] = out_dicts['veg_dict'][var]
new[-1, :, :] = bare
new /= new.sum(axis=0)
new[:, ymask, xmask] = FILLVALUE_F
out_dicts['veg_dict'][var] = new
# add dummy values for other veg vars
# double root_depth(veg_class, root_zone, nj, ni) ;
# double root_fract(veg_class, root_zone, nj, ni) ;
# double LAI(veg_class, month, nj, ni) ;
for var in ['root_depth', 'root_fract', 'LAI']:
shape = (nveg_clases, ) + out_dicts['veg_dict'][var].shape[1:]
new = np.zeros(shape) + FILLVALUE_F
new[:-1, :, :] = out_dicts['veg_dict'][var]
new[-1, :, :] = 0
out_dicts['veg_dict'][var] = np.ma.masked_values(new, FILLVALUE_F)
# Distribute the veglib variables
# 1st - the 1d vars
# double lib_overstory(veg_class) ; --> (veg_class, nj, ni)
for var in ['overstory', 'rarc', 'rmin', 'wind_h', 'RGL', 'rad_atten',
'rad_atten', 'wind_atten', 'trunk_ratio', 'snow_albedo']:
lib_var = 'lib_{0}'.format(var)
new = np.zeros((nveg_clases, ysize, xsize)) + FILLVALUE_F
new[:-1, yi, xi] = veglib_dict[lib_var][:, np.newaxis]
new[-1, yi, xi] = 0
new[:, ymask, xmask] = fill_val
out_dicts['veg_dict'][var] = np.ma.masked_values(new, FILLVALUE_F)
# 2nd - the 2d vars
for var in ['albedo', 'veg_rough', 'displacement']:
lib_var = 'lib_{0}'.format(var)
shape = (nveg_clases, veglib_dict[lib_var].shape[1], ysize, xsize)
new = np.zeros(shape) + FILLVALUE_F
new[:-1, :, yi, xi] = veglib_dict[lib_var][:, :, np.newaxis]
new[-1, :, yi, xi] = 0
for y, x in pyzip(ymask, xmask):
new[:, :, y, x] = fill_val
out_dicts['veg_dict'][var] = np.ma.masked_values(new, FILLVALUE_F)
# 3rd - remove the redundant vars
# double lib_LAI(veg_class, month) ;
# removed from file
return out_dicts
def test_pyrange():
x = pyrange(5)
assert list(x) == [0, 1, 2, 3, 4]
