def write_netcdf_convcond_v1():
mlp_str = 'convcond_X-StandardScaler-qTindi_Y-SimpleY-qTindi_' +\
'Ntrnex100000_r_100R_mom0.9reg1e-05_Niter10000_v3'
datasource = './data/convcond_training_v3.pkl'
# Set output filename
filename = '/Users/jgdwyer/neural_weights_convcond_v1.nc'
# Load ANN and preprocessors
mlp, _, errors, x_ppi, y_ppi, x_pp, y_pp, lat, lev, dlev = \
pickle.load(open('./data/regressors/' + mlp_str + '.pkl', 'rb'))
# Need to first transform some data for preprocessors to be able to export params
x_unscl, y_unscl, _, _, _, _, _, _ = nnload.LoadData(datasource,
minlev=min(lev))
x_scl = nnload.transform_data(x_ppi, x_pp, x_unscl)
y_scl = nnload.transform_data(y_ppi, y_pp, y_unscl)
# Also need to first use the predict method to be able to export ANN params
_ = mlp.predict(x_scl)
# Grab weights and input normalization
w1 = mlp.get_parameters()[0].weights
w2 = mlp.get_parameters()[1].weights
b1 = mlp.get_parameters()[0].biases
b2 = mlp.get_parameters()[1].biases
xscale_mean = x_pp.mean_
xscale_stnd = x_pp.scale_
Nlev = len(lev)
yscale_absmax = np.zeros(b2.shape)
yscale_absmax[:Nlev] = y_pp[0]
yscale_absmax[Nlev:] = y_pp[1]
# Write weights to file
ncfile = Dataset(filename, 'w')
# Write the dimensions
ncfile.createDimension('N_in', w1.shape[0])
ncfile.createDimension('N_h1', w1.shape[1])
ncfile.createDimension('N_out', w2.shape[1])
# Create variable entries in the file
nc_w1 = ncfile.createVariable('w1',
np.dtype('float64').char,
('N_h1', 'N_in')) # Reverse dims
nc_w2 = ncfile.createVariable('w2',
np.dtype('float64').char, ('N_out', 'N_h1'))
nc_b1 = ncfile.createVariable('b1', np.dtype('float64').char, ('N_h1'))
nc_b2 = ncfile.createVariable('b2', np.dtype('float64').char, ('N_out'))
nc_xscale_mean = ncfile.createVariable('xscale_mean',
np.dtype('float64').char, ('N_in'))
nc_xscale_stnd = ncfile.createVariable('xscale_stnd',
np.dtype('float64').char, ('N_in'))
nc_yscale_absmax = ncfile.createVariable('yscale_absmax',
np.dtype('float64').char,
('N_out'))
# Write variables and close file - transpose because fortran reads it in
# "backwards"
nc_w1[:] = w1.T
nc_w2[:] = w2.T
nc_b1[:] = b1
nc_b2[:] = b2
nc_xscale_mean[:] = xscale_mean
nc_xscale_stnd[:] = xscale_stnd
nc_yscale_absmax[:] = yscale_absmax
# Write global file attributes
ncfile.description = mlp_str
ncfile.close()
def PreprocessData(x_ppi, x, y_ppi, y, pp_str, N_trn_exs):
"""Transform data according to input preprocessor requirements and make
make preprocessor string for saving"""
x_pp = nnload.init_pp(x_ppi, x)
x = nnload.transform_data(x_ppi, x_pp, x)
y_pp = nnload.init_pp(y_ppi, y)
y = nnload.transform_data(y_ppi, y_pp, y)
# Make preprocessor string for saving
pp_str = pp_str + 'X-' + x_ppi['name'] + '-' + x_ppi['method'][:6] + '_'
pp_str = pp_str + 'Y-' + y_ppi['name'] + '-' + y_ppi['method'][:6] + '_'
# Add number of training examples to string
pp_str = pp_str + 'Ntrnex' + str(N_trn_exs) + '_'
return x_pp, x, y_pp, y, pp_str
def PreprocessData(x_ppi, x, y_ppi, y, pp_str, N_trn_exs):
"""Transform data according to input preprocessor requirements and make
make preprocessor string for saving"""
x_pp = nnload.init_pp(x_ppi, x)
x = nnload.transform_data(x_ppi, x_pp, x)
y_pp = nnload.init_pp(y_ppi, y)
y = nnload.transform_data(y_ppi, y_pp, y)
# Make preprocessor string for saving
pp_str = pp_str + 'X-' + x_ppi['name'] + '-' + x_ppi['method'][:6] + '_'
pp_str = pp_str + 'Y-' + y_ppi['name'] + '-' + y_ppi['method'][:6] + '_'
# Add number of training examples to string
pp_str = pp_str + 'Ntrnex' + str(N_trn_exs) + '_'
return x_pp, x, y_pp, y, pp_str
def verify_netcdf_weights():
r_str = 'convcond_X-StandardScaler-qTindi_Y-SimpleY-qTindi_' +\
'Ntrnex100000_r_100R_mom0.9reg1e-05_Niter10000_v3'
nc_str = '/Users/jgdwyer/neural_weights_convcond_v1.nc'
# Load unscaled data
x, y, cv, Pout, lat, lev, dlev, timestep = \
nnload.LoadData('./data/convcond_testing_v3.pkl',
0.2, all_lats=True, indlat=None, rainonly=False)
# Load preprocessers
r_mlp_eval, _, errors, x_ppi, y_ppi, x_pp, y_pp, lat2, lev2, dlev = \
pickle.load(open('./data/regressors/' + r_str + '.pkl', 'rb'))
print('Loading predictor: ' + r_str)
print('Loading ncfile :' + nc_str)
# Load netcdf files
ncfile = Dataset(nc_str, 'r')
yscale_absmax = ncfile['yscale_absmax'][:]
yscale_absmax = yscale_absmax[:, None].T
xscale_mean = ncfile['xscale_mean'][:]
xscale_mean = xscale_mean[:, None].T
xscale_std = ncfile['xscale_stnd'][:]
xscale_std = xscale_std[:, None].T
print(x_ppi)
# Scaled variables as calculated by NN weights
xs = nnload.transform_data(x_ppi, x_pp, x)
ys = nnload.transform_data(y_ppi, y_pp, y)
# Scaled variables as calculated by hand from netcdf files
xs_byhand = (x - xscale_mean) / xscale_std
ys_byhand = y / yscale_absmax
print('Difference between x-scaling methods: {:.1f}'.format(
np.sum(np.abs(xs - xs_byhand))))
print('Difference between y-scaling methods: {:.1f}'.format(
np.sum(np.abs(ys - ys_byhand))))
# Now check that transformation is done correctly
# Load NN weights
w1 = ncfile['w1'][:].T
w2 = ncfile['w2'][:].T
b1 = ncfile['b1'][:]
b2 = ncfile['b2'][:]
yps_byhand = np.dot(xs_byhand, w1) + b1
yps_byhand[yps_byhand < 0] = 0
yps_byhand = np.dot(yps_byhand, w2) + b2
yps = r_mlp_eval.predict(xs)
print('Difference between predicted tendencies: {:.1f}'.format(
np.sum(np.abs(yps - yps_byhand))))
def compare_convcond_prediction(cv_str, cvcd_str, minlev):
cv_mlp, _, errors, x_ppi, y_ppi, x_pp, y_pp, lat, lev, _ = \
pickle.load(open('./data/regressors/' + cv_str + '.pkl', 'rb'))
cvcd_mlp, _, errors, x_ppi_check, y_ppi_check, x_pp, y_pp, lat, lev, _ = \
pickle.load(open('./data/regressors/' + cvcd_str + '.pkl', 'rb'))
# Check that preprocessers are the same
if ((x_ppi != x_ppi_check) or (y_ppi != y_ppi_check)):
raise ValueError('Preprocessing schemes different for conv only and ' +
'conv+cond! This means that comparing the two in ' +
'scaled space may give different results')
# Load data
x_unscl, ytcv_unscl, _, _, _, _, _, _ = \
nnload.LoadData('./data/conv_testing_v3.pkl', minlev=minlev,
N_trn_exs=10000, randseed=True)
xcvcd_unscl, ytcvcd_unscl, _, _, _, _, _, _ = \
nnload.LoadData('./data/convcond_testing_v3.pkl', minlev=minlev,
N_trn_exs=10000, randseed=True)
# Check that x values are the same to make sure random seeds are same
if np.sum(np.abs(x_unscl - xcvcd_unscl)) > 0.0:
raise ValueError('Data loaded in different order!')
# Convert true y-values to scaled by applying an inverse transformation
ytcv_scl = nnload.transform_data(y_ppi, y_pp, ytcv_unscl)
ytcvcd_scl = nnload.transform_data(y_ppi, y_pp, ytcvcd_unscl)
# Derived true y-values for cond only
ytcd_scl = ytcvcd_scl - ytcvcd_scl
# Calculate predicted y values for conv and convcond
ypcv_scl = cv_mlp.predict(x_scl)
ypcvcd_scl = cvcd_mlp.predict(x_scl)
# Add true cond values to ycv_true and ycv_pred
v = 'q'
mse_cvcd_predictboth = nnload.calc_mse(nnload.unpack(ypcvcd_scl, v),
nnload.unpack(ytcvcd_scl, v),
relflag=True)
mse_cv = nnload.calc_mse(nnload.unpack(ypcv_scl, v),
nnload.unpack(ytcv_scl, v),
relflag=True)
print('MSE predicting convection and condensation in one step: {:.5f}'.
format(mse_cvcd_predictboth))
print('MSE predicting convection only (no condensation): {:.5f}'.format(
mse_cv))
def plot_neural_fortran(training_file, mlp_str, latind=None, timeind=None,
ensemble=False):
# mlp_str = 'X-StandardScaler-qTindi_Y-SimpleY-qTindi_' + \
# 'Ntrnex100000_r_100R_mom0.9reg1e-06_Niter10000_v3'
mlp, _, errors, x_ppi, y_ppi, x_pp, y_pp, lat, lev, dlev = \
pickle.load(open('./data/regressors/' + mlp_str + '.pkl', 'rb'))
x_unscl, ytrue_unscl, y_dbm_unscl, Ptrue, P_dbm, ten, qen = \
nnload.load_netcdf_onepoint(training_file, min(lev), latind=latind,
timeind=timeind, ensemble=ensemble)
ind = 0
x_scl = nnload.transform_data(x_ppi, x_pp, x_unscl)
ypred_scl = mlp.predict(x_scl)
ypred_unscl = nnload.inverse_transform_data(y_ppi, y_pp, ypred_scl)
Ppred = nnatmos.calc_precip(nnload.unpack(ypred_unscl, 'q'), dlev)
f, (a1, a2) = plt.subplots(1, 2)
a1.plot(unpack(ytrue_unscl, 'T')[ind, :], lev, label='GCM dT')
a1.plot(unpack(ypred_unscl, 'T')[ind, :], lev, label='NN dT')
a1.plot(unpack(y_dbm_unscl, 'T')[ind, :], lev, label='DBM dT')
if ensemble:
for key in ten:
a1.plot(ten[key], lev, color='gray')
a1.set_xlabel('K/day')
a1.set_ylim(1, 0)
a1.legend()
a2.plot(unpack(ytrue_unscl, 'q')[ind, :], lev, label='GCM dq')
a2.plot(unpack(ypred_unscl, 'q')[ind, :], lev, label='NN dq')
a2.plot(unpack(y_dbm_unscl, 'q')[ind, :], lev, label='DBM dq')
if ensemble:
for key in qen:
a2.plot(qen[key], lev, color='gray')
a2.set_xlabel('g/kg/day')
a2.set_ylim(1, 0)
a2.legend()
f.savefig('./figs/sampletest/out.png', bbox_inches='tight')
# Plot inputs
f, (a1, a2) = plt.subplots(1, 2)
a1.plot(unpack(x_unscl, 'T')[ind, :].T, lev, label='input T [K]')
a1.set_ylim(1, 0)
q0 = unpack(x_unscl, 'q')[ind, :]*1000 # now g/kg
time_step = 20*60
dq = unpack(ypred_unscl, 'q')[ind, :]*time_step/3600/24 # now g/kg
a2.plot(q0, lev, label='input q [kg/kg]')
a2.plot(q0 + dq, lev, label='q after')
a2.set_ylim(1, 0)
plt.xlabel('g/kg')
plt.legend()
print('GCM Precip is: {:.2f}'.format(Ptrue[ind]))
print('MLP Precip is: {:.2f}'.format(Ppred[ind]))
print('DBM Precip is: {:.2f}'.format(P_dbm[ind]))
f.savefig('./figs/sampletest/in.png', bbox_inches='tight')
def write_netcdf_ensemble1():
ntrns = np.arange(125000, 125010)
base1 = 'X-StandardScaler-qTindi_Y-SimpleY-qTindi_Ntrnex'
base2 = '_r_50R_mom0.9reg1e-06_Niter3000_v3'
mlp_str = [base1 + str(ntrn) + base2 for ntrn in ntrns]
N_e = len(mlp_str)
datasource = './data/conv_training_v3.pkl'
# Set output filename
filename = '/Users/jgdwyer/neural_weights_ensemble1.nc'
# Load ANN and preprocessors
yscale_absmax = np.zeros((32, len(mlp_str)))
w1 = np.zeros((32, 50, N_e))
w2 = np.zeros((50, 32, N_e))
b1 = np.zeros((50, N_e))
b2 = np.zeros((32, N_e))
xscale_mean = np.zeros((32, N_e))
xscale_stnd = np.zeros((32, N_e))
yscale_absmax = np.zeros((32, N_e))
for i in range(len(mlp_str)):
mlp, _, errors, x_ppi, y_ppi, x_pp, y_pp, lat, lev, dlev = \
pickle.load(open('./data/regressors/' + mlp_str[i] + '.pkl', 'rb'))
# Need to transform some data for preprocessors to be able to export
# params
x_unscl, y_unscl, _, _, _, _, _, _ = nnload.LoadData(datasource,
minlev=min(lev))
x_scl = nnload.transform_data(x_ppi, x_pp, x_unscl)
y_scl = nnload.transform_data(y_ppi, y_pp, y_unscl)
# Also need to use the predict method to be able to export ANN params
_ = mlp.predict(x_scl)
# Grab weights and input normalization
w1[:, :, i] = mlp.get_parameters()[0].weights
w2[:, :, i] = mlp.get_parameters()[1].weights
b1[:, i] = mlp.get_parameters()[0].biases
b2[:, i] = mlp.get_parameters()[1].biases
xscale_mean[:, i] = x_pp.mean_
xscale_stnd[:, i] = x_pp.scale_
Nlev = len(lev)
yscale_absmax[:Nlev, i] = y_pp[0]
yscale_absmax[Nlev:, i] = y_pp[1]
# Write weights to file
ncfile = Dataset(filename, 'w')
# Write the dimensions
ncfile.createDimension('N_in', w1.shape[0])
ncfile.createDimension('N_h1', w1.shape[1])
ncfile.createDimension('N_out', w2.shape[1])
ncfile.createDimension('N_e', N_e)
# Create variable entries in the file
# Variables need to "reversed" to be read in by Fortran GCM code
nc_w1 = ncfile.createVariable('w1',
np.dtype('float64').char,
('N_e', 'N_h1', 'N_in'))
nc_w2 = ncfile.createVariable('w2',
np.dtype('float64').char,
('N_e', 'N_out', 'N_h1'))
nc_b1 = ncfile.createVariable('b1',
np.dtype('float64').char, ('N_e', 'N_h1'))
nc_b2 = ncfile.createVariable('b2',
np.dtype('float64').char, ('N_e', 'N_out'))
nc_xscale_mean = ncfile.createVariable('xscale_mean',
np.dtype('float64').char,
('N_e', 'N_in'))
nc_xscale_stnd = ncfile.createVariable('xscale_stnd',
np.dtype('float64').char,
('N_e', 'N_in'))
nc_yscale_absmax = ncfile.createVariable('yscale_absmax',
np.dtype('float64').char,
('N_e', 'N_out'))
# Write variables and close file - transpose because fortran reads it in
# "backwards"
nc_w1[:] = np.transpose(w1, (2, 1, 0))
nc_w2[:] = np.transpose(w2, (2, 1, 0))
nc_b1[:] = b1.T
nc_b2[:] = b2.T
nc_xscale_mean[:] = xscale_mean.T
nc_xscale_stnd[:] = xscale_stnd.T
nc_yscale_absmax[:] = yscale_absmax.T
ncfile.close()