def sample(self, batch_size):
sample = Variable(torch.randn(batch_size, self.z_dim))
recon_x = self.decoder(sample).detach().numpy()
result = descale(torch.Tensor(recon_x), self.state_low,
self.state_high, self.action_low, self.action_high,
self.reward_low, self.reward_high)
return (torch.FloatTensor(result[:, 0:3]).to(self.device),
torch.FloatTensor(result[:, 3]).unsqueeze(1).to(self.device),
torch.FloatTensor(result[:, 4:7]).to(self.device),
torch.FloatTensor(result[:, -2]).unsqueeze(1).to(self.device),
torch.FloatTensor(
np.random.choice(2, batch_size,
p=[1 / 200.0, 199 / 200.0
])).unsqueeze(1).to(self.device))
def driver(inputdir,
outputdir,
datadir,
plotdir,
preddir,
trainflag,
validflag,
testflag,
normalize,
fmean,
fstdev,
scale,
fmin,
fmax,
scalelims,
fsize,
rmse_file,
r2_file,
inD,
outD,
ilog,
olog,
TFRfile,
batch_size,
ncores,
buffer_size,
gridsearch,
architectures,
layers,
lay_params,
activations,
act_params,
nodes,
lengthscale,
max_lr,
clr_mode,
clr_steps,
epochs,
patience,
weight_file,
resume,
plot_cases,
fxvals,
xlabel,
ylabel,
filters=None,
filtconv=1.):
"""
Driver function to handle model training and evaluation.
Inputs
------
inputdir : string. Path/to/directory of inputs.
outputdir : string. Path/to/directory of outputs.
datadir : string. Path/to/directory of data.
plotdir : string. Path/to/directory of plots.
preddir : string. Path/to/directory of predictions.
trainflag : bool. Determines whether to train the NN model.
validflag : bool. Determines whether to validate the NN model.
testflag : bool. Determines whether to test the NN model.
normalize : bool. Determines whether to normalize the data.
fmean : string. Path/to/file of mean of training data.
fstdev : string. Path/to/file of stdev of training data.
scale : bool. Determines whether to scale the data.
fmin : string. Path/to/file of minima of training data.
fmax : string. Path/to/file of maxima of training data.
scalelims : list, floats. [min, max] of range of scaled data.
rmse_file : string. Prefix for savefiles for RMSE calculations.
r2_file : string. Prefix for savefiles for R2 calculations.
inD : int. Dimensionality of the input data.
outD : int. Dimensionality of the output data.
ilog : bool. Determines whether to take the log10 of intput data.
olog : bool. Determines whether to take the log10 of output data.
TFRfile : string. Prefix for TFRecords files.
batch_size : int. Size of batches for training/validating/testing.
ncores : int. Number of cores to use to load data cases.
buffer_size: int. Number of data cases to pre-load in memory.
gridsearch : bool. Determines whether to perform a grid search over
`architectures`.
architectures: list. Model architectures to consider.
layers : list, str. Types of hidden layers.
lay_params : list, ints. Parameters for the layer type
E.g., kernel size
activations: list, str. Activation functions for each hidden layer.
act_params : list, floats. Parameters for the activation functions.
nodes : list, ints. For layers with nodes, number of nodes per layer.
lengthscale: float. Minimum learning rat.e
max_lr : float. Maximum learning rate.
clr_mode : string. Sets the cyclical learning rate function.
clr_steps : int. Number of steps per cycle of the learning rate.
epochs : int. Max number of iterations through dataset for training.
patience : int. If no model improvement after `patience` epochs,
halts training.
weight_file: string. Path/to/file where NN weights are saved.
resume : bool. Determines whether to resume training.
plot_cases : list, ints. Cases from test set to plot.
fxvals : string. Path/to/file of X-axis values to correspond to
predicted Y values.
xlabel : string. X-axis label for plotting.
ylabel : string. Y-axis label for plotting.
filters : list, strings. Paths/to/filter files. Default: None
If specified, will compute RMSE/R2 stats over the
integrated filter bandpasses.
filtconv : float. Conversion factor for filter file x-axis values to
desired unit. Default: 1.0
"""
# Get file names, calculate number of cases per file
print('Loading files & calculating total number of batches...')
try:
datsize = np.load(inputdir + fsize)
num_train = datsize[0]
num_valid = datsize[1]
num_test = datsize[2]
except:
ftrain = glob.glob(datadir + 'train' + os.sep + '*.npy')
fvalid = glob.glob(datadir + 'valid' + os.sep + '*.npy')
ftest = glob.glob(datadir + 'test' + os.sep + '*.npy')
num_train = U.data_set_size(ftrain, ncores)
num_valid = U.data_set_size(fvalid, ncores)
num_test = U.data_set_size(ftest, ncores)
np.save(inputdir + fsize,
np.array([num_train, num_valid, num_test], dtype=int))
del ftrain, fvalid, ftest
print("Data set sizes")
print("Training data:", num_train)
print("Validation data:", num_valid)
print("Testing data:", num_test)
print("Total data:", num_train + num_valid + num_test)
train_batches = num_train // batch_size
valid_batches = num_valid // batch_size
test_batches = num_test // batch_size
# Update `clr_steps`
if clr_steps == "range test":
clr_steps = train_batches * epochs
rng_test = True
else:
clr_steps = train_batches * int(clr_steps)
rng_test = False
# Get mean/stdev for normalizing
if normalize:
print('\nNormalizing the data...')
try:
mean = np.load(inputdir + fmean)
stdev = np.load(inputdir + fstdev)
except:
print("Calculating the mean and standard deviation of the data " +\
"using Welford's method.")
# Compute stats
ftrain = glob.glob(datadir + 'train' + os.sep + '*.npy')
mean, stdev, datmin, datmax = S.mean_stdev(ftrain, inD, ilog, olog)
np.save(inputdir + fmean, mean)
np.save(inputdir + fstdev, stdev)
np.save(inputdir + fmin, datmin)
np.save(inputdir + fmax, datmax)
del datmin, datmax, ftrain
print("mean :", mean)
print("stdev:", stdev)
# Slice desired indices
x_mean, y_mean = mean[:inD], mean[inD:]
x_std, y_std = stdev[:inD], stdev[inD:]
# Memory cleanup -- no longer need full mean/stdev arrays
del mean, stdev
else:
x_mean = 0.
x_std = 1.
y_mean = 0.
y_std = 1.
if olog:
# To properly calculate RMSE & R2 for log-scaled output
try:
y_mean_delog = np.load(inputdir +
fmean.replace(".npy", "_delog.npy"))
except:
ftrain = glob.glob(datadir + 'train' + os.sep + '*.npy')
mean_delog = S.mean_stdev(ftrain, inD, ilog, False)[0]
del ftrain
y_mean_delog = mean_delog[inD:]
np.save(inputdir + fmean.replace(".npy", "_delog.npy"),
y_mean_delog)
else:
y_mean_delog = y_mean
# Get min/max values for scaling
if scale:
print('\nScaling the data...')
try:
datmin = np.load(inputdir + fmin)
datmax = np.load(inputdir + fmax)
except:
ftrain = glob.glob(datadir + 'train' + os.sep + '*.npy')
mean, stdev, datmin, datmax = S.mean_stdev(ftrain, inD, ilog, olog)
np.save(inputdir + fmean, mean)
np.save(inputdir + fstdev, stdev)
np.save(inputdir + fmin, datmin)
np.save(inputdir + fmax, datmax)
del mean, stdev, ftrain
print("min :", datmin)
print("max :", datmax)
# Slice desired indices
x_min, y_min = datmin[:inD], datmin[inD:]
x_max, y_max = datmax[:inD], datmax[inD:]
# Memory cleanup -- no longer need min/max arrays
del datmin, datmax
# Normalize min/max values
if normalize:
x_min = U.normalize(x_min, x_mean, x_std)
x_max = U.normalize(x_max, x_mean, x_std)
y_min = U.normalize(y_min, y_mean, y_std)
y_max = U.normalize(y_max, y_mean, y_std)
else:
x_min = 0.
x_max = 1.
y_min = 0.
y_max = 1.
scalelims = [0., 1.]
# Get TFRecord file names
print('\nLoading TFRecords file names...')
TFRpath = inputdir + 'TFRecords' + os.sep + TFRfile
ftrain_TFR = glob.glob(TFRpath + 'train*.tfrecords')
fvalid_TFR = glob.glob(TFRpath + 'valid*.tfrecords')
ftest_TFR = glob.glob(TFRpath + 'test*.tfrecords')
if len(ftrain_TFR) == 0 or len(fvalid_TFR) == 0 or len(ftest_TFR) == 0:
# Doesn't exist -- make them
print("\nSome TFRecords files do not exist yet.")
ftrain = glob.glob(datadir + 'train' + os.sep + '*.npy')
fvalid = glob.glob(datadir + 'valid' + os.sep + '*.npy')
ftest = glob.glob(datadir + 'test' + os.sep + '*.npy')
if len(ftrain_TFR) == 0:
print("Making TFRecords for training data...")
U.make_TFRecord(
inputdir + 'TFRecords' + os.sep + TFRfile + 'train.tfrecords',
ftrain, inD, ilog, olog, batch_size, train_batches)
if len(fvalid_TFR) == 0:
print("\nMaking TFRecords for validation data...")
U.make_TFRecord(
inputdir + 'TFRecords' + os.sep + TFRfile + 'valid.tfrecords',
fvalid, inD, ilog, olog, batch_size, valid_batches)
if len(ftest_TFR) == 0:
print("\nMaking TFRecords for test data...")
U.make_TFRecord(
inputdir + 'TFRecords' + os.sep + TFRfile + 'test.tfrecords',
ftest, inD, ilog, olog, batch_size, test_batches)
print("\nTFRecords creation complete.")
# Free memory
del ftrain, fvalid, ftest
# Get TFR file names for real this time
ftrain_TFR = glob.glob(TFRpath + 'train*.tfrecords')
fvalid_TFR = glob.glob(TFRpath + 'valid*.tfrecords')
ftest_TFR = glob.glob(TFRpath + 'test*.tfrecords')
# Load the xvals
if fxvals is not None:
xvals = np.load(fxvals)
else:
xvals = None
# Perform grid search
if gridsearch:
# Train a model for each architecture, w/ unique directories
print("\nPerforming a grid search.\n")
maxlen = 0
for i, arch in enumerate(architectures):
if len(arch) > maxlen:
maxlen = len(arch)
archdir = os.path.join(outputdir, arch, '')
wsplit = weight_file.rsplit(os.sep, 1)[1].rsplit('.', 1)
wfile = ''.join([archdir, wsplit[0], '_', arch, '.', wsplit[1]])
U.make_dir(archdir)
nn = NNModel(ftrain_TFR,
fvalid_TFR,
ftest_TFR,
inD,
outD,
olog,
x_mean,
x_std,
y_mean,
y_std,
x_min,
x_max,
y_min,
y_max,
scalelims,
ncores,
buffer_size,
batch_size,
[train_batches, valid_batches, test_batches],
layers[i],
lay_params[i],
activations[i],
act_params[i],
nodes[i],
lengthscale,
max_lr,
clr_mode,
clr_steps,
wfile,
stop_file='./STOP',
resume=resume,
train_flag=True,
shuffle=True)
nn.train(train_batches, valid_batches, epochs, patience)
P.loss(nn, archdir)
# Print/save out the minmium validation loss for each architecture
minvl = np.ones(len(architectures)) * np.inf
print('Grid search summary')
print('-------------------')
with open(outputdir + 'gridsearch.txt', 'w') as foo:
foo.write('Grid search summary\n')
foo.write('-------------------\n')
for i, arch in enumerate(architectures):
archdir = os.path.join(outputdir, arch, '')
history = np.load(archdir + 'history.npz')
minvl[i] = np.amin(history['val_loss'])
print(arch.ljust(maxlen, ' ') + ': ' + str(minvl[i]))
with open(outputdir + 'gridsearch.txt', 'a') as foo:
foo.write(arch.ljust(maxlen, ' ') + ': ' \
+ str(minvl[i]) + '\n')
return
# Train a model
if trainflag:
print('\nBeginning model training.\n')
nn = NNModel(ftrain_TFR,
fvalid_TFR,
ftest_TFR,
inD,
outD,
olog,
x_mean,
x_std,
y_mean,
y_std,
x_min,
x_max,
y_min,
y_max,
scalelims,
ncores,
buffer_size,
batch_size, [train_batches, valid_batches, test_batches],
layers,
lay_params,
activations,
act_params,
nodes,
lengthscale,
max_lr,
clr_mode,
clr_steps,
weight_file,
stop_file='./STOP',
train_flag=True,
shuffle=True,
resume=resume)
nn.train(train_batches, valid_batches, epochs, patience)
# Plot the loss
P.loss(nn, plotdir)
# Call new model with shuffle=False
nn = NNModel(ftrain_TFR,
fvalid_TFR,
ftest_TFR,
inD,
outD,
olog,
x_mean,
x_std,
y_mean,
y_std,
x_min,
x_max,
y_min,
y_max,
scalelims,
ncores,
buffer_size,
batch_size, [train_batches, valid_batches, test_batches],
layers,
lay_params,
activations,
act_params,
nodes,
lengthscale,
max_lr,
clr_mode,
clr_steps,
weight_file,
stop_file='./STOP',
train_flag=False,
shuffle=False,
resume=False)
if '.h5' in weight_file or '.hdf5' in weight_file:
nn.model.load_weights(weight_file) # Load the model
# Save in ONNX format
try:
onnx_model = keras2onnx.convert_keras(nn.model)
onnx.save_model(onnx_model,
nn.weight_file.rsplit('.', 1)[0] + '.onnx')
except Exception as e:
print("Unable to convert the Keras model to ONNX:")
print(e)
else:
nn.model = onnx_to_keras(onnx.load_model(weight_file), ['input_1'])
# Validate model
if (validflag or trainflag) and not rng_test:
print('\nValidating the model...\n')
# Y values
print(' Predicting...')
fvalpred = nn.Yeval('pred',
'valid',
preddir,
denorm=(normalize == False and scale == False))
fvalpred = glob.glob(fvalpred + '*')
print(' Loading the true Y values...')
fvaltrue = nn.Yeval('true',
'valid',
preddir,
denorm=(normalize == False and scale == False))
fvaltrue = glob.glob(fvaltrue + '*')
### RMSE & R2
print('\n Calculating RMSE & R2...')
if not normalize and not scale:
val_stats = S.rmse_r2(fvalpred,
fvaltrue,
y_mean,
olog=olog,
y_mean_delog=y_mean_delog,
x_vals=xvals,
filters=filters,
filtconv=filtconv)
else:
val_stats = S.rmse_r2(fvalpred, fvaltrue, y_mean, y_std, y_min,
y_max, scalelims, olog, y_mean_delog, xvals,
filters, filtconv)
# RMSE
if np.any(val_stats[0] != -1) and np.any(val_stats[1] != -1):
print(' Normalized RMSE : ', val_stats[0])
print(' Mean normalized RMSE : ', np.mean(val_stats[0]))
print(' Denormalized RMSE : ', val_stats[1])
print(' Mean denormalized RMSE: ', np.mean(val_stats[1]))
np.savez(outputdir + rmse_file + '_val_norm.npz',
rmse=val_stats[0],
rmse_mean=np.mean(val_stats[0]))
saveRMSEnorm = True
saveRMSEdenorm = True
elif np.any(val_stats[0] != -1):
print(' RMSE : ', val_stats[0])
print(' Mean RMSE: ', np.mean(val_stats[0]))
saveRMSEnorm = True
saveRMSEdenorm = False
elif np.any(val_stats[1] != -1):
print(' RMSE : ', val_stats[1])
print(' Mean RMSE: ', np.mean(val_stats[1]))
saveRMSEnorm = False
saveRMSEdenorm = True
else:
print(" No files passed in to compute RMSE.")
saveRMSEnorm = False
saveRMSEdenorm = False
if saveRMSEnorm:
P.plot(''.join([plotdir, rmse_file, '_val_norm.png']), xvals,
val_stats[0], xlabel, 'RMSE')
np.savez(outputdir + rmse_file + '_val_norm.npz',
rmse=val_stats[0],
rmse_mean=np.mean(val_stats[0]))
if saveRMSEdenorm:
P.plot(''.join([plotdir, rmse_file, '_val_denorm.png']), xvals,
val_stats[1], xlabel, 'RMSE')
np.savez(outputdir + rmse_file + '_val_denorm.npz',
rmse=val_stats[1],
rmse_mean=np.mean(val_stats[1]))
# R2
if np.any(val_stats[2] != -1) and np.any(val_stats[3] != -1):
print(' Normalized R2 : ', val_stats[2])
print(' Mean normalized R2 : ', np.mean(val_stats[2]))
print(' Denormalized R2 : ', val_stats[3])
print(' Mean denormalized R2: ', np.mean(val_stats[3]))
saveR2norm = True
saveR2denorm = True
elif np.any(val_stats[2] != -1):
print(' R2 : ', val_stats[2])
print(' Mean R2: ', np.mean(val_stats[2]))
saveR2norm = True
saveR2denorm = False
elif np.any(val_stats[3] != -1):
print(' R2 : ', val_stats[3])
print(' Mean R2: ', np.mean(val_stats[3]))
saveR2norm = False
saveR2denorm = True
else:
print(" No files passed in to compute R2.")
saveR2norm = False
saveR2denorm = False
if saveR2norm:
P.plot(''.join([plotdir, r2_file, '_val_norm.png']), xvals,
val_stats[2], xlabel, '$R^2$')
np.savez(outputdir + r2_file + '_val_norm.npz',
r2=val_stats[2],
r2_mean=np.mean(val_stats[2]))
if saveR2denorm:
P.plot(''.join([plotdir, r2_file, '_val_denorm.png']), xvals,
val_stats[3], xlabel, '$R^2$')
np.savez(outputdir + r2_file + '_val_denorm.npz',
r2=val_stats[3],
r2_mean=np.mean(val_stats[3]))
# Evaluate model on test set
if testflag and not rng_test:
print('\nTesting the model...\n')
# Y values
print(' Predicting...')
ftestpred = nn.Yeval('pred',
'test',
preddir,
denorm=(normalize == False and scale == False))
ftestpred = glob.glob(ftestpred + '*')
print(' Loading the true Y values...')
ftesttrue = nn.Yeval('true',
'test',
preddir,
denorm=(normalize == False and scale == False))
ftesttrue = glob.glob(ftesttrue + '*')
### RMSE & R2
print('\n Calculating RMSE & R2...')
if not normalize and not scale:
test_stats = S.rmse_r2(ftestpred,
ftesttrue,
y_mean,
olog=olog,
y_mean_delog=y_mean_delog,
x_vals=xvals,
filters=filters,
filtconv=filtconv)
else:
test_stats = S.rmse_r2(ftestpred, ftesttrue, y_mean, y_std, y_min,
y_max, scalelims, olog, y_mean_delog, xvals,
filters, filtconv)
# RMSE
if np.any(test_stats[0] != -1) and np.any(test_stats[1] != -1):
print(' Normalized RMSE : ', test_stats[0])
print(' Mean normalized RMSE : ', np.mean(test_stats[0]))
print(' Denormalized RMSE : ', test_stats[1])
print(' Mean denormalized RMSE: ', np.mean(test_stats[1]))
np.savez(outputdir + rmse_file + '_val_norm.npz',
rmse=test_stats[0],
rmse_mean=np.mean(test_stats[0]))
saveRMSEnorm = True
saveRMSEdenorm = True
elif np.any(test_stats[0] != -1):
print(' RMSE : ', test_stats[0])
print(' Mean RMSE: ', np.mean(test_stats[0]))
saveRMSEnorm = True
saveRMSEdenorm = False
elif np.any(test_stats[1] != -1):
print(' RMSE : ', test_stats[1])
print(' Mean RMSE: ', np.mean(test_stats[1]))
saveRMSEnorm = False
saveRMSEdenorm = True
else:
print(" No files passed in to compute RMSE.")
saveRMSEnorm = False
saveRMSEdenorm = False
if saveRMSEnorm:
P.plot(''.join([plotdir, rmse_file, '_test_norm.png']), xvals,
test_stats[0], xlabel, 'RMSE')
np.savez(outputdir + rmse_file + '_test_norm.npz',
rmse=test_stats[0],
rmse_mean=np.mean(test_stats[0]))
if saveRMSEdenorm:
P.plot(''.join([plotdir, rmse_file, '_test_denorm.png']), xvals,
test_stats[1], xlabel, 'RMSE')
np.savez(outputdir + rmse_file + '_test_denorm.npz',
rmse=test_stats[1],
rmse_mean=np.mean(test_stats[1]))
# R2
if np.any(test_stats[2] != -1) and np.any(test_stats[3] != -1):
print(' Normalized R2 : ', test_stats[2])
print(' Mean normalized R2 : ', np.mean(test_stats[2]))
print(' Denormalized R2 : ', test_stats[3])
print(' Mean denormalized R2: ', np.mean(test_stats[3]))
saveR2norm = True
saveR2denorm = True
elif np.any(test_stats[2] != -1):
print(' R2 : ', test_stats[2])
print(' Mean R2: ', np.mean(test_stats[2]))
saveR2norm = True
saveR2denorm = False
elif np.any(test_stats[3] != -1):
print(' R2 : ', test_stats[3])
print(' Mean R2: ', np.mean(test_stats[3]))
saveR2norm = False
saveR2denorm = True
else:
print(" No files passed in to compute R2.")
saveR2norm = False
saveR2denorm = False
if saveR2norm:
P.plot(''.join([plotdir, r2_file, '_test_norm.png']), xvals,
test_stats[2], xlabel, '$R^2$')
np.savez(outputdir + r2_file + '_test_norm.npz',
r2=test_stats[2],
r2_mean=np.mean(test_stats[2]))
if saveR2denorm:
P.plot(''.join([plotdir, r2_file, '_test_denorm.png']), xvals,
test_stats[3], xlabel, '$R^2$')
np.savez(outputdir + r2_file + '_test_denorm.npz',
r2=test_stats[3],
r2_mean=np.mean(test_stats[3]))
# Plot requested cases
if not rng_test:
predfoo = sorted(glob.glob(preddir + 'test' + os.sep + 'pred*'))
truefoo = sorted(glob.glob(preddir + 'test' + os.sep + 'true*'))
if len(predfoo) > 0 and len(truefoo) > 0:
print("\nPlotting the requested cases...")
nplot = 0
for v in plot_cases:
fname = plotdir + 'spec' + str(v) + '_pred-vs-true.png'
predspec = np.load(predfoo[v // batch_size])[v % batch_size]
predspec = U.denormalize(
U.descale(predspec, y_min, y_max, scalelims), y_mean,
y_std)
truespec = np.load(truefoo[v // batch_size])[v % batch_size]
truespec = U.denormalize(
U.descale(truespec, y_min, y_max, scalelims), y_mean,
y_std)
if olog:
predspec[olog] = 10**predspec[olog]
truespec[olog] = 10**truespec[olog]
P.plot_spec(fname, predspec, truespec, xvals, xlabel, ylabel)
nplot += 1
print(" Plot " + str(nplot) + "/" + str(len(plot_cases)),
end='\r')
print("")
else:
raise Exception("No predictions found in " + preddir + "test.")
return
def Yeval(self, mode, dataset, preddir, denorm=False):
"""
Saves out .NPY files of the true or predicted Y values for a
specified data set.
Inputs
------
mode : string. 'pred' or 'true'. Specifies whether to save the true
or predicted Y values of `dataset`.
dataset: string. 'train', 'valid', or 'test'. Specifies the data set to
make predictions on.
preddir: string. Path/to/directory where predictions will be saved.
denorm : bool. Determines whether to denormalize the predicted values.
"""
if self.shuffle:
raise ValueError("This model has shuffled TFRecords.\nCreate a " +\
"new NNModel object with shuffle=False and try again.")
if dataset == 'train':
X = self.X
Y = self.Y
num_batches = self.train_batches
elif dataset == 'valid':
X = self.Xval
Y = self.Yval
num_batches = self.valid_batches
elif dataset == 'test':
X = self.Xte
Y = self.Yte
num_batches = self.test_batches
else:
raise ValueError("Invalid specification for `dataset` parameter " +\
"of NNModel.Yeval().\nAllowed options: 'train', 'valid'," +\
" or 'test'\nPlease correct this and try again.")
# Prefix for the savefiles
if mode == 'pred' or mode == 'true':
fname = ''.join([preddir, dataset, os.sep, mode])
else:
raise ValueError("Invalid specification for `mode` parameter of " +\
"NNModel.Yeval().\nAllowed options: 'pred' or " +\
"'true'\nPlease correct this and try again.")
if denorm:
fname = ''.join([fname, '-denorm_'])
else:
fname = ''.join([fname, '-norm_'])
U.make_dir(preddir + dataset) # Ensure the directory exists
# Save out the Y values
y_batch = np.zeros((self.batch_size, self.outD))
for i in range(num_batches):
foo = ''.join([fname, str(i).zfill(len(str(num_batches))), '.npy'])
if mode == 'pred': # Predicted Y values
x_batch = K.eval(X)
y_batch = self.model.predict(x_batch)
else: # True Y values
y_batch = K.eval(Y)
if denorm:
y_batch = U.denormalize(
U.descale(y_batch, self.y_min, self.y_max, self.scalelims),
self.y_mean, self.y_std)
if self.olog:
y_batch[:, self.olog] = 10**y_batch[:, self.olog]
np.save(foo, y_batch)
print(''.join([' Batch ',
str(i + 1), '/',
str(num_batches)]),
end='\r')
print('')
return fname
def eval(params,
nn,
x_mean,
x_std,
y_mean,
y_std,
x_min,
x_max,
y_min,
y_max,
scalelims,
xvals=None,
starspec=None,
factor=None,
filters=None,
ifilt=None,
ilog=False,
olog=False,
kll=None,
count=0,
burnin=0):
"""
Evaluates the model for given inputs.
Integrates the output according to filters.
Inputs
------
params : array. Parameters to be predicted on.
nn : object. Trained NN model.
x_mean : array. Mean of inputs.
x_std : array. Standard deviation of inputs.
y_mean : array. Mean of outputs.
y_std : array. Standard deviation of outputs.
x_min : array. Minima of inputs.
x_max : array. Maxima of inputs.
y_min : array. Minima of outputs.
y_max : array. Maxima of outputs.
scalelims: list. [Lower, upper] bounds for scaling.
xvals : array. (filler) X-axis values associated with the NN output.
starspec : array. (optional) Stellar spectrum at `xvals`.
factor : float. (optional) Multiplication factor to convert the
de-normalized NN output.
filters : list, arrays. (filler) Transmission of filters.
ifilt : array. (filler) `xvals` indices where the filters are nonzero.
ilog : bool. True if the NN input is the log10 of the inputs.
olog : bool. True if the NN output is the log10 of the outputs.
kll : object. (optional) Streaming quantiles calculator.
count : array. Ensures that burned iterations do not
contribute to quantiles.
Must be specified if KLL is not None.
burnin : int. Number of burned iterations.
Must be specified if KLL is not None.
Outputs
-------
results: array. Integrated predicted values.
"""
# Input must be 2D
if len(params.shape) == 1:
params = np.expand_dims(params, 0)
# Log-scale
if ilog:
params[:, ilog] = np.log10(params[:, ilog])
# Normalize & scale
pars = U.scale(U.normalize(params, x_mean, x_std), x_min, x_max, scalelims)
# Predict
pred = nn.predict(pars)
# Post-process
# Descale & denormalize
pred = U.denormalize(U.descale(pred, y_min, y_max, scalelims), y_mean,
y_std)
# De-log
if olog:
pred[:, olog] = 10**pred[:, olog]
# Divide by stellar spectrum
if starspec is not None:
pred = pred / starspec
# Multiply by any conversion factors, e.g., R_p/R_s, unit conversion
if factor is not None:
pred *= factor
# Update the streaming quantiles
if kll is not None:
count += 1
if count[0] > burnin:
kll.update(pred)
return pred
def rmse_r2(fpred,
ftrue,
y_mean,
y_std=None,
y_min=None,
y_max=None,
scalelims=None,
olog=False,
y_mean_delog=None,
filters=None,
x_vals=None,
filt2um=1.0):
"""
Calculates the root mean squared error (RMSE) and R-squared for a data set.
Data must be saved in .NPY file(s), and `fpred` and `ftrue` must exactly
correspond.
Default behavior: compute RMSE/R2 for the raw predictions.
If y_std, y_min, y_max, scalelims, and olog are specified,
it will compute RMSE/R2 for both the raw and denormalized predictions.
If filters and x_vals are specified, then the RMSE/R2 will be computed on
the integrated filter bandpasses, rather than for each output parameter.
Inputs
------
fpred: list, strings. .NPY files to compute RMSE, predicted values.
ftrue: list, strings. .NPY files to compute RMSE, true values.
y_mean: array. Training set mean value of each output parameter.
y_std : array. Training set standard deviation of each output parameter.
y_min : array. Training set minimum of each output parameter.
y_max : array. Training set maximum of each output parameter.
scalelims: tuple/list/array. Min/max that the normalized data was scaled to.
olog : bool. Determines if the denormalized values are the log10 of the
true output parameters.
y_mean_delog: array. Mean of the training set, not log-scaled.
filters: list, strings. Filter bandpasses to integrate over.
Must be 2-column file: wavelength then transmission.
x_vals : array. X values corresponding to the Y values.
filt2um: float. Conversion factor for filter's wavelengths to
microns.
Outputs
-------
rmse_norm : array. RMSE for each parameter, normalized data.
rmse_denorm: array. RMSE for each parameter, denormalized data.
r2_norm : array. R2 for each parameter, normalized data.
r2_denorm : array. R2 for each parameter, denormalized data.
Notes
-----
Data will only be denormalized if y_std, y_min, y_max, scalelims, and olog
are all not None.
"""
if len(fpred) != len(ftrue):
raise Exception("The prediction/true file structures do not match.\n" +\
"Ensure that each set of files follows the same " +\
"exact structure and order.\nSee NNModel.Yeval().")
# Integrate over filter bandpasses?
if filters is not None and x_vals is not None:
integ = True
# Load filters
nfilters = len(filters)
filttran = []
ifilt = np.zeros((nfilters, 2), dtype=int)
meanwn = []
for i in range(nfilters):
datfilt = np.loadtxt(filters[i])
# Convert filter wavelenths to microns, then convert um -> cm-1
finterp = si.interp1d(10000. / (filt2um * datfilt[:, 0]),
datfilt[:, 1],
bounds_error=False,
fill_value=0)
# Interpolate and normalize
tranfilt = finterp(x_vals)
tranfilt = tranfilt / np.trapz(tranfilt, x_vals)
meanwn.append(np.sum(x_vals * tranfilt) / sum(tranfilt))
# Find non-zero indices for faster integration
nonzero = np.where(tranfilt != 0)
ifilt[i, 0] = max(nonzero[0][0] - 1, 0)
ifilt[i, 1] = min(nonzero[0][-1] + 1, len(x_vals) - 1)
filttran.append(tranfilt[ifilt[i, 0]:ifilt[i, 1]]) # Store filter
else:
integ = False
if not olog:
y_mean_delog = y_mean
else:
if y_mean_delog is None:
raise ValueError("Must give the non-log-scaled training set mean.")
if all(v is not None for v in [y_std, y_min, y_max, scalelims]) or olog:
denorm = True
else:
denorm = False
# Variables for computing RMSE & R2
n = 0 # number of cases seen
mss = 0 # Model sum of squares
tss = 0 # True sum of squares
rss = 0 # Residual sum of squares -- squared error
if denorm:
mss_denorm = 0
tss_denorm = 0
rss_denorm = 0
# By definition, R2 = 1 - rss / tss
# If mss + rss = tss then R2 = mss / tss
### Helper functions to avoid repeating code
def squared_diffs(pred, true, y_mean):
"""
Computes squared differences for RMSE/R2 calculations.
"""
mss = np.sum((pred - y_mean)**2, axis=0)
tss = np.sum((true - y_mean)**2, axis=0)
rss = np.sum((true - pred)**2, axis=0)
return mss, tss, rss
def integ_spec(pred, true, y_mean, x_vals, filttran, ifilt):
"""
Integrates and predicted and true spectrum according to filter
bandpasses.
"""
nfilters = len(filttran)
pred_res = np.zeros((pred.shape[0], nfilters))
true_res = np.zeros((true.shape[0], nfilters))
y_mean_res = np.zeros(nfilters)
for i in range(nfilters):
pred_spec = pred[:, ifilt[i, 0]:ifilt[i, 1]]
true_spec = true[:, ifilt[i, 0]:ifilt[i, 1]]
xval_spec = x_vals[ifilt[i, 0]:ifilt[i, 1]]
y_mean_spec = y_mean[ifilt[i, 0]:ifilt[i, 1]]
pred_res[:, i] = np.trapz(pred_spec * filttran[i],
xval_spec,
axis=-1)
true_res[:, i] = np.trapz(true_spec * filttran[i],
xval_spec,
axis=-1)
y_mean_res[i] = np.trapz(y_mean_spec * filttran[i], xval_spec)
return pred_res, true_res, y_mean_res
# Compute RMSE & R2
for j in range(len(fpred)):
# Load batch
pred = np.load(fpred[j])
true = np.load(ftrue[j])
# Add contributions to RMSE/R2
if integ:
pred_res, true_res, y_mean_res = integ_spec(
pred, true, y_mean, x_vals, filttran, ifilt)
contribs = squared_diffs(pred_res, true_res, y_mean_res)
else:
contribs = squared_diffs(pred, true, y_mean)
n += pred.shape[0]
mss += contribs[0]
tss += contribs[1]
rss += contribs[2]
if denorm:
# Calculate this for the denormalized values
pred = U.denormalize(U.descale(pred, y_min, y_max, scalelims),
y_mean, y_std)
true = U.denormalize(U.descale(true, y_min, y_max, scalelims),
y_mean, y_std)
if olog:
pred[:, olog] = 10**pred[:, olog]
true[:, olog] = 10**true[:, olog]
if integ:
pred_res, true_res, y_mean_res = integ_spec(
pred, true, y_mean_delog, x_vals, filttran, ifilt)
contribs = squared_diffs(pred_res, true_res, y_mean_res)
else:
contribs = squared_diffs(pred, true, y_mean_delog)
mss_denorm += contribs[0]
tss_denorm += contribs[1]
rss_denorm += contribs[2]
print(" Batch " + str(j + 1) + "/" + str(len(fpred)), end='\r')
print('')
rmse = (rss / n)**0.5
r2 = 1 - rss / tss
if denorm:
rmse_denorm = (rss_denorm / n)**0.5
r2_denorm = 1 - rss_denorm / tss_denorm
else:
rmse_denorm = -1
r2_denorm = -1
return rmse, rmse_denorm, r2, r2_denorm
def eval_Smith_bandinteg(params,
nn,
x_mean,
x_std,
y_mean,
y_std,
x_min,
x_max,
y_min,
y_max,
scalelims,
wavenum=None,
starspec=None,
factor=None,
filters=None,
ifilt=None,
ilog=False,
olog=False,
kll=None,
count=0,
burnin=0):
"""
Evaluates the model for given inputs.
Integrates the output according to filters.
Inputs
------
params : array. Parameters to be predicted on.
nn : object. Trained NN model.
x_mean : array. Mean of inputs.
x_std : array. Standard deviation of inputs.
y_mean : array. Mean of outputs.
y_std : array. Standard deviation of outputs.
x_min : array. Minima of inputs.
x_max : array. Maxima of inputs.
y_min : array. Minima of outputs.
y_max : array. Maxima of outputs.
scalelims: list. [Lower, upper] bounds for scaling.
wavenum : array. (filler) Wavenumbers (cm-1) associated with the NN output.
starspec : array. (optional) Stellar spectrum at `wavenum`.
factor : float. (optional) Multiplication factor to convert the
de-normalized NN output.
filters : list, arrays. (filler) Transmission of filters.
ifilt : array. (filler) `wavenum` indices where the filters are nonzero.
ilog : bool. True if the NN input is the log10 of the inputs.
olog : bool. True if the NN output is the log10 of the outputs.
kll : object. (optional) Streaming quantiles calculator.
count : array. Ensures that burned iterations do not
contribute to quantiles.
Must be specified if KLL is not None.
burnin : int. Number of burned iterations.
Must be specified if KLL is not None.
Outputs
-------
results: array. Integrated predicted values.
"""
# Input must be 2D
if len(params.shape) == 1:
params = np.expand_dims(params, 0)
if ilog:
params = np.log10(params)
# Normalize & scale
pars = U.scale(U.normalize(params, x_mean, x_std), x_min, x_max, scalelims)
# Predict
pred = nn.predict(pars)
# Post-process
# Descale & denormalize
pred = U.denormalize(U.descale(pred, y_min, y_max, scalelims), y_mean,
y_std)
# De-log
if olog:
pred = 10**pred
# Enforce cloudtop pressure < surface pressure
pred[params[:, -1] >= params[:, 3]] = -100
# Update the streaming quantiles for valid models
if kll is not None:
count += 1
if count[0] > burnin:
kll.update(pred[params[:, -1] < params[:, 3]])
if filters is not None and wavenum is not None:
nfilters = len(filters)
results = np.zeros((pars.shape[0], nfilters))
for i in range(nfilters):
results[:, i] = np.trapz(pred[:, ifilt[i, 0]:ifilt[i, 1]] *
filters[i],
wavenum[ifilt[i, 0]:ifilt[i, 1]],
axis=-1)
return results
else:
return pred