def load_data(dataset):
if dataset == 'mnist':
(train_image, train_label, valid_image, valid_label, test_image,
test_label) = load_dataset.load_mnist(num_train=1000,
mean_subtraction=True,
random_roated_labels=False)
elif dataset == 'cifar':
(train_image, train_label, valid_image, valid_label, test_image,
test_label) = load_dataset.load_cifar10(num_train=1000,
mean_subtraction=True,
random_roated_labels=False)
elif dataset == 'stl10':
(train_image, train_label, valid_image, valid_label, test_image,
test_label) = load_dataset.load_stl10(num_train=1000,
mean_subtraction=True,
random_roated_labels=False)
else:
raise NotImplementedError
return train_image, train_label, valid_image, valid_label, test_image, test_label
import numpy as np
import tensorflow as tf
# tf.enable_eager_execution()
# import gpr
import load_dataset
# import nngp
from gpflow import settings
import tqdm
m = 20000
(train_image, train_label, valid_image, valid_label, test_image,
test_label) = load_dataset.load_mnist(num_train=m,
mean_subtraction=True,
random_roated_labels=False)
# %%
def pi(y, f):
return 1 / (1 + tf.exp(-y * f))
def GP_prob_iteration(fs_old, ys, K):
N = ys.get_shape().as_list()[0]
# print(N,type(N))
# W = tf.diag([pi(1,fs_old[i])*(1-pi(1,fs_old[i])) for i,y in enumerate(ys)])
W = tf.diag(tf.map_fn(lambda f: pi(1, f) * (1 - pi(1, f)), fs_old))
B = tf.eye(N, dtype="float64") + tf.matmul(W**0.5, tf.matmul(K, W**0.5))
L = tf.cholesky(B)
from base_function import Sigmoid, Linear, show_W, add_dummy_variable, data_T, OneofK from nnet_init import W_init, Activation_function from load_dataset import load_mnist from nn_tr import nnet_calculation, Back_propagation import numpy as np X, y, X_test, y_test = load_mnist(only_binary=False) print "dataset reading complete,X.shape:", np.shape(X)[0], np.shape(X)[1] print "dataset reading complete,y.shape:", np.shape(y)[0] print "adding dummy variable to X.." X = add_dummy_variable(X) print "dummy variable adding complete,X.shape:", np.shape(X)[0], np.shape(X)[1] print 'data tranposing..' X, y = data_T(X, y) print "data tranposing complete,X.shape:", np.shape(X)[0], np.shape(X)[1] print "data tranposing complete,y.shape:", np.shape(y)[0] print 'one of k coding to y' y = OneofK(y) print 'one of k coding complete,y.shape:', np.shape(y) print "initial weight.." W = W_init([100], X) print "weight initialing complete" print 'weight shape takes the form as follow' print 'W_shape:', np.shape(W) print 'initialing activation function..' AF = Activation_function([Sigmoid, Sigmoid, Sigmoid]) print 'activation function initialing complete'
y_1 = 6
y_2 = 8
"""Loading data
Variables
---------
X_train : list of lists, (n_samples, n_features) = (60000,784)
The features of training set
y_train : array, shape(n_samples,) = (60000,)
The labels of training set range from 0 to 9
X_test : list of lists, (n_sampels, n_features) = (10000, 784)
The features of test set
y_test : array, shape(n_samples,) = (10000,)
"""
X_train, y_train, X_test, y_test = load_mnist(only_binary=True,
y_1=y_1,
y_2=y_2) #
#Separating X_train according to class label
X_train_0 = X_train[y_train == y_1, :]
X_train_1 = X_train[y_train == y_2, :]
#Training and Predicting
w_star, w_0 = FDA_train(X_train_0, X_train_1)
y_pred = FDA_test(X_test, w_star, w_0)
#one hot encoding of y_test(1-of-K code scheme)
y_test = y_test[:, np.newaxis]
y_0 = (y_test == y_1)
y_1 = (y_test == y_2)
y_test = np.hstack((y_0, y_1)).astype(int)
def run_nngp_eval(hparams, run_dir):
"""Runs experiments."""
tf.gfile.MakeDirs(run_dir)
# Write hparams to experiment directory.
with tf.gfile.GFile(run_dir + '/hparams', mode='w') as f:
f.write(hparams.to_proto().SerializeToString())
tf.logging.info('Starting job.')
tf.logging.info('Hyperparameters')
tf.logging.info('---------------------')
tf.logging.info(hparams)
tf.logging.info('---------------------')
tf.logging.info('Loading data')
# Get the sets of images and labels for training, validation, and
# # test on dataset.
if FLAGS.dataset == 'mnist':
(
train_image,
train_label, #valid_image, valid_label,
test_image,
test_label) = load_dataset.load_mnist(num_train=FLAGS.num_train,
mean_subtraction=True,
random_rotated_labels=False)
elif FLAGS.dataset == 'cifar10':
(
train_image,
train_label, #valid_image, valid_label,
test_image,
test_label) = load_dataset.load_cifar10(num_train=FLAGS.num_train,
mean_subtraction=True)
else:
raise NotImplementedError
tf.logging.info('Building Model')
if hparams.nonlinearity == 'tanh':
nonlin_fn = tf.tanh
elif hparams.nonlinearity == 'relu':
nonlin_fn = tf.nn.relu
else:
raise NotImplementedError
with tf.Session() as sess:
# Construct NNGP kernel
nngp_kernel = nngp.NNGPKernel(
depth=hparams.depth,
weight_var=hparams.weight_var,
bias_var=hparams.bias_var,
mu_2=hparams.mu_2,
nonlin_fn=nonlin_fn,
grid_path=FLAGS.grid_path,
n_gauss=FLAGS.n_gauss,
n_var=FLAGS.n_var,
n_corr=FLAGS.n_corr,
max_gauss=FLAGS.max_gauss,
max_var=FLAGS.max_var,
use_fixed_point_norm=FLAGS.use_fixed_point_norm)
# Construct Gaussian Process Regression model
model = gpr.GaussianProcessRegression(train_image,
train_label,
kern=nngp_kernel)
start_time = time.time()
tf.logging.info('Training')
# For large number of training points, we do not evaluate on full set to
# save on training evaluation time.
if FLAGS.num_train <= 5000:
acc_train, mse_train, var_train, norm_train, final_eps = do_eval(
sess, model, train_image[:FLAGS.num_eval],
train_label[:FLAGS.num_eval])
tf.logging.info('Evaluation of training set (%d examples) took '
'%.3f secs' %
(min(FLAGS.num_train,
FLAGS.num_eval), time.time() - start_time))
else:
acc_train, mse_train, var_train, norm_train, final_eps = do_eval(
sess, model, train_image[:1000], train_label[:1000])
tf.logging.info('Evaluation of training set (%d examples) took '
'%.3f secs' % (1000, time.time() - start_time))
# start_time = time.time()
# tf.logging.info('Validation')
# acc_valid, mse_valid, var_valid, norm_valid, _ = do_eval(
# sess, model, valid_image[:FLAGS.num_eval],
# valid_label[:FLAGS.num_eval])
# tf.logging.info('Evaluation of valid set (%d examples) took %.3f secs'%(
# FLAGS.num_eval, time.time() - start_time))
start_time = time.time()
tf.logging.info('Test')
acc_test, mse_test, var_test, norm_test, _ = do_eval(
sess,
model,
test_image[:FLAGS.num_eval],
test_label[:FLAGS.num_eval],
save_pred=False)
tf.logging.info('Evaluation of test set (%d examples) took %.3f secs' %
(FLAGS.num_eval, time.time() - start_time))
metrics = {
'train_acc': float(acc_train),
'train_mse': float(mse_train),
'train_norm': float(norm_train),
# 'valid_acc': float(acc_valid),
# 'valid_mse': float(mse_valid),
# 'valid_norm': float(norm_valid),
'test_acc': float(acc_test),
'test_mse': float(mse_test),
'test_norm': float(norm_test),
'stability_eps': float(final_eps),
}
record_results = [
# FLAGS.num_train, hparams.nonlinearity, hparams.weight_var,
# hparams.bias_var, hparams.mu_2, hparams.depth, acc_train, acc_valid, acc_test,
# mse_train, mse_valid, mse_test, final_eps
FLAGS.num_train,
hparams.nonlinearity,
hparams.weight_var,
hparams.bias_var,
hparams.mu_2,
hparams.depth,
acc_train,
acc_test,
var_test,
norm_train
]
if nngp_kernel.use_fixed_point_norm:
metrics['var_fixed_point'] = float(nngp_kernel.var_fixed_point_np[0])
record_results.append(nngp_kernel.var_fixed_point_np[0])
# Store data
result_file = os.path.join(run_dir,
'results' + str(hparams.depth) + '.csv')
with tf.gfile.Open(result_file, 'a') as f:
filewriter = csv.writer(f)
filewriter.writerow(record_results)
with tf.Session() as sess:
varss = np.array(
[x[0] for x in sess.run(nngp_kernel.layer_qaa_dict).values()])
save_string = str(hparams.depth) + "_" + str(
hparams.weight_var) + '_' + str(hparams.mu_2)
np.save('results/vars/' + save_string, varss)
return metrics
def run_nngp_eval(hparams, run_dir):
"""Runs experiments."""
# Write hparams to experiment directory.
tf.gfile.MakeDirs(run_dir)
with tf.gfile.GFile(run_dir + "/hparams", mode="w") as f:
f.write(hparams.to_proto().SerializeToString())
tf.logging.info("Starting job.")
tf.logging.info("Hyperparameters")
tf.logging.info("---------------------")
tf.logging.info(hparams)
tf.logging.info("---------------------")
tf.logging.info("Loading data")
# Get the sets of images and labels for training, validation, and test on dataset.
if FLAGS.dataset == "mnist":
(
train_image,
train_label,
valid_image,
valid_label,
test_image,
test_label,
) = load_dataset.load_mnist(num_train=FLAGS.num_train,
mean_subtraction=True,
random_roated_labels=False)
else:
raise NotImplementedError
tf.logging.info("Building Model")
if hparams.nonlinearity == "tanh":
nonlin_fn = tf.tanh
elif hparams.nonlinearity == "relu":
nonlin_fn = tf.nn.relu
else:
raise NotImplementedError
with tf.Session() as sess:
# Construct NNGP kernel
nngp_kernel = nngp.NNGPKernel(
depth=hparams.depth,
weight_var=hparams.weight_var,
bias_var=hparams.bias_var,
nonlin_fn=nonlin_fn,
grid_path=FLAGS.grid_path,
n_gauss=FLAGS.n_gauss,
n_var=FLAGS.n_var,
n_corr=FLAGS.n_corr,
max_gauss=FLAGS.max_gauss,
max_var=FLAGS.max_var,
use_fixed_point_norm=FLAGS.use_fixed_point_norm,
)
# Construct Gaussian Process Regression model
model = gpr.GaussianProcessRegression(train_image,
train_label,
kern=nngp_kernel)
# 1.Training
start_time = time.time()
tf.logging.info("Training")
# For large number of training points, we do not evaluate on full set to
# save on training evaluation time.
if FLAGS.num_train <= 5000:
acc_train, mse_train, norm_train, final_eps = do_eval(
sess,
model,
train_image[:FLAGS.num_eval],
train_label[:FLAGS.num_eval],
)
tf.logging.info(
"Evaluation of training set (%d examples) took %.3f secs" %
(min(FLAGS.num_train,
FLAGS.num_eval), time.time() - start_time))
else:
acc_train, mse_train, norm_train, final_eps = do_eval(
sess, model, train_image[:1000], train_label[:1000])
tf.logging.info(
"Evaluation of training set (%d examples) took %.3f secs" %
(1000, time.time() - start_time))
# 2.Validation
start_time = time.time()
tf.logging.info("Validation")
acc_valid, mse_valid, norm_valid, _ = do_eval(
sess, model, valid_image[:FLAGS.num_eval],
valid_label[:FLAGS.num_eval])
tf.logging.info(
"Evaluation of valid set (%d examples) took %.3f secs" %
(FLAGS.num_eval, time.time() - start_time))
# 3.Test
start_time = time.time()
tf.logging.info("Test")
acc_test, mse_test, norm_test, _ = do_eval(
sess,
model,
test_image[:FLAGS.num_eval],
test_label[:FLAGS.num_eval],
save_pred=False,
)
tf.logging.info("Evaluation of test set (%d examples) took %.3f secs" %
(FLAGS.num_eval, time.time() - start_time))
metrics = {
"train_acc": float(acc_train),
"train_mse": float(mse_train),
"train_norm": float(norm_train),
"valid_acc": float(acc_valid),
"valid_mse": float(mse_valid),
"valid_norm": float(norm_valid),
"test_acc": float(acc_test),
"test_mse": float(mse_test),
"test_norm": float(norm_test),
"stability_eps": float(final_eps),
}
record_results = [
FLAGS.num_train,
hparams.nonlinearity,
hparams.weight_var,
hparams.bias_var,
hparams.depth,
acc_train,
acc_valid,
acc_test,
mse_train,
mse_valid,
mse_test,
final_eps,
]
if nngp_kernel.use_fixed_point_norm:
metrics["var_fixed_point"] = float(nngp_kernel.var_fixed_point_np[0])
record_results.append(nngp_kernel.var_fixed_point_np[0])
# Store data
result_file = os.path.join(run_dir, "results.csv")
with tf.gfile.Open(result_file, "a") as f:
filewriter = csv.writer(f)
filewriter.writerow(record_results)
return metrics
def run_nngp_eval(hparams, run_dir):
"""Runs experiments."""
tf.gfile.MakeDirs(run_dir)
# Write hparams to experiment directory.
with tf.gfile.GFile(run_dir + '/hparams', mode='w') as f:
f.write(hparams.to_proto().SerializeToString())
tf.logging.info('Starting job.')
tf.logging.info('Hyperparameters')
tf.logging.info('---------------------')
tf.logging.info(hparams)
tf.logging.info('---------------------')
tf.logging.info('Loading data')
# Get the sets of images and labels for training, validation, and
# # test on dataset.
if FLAGS.dataset == 'mnist':
(train_image, train_label, valid_image, valid_label, test_image,
test_label) = load_dataset.load_mnist(num_train=FLAGS.num_train,
mean_subtraction=True,
random_roated_labels=False)
elif FLAGS.dataset == 'qi':
train_image, train_label, test_image, test_label, y_mu, y_std, train_label_normal = load_data(
)
else:
raise NotImplementedError
tf.logging.info('train X shape: %s' % (str(train_image.shape)))
tf.logging.info('train y shape: %s' % (str(train_label.shape)))
tf.logging.info('test X shape: %s' % (str(test_image.shape)))
tf.logging.info('test y shape: %s' % (str(test_label.shape)))
#return train_image, train_label, valid_image, valid_label, test_image,test_label, mu_y,std_y
tf.logging.info('Building Model')
if hparams.nonlinearity == 'tanh':
nonlin_fn = tf.tanh
elif hparams.nonlinearity == 'relu':
nonlin_fn = tf.nn.relu
else:
raise NotImplementedError
with tf.Session() as sess:
# Construct NNGP kernel
nngp_kernel = nngp.NNGPKernel(
depth=hparams.depth,
weight_var=hparams.weight_var,
bias_var=hparams.bias_var,
nonlin_fn=nonlin_fn,
grid_path=FLAGS.grid_path,
n_gauss=FLAGS.n_gauss,
n_var=FLAGS.n_var,
n_corr=FLAGS.n_corr,
max_gauss=FLAGS.max_gauss,
max_var=FLAGS.max_var,
use_fixed_point_norm=FLAGS.use_fixed_point_norm)
# Construct Gaussian Process Regression model
model = gpr.GaussianProcessRegression(train_image,
train_label_normal,
kern=nngp_kernel)
if True:
start_time = time.time()
tf.logging.info('Training')
rmse_train = do_eval_qi(sess, model, train_image, train_label,
y_mu, y_std)
tf.logging.info('Evaluation of training set (%d examples) took '
'%.3f secs' %
((train_image.shape[0]), time.time() - start_time))
start_time = time.time()
tf.logging.info('Test')
rmse_test = do_eval_qi(sess,
model,
test_image,
test_label,
y_mu,
y_std,
save_pred=False)
tf.logging.info('Evaluation of test set (%d examples) took %.3f secs' %
(test_image.shape[0], time.time() - start_time))
## For large number of training points, we do not evaluate on full set to
## save on training evaluation time.
#if FLAGS.num_train <= 5000:
# acc_train, mse_train, norm_train, final_eps = do_eval(
# sess, model, train_image[:FLAGS.num_eval],
# train_label[:FLAGS.num_eval])
# tf.logging.info('Evaluation of training set (%d examples) took '
# '%.3f secs'%(
# min(FLAGS.num_train, FLAGS.num_eval),
# time.time() - start_time))
#else:
# acc_train, mse_train, norm_train, final_eps = do_eval(
# sess, model, train_image[:1000], train_label[:1000])
# tf.logging.info('Evaluation of training set (%d examples) took '
# '%.3f secs'%(1000, time.time() - start_time))
#start_time = time.time()
#tf.logging.info('Validation')
#acc_valid, mse_valid, norm_valid, _ = do_eval(
# sess, model, valid_image[:FLAGS.num_eval],
# valid_label[:FLAGS.num_eval])
#tf.logging.info('Evaluation of valid set (%d examples) took %.3f secs'%(
# FLAGS.num_eval, time.time() - start_time))
#start_time = time.time()
#tf.logging.info('Test')
#acc_test, mse_test, norm_test, _ = do_eval(
# sess,
# model,
# test_image[:FLAGS.num_eval],
# test_label[:FLAGS.num_eval],
# save_pred=False)
#tf.logging.info('Evaluation of test set (%d examples) took %.3f secs'%(
# FLAGS.num_eval, time.time() - start_time))
metrics = {
'train_rmse': float(rmse_train),
'test_rmse': float(rmse_test),
}
record_results = [
[
"FLAGS.train_test_split", "hparams.nonlinearity",
"hparams.weight_var", "hparams.bias_var", "hparams.depth",
"rmse_train", "rmse_test"
],
[
FLAGS.train_test_split, hparams.nonlinearity, hparams.weight_var,
hparams.bias_var, hparams.depth, rmse_train, rmse_test
],
]
if nngp_kernel.use_fixed_point_norm:
metrics['var_fixed_point'] = float(nngp_kernel.var_fixed_point_np[0])
record_results[0].append("nngp_kernel.var_fixed_point_np")
record_results[1].append(nngp_kernel.var_fixed_point_np[0])
# Store data
result_file = os.path.join(run_dir, 'results.csv')
with tf.gfile.Open(result_file, 'a') as f:
filewriter = csv.writer(f)
filewriter.writerow(record_results)
return metrics
def run_nngp_eval(args):
"""Runs experiments."""
run_dir = args.experiment_dir
os.makedirs(run_dir, exist_ok=True)
hparams = {
'nonlinearity': args.nonlinearity,
'weight_var': args.weight_var,
'bias_var': args.bias_var,
'depth': args.depth
}
# Write hparams to experiment directory.
with open(run_dir + '/hparams.txt', mode='w') as f:
f.write(json.dumps(hparams))
logging.info('Starting job.')
logging.info('Hyperparameters')
logging.info('---------------------')
logging.info(hparams)
logging.info('---------------------')
logging.info('Loading data')
# Get the sets of images and labels for training, validation, and
# # test on dataset.
if args.dataset == 'mnist':
(train_image, train_label, valid_image, valid_label, test_image,
test_label) = load_dataset.load_mnist(args,
num_train=args.num_train,
mean_subtraction=True,
random_roated_labels=False)
else:
raise NotImplementedError
logging.info('Building Model')
if hparams['nonlinearity'] == 'tanh':
nonlin_fn = lambda x: np.tanh(x)
elif hparams['nonlinearity'] == 'relu':
def relu(x):
return x * (x > 0)
nonlin_fn = relu
else:
raise NotImplementedError
# Construct NNGP kernel
nngp_kernel = nngp.NNGPKernel(depth=hparams['depth'],
weight_var=hparams['weight_var'],
bias_var=hparams['bias_var'],
nonlin_fn=nonlin_fn,
grid_path=args.grid_path,
n_gauss=args.n_gauss,
n_var=args.n_var,
n_corr=args.n_corr,
max_gauss=args.max_gauss,
max_var=args.max_var,
use_precomputed_grid=True)
# Construct Gaussian Process Regression model
model = gpr.GaussianProcessRegression(train_image,
train_label,
kern=nngp_kernel)
start_time = time.time()
logging.info('Training')
# For large number of training points, we do not evaluate on full set to
# save on training evaluation time.
train_size = args.num_eval if args.num_train <= 5000 else 1000
acc_train, mse_train, norm_train, final_eps = do_eval(
args, model, train_image[:train_size], train_label[:train_size])
logging.info('Evaluation of training set ({0} examples) took '
'{1:.3f} secs'.format(min(args.num_train, args.num_eval),
time.time() - start_time))
start_time = time.time()
logging.info('Validation')
acc_valid, mse_valid, norm_valid, _ = do_eval(args, model,
valid_image[:args.num_eval],
valid_label[:args.num_eval])
logging.info(
'Evaluation of valid set ({0} examples) took {1:.3f} secs'.format(
args.num_eval,
time.time() - start_time))
start_time = time.time()
logging.info('Test')
acc_test, mse_test, norm_test, _ = do_eval(args, model,
test_image[:args.num_eval],
test_label[:args.num_eval])
logging.info(
'Evaluation of valid set ({0} examples) took {1:.3f} secs'.format(
args.num_eval,
time.time() - start_time))
metrics = {
'train_acc': float(acc_train),
'train_mse': float(mse_train),
'train_norm': float(norm_train),
'valid_acc': float(acc_valid),
'valid_mse': float(mse_valid),
'valid_norm': float(norm_valid),
'test_acc': float(acc_test),
'test_mse': float(mse_test),
'test_norm': float(norm_test),
'stability_eps': float(final_eps),
}
record_results = [
args.num_train, hparams['nonlinearity'], hparams['weight_var'],
hparams['bias_var'], hparams['depth'], acc_train, acc_valid, acc_test,
mse_train, mse_valid, mse_test, final_eps
]
# Store data
result_file = os.path.join(run_dir, 'results.csv')
with open(result_file, 'a') as f:
f.write(json.dumps(record_results))
return metrics
def run_nngp_eval(hparams, run_dir):
"""Runs experiments."""
tf.gfile.MakeDirs(run_dir)
# Write hparams to experiment directory.
with tf.gfile.GFile(run_dir + '/hparams', mode='w') as f:
f.write(hparams.to_proto().SerializeToString())
tf.logging.info('Starting job.')
tf.logging.info('Hyperparameters')
tf.logging.info('---------------------')
tf.logging.info(hparams)
tf.logging.info('---------------------')
tf.logging.info('Loading data')
# Get the sets of images and labels for training, validation, and
# # test on dataset.
if FLAGS.dataset == 'mnist':
(train_image, train_label, valid_image, valid_label, test_image,
test_label) = load_dataset.load_mnist(num_train=FLAGS.num_train,
mean_subtraction=True,
random_roated_labels=False)
elif FLAGS.dataset == 'cifar':
(train_image, train_label, valid_image, valid_label, test_image,
test_label) = load_dataset.load_cifar10(num_train=FLAGS.num_train,
mean_subtraction=True)
elif FLAGS.dataset == 'stl10':
(train_image, train_label, valid_image, valid_label, test_image,
test_label) = load_dataset.load_stl10(num_train=FLAGS.num_train,
mean_subtraction=True)
else:
raise NotImplementedError
tf.logging.info('Building Model')
if hparams.nonlinearity == 'tanh':
nonlin_fn = tf.tanh
elif hparams.nonlinearity == 'relu':
nonlin_fn = tf.nn.relu
else:
raise NotImplementedError
session_conf = tf.ConfigProto(intra_op_parallelism_threads=12,
inter_op_parallelism_threads=1)
with tf.Session(config=session_conf) as sess:
# Construct NNGP kernel
nngp_kernel = nngp.NNGPKernel(
depth=hparams.depth,
weight_var=hparams.weight_var,
bias_var=hparams.bias_var,
nonlin_fn=nonlin_fn,
grid_path=FLAGS.grid_path,
n_gauss=FLAGS.n_gauss,
n_var=FLAGS.n_var,
n_corr=FLAGS.n_corr,
max_gauss=FLAGS.max_gauss,
max_var=FLAGS.max_var,
use_fixed_point_norm=FLAGS.use_fixed_point_norm)
# Construct Gaussian Process Regression model
model = gpr.GaussianProcessRegression(train_image,
train_label,
kern=nngp_kernel)
start_time = time.time()
tf.logging.info('Training')
# For large number of training points, we do not evaluate on full set to
# save on training evaluation time.
if FLAGS.num_train <= 5000:
acc_train, mse_train, norm_train, final_eps = do_eval(
sess, model, train_image[:FLAGS.num_eval],
train_label[:FLAGS.num_eval])
tf.logging.info('Evaluation of training set (%d examples) took '
'%.3f secs' %
(min(FLAGS.num_train,
FLAGS.num_eval), time.time() - start_time))
else:
acc_train, mse_train, norm_train, final_eps = do_eval(
sess, model, train_image[:1000], train_label[:1000])
tf.logging.info('Evaluation of training set (%d examples) took '
'%.3f secs' % (1000, time.time() - start_time))
vfile = "validation_{0}_{1}_{2}_{3}_{4}_{5}.npy".format(
FLAGS.dataset, FLAGS.num_train, FLAGS.num_eval, hparams.depth,
hparams.weight_var, hparams.bias_var)
start_time = time.time()
tf.logging.info('Validation')
acc_valid, mse_valid, norm_valid, _ = do_eval(
sess,
model,
valid_image[:FLAGS.num_eval],
valid_label[:FLAGS.num_eval],
save_pred=True,
fname=vfile)
tf.logging.info(
'Evaluation of valid set (%d examples) took %.3f secs' %
(FLAGS.num_eval, time.time() - start_time))
tfile = "test_{0}_{1}_{2}_{3}_{4}_{5}.npy".format(
FLAGS.dataset, FLAGS.num_train, FLAGS.num_eval, hparams.depth,
hparams.weight_var, hparams.bias_var)
start_time = time.time()
tf.logging.info('Test')
acc_test, mse_test, norm_test, _ = do_eval(sess,
model,
test_image[:FLAGS.num_eval],
test_label[:FLAGS.num_eval],
save_pred=True,
fname=tfile)
tf.logging.info('Evaluation of test set (%d examples) took %.3f secs' %
(FLAGS.num_eval, time.time() - start_time))
metrics = {
'train_acc': float(acc_train),
'train_mse': float(mse_train),
'train_norm': float(norm_train),
'valid_acc': float(acc_valid),
'valid_mse': float(mse_valid),
'valid_norm': float(norm_valid),
'test_acc': float(acc_test),
'test_mse': float(mse_test),
'test_norm': float(norm_test),
'stability_eps': float(final_eps),
}
record_results = [
FLAGS.num_train, hparams.nonlinearity, hparams.weight_var,
hparams.bias_var, hparams.depth, acc_train, acc_valid, acc_test,
mse_train, mse_valid, mse_test, final_eps
]
if nngp_kernel.use_fixed_point_norm:
metrics['var_fixed_point'] = float(nngp_kernel.var_fixed_point_np[0])
record_results.append(nngp_kernel.var_fixed_point_np[0])
# Store data
rfile = "results_{0}_{1}_{2}_{3}_{4}_{5}.csv".format(FLAGS.dataset, \
FLAGS.num_train, \
FLAGS.num_eval, \
hparams.depth, \
hparams.weight_var, \
hparams.bias_var)
result_file = os.path.join(run_dir, rfile)
with tf.gfile.Open(result_file, 'a') as f:
filewriter = csv.writer(f)
filewriter.writerow(record_results)
return metrics