def load_data(dataset):
if dataset == 'mnist':
(train_image, train_label, valid_image, valid_label, test_image,
test_label) = load_dataset.load_mnist(num_train=1000,
mean_subtraction=True,
random_roated_labels=False)
elif dataset == 'cifar':
(train_image, train_label, valid_image, valid_label, test_image,
test_label) = load_dataset.load_cifar10(num_train=1000,
mean_subtraction=True,
random_roated_labels=False)
elif dataset == 'stl10':
(train_image, train_label, valid_image, valid_label, test_image,
test_label) = load_dataset.load_stl10(num_train=1000,
mean_subtraction=True,
random_roated_labels=False)
else:
raise NotImplementedError
return train_image, train_label, valid_image, valid_label, test_image, test_label
import numpy as np
from load_dataset import load_cifar10
from os import listdir
if __name__ == '__main__':
DIRECTORY = '/tmp/nngp/'
files = [f for f in listdir(DIRECTORY)]
(_, _, _, _, _, test_label) = load_cifar10(10000, mean_subtraction=True)
max_acc = 0
max_acc_f = ''
min_mse = 1e12
min_mse_f = ''
for f in files:
if 'test_cifar_10000_' in f:
res = np.load(DIRECTORY + f)
accuracy = np.sum(
np.argmax(res, axis=1) == np.argmax(test_label,
axis=1)) / float(len(res))
i, j = test_label.shape
mse = np.sum((res - test_label)**2) / (i * j)
print(f)
print("Accuracy:")
print(accuracy)
print("Best MSE:")
print(mse)
def run_nngp_eval(hparams, run_dir):
"""Runs experiments."""
tf.gfile.MakeDirs(run_dir)
# Write hparams to experiment directory.
with tf.gfile.GFile(run_dir + '/hparams', mode='w') as f:
f.write(hparams.to_proto().SerializeToString())
tf.logging.info('Starting job.')
tf.logging.info('Hyperparameters')
tf.logging.info('---------------------')
tf.logging.info(hparams)
tf.logging.info('---------------------')
tf.logging.info('Loading data')
# Get the sets of images and labels for training, validation, and
# # test on dataset.
if FLAGS.dataset == 'mnist':
(
train_image,
train_label, #valid_image, valid_label,
test_image,
test_label) = load_dataset.load_mnist(num_train=FLAGS.num_train,
mean_subtraction=True,
random_rotated_labels=False)
elif FLAGS.dataset == 'cifar10':
(
train_image,
train_label, #valid_image, valid_label,
test_image,
test_label) = load_dataset.load_cifar10(num_train=FLAGS.num_train,
mean_subtraction=True)
else:
raise NotImplementedError
tf.logging.info('Building Model')
if hparams.nonlinearity == 'tanh':
nonlin_fn = tf.tanh
elif hparams.nonlinearity == 'relu':
nonlin_fn = tf.nn.relu
else:
raise NotImplementedError
with tf.Session() as sess:
# Construct NNGP kernel
nngp_kernel = nngp.NNGPKernel(
depth=hparams.depth,
weight_var=hparams.weight_var,
bias_var=hparams.bias_var,
mu_2=hparams.mu_2,
nonlin_fn=nonlin_fn,
grid_path=FLAGS.grid_path,
n_gauss=FLAGS.n_gauss,
n_var=FLAGS.n_var,
n_corr=FLAGS.n_corr,
max_gauss=FLAGS.max_gauss,
max_var=FLAGS.max_var,
use_fixed_point_norm=FLAGS.use_fixed_point_norm)
# Construct Gaussian Process Regression model
model = gpr.GaussianProcessRegression(train_image,
train_label,
kern=nngp_kernel)
start_time = time.time()
tf.logging.info('Training')
# For large number of training points, we do not evaluate on full set to
# save on training evaluation time.
if FLAGS.num_train <= 5000:
acc_train, mse_train, var_train, norm_train, final_eps = do_eval(
sess, model, train_image[:FLAGS.num_eval],
train_label[:FLAGS.num_eval])
tf.logging.info('Evaluation of training set (%d examples) took '
'%.3f secs' %
(min(FLAGS.num_train,
FLAGS.num_eval), time.time() - start_time))
else:
acc_train, mse_train, var_train, norm_train, final_eps = do_eval(
sess, model, train_image[:1000], train_label[:1000])
tf.logging.info('Evaluation of training set (%d examples) took '
'%.3f secs' % (1000, time.time() - start_time))
# start_time = time.time()
# tf.logging.info('Validation')
# acc_valid, mse_valid, var_valid, norm_valid, _ = do_eval(
# sess, model, valid_image[:FLAGS.num_eval],
# valid_label[:FLAGS.num_eval])
# tf.logging.info('Evaluation of valid set (%d examples) took %.3f secs'%(
# FLAGS.num_eval, time.time() - start_time))
start_time = time.time()
tf.logging.info('Test')
acc_test, mse_test, var_test, norm_test, _ = do_eval(
sess,
model,
test_image[:FLAGS.num_eval],
test_label[:FLAGS.num_eval],
save_pred=False)
tf.logging.info('Evaluation of test set (%d examples) took %.3f secs' %
(FLAGS.num_eval, time.time() - start_time))
metrics = {
'train_acc': float(acc_train),
'train_mse': float(mse_train),
'train_norm': float(norm_train),
# 'valid_acc': float(acc_valid),
# 'valid_mse': float(mse_valid),
# 'valid_norm': float(norm_valid),
'test_acc': float(acc_test),
'test_mse': float(mse_test),
'test_norm': float(norm_test),
'stability_eps': float(final_eps),
}
record_results = [
# FLAGS.num_train, hparams.nonlinearity, hparams.weight_var,
# hparams.bias_var, hparams.mu_2, hparams.depth, acc_train, acc_valid, acc_test,
# mse_train, mse_valid, mse_test, final_eps
FLAGS.num_train,
hparams.nonlinearity,
hparams.weight_var,
hparams.bias_var,
hparams.mu_2,
hparams.depth,
acc_train,
acc_test,
var_test,
norm_train
]
if nngp_kernel.use_fixed_point_norm:
metrics['var_fixed_point'] = float(nngp_kernel.var_fixed_point_np[0])
record_results.append(nngp_kernel.var_fixed_point_np[0])
# Store data
result_file = os.path.join(run_dir,
'results' + str(hparams.depth) + '.csv')
with tf.gfile.Open(result_file, 'a') as f:
filewriter = csv.writer(f)
filewriter.writerow(record_results)
with tf.Session() as sess:
varss = np.array(
[x[0] for x in sess.run(nngp_kernel.layer_qaa_dict).values()])
save_string = str(hparams.depth) + "_" + str(
hparams.weight_var) + '_' + str(hparams.mu_2)
np.save('results/vars/' + save_string, varss)
return metrics
def run_nngp_eval(hparams, run_dir):
"""Runs experiments."""
tf.gfile.MakeDirs(run_dir)
# Write hparams to experiment directory.
with tf.gfile.GFile(run_dir + '/hparams', mode='w') as f:
f.write(hparams.to_proto().SerializeToString())
tf.logging.info('Starting job.')
tf.logging.info('Hyperparameters')
tf.logging.info('---------------------')
tf.logging.info(hparams)
tf.logging.info('---------------------')
tf.logging.info('Loading data')
# Get the sets of images and labels for training, validation, and
# # test on dataset.
if FLAGS.dataset == 'mnist':
(train_image, train_label, valid_image, valid_label, test_image,
test_label) = load_dataset.load_mnist(num_train=FLAGS.num_train,
mean_subtraction=True,
random_roated_labels=False)
elif FLAGS.dataset == 'cifar':
(train_image, train_label, valid_image, valid_label, test_image,
test_label) = load_dataset.load_cifar10(num_train=FLAGS.num_train,
mean_subtraction=True)
elif FLAGS.dataset == 'stl10':
(train_image, train_label, valid_image, valid_label, test_image,
test_label) = load_dataset.load_stl10(num_train=FLAGS.num_train,
mean_subtraction=True)
else:
raise NotImplementedError
tf.logging.info('Building Model')
if hparams.nonlinearity == 'tanh':
nonlin_fn = tf.tanh
elif hparams.nonlinearity == 'relu':
nonlin_fn = tf.nn.relu
else:
raise NotImplementedError
session_conf = tf.ConfigProto(intra_op_parallelism_threads=12,
inter_op_parallelism_threads=1)
with tf.Session(config=session_conf) as sess:
# Construct NNGP kernel
nngp_kernel = nngp.NNGPKernel(
depth=hparams.depth,
weight_var=hparams.weight_var,
bias_var=hparams.bias_var,
nonlin_fn=nonlin_fn,
grid_path=FLAGS.grid_path,
n_gauss=FLAGS.n_gauss,
n_var=FLAGS.n_var,
n_corr=FLAGS.n_corr,
max_gauss=FLAGS.max_gauss,
max_var=FLAGS.max_var,
use_fixed_point_norm=FLAGS.use_fixed_point_norm)
# Construct Gaussian Process Regression model
model = gpr.GaussianProcessRegression(train_image,
train_label,
kern=nngp_kernel)
start_time = time.time()
tf.logging.info('Training')
# For large number of training points, we do not evaluate on full set to
# save on training evaluation time.
if FLAGS.num_train <= 5000:
acc_train, mse_train, norm_train, final_eps = do_eval(
sess, model, train_image[:FLAGS.num_eval],
train_label[:FLAGS.num_eval])
tf.logging.info('Evaluation of training set (%d examples) took '
'%.3f secs' %
(min(FLAGS.num_train,
FLAGS.num_eval), time.time() - start_time))
else:
acc_train, mse_train, norm_train, final_eps = do_eval(
sess, model, train_image[:1000], train_label[:1000])
tf.logging.info('Evaluation of training set (%d examples) took '
'%.3f secs' % (1000, time.time() - start_time))
vfile = "validation_{0}_{1}_{2}_{3}_{4}_{5}.npy".format(
FLAGS.dataset, FLAGS.num_train, FLAGS.num_eval, hparams.depth,
hparams.weight_var, hparams.bias_var)
start_time = time.time()
tf.logging.info('Validation')
acc_valid, mse_valid, norm_valid, _ = do_eval(
sess,
model,
valid_image[:FLAGS.num_eval],
valid_label[:FLAGS.num_eval],
save_pred=True,
fname=vfile)
tf.logging.info(
'Evaluation of valid set (%d examples) took %.3f secs' %
(FLAGS.num_eval, time.time() - start_time))
tfile = "test_{0}_{1}_{2}_{3}_{4}_{5}.npy".format(
FLAGS.dataset, FLAGS.num_train, FLAGS.num_eval, hparams.depth,
hparams.weight_var, hparams.bias_var)
start_time = time.time()
tf.logging.info('Test')
acc_test, mse_test, norm_test, _ = do_eval(sess,
model,
test_image[:FLAGS.num_eval],
test_label[:FLAGS.num_eval],
save_pred=True,
fname=tfile)
tf.logging.info('Evaluation of test set (%d examples) took %.3f secs' %
(FLAGS.num_eval, time.time() - start_time))
metrics = {
'train_acc': float(acc_train),
'train_mse': float(mse_train),
'train_norm': float(norm_train),
'valid_acc': float(acc_valid),
'valid_mse': float(mse_valid),
'valid_norm': float(norm_valid),
'test_acc': float(acc_test),
'test_mse': float(mse_test),
'test_norm': float(norm_test),
'stability_eps': float(final_eps),
}
record_results = [
FLAGS.num_train, hparams.nonlinearity, hparams.weight_var,
hparams.bias_var, hparams.depth, acc_train, acc_valid, acc_test,
mse_train, mse_valid, mse_test, final_eps
]
if nngp_kernel.use_fixed_point_norm:
metrics['var_fixed_point'] = float(nngp_kernel.var_fixed_point_np[0])
record_results.append(nngp_kernel.var_fixed_point_np[0])
# Store data
rfile = "results_{0}_{1}_{2}_{3}_{4}_{5}.csv".format(FLAGS.dataset, \
FLAGS.num_train, \
FLAGS.num_eval, \
hparams.depth, \
hparams.weight_var, \
hparams.bias_var)
result_file = os.path.join(run_dir, rfile)
with tf.gfile.Open(result_file, 'a') as f:
filewriter = csv.writer(f)
filewriter.writerow(record_results)
return metrics