def testComputeQmapGridRelu(self):
"""Test checks the compute_qmap_grid function.
(i) Checks sizes are appropriate and (ii) checks
accuracy of the numerical values generated by the
grid by comparing against the analytically known
form for Relu (Cho and Saul, '09).
"""
n_gauss, n_var, n_corr = 301, 33, 31
kernel = nngp.NNGPKernel(nonlin_fn=tf.nn.relu,
n_gauss=n_gauss,
n_var=n_var,
n_corr=n_corr)
var_aa_grid = kernel.var_aa_grid
corr_ab_grid = kernel.corr_ab_grid
qaa_grid = kernel.qaa_grid
qab_grid = kernel.qab_grid
qaa_exact = 0.5 * var_aa_grid
qab_exact = self.ExactQabArcCos(var_aa_grid, corr_ab_grid)
with self.test_session() as sess:
self.assertEqual(var_aa_grid.shape.as_list(), [n_var])
self.assertEqual(corr_ab_grid.shape.as_list(), [n_corr])
self.assertAllClose(sess.run(qaa_exact),
sess.run(qaa_grid),
rtol=1e-6)
self.assertAllClose(sess.run(qab_exact),
sess.run(qab_grid),
rtol=1e-6,
atol=2e-2)
def testComputeQmapGridReluLogSpacing(self):
n_gauss, n_var, n_corr = 301, 33, 31
kernel = nngp.NNGPKernel(nonlin_fn=tf.nn.relu,
n_gauss=n_gauss,
n_var=n_var,
n_corr=n_corr)
var_aa_grid = kernel.var_aa_grid
corr_ab_grid = kernel.corr_ab_grid
qaa_grid = kernel.qaa_grid
qab_grid = kernel.qab_grid
qaa_exact = 0.5 * var_aa_grid
qab_exact = self.ExactQabArcCos(var_aa_grid, corr_ab_grid)
with self.test_session() as sess:
self.assertEqual(var_aa_grid.shape.as_list(), [n_var])
self.assertEqual(corr_ab_grid.shape.as_list(), [n_corr])
self.assertAllClose(sess.run(qaa_exact),
sess.run(qaa_grid),
rtol=1e-6,
atol=2e-2)
self.assertAllClose(sess.run(qab_exact),
sess.run(qab_grid),
rtol=1e-6,
atol=2e-2)
def testComputeQmapGridRelu(self):
"""Test checks the compute_qmap_grid function.
(i) Checks sizes are appropriate and (ii) checks
accuracy of the numerical values generated by the
grid by comparing against the analytically known
form for Relu (Cho and Saul, '09).
"""
n_gauss, n_var, n_corr = 301, 33, 31
kernel = nngp.NNGPKernel(nonlin_fn=lambda x: x * (x > 0),
n_gauss=n_gauss,
n_var=n_var,
n_corr=n_corr)
var_aa_grid = kernel.var_aa_grid
corr_ab_grid = kernel.corr_ab_grid
qaa_grid = kernel.qaa_grid
qab_grid = kernel.qab_grid
qaa_exact = 0.5 * var_aa_grid
qab_exact = self.ExactQabArcCos(var_aa_grid, corr_ab_grid)
self.assertEqual(list(var_aa_grid.shape), [n_var])
self.assertEqual(list(corr_ab_grid.shape), [n_corr])
self.assertTrue(np.allclose(qaa_exact, qaa_grid, rtol=1e-6))
self.assertTrue(np.allclose(qab_exact, qab_grid, rtol=1e-6, atol=2e-2))
def testComputeQmapGridEvenNGauss(self):
n_gauss, n_var, n_corr = 102, 33, 31
with self.assertRaises(ValueError):
nngp.NNGPKernel(nonlin_fn=tf.nn.relu,
n_gauss=n_gauss,
n_var=n_var,
n_corr=n_corr)
def testGetVarFixedPoint(self):
n_gauss, n_var, n_corr = 101, 33, 31
weight_var, bias_var = 1.9, 0.2
analytic_fixed_point = bias_var / (1. - weight_var / 2)
kernel = nngp.NNGPKernel(nonlin_fn=tf.nn.relu,
weight_var=weight_var,
bias_var=bias_var,
n_gauss=n_gauss,
n_var=n_var,
n_corr=n_corr)
fixed_point, _ = kernel.get_var_fixed_point()
self.assertAllClose(analytic_fixed_point, fixed_point[0], atol=1e-4)
def testComputeQmapGridReluLogSpacing(self):
"""Test checks the compute_qmap_grid function with log_spacing=True.
"""
n_gauss, n_var, n_corr = 301, 33, 31
kernel = nngp.NNGPKernel(nonlin_fn=lambda x: x * (x > 0),
n_gauss=n_gauss,
n_var=n_var,
n_corr=n_corr)
var_aa_grid = kernel.var_aa_grid
corr_ab_grid = kernel.corr_ab_grid
qaa_grid = kernel.qaa_grid
qab_grid = kernel.qab_grid
qaa_exact = 0.5 * var_aa_grid
qab_exact = self.ExactQabArcCos(var_aa_grid, corr_ab_grid)
self.assertEqual(list(var_aa_grid.shape), [n_var])
self.assertEqual(list(corr_ab_grid.shape), [n_corr])
self.assertTrue(np.allclose(qaa_exact, qaa_grid, rtol=1e-6, atol=2e-2))
self.assertTrue(np.allclose(qab_exact, qab_grid, rtol=1e-6, atol=2e-2))
def run_nngp_eval(hparams, run_dir):
"""Runs experiments."""
tf.gfile.MakeDirs(run_dir)
# Write hparams to experiment directory.
with tf.gfile.GFile(run_dir + '/hparams', mode='w') as f:
f.write(hparams.to_proto().SerializeToString())
tf.logging.info('Starting job.')
tf.logging.info('Hyperparameters')
tf.logging.info('---------------------')
tf.logging.info(hparams)
tf.logging.info('---------------------')
tf.logging.info('Loading data')
# Get the sets of images and labels for training, validation, and
# # test on dataset.
if FLAGS.dataset == 'mnist':
(
train_image,
train_label, #valid_image, valid_label,
test_image,
test_label) = load_dataset.load_mnist(num_train=FLAGS.num_train,
mean_subtraction=True,
random_rotated_labels=False)
elif FLAGS.dataset == 'cifar10':
(
train_image,
train_label, #valid_image, valid_label,
test_image,
test_label) = load_dataset.load_cifar10(num_train=FLAGS.num_train,
mean_subtraction=True)
else:
raise NotImplementedError
tf.logging.info('Building Model')
if hparams.nonlinearity == 'tanh':
nonlin_fn = tf.tanh
elif hparams.nonlinearity == 'relu':
nonlin_fn = tf.nn.relu
else:
raise NotImplementedError
with tf.Session() as sess:
# Construct NNGP kernel
nngp_kernel = nngp.NNGPKernel(
depth=hparams.depth,
weight_var=hparams.weight_var,
bias_var=hparams.bias_var,
mu_2=hparams.mu_2,
nonlin_fn=nonlin_fn,
grid_path=FLAGS.grid_path,
n_gauss=FLAGS.n_gauss,
n_var=FLAGS.n_var,
n_corr=FLAGS.n_corr,
max_gauss=FLAGS.max_gauss,
max_var=FLAGS.max_var,
use_fixed_point_norm=FLAGS.use_fixed_point_norm)
# Construct Gaussian Process Regression model
model = gpr.GaussianProcessRegression(train_image,
train_label,
kern=nngp_kernel)
start_time = time.time()
tf.logging.info('Training')
# For large number of training points, we do not evaluate on full set to
# save on training evaluation time.
if FLAGS.num_train <= 5000:
acc_train, mse_train, var_train, norm_train, final_eps = do_eval(
sess, model, train_image[:FLAGS.num_eval],
train_label[:FLAGS.num_eval])
tf.logging.info('Evaluation of training set (%d examples) took '
'%.3f secs' %
(min(FLAGS.num_train,
FLAGS.num_eval), time.time() - start_time))
else:
acc_train, mse_train, var_train, norm_train, final_eps = do_eval(
sess, model, train_image[:1000], train_label[:1000])
tf.logging.info('Evaluation of training set (%d examples) took '
'%.3f secs' % (1000, time.time() - start_time))
# start_time = time.time()
# tf.logging.info('Validation')
# acc_valid, mse_valid, var_valid, norm_valid, _ = do_eval(
# sess, model, valid_image[:FLAGS.num_eval],
# valid_label[:FLAGS.num_eval])
# tf.logging.info('Evaluation of valid set (%d examples) took %.3f secs'%(
# FLAGS.num_eval, time.time() - start_time))
start_time = time.time()
tf.logging.info('Test')
acc_test, mse_test, var_test, norm_test, _ = do_eval(
sess,
model,
test_image[:FLAGS.num_eval],
test_label[:FLAGS.num_eval],
save_pred=False)
tf.logging.info('Evaluation of test set (%d examples) took %.3f secs' %
(FLAGS.num_eval, time.time() - start_time))
metrics = {
'train_acc': float(acc_train),
'train_mse': float(mse_train),
'train_norm': float(norm_train),
# 'valid_acc': float(acc_valid),
# 'valid_mse': float(mse_valid),
# 'valid_norm': float(norm_valid),
'test_acc': float(acc_test),
'test_mse': float(mse_test),
'test_norm': float(norm_test),
'stability_eps': float(final_eps),
}
record_results = [
# FLAGS.num_train, hparams.nonlinearity, hparams.weight_var,
# hparams.bias_var, hparams.mu_2, hparams.depth, acc_train, acc_valid, acc_test,
# mse_train, mse_valid, mse_test, final_eps
FLAGS.num_train,
hparams.nonlinearity,
hparams.weight_var,
hparams.bias_var,
hparams.mu_2,
hparams.depth,
acc_train,
acc_test,
var_test,
norm_train
]
if nngp_kernel.use_fixed_point_norm:
metrics['var_fixed_point'] = float(nngp_kernel.var_fixed_point_np[0])
record_results.append(nngp_kernel.var_fixed_point_np[0])
# Store data
result_file = os.path.join(run_dir,
'results' + str(hparams.depth) + '.csv')
with tf.gfile.Open(result_file, 'a') as f:
filewriter = csv.writer(f)
filewriter.writerow(record_results)
with tf.Session() as sess:
varss = np.array(
[x[0] for x in sess.run(nngp_kernel.layer_qaa_dict).values()])
save_string = str(hparams.depth) + "_" + str(
hparams.weight_var) + '_' + str(hparams.mu_2)
np.save('results/vars/' + save_string, varss)
return metrics
def run_nngp_eval(hparams, run_dir):
"""Runs experiments."""
# Write hparams to experiment directory.
tf.gfile.MakeDirs(run_dir)
with tf.gfile.GFile(run_dir + "/hparams", mode="w") as f:
f.write(hparams.to_proto().SerializeToString())
tf.logging.info("Starting job.")
tf.logging.info("Hyperparameters")
tf.logging.info("---------------------")
tf.logging.info(hparams)
tf.logging.info("---------------------")
tf.logging.info("Loading data")
# Get the sets of images and labels for training, validation, and test on dataset.
if FLAGS.dataset == "mnist":
(
train_image,
train_label,
valid_image,
valid_label,
test_image,
test_label,
) = load_dataset.load_mnist(num_train=FLAGS.num_train,
mean_subtraction=True,
random_roated_labels=False)
else:
raise NotImplementedError
tf.logging.info("Building Model")
if hparams.nonlinearity == "tanh":
nonlin_fn = tf.tanh
elif hparams.nonlinearity == "relu":
nonlin_fn = tf.nn.relu
else:
raise NotImplementedError
with tf.Session() as sess:
# Construct NNGP kernel
nngp_kernel = nngp.NNGPKernel(
depth=hparams.depth,
weight_var=hparams.weight_var,
bias_var=hparams.bias_var,
nonlin_fn=nonlin_fn,
grid_path=FLAGS.grid_path,
n_gauss=FLAGS.n_gauss,
n_var=FLAGS.n_var,
n_corr=FLAGS.n_corr,
max_gauss=FLAGS.max_gauss,
max_var=FLAGS.max_var,
use_fixed_point_norm=FLAGS.use_fixed_point_norm,
)
# Construct Gaussian Process Regression model
model = gpr.GaussianProcessRegression(train_image,
train_label,
kern=nngp_kernel)
# 1.Training
start_time = time.time()
tf.logging.info("Training")
# For large number of training points, we do not evaluate on full set to
# save on training evaluation time.
if FLAGS.num_train <= 5000:
acc_train, mse_train, norm_train, final_eps = do_eval(
sess,
model,
train_image[:FLAGS.num_eval],
train_label[:FLAGS.num_eval],
)
tf.logging.info(
"Evaluation of training set (%d examples) took %.3f secs" %
(min(FLAGS.num_train,
FLAGS.num_eval), time.time() - start_time))
else:
acc_train, mse_train, norm_train, final_eps = do_eval(
sess, model, train_image[:1000], train_label[:1000])
tf.logging.info(
"Evaluation of training set (%d examples) took %.3f secs" %
(1000, time.time() - start_time))
# 2.Validation
start_time = time.time()
tf.logging.info("Validation")
acc_valid, mse_valid, norm_valid, _ = do_eval(
sess, model, valid_image[:FLAGS.num_eval],
valid_label[:FLAGS.num_eval])
tf.logging.info(
"Evaluation of valid set (%d examples) took %.3f secs" %
(FLAGS.num_eval, time.time() - start_time))
# 3.Test
start_time = time.time()
tf.logging.info("Test")
acc_test, mse_test, norm_test, _ = do_eval(
sess,
model,
test_image[:FLAGS.num_eval],
test_label[:FLAGS.num_eval],
save_pred=False,
)
tf.logging.info("Evaluation of test set (%d examples) took %.3f secs" %
(FLAGS.num_eval, time.time() - start_time))
metrics = {
"train_acc": float(acc_train),
"train_mse": float(mse_train),
"train_norm": float(norm_train),
"valid_acc": float(acc_valid),
"valid_mse": float(mse_valid),
"valid_norm": float(norm_valid),
"test_acc": float(acc_test),
"test_mse": float(mse_test),
"test_norm": float(norm_test),
"stability_eps": float(final_eps),
}
record_results = [
FLAGS.num_train,
hparams.nonlinearity,
hparams.weight_var,
hparams.bias_var,
hparams.depth,
acc_train,
acc_valid,
acc_test,
mse_train,
mse_valid,
mse_test,
final_eps,
]
if nngp_kernel.use_fixed_point_norm:
metrics["var_fixed_point"] = float(nngp_kernel.var_fixed_point_np[0])
record_results.append(nngp_kernel.var_fixed_point_np[0])
# Store data
result_file = os.path.join(run_dir, "results.csv")
with tf.gfile.Open(result_file, "a") as f:
filewriter = csv.writer(f)
filewriter.writerow(record_results)
return metrics
def run_nngp_eval(hparams, run_dir):
"""Runs experiments."""
tf.gfile.MakeDirs(run_dir)
# Write hparams to experiment directory.
with tf.gfile.GFile(run_dir + '/hparams', mode='w') as f:
f.write(hparams.to_proto().SerializeToString())
tf.logging.info('Starting job.')
tf.logging.info('Hyperparameters')
tf.logging.info('---------------------')
tf.logging.info(hparams)
tf.logging.info('---------------------')
tf.logging.info('Loading data')
# Get the sets of images and labels for training, validation, and
# # test on dataset.
if FLAGS.dataset == 'mnist':
(train_image, train_label, valid_image, valid_label, test_image,
test_label) = load_dataset.load_mnist(num_train=FLAGS.num_train,
mean_subtraction=True,
random_roated_labels=False)
elif FLAGS.dataset == 'qi':
train_image, train_label, test_image, test_label, y_mu, y_std, train_label_normal = load_data(
)
else:
raise NotImplementedError
tf.logging.info('train X shape: %s' % (str(train_image.shape)))
tf.logging.info('train y shape: %s' % (str(train_label.shape)))
tf.logging.info('test X shape: %s' % (str(test_image.shape)))
tf.logging.info('test y shape: %s' % (str(test_label.shape)))
#return train_image, train_label, valid_image, valid_label, test_image,test_label, mu_y,std_y
tf.logging.info('Building Model')
if hparams.nonlinearity == 'tanh':
nonlin_fn = tf.tanh
elif hparams.nonlinearity == 'relu':
nonlin_fn = tf.nn.relu
else:
raise NotImplementedError
with tf.Session() as sess:
# Construct NNGP kernel
nngp_kernel = nngp.NNGPKernel(
depth=hparams.depth,
weight_var=hparams.weight_var,
bias_var=hparams.bias_var,
nonlin_fn=nonlin_fn,
grid_path=FLAGS.grid_path,
n_gauss=FLAGS.n_gauss,
n_var=FLAGS.n_var,
n_corr=FLAGS.n_corr,
max_gauss=FLAGS.max_gauss,
max_var=FLAGS.max_var,
use_fixed_point_norm=FLAGS.use_fixed_point_norm)
# Construct Gaussian Process Regression model
model = gpr.GaussianProcessRegression(train_image,
train_label_normal,
kern=nngp_kernel)
if True:
start_time = time.time()
tf.logging.info('Training')
rmse_train = do_eval_qi(sess, model, train_image, train_label,
y_mu, y_std)
tf.logging.info('Evaluation of training set (%d examples) took '
'%.3f secs' %
((train_image.shape[0]), time.time() - start_time))
start_time = time.time()
tf.logging.info('Test')
rmse_test = do_eval_qi(sess,
model,
test_image,
test_label,
y_mu,
y_std,
save_pred=False)
tf.logging.info('Evaluation of test set (%d examples) took %.3f secs' %
(test_image.shape[0], time.time() - start_time))
## For large number of training points, we do not evaluate on full set to
## save on training evaluation time.
#if FLAGS.num_train <= 5000:
# acc_train, mse_train, norm_train, final_eps = do_eval(
# sess, model, train_image[:FLAGS.num_eval],
# train_label[:FLAGS.num_eval])
# tf.logging.info('Evaluation of training set (%d examples) took '
# '%.3f secs'%(
# min(FLAGS.num_train, FLAGS.num_eval),
# time.time() - start_time))
#else:
# acc_train, mse_train, norm_train, final_eps = do_eval(
# sess, model, train_image[:1000], train_label[:1000])
# tf.logging.info('Evaluation of training set (%d examples) took '
# '%.3f secs'%(1000, time.time() - start_time))
#start_time = time.time()
#tf.logging.info('Validation')
#acc_valid, mse_valid, norm_valid, _ = do_eval(
# sess, model, valid_image[:FLAGS.num_eval],
# valid_label[:FLAGS.num_eval])
#tf.logging.info('Evaluation of valid set (%d examples) took %.3f secs'%(
# FLAGS.num_eval, time.time() - start_time))
#start_time = time.time()
#tf.logging.info('Test')
#acc_test, mse_test, norm_test, _ = do_eval(
# sess,
# model,
# test_image[:FLAGS.num_eval],
# test_label[:FLAGS.num_eval],
# save_pred=False)
#tf.logging.info('Evaluation of test set (%d examples) took %.3f secs'%(
# FLAGS.num_eval, time.time() - start_time))
metrics = {
'train_rmse': float(rmse_train),
'test_rmse': float(rmse_test),
}
record_results = [
[
"FLAGS.train_test_split", "hparams.nonlinearity",
"hparams.weight_var", "hparams.bias_var", "hparams.depth",
"rmse_train", "rmse_test"
],
[
FLAGS.train_test_split, hparams.nonlinearity, hparams.weight_var,
hparams.bias_var, hparams.depth, rmse_train, rmse_test
],
]
if nngp_kernel.use_fixed_point_norm:
metrics['var_fixed_point'] = float(nngp_kernel.var_fixed_point_np[0])
record_results[0].append("nngp_kernel.var_fixed_point_np")
record_results[1].append(nngp_kernel.var_fixed_point_np[0])
# Store data
result_file = os.path.join(run_dir, 'results.csv')
with tf.gfile.Open(result_file, 'a') as f:
filewriter = csv.writer(f)
filewriter.writerow(record_results)
return metrics
def run_nngp_eval(args):
"""Runs experiments."""
run_dir = args.experiment_dir
os.makedirs(run_dir, exist_ok=True)
hparams = {
'nonlinearity': args.nonlinearity,
'weight_var': args.weight_var,
'bias_var': args.bias_var,
'depth': args.depth
}
# Write hparams to experiment directory.
with open(run_dir + '/hparams.txt', mode='w') as f:
f.write(json.dumps(hparams))
logging.info('Starting job.')
logging.info('Hyperparameters')
logging.info('---------------------')
logging.info(hparams)
logging.info('---------------------')
logging.info('Loading data')
# Get the sets of images and labels for training, validation, and
# # test on dataset.
if args.dataset == 'mnist':
(train_image, train_label, valid_image, valid_label, test_image,
test_label) = load_dataset.load_mnist(args,
num_train=args.num_train,
mean_subtraction=True,
random_roated_labels=False)
else:
raise NotImplementedError
logging.info('Building Model')
if hparams['nonlinearity'] == 'tanh':
nonlin_fn = lambda x: np.tanh(x)
elif hparams['nonlinearity'] == 'relu':
def relu(x):
return x * (x > 0)
nonlin_fn = relu
else:
raise NotImplementedError
# Construct NNGP kernel
nngp_kernel = nngp.NNGPKernel(depth=hparams['depth'],
weight_var=hparams['weight_var'],
bias_var=hparams['bias_var'],
nonlin_fn=nonlin_fn,
grid_path=args.grid_path,
n_gauss=args.n_gauss,
n_var=args.n_var,
n_corr=args.n_corr,
max_gauss=args.max_gauss,
max_var=args.max_var,
use_precomputed_grid=True)
# Construct Gaussian Process Regression model
model = gpr.GaussianProcessRegression(train_image,
train_label,
kern=nngp_kernel)
start_time = time.time()
logging.info('Training')
# For large number of training points, we do not evaluate on full set to
# save on training evaluation time.
train_size = args.num_eval if args.num_train <= 5000 else 1000
acc_train, mse_train, norm_train, final_eps = do_eval(
args, model, train_image[:train_size], train_label[:train_size])
logging.info('Evaluation of training set ({0} examples) took '
'{1:.3f} secs'.format(min(args.num_train, args.num_eval),
time.time() - start_time))
start_time = time.time()
logging.info('Validation')
acc_valid, mse_valid, norm_valid, _ = do_eval(args, model,
valid_image[:args.num_eval],
valid_label[:args.num_eval])
logging.info(
'Evaluation of valid set ({0} examples) took {1:.3f} secs'.format(
args.num_eval,
time.time() - start_time))
start_time = time.time()
logging.info('Test')
acc_test, mse_test, norm_test, _ = do_eval(args, model,
test_image[:args.num_eval],
test_label[:args.num_eval])
logging.info(
'Evaluation of valid set ({0} examples) took {1:.3f} secs'.format(
args.num_eval,
time.time() - start_time))
metrics = {
'train_acc': float(acc_train),
'train_mse': float(mse_train),
'train_norm': float(norm_train),
'valid_acc': float(acc_valid),
'valid_mse': float(mse_valid),
'valid_norm': float(norm_valid),
'test_acc': float(acc_test),
'test_mse': float(mse_test),
'test_norm': float(norm_test),
'stability_eps': float(final_eps),
}
record_results = [
args.num_train, hparams['nonlinearity'], hparams['weight_var'],
hparams['bias_var'], hparams['depth'], acc_train, acc_valid, acc_test,
mse_train, mse_valid, mse_test, final_eps
]
# Store data
result_file = os.path.join(run_dir, 'results.csv')
with open(result_file, 'a') as f:
f.write(json.dumps(record_results))
return metrics
def run_nngp_eval(hparams, run_dir):
"""Runs experiments."""
tf.gfile.MakeDirs(run_dir)
# Write hparams to experiment directory.
with tf.gfile.GFile(run_dir + '/hparams', mode='w') as f:
f.write(hparams.to_proto().SerializeToString())
tf.logging.info('Starting job.')
tf.logging.info('Hyperparameters')
tf.logging.info('---------------------')
tf.logging.info(hparams)
tf.logging.info('---------------------')
tf.logging.info('Loading data')
# Get the sets of images and labels for training, validation, and
# # test on dataset.
if FLAGS.dataset == 'mnist':
(train_image, train_label, valid_image, valid_label, test_image,
test_label) = load_dataset.load_mnist(num_train=FLAGS.num_train,
mean_subtraction=True,
random_roated_labels=False)
elif FLAGS.dataset == 'cifar':
(train_image, train_label, valid_image, valid_label, test_image,
test_label) = load_dataset.load_cifar10(num_train=FLAGS.num_train,
mean_subtraction=True)
elif FLAGS.dataset == 'stl10':
(train_image, train_label, valid_image, valid_label, test_image,
test_label) = load_dataset.load_stl10(num_train=FLAGS.num_train,
mean_subtraction=True)
else:
raise NotImplementedError
tf.logging.info('Building Model')
if hparams.nonlinearity == 'tanh':
nonlin_fn = tf.tanh
elif hparams.nonlinearity == 'relu':
nonlin_fn = tf.nn.relu
else:
raise NotImplementedError
session_conf = tf.ConfigProto(intra_op_parallelism_threads=12,
inter_op_parallelism_threads=1)
with tf.Session(config=session_conf) as sess:
# Construct NNGP kernel
nngp_kernel = nngp.NNGPKernel(
depth=hparams.depth,
weight_var=hparams.weight_var,
bias_var=hparams.bias_var,
nonlin_fn=nonlin_fn,
grid_path=FLAGS.grid_path,
n_gauss=FLAGS.n_gauss,
n_var=FLAGS.n_var,
n_corr=FLAGS.n_corr,
max_gauss=FLAGS.max_gauss,
max_var=FLAGS.max_var,
use_fixed_point_norm=FLAGS.use_fixed_point_norm)
# Construct Gaussian Process Regression model
model = gpr.GaussianProcessRegression(train_image,
train_label,
kern=nngp_kernel)
start_time = time.time()
tf.logging.info('Training')
# For large number of training points, we do not evaluate on full set to
# save on training evaluation time.
if FLAGS.num_train <= 5000:
acc_train, mse_train, norm_train, final_eps = do_eval(
sess, model, train_image[:FLAGS.num_eval],
train_label[:FLAGS.num_eval])
tf.logging.info('Evaluation of training set (%d examples) took '
'%.3f secs' %
(min(FLAGS.num_train,
FLAGS.num_eval), time.time() - start_time))
else:
acc_train, mse_train, norm_train, final_eps = do_eval(
sess, model, train_image[:1000], train_label[:1000])
tf.logging.info('Evaluation of training set (%d examples) took '
'%.3f secs' % (1000, time.time() - start_time))
vfile = "validation_{0}_{1}_{2}_{3}_{4}_{5}.npy".format(
FLAGS.dataset, FLAGS.num_train, FLAGS.num_eval, hparams.depth,
hparams.weight_var, hparams.bias_var)
start_time = time.time()
tf.logging.info('Validation')
acc_valid, mse_valid, norm_valid, _ = do_eval(
sess,
model,
valid_image[:FLAGS.num_eval],
valid_label[:FLAGS.num_eval],
save_pred=True,
fname=vfile)
tf.logging.info(
'Evaluation of valid set (%d examples) took %.3f secs' %
(FLAGS.num_eval, time.time() - start_time))
tfile = "test_{0}_{1}_{2}_{3}_{4}_{5}.npy".format(
FLAGS.dataset, FLAGS.num_train, FLAGS.num_eval, hparams.depth,
hparams.weight_var, hparams.bias_var)
start_time = time.time()
tf.logging.info('Test')
acc_test, mse_test, norm_test, _ = do_eval(sess,
model,
test_image[:FLAGS.num_eval],
test_label[:FLAGS.num_eval],
save_pred=True,
fname=tfile)
tf.logging.info('Evaluation of test set (%d examples) took %.3f secs' %
(FLAGS.num_eval, time.time() - start_time))
metrics = {
'train_acc': float(acc_train),
'train_mse': float(mse_train),
'train_norm': float(norm_train),
'valid_acc': float(acc_valid),
'valid_mse': float(mse_valid),
'valid_norm': float(norm_valid),
'test_acc': float(acc_test),
'test_mse': float(mse_test),
'test_norm': float(norm_test),
'stability_eps': float(final_eps),
}
record_results = [
FLAGS.num_train, hparams.nonlinearity, hparams.weight_var,
hparams.bias_var, hparams.depth, acc_train, acc_valid, acc_test,
mse_train, mse_valid, mse_test, final_eps
]
if nngp_kernel.use_fixed_point_norm:
metrics['var_fixed_point'] = float(nngp_kernel.var_fixed_point_np[0])
record_results.append(nngp_kernel.var_fixed_point_np[0])
# Store data
rfile = "results_{0}_{1}_{2}_{3}_{4}_{5}.csv".format(FLAGS.dataset, \
FLAGS.num_train, \
FLAGS.num_eval, \
hparams.depth, \
hparams.weight_var, \
hparams.bias_var)
result_file = os.path.join(run_dir, rfile)
with tf.gfile.Open(result_file, 'a') as f:
filewriter = csv.writer(f)
filewriter.writerow(record_results)
return metrics
