def test_matches_old_dkern(self, D=7):
with self.test_context() as sess:
for L in [0, 1, 3, 6]: # , 3, 6]:
shape = [3, D, D]
ka = ck.DeepKernel(
shape,
[[3, 3]] * L,
ck.ExReLU(),
var_weight=1.2,
var_bias=0.8,
padding="SAME", # strides=1,
data_format="NCHW")
kb = ck.dkern.DeepKernelTesting(shape,
block_sizes=[-1] * L,
block_strides=[-1] * L,
kernel_size=3,
recurse_kern=ck.ExReLU(),
var_weight=1.2,
var_bias=0.8)
x = tf.constant(np.random.randn(12, *shape),
dtype=settings.float_type)
x2 = tf.constant(np.random.randn(15, *shape),
dtype=settings.float_type)
Kxx = [ka.K(x), kb.K(x)]
Kxx2 = [ka.K(x, x2), kb.K(x, x2)]
Kxdiag = [ka.Kdiag(x), kb.Kdiag(x)]
self.assertAllClose(*sess.run(Kxx))
self.assertAllClose(*sess.run(Kxx2))
self.assertAllClose(*sess.run(Kxdiag))
def create_kern(ps):
if ps['seed'] == 1234:
return dkern.DeepKernel(
[1, 28, 28],
filter_sizes=[[5, 5], [2, 2], [5, 5], [2, 2]],
recurse_kern=dkern.ExReLU(multiply_by_sqrt2=True),
var_weight=1.,
var_bias=1.,
padding=["VALID", "SAME", "VALID", "SAME"],
strides=[[1, 1]] * 4,
data_format="NCHW",
skip_freq=-1,
)
if 'skip_freq' not in ps:
ps['skip_freq'] = -1
if ps['nlin'] == 'ExReLU':
recurse_kern = dkern.ExReLU(multiply_by_sqrt2=True)
else:
recurse_kern = dkern.ExErf()
return dkern.DeepKernel(
[1, 28, 28],
filter_sizes=[[ps['filter_sizes'], ps['filter_sizes']]] * ps['n_layers'],
recurse_kern=recurse_kern,
var_weight=ps['var_weight'],
var_bias=ps['var_bias'],
padding=ps['padding'],
strides=[[ps['strides'], ps['strides']]] * ps['n_layers'],
data_format="NCHW",
skip_freq=ps['skip_freq'],
)
def test_relu(self):
"Test very iffy; but I can't get it to pass"
with self.test_context() as sess:
self.compare_exkern(sess,
ck.ExReLU(),
Hdiag=100,
Hk=100,
rtol=0.001)
def test_step(self):
"Test very iffy; but I can't get it to pass"
with self.test_context() as sess:
self.compare_exkern(sess,
ck.ExReLU(exponent=0),
Hdiag=100,
Hk=200,
rtol=0.01)
def kernel_matrix(X,
depth=5,
image_size=28,
number_channels=1,
n_blocks=3,
sigmaw=1.0,
sigmab=1.0,
n_gpus=1):
# resnet_n=5
block_depth = (depth - 2) // (n_blocks * 2)
# resnet_n_plain = resnet_n % 100
with tf.device("cpu:0"):
kern = dkern.resnet.ResnetKernel(
input_shape=[number_channels, image_size, image_size],
# block_sizes=[resnet_n_plain]*depth,
block_sizes=[block_depth] * n_blocks,
block_strides=[1, 2, 2, 2, 2, 2, 2][:n_blocks],
var_weight=sigmaw**
2, # scipy.stats.truncnorm.std(a=-2, b=2, loc=0., scale=1.)**2,
var_bias=sigmab**2,
kernel_size=3,
conv_stride=1,
recurse_kern=(dkern.ExReLU() if depth < 100 else dkern.ExErf()),
data_format='NCHW',
)
# kern
#N_train=100; N_vali=1000
#X, Y, Xv, _, Xt, _ = mnist_1hot_all()
## Xv = np.concatenate([X[N_train:, :], Xv], axis=0)[:N_vali, :]
#X = X[:N_train]
#Y = Y[:N_train]
#
# Y.shape
#
# ys = [int((np.argmax(labels)>5))*2.0-1 for labels in Y]
# sess.close()
#sess = gpflow.get_default_session()
#N = X.shape[0]
#out = create_array_dataset(False, N,N)
#K=compute_big_K(out,sess,kern,100,X,n_gpus=n_gpus)
sess = gpflow.get_default_session()
K = compute_big_K(sess, kern, 400, X, n_gpus=n_gpus)
#K += 1e-6 * np.eye(len(X))
# print(K)
return K
def main(_):
FLAGS = tf.app.flags.FLAGS
np.random.seed(FLAGS.seed)
tf.set_random_seed(FLAGS.seed)
DEPTH = FLAGS.depth
path = FLAGS.path
if path is None:
raise ValueError("Please provide a value for `FLAGS.path`")
def file_for(name, fmt):
return os.path.join(path, "{}_{:02d}.{}".format(name, FLAGS.seed, fmt))
params_file = file_for('params', 'pkl.gz')
gram_file = file_for('gram', 'npy')
Kxvx_file = file_for('Kxvx', 'npy')
Kxtx_file = file_for('Kxtx', 'npy')
Kv_diag_file = file_for('Kv_diag', 'npy')
Kt_diag_file = file_for('Kt_diag', 'npy')
# Copied from https://github.com/tensorflow/models/blob/d089975f/official/resnet/cifar10_main.py#L167-L170
if DEPTH % 6 != 2:
raise ValueError('DEPTH must be 6n + 2:', DEPTH)
block_depth = (DEPTH - 2) // 6
params = {'depth': DEPTH}
print("Params:", params)
pu.dump(params, params_file)
with tf.device("cpu:0"):
kern = dkern.resnet.ResnetKernel(
input_shape=[1, 28, 28],
block_sizes=[block_depth] * FLAGS.n_blocks,
block_strides=[1, 2, 2, 2, 2, 2, 2][:FLAGS.n_blocks],
kernel_size=3,
conv_stride=1,
recurse_kern=dkern.ExReLU(),
var_weight=
1., # scipy.stats.truncnorm.std(a=-2, b=2, loc=0., scale=1.)**2,
var_bias=0.,
data_format='NCHW',
)
save_kernels(kern,
FLAGS.N_train,
FLAGS.N_vali,
FLAGS.n_gpus,
gram_file,
Kxvx_file,
Kxtx_file,
n_max=FLAGS.n_max,
Kv_diag_file=Kv_diag_file,
Kt_diag_file=Kt_diag_file)
def kernel_matrix(X,
X2=None,
image_size=28,
number_channels=1,
filter_sizes=[[5, 5], [2, 2], [5, 5], [2, 2]],
padding=["VALID", "SAME", "VALID", "SAME"],
strides=[[1, 1]] * 4,
sigmaw=1.0,
sigmab=1.0,
n_gpus=1):
with tf.device("cpu:0"):
kern = dkern.DeepKernel(
#[number_channels, image_size, image_size],
([number_channels, image_size, image_size]
if n_gpus > 0 else [image_size, image_size, number_channels]),
filter_sizes=filter_sizes,
recurse_kern=dkern.ExReLU(multiply_by_sqrt2=False),
var_weight=sigmaw**2,
var_bias=sigmab**2,
padding=padding,
strides=strides,
#data_format="NCHW",
data_format=(
"NCHW" if n_gpus > 0 else "NHWC"
), #but don't need to change inputs dkern transposes the inputs itself apparently :P
skip_freq=-1, # no residual connections
)
# kern
# N_train=20000; N_vali=1000
# X, Y, Xv, _, Xt, _ = mnist_1hot_all()
# # Xv = np.concatenate([X[N_train:, :], Xv], axis=0)[:N_vali, :]
# X = X[:N_train]
# Y = Y[:N_train]
#
# Y.shape
#
# ys = [int((np.argmax(labels)>5))*2.0-1 for labels in Y]
# sess.close()
sess = gpflow.get_default_session()
K = compute_big_K(sess, kern, 400, X, X2, n_gpus=n_gpus)
sess.close()
return K
def test_equivalent_BNN(self, L=1, n_random_tests=4):
s = settings.get_settings()
s.dtypes.float_type = 'float32'
with self.test_context() as sess, settings.temp_settings(s):
shape = [3, 12, 10]
X = tf.ones([1] + shape, dtype=settings.float_type)
kb = ck.DeepKernel(shape, [[3, 3]] * L,
ck.ExReLU(),
var_weight=1.2,
var_bias=0.8,
data_format="NCHW")
tf_y_bnn = kb.equivalent_BNN(X, n_samples=2, n_filters=7)
W0, b0 = (list(t[0] for t in t_list) for t_list in [kb._W, kb._b])
W1, b1 = (list(t[1] for t in t_list) for t_list in [kb._W, kb._b])
tf_y0 = cnn_from_params(W0, b0, kb, X)
tf_y1 = cnn_from_params(W1, b1, kb, X)
for _ in range(n_random_tests):
y_bnn, y0, y1 = sess.run([tf_y_bnn, tf_y0, tf_y1])
self.assertAllClose(y_bnn[0:1], y0)
self.assertAllClose(y_bnn[1:2], y1)
def test_matches_relu_fc(self, D=7):
with self.test_context() as sess:
for L in [0, 1, 3, 6]:
ka = DeepArcCosine(D, L, variance=1.2, bias_variance=0.8)
kb = ck.DeepKernel([D], [[]] * L,
ck.ExReLU(),
var_weight=1.2,
var_bias=0.8,
data_format="NC")
x = tf.constant(np.random.randn(12, D),
dtype=settings.float_type)
x2 = tf.constant(np.random.randn(15, D),
dtype=settings.float_type)
Kxx = [ka.K(x), kb.K(x)]
Kxx2 = [ka.K(x, x2), kb.K(x, x2)]
Kxdiag = [ka.Kdiag(x), kb.Kdiag(x)]
self.assertAllClose(*sess.run(Kxx))
self.assertAllClose(*sess.run(Kxx2))
self.assertAllClose(*sess.run(Kxdiag))
@gpflow.decors.params_as_tensors
def _build_likelihood(self):
# Get around fast_1sample_equivalent_BNN not getting tensors from param
Ws_tensors = list(self.Ws[i] for i in range(len(self.Ws)))
bs_tensors = list(self.bs[i] for i in range(len(self.bs)))
logits = self._kern.fast_1sample_equivalent_BNN(
tf.reshape(self.X, [-1] + self._kern.input_shape),
Ws=Ws_tensors, bs=bs_tensors)
return tf.losses.sparse_softmax_cross_entropy(labels=self.Y, logits=logits)
@gpflow.decors.autoflow((settings.float_type, [None, None]))
def predict_y(self, Xnew):
return self._build_predict_y(Xnew), tf.constant(0.0, dtype=settings.float_type)
@gpflow.decors.params_as_tensors
def _build_predict_y(self, Xnew):
Ws_tensors = list(self.Ws[i] for i in range(len(self.Ws)))
bs_tensors = list(self.bs[i] for i in range(len(self.bs)))
logits = self._kern.fast_1sample_equivalent_BNN(
tf.reshape(Xnew, [-1] + self._kern.input_shape),
Ws=Ws_tensors, bs=bs_tensors)
return tf.nn.softmax(logits)
if __name__ == '__main__':
import deep_ckern as dk, numpy as np
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
k = dk.DeepKernel([1, 16, 16], [[3, 3]]*5, dk.ExReLU(),[1,3])
X = np.random.randn(3, 16**2)
k.compute_K(X, X)
n_blocks=3
sigmaw=1.0
sigmab=0.0
# def kernel_matrix(X,depth=32,image_size=28,number_channels=1,n_blocks=3, sigmaw=1.0,sigmab=1.0):
block_depth = (depth - 2) // 6
with tf.device("cpu:0"):
kern = dkern.resnet.ResnetKernel(
input_shape=[number_channels, image_size, image_size],
block_sizes=[block_depth]*n_blocks,
block_strides=[1, 2, 2, 2, 2, 2, 2][:n_blocks],
var_weight=sigmaw**2, # scipy.stats.truncnorm.std(a=-2, b=2, loc=0., scale=1.)**2,
var_bias=sigmab**2,
kernel_size=3,
conv_stride=1,
recurse_kern=dkern.ExReLU(),
data_format='NCHW',
)
# kern
N_train=10000; N_vali=1000
X, Y, Xv, _, Xt, _ = mnist_1hot_all()
# Xv = np.concatenate([X[N_train:, :], Xv], axis=0)[:N_vali, :]
X = X[:N_train]
Y = Y[:N_train]
#
# Y.shape
#
# ys = [int((np.argmax(labels)>5))*2.0-1 for labels in Y]