def test_seq(self):
X = K.placeholder((None, 28, 28, 1))
f = N.Sequence([
N.Conv(8, (3, 3), strides=1, pad='same'),
N.Dimshuffle(pattern=(0, 3, 1, 2)),
N.Flatten(outdim=2),
N.Noise(level=0.3, noise_dims=None, noise_type='gaussian'),
N.Dense(128, activation=tf.nn.relu),
N.Dropout(level=0.3, noise_dims=None),
N.Dense(10, activation=tf.nn.softmax)
])
y = f(X)
yT = f.T(y)
f1 = K.function(X, y, defaults={K.is_training(): True})
f2 = K.function(X, yT, defaults={K.is_training(): False})
f = cPickle.loads(cPickle.dumps(f))
y = f(X)
yT = f.T(y)
f3 = K.function(X, y, defaults={K.is_training(): True})
f4 = K.function(X, yT, defaults={K.is_training(): False})
x = np.random.rand(12, 28, 28, 1)
self.assertEquals(f1(x).shape, (2688, 10))
self.assertEquals(f3(x).shape, (2688, 10))
self.assertEqual(np.round(f1(x).sum(), 4), np.round(f3(x).sum(), 4))
self.assertEquals(y.shape.as_list(), (None, 10))
self.assertEquals(f2(x).shape, (12, 28, 28, 1))
self.assertEquals(f4(x).shape, (12, 28, 28, 1))
self.assertEqual(str(f2(x).sum())[:4], str(f4(x).sum())[:4])
self.assertEquals(yT.shape.as_list(), (None, 28, 28, 1))
def test_dropout(self):
x = K.placeholder((4, 6))
f1 = N.Dropout(level=0.5, noise_dims=0, rescale=True)
y = f1(x)
f = K.function(x, y, defaults={K.is_training(): True})
z = f(np.ones((4, 6)))
z = z.tolist()
self.assertTrue(all(i == z[0] for i in z))
f1 = N.Dropout(level=0.5, noise_dims=1, rescale=True)
y = f1(x)
f = K.function(x, y, defaults={K.is_training(): True})
z = f(np.ones((4, 6)))
z = z.T.tolist()
self.assertTrue(all(i == z[0] for i in z))
def test_noise(self):
x = K.placeholder((2, 3))
f1 = N.Noise(level=0.5, noise_dims=0, noise_type='gaussian')
y = f1(x)
f = K.function(x, y, defaults={K.is_training(): True})
z = f(np.ones((2, 3)))
z = z.tolist()
self.assertTrue(all(i == z[0] for i in z))
f1 = N.Noise(level=0.5, noise_dims=1, noise_type='gaussian')
y = f1(x)
f = K.function(x, y, defaults={K.is_training(): True})
z = f(np.ones((2, 3)))
z = z.T.tolist()
self.assertTrue(all(i == z[0] for i in z))
def _apply(self, x, **kwargs):
is_training = False
for i in as_tuple(x):
if K.is_variable(i) and K.is_training():
is_training = True
if is_training:
self.ops = [self.training]
return self.training(x, **_shrink_kwargs(self.training, kwargs))
else:
self.ops = [self.deploying]
return self.deploying(x, **_shrink_kwargs(self.deploying, kwargs))
def _apply(self, x):
input_shape = K.get_shape(x)
# calculate statistics
mean, logsigma = self.get_mean_logsigma(x)
# variational output
output = mean
if K.is_training():
output = self.sampling(x)
# set shape for output
K.add_shape(output, input_shape[:-1] + (self.num_units, ))
return output
def _apply(self, x, **kwargs):
if self.debug:
print('**************** Sequences: %s ****************' %
self.name)
print('Is training:', K.is_training())
print('First input:', K.get_shape(x))
for op in self.ops:
x = op(x, **_shrink_kwargs(op, kwargs))
# print after finnish the op
if self.debug:
print(' ', str(op), '->',
[K.get_shape(i)
for i in x] if isinstance(x,
(tuple,
list)) else K.get_shape(x))
# end debug
if self.debug:
print()
return x
def _apply(self, X, noise=0):
ndim = X.shape.ndims
# if is training, normalize input by its own mean and std
mean, var = tf.nn.moments(X, axes=self.axes)
# prepare dimshuffle pattern inserting broadcastable axes as needed
param_axes = iter(range(ndim - len(self.axes)))
pattern = ['x' if input_axis in self.axes else next(param_axes)
for input_axis in range(ndim)]
# apply dimshuffle pattern to all parameters
beta = 0 if self.beta_init is None else \
K.dimshuffle(self.get('beta'), pattern)
gamma = 1 if self.gamma_init is None else \
K.dimshuffle(self.get('gamma'), pattern)
# ====== if trainign: use local mean and var ====== #
def training_fn():
running_mean = ((1 - self.alpha) * self.get('mean') +
self.alpha * mean)
running_var = ((1 - self.alpha) * self.get('var') +
self.alpha * var)
with tf.control_dependencies([
tf.assign(self.get('mean'), running_mean),
tf.assign(self.get('var'), running_var)]):
return tf.identity(mean), tf.identity(var)
# ====== if inference: use global mean and var ====== #
def infer_fn():
return self.get('mean'), self.get('var')
mean, var = tf.cond(K.is_training(), training_fn, infer_fn)
inv_std = tf.rsqrt(var + self.epsilon)
normalized = (X - K.dimshuffle(mean, pattern)) * \
(gamma * K.dimshuffle(inv_std, pattern))
# ====== applying noise if required ====== #
if self.noise_level is not None:
normalized = K.rand.apply_noise(normalized,
level=self.noise_level, noise_dims=self.noise_dims,
noise_type='gaussian')
# add beta
normalized = normalized + beta
# activated output
return self.activation(normalized)
def _apply(self, x):
input_shape = K.get_shape(x)
is_training = K.is_training()
ndim = K.ndim(x)
# if is training, normalize input by its own mean and std
if not is_training:
mean = self.mean
inv_std = self.inv_std
else:
mean = K.mean(x, self.axes)
inv_std = K.inv(K.sqrt(K.var(x, self.axes) + self.epsilon))
# set a default update for them:
running_mean = ((1 - self.alpha) * self.mean + self.alpha * mean)
running_inv_std = ((1 - self.alpha) * self.inv_std +
self.alpha * inv_std)
# prepare dimshuffle pattern inserting broadcastable axes as needed
param_axes = iter(range(ndim - len(self.axes)))
pattern = [
'x' if input_axis in self.axes else next(param_axes)
for input_axis in range(ndim)
]
# apply dimshuffle pattern to all parameters
beta = 0 if not hasattr(self, 'beta') else K.dimshuffle(
self.beta, pattern)
gamma = 1 if not hasattr(self, 'gamma') else K.dimshuffle(
self.gamma, pattern)
# normalize
normalized = (x - K.dimshuffle(mean, pattern)) * \
(gamma * K.dimshuffle(inv_std, pattern)) + beta
# set shape for output
K.add_shape(normalized, input_shape)
# activated output
output = self.activation(normalized)
# add updates for final output
if is_training:
add_updates(output, self.mean, running_mean)
add_updates(output, self.inv_std, running_inv_std)
return output
def convolutional_vae(X, saved_states, **kwargs):
""" convolutional_vae
Return
------
[y_encoder, y_decoder]
States
------
[f_inference (encoder), f_generative (decoder)]
"""
n = kwargs.get('n', 10)
batch_size = K.get_shape(X)[0]
if batch_size is None:
raise ValueError("You must specify batch_size dimension for the input placeholder.")
# ====== init ====== #
if saved_states is None:
# Encoder
f_inference = N.Sequence([
N.Reshape(shape=(-1, 28, 28, 1)),
N.Conv(num_filters=32, filter_size=3, strides=1, pad='valid',
b_init=init_ops.constant_initializer(0.), activation=K.elu),
N.Conv(num_filters=64, filter_size=5, strides=2, pad='same',
b_init=init_ops.constant_initializer(0.), activation=K.elu),
N.Dropout(level=0.1),
N.Flatten(outdim=2),
N.Dense(num_units=n * 2, b_init=None),
N.BatchNorm(axes=0)
], debug=True, name='Encoder')
# Decoder
f_generative = N.Sequence([
N.Dimshuffle(pattern=(0, 'x', 'x', 1)),
N.TransposeConv(num_filters=64, filter_size=3, strides=1, pad='valid',
b_init=init_ops.constant_initializer(0.), activation=K.elu),
N.TransposeConv(num_filters=32, filter_size=5, strides=2, pad='same',
b_init=init_ops.constant_initializer(0.), activation=K.elu),
N.TransposeConv(num_filters=1, filter_size=13, strides=3, pad='valid',
b_init=None),
N.BatchNorm(activation=K.linear),
N.Flatten(outdim=3)
], debug=True, name="Decoder")
else:
f_inference, f_generative = saved_states
# ====== Perfrom ====== #
# Encoder
y_encoder = f_inference(K.cast(X, 'float32'))
mu = y_encoder[:, :n]
sigma = K.softplus(y_encoder[:, n:])
qz = Normal(mu=mu, sigma=sigma, name='Normal_qz')
# Decoder
z = Normal(mu=K.zeros(shape=(batch_size, n)),
sigma=K.ones(shape=(batch_size, n)), name="Normal_pz")
logits = f_generative(z)
X_reconstruct = Bernoulli(logits=logits)
# inference
params = f_inference.parameters + f_generative.parameters
inference = ed.KLqp(latent_vars={z: qz}, data={X_reconstruct: X})
# ====== get cost for training ====== #
# Bind p(x, z) and q(z | x) to the same placeholder for x.
if K.is_training():
import tensorflow as tf
inference.initialize()
if True:
optimizer = tf.train.AdamOptimizer(0.01, epsilon=1.0)
updates = optimizer.apply_gradients(
optimizer.compute_gradients(inference.loss, var_list=params))
init = tf.global_variables_initializer()
init.run()
f_train = K.function(X, inference.loss, updates)
else:
optimizer = tf.train.AdamOptimizer(0.01, epsilon=1.0)
inference.initialize(optimizer=optimizer, var_list=params)
init = tf.global_variables_initializer()
init.run()
f_train = lambda x: inference.update(feed_dict={X: x})['loss']
samples = K.sigmoid(logits)
return (samples, z, qz), (f_inference, f_generative)
def convolutional_vae(X, saved_states, **kwargs):
""" convolutional_vae
Return
------
[y_encoder, y_decoder]
States
------
[f_inference (encoder), f_generative (decoder)]
"""
n = kwargs.get('n', 10)
batch_size = K.get_shape(X)[0]
if batch_size is None:
raise ValueError(
"You must specify batch_size dimension for the input placeholder.")
# ====== init ====== #
if saved_states is None:
# Encoder
f_inference = N.Sequence([
N.Reshape(shape=(-1, 28, 28, 1)),
N.Conv(num_filters=32,
filter_size=3,
strides=1,
pad='valid',
b_init=init_ops.constant_initializer(0.),
activation=K.elu),
N.Conv(num_filters=64,
filter_size=5,
strides=2,
pad='same',
b_init=init_ops.constant_initializer(0.),
activation=K.elu),
N.Dropout(level=0.1),
N.Flatten(outdim=2),
N.Dense(num_units=n * 2, b_init=None),
N.BatchNorm(axes=0)
],
debug=True,
name='Encoder')
# Decoder
f_generative = N.Sequence([
N.Dimshuffle(pattern=(0, 'x', 'x', 1)),
N.TransposeConv(num_filters=64,
filter_size=3,
strides=1,
pad='valid',
b_init=init_ops.constant_initializer(0.),
activation=K.elu),
N.TransposeConv(num_filters=32,
filter_size=5,
strides=2,
pad='same',
b_init=init_ops.constant_initializer(0.),
activation=K.elu),
N.TransposeConv(num_filters=1,
filter_size=13,
strides=3,
pad='valid',
b_init=None),
N.BatchNorm(activation=K.linear),
N.Flatten(outdim=3)
],
debug=True,
name="Decoder")
else:
f_inference, f_generative = saved_states
# ====== Perfrom ====== #
# Encoder
y_encoder = f_inference(K.cast(X, 'float32'))
mu = y_encoder[:, :n]
sigma = K.softplus(y_encoder[:, n:])
qz = Normal(mu=mu, sigma=sigma, name='Normal_qz')
# Decoder
z = Normal(mu=K.zeros(shape=(batch_size, n)),
sigma=K.ones(shape=(batch_size, n)),
name="Normal_pz")
logits = f_generative(z)
X_reconstruct = Bernoulli(logits=logits)
# inference
params = f_inference.parameters + f_generative.parameters
inference = ed.KLqp(latent_vars={z: qz}, data={X_reconstruct: X})
# ====== get cost for training ====== #
# Bind p(x, z) and q(z | x) to the same placeholder for x.
if K.is_training():
import tensorflow as tf
inference.initialize()
if True:
optimizer = tf.train.AdamOptimizer(0.01, epsilon=1.0)
updates = optimizer.apply_gradients(
optimizer.compute_gradients(inference.loss, var_list=params))
init = tf.global_variables_initializer()
init.run()
f_train = K.function(X, inference.loss, updates)
else:
optimizer = tf.train.AdamOptimizer(0.01, epsilon=1.0)
inference.initialize(optimizer=optimizer, var_list=params)
init = tf.global_variables_initializer()
init.run()
f_train = lambda x: inference.update(feed_dict={X: x})['loss']
samples = K.sigmoid(logits)
return (samples, z, qz), (f_inference, f_generative)