def test_shape(self):
x = K.variable(np.ones((25, 8, 12)))
def test_func(func):
y = func(x)
yT = func.T(func(x))
self.assertEquals(K.eval(y).shape, tuple(y.shape.as_list()))
self.assertEquals(K.eval(yT).shape, (25, 8, 12))
self.assertEquals(K.eval(yT).shape, tuple(yT.shape.as_list()))
test_func(N.Flatten(outdim=2))
test_func(N.Flatten(outdim=1))
test_func(N.Reshape((25, 4, 2, 6, 2)))
test_func(N.Dimshuffle((2, 0, 1)))
def ladder1(X, y, states, **kwargs):
noise = kwargs.get('noise', 0.3)
# hyperparameters that denote the importance of each layer
denoising_cost = [1000.0, 10.0, 0.10, 0.10, 0.10]
if states is None:
#
f_encoder = N.Sequence([
N.Flatten(outdim=2),
N.Dense(num_units=1024, b_init=None),
N.BatchNorm(
axes=0, noise_level=noise, noise_dims=None, activation=K.relu),
N.Dense(num_units=512, b_init=None),
N.BatchNorm(
axes=0, noise_level=noise, noise_dims=None, activation=K.relu),
N.Dense(num_units=256, b_init=None),
N.BatchNorm(
axes=0, noise_level=noise, noise_dims=None, activation=K.relu),
N.Dense(num_units=128, b_init=None),
N.BatchNorm(
axes=0, noise_level=noise, noise_dims=None, activation=K.relu),
N.Dense(num_units=10, activation=K.softmax),
],
all_layers=True,
debug=True,
name='Encoder')
#
f_decoder = N.Sequence([
N.Dense(num_units=128, b_init=None),
N.BatchNorm(axes=0, activation=K.relu),
N.Dense(num_units=256, b_init=None),
N.BatchNorm(axes=0, activation=K.relu),
N.Dense(num_units=512, b_init=None),
N.BatchNorm(axes=0, activation=K.relu),
N.Dense(num_units=1024, b_init=None),
N.BatchNorm(axes=0, activation=K.relu),
N.Reshape(shape=(-1, 28, 28)),
],
all_layers=True,
debug=True,
name='Decoder')
else:
f_encoder, f_decoder = states
y_encoder_clean = f_encoder(X, noise=-1)[2::2]
y_encoder_corrp = f_encoder(X, noise=1)[2::2]
print(len(y_encoder_clean), len(y_encoder_corrp))
exit()
return (None, None), [f_encoder, f_decoder]
N.Dense(num_units=512),
N.BatchNorm(axes=0, activation=K.relu),
N.Dense(num_units=512),
N.BatchNorm(axes=0, activation=K.relu),
N.Dense(num_units=args.dim)
], debug=True, name='EncoderNetwork')
f_decoder = N.Sequence([
N.Dropout(level=LATENT_DROPOUT, noise_type='uniform'),
N.Noise(level=1.0, noise_type='gaussian'),
N.Dense(num_units=512, activation=K.relu),
N.BatchNorm(axes=0, activation=K.relu),
N.Dense(num_units=512, activation=K.relu),
N.BatchNorm(axes=0, activation=K.relu),
N.Dense(num_units=np.prod(input_shape[1:]), activation=K.linear),
N.Reshape(shape=([0],) + input_shape[1:])
], debug=True, name='DecoderNetwork')
# ===========================================================================
# Create model and objectives
# ===========================================================================
Z = f_encoder(X)
X_logits = f_decoder(Z)
X_probas = tf.nn.sigmoid(X_logits)
f_X = K.function(inputs=X, outputs=X_probas,
training=True)
X_samples = f_decoder(tf.random_normal(shape=(25, args.dim),
dtype=X_probas.dtype))
f_samples = K.function(inputs=[], outputs=X_samples, training=False)
# ====== `distortion` is the negative log likelihood ====== #
def convolutional_vae(X, saved_states, **kwargs):
""" convolutional_vae
Return
------
[y_encoder, y_decoder]
States
------
[f_inference (encoder), f_generative (decoder)]
"""
n = kwargs.get('n', 10)
batch_size = K.get_shape(X)[0]
if batch_size is None:
raise ValueError(
"You must specify batch_size dimension for the input placeholder.")
# ====== init ====== #
if saved_states is None:
# Encoder
f_inference = N.Sequence([
N.Reshape(shape=(-1, 28, 28, 1)),
N.Conv(num_filters=32,
filter_size=3,
strides=1,
pad='valid',
b_init=init_ops.constant_initializer(0.),
activation=K.elu),
N.Conv(num_filters=64,
filter_size=5,
strides=2,
pad='same',
b_init=init_ops.constant_initializer(0.),
activation=K.elu),
N.Dropout(level=0.1),
N.Flatten(outdim=2),
N.Dense(num_units=n * 2, b_init=None),
N.BatchNorm(axes=0)
],
debug=True,
name='Encoder')
# Decoder
f_generative = N.Sequence([
N.Dimshuffle(pattern=(0, 'x', 'x', 1)),
N.TransposeConv(num_filters=64,
filter_size=3,
strides=1,
pad='valid',
b_init=init_ops.constant_initializer(0.),
activation=K.elu),
N.TransposeConv(num_filters=32,
filter_size=5,
strides=2,
pad='same',
b_init=init_ops.constant_initializer(0.),
activation=K.elu),
N.TransposeConv(num_filters=1,
filter_size=13,
strides=3,
pad='valid',
b_init=None),
N.BatchNorm(activation=K.linear),
N.Flatten(outdim=3)
],
debug=True,
name="Decoder")
else:
f_inference, f_generative = saved_states
# ====== Perfrom ====== #
# Encoder
y_encoder = f_inference(K.cast(X, 'float32'))
mu = y_encoder[:, :n]
sigma = K.softplus(y_encoder[:, n:])
qz = Normal(mu=mu, sigma=sigma, name='Normal_qz')
# Decoder
z = Normal(mu=K.zeros(shape=(batch_size, n)),
sigma=K.ones(shape=(batch_size, n)),
name="Normal_pz")
logits = f_generative(z)
X_reconstruct = Bernoulli(logits=logits)
# inference
params = f_inference.parameters + f_generative.parameters
inference = ed.KLqp(latent_vars={z: qz}, data={X_reconstruct: X})
# ====== get cost for training ====== #
# Bind p(x, z) and q(z | x) to the same placeholder for x.
if K.is_training():
import tensorflow as tf
inference.initialize()
if True:
optimizer = tf.train.AdamOptimizer(0.01, epsilon=1.0)
updates = optimizer.apply_gradients(
optimizer.compute_gradients(inference.loss, var_list=params))
init = tf.global_variables_initializer()
init.run()
f_train = K.function(X, inference.loss, updates)
else:
optimizer = tf.train.AdamOptimizer(0.01, epsilon=1.0)
inference.initialize(optimizer=optimizer, var_list=params)
init = tf.global_variables_initializer()
init.run()
f_train = lambda x: inference.update(feed_dict={X: x})['loss']
samples = K.sigmoid(logits)
return (samples, z, qz), (f_inference, f_generative)