def test_computational_graph3(self):
# validate the number of updates found by ComputationGraph
X = K.placeholder(shape=(None, 28, 28, 3))
f = N.Sequence([
N.Conv(32, 3, pad='same', activation=K.linear),
N.BatchNorm(activation=K.relu),
N.Flatten(outdim=2),
N.Dense(16),
N.BatchNorm(),
N.Dense(10)
])
K.set_training(True)
y_train = f(X)
K.set_training(False)
y_score = f(X)
self.assertTrue(
K.get_shape(y_train) == K.get_shape(y_score)
and K.get_shape(y_score) == (None, 10))
cc_train = K.ComputationGraph(y_train)
cc_score = K.ComputationGraph(y_score)
self.assertTrue(len(cc_score.updates) == 0)
self.assertTrue(len(cc_train.updates) == 4)
# create real function
fn_train = K.function(X, y_train)
fn_score = K.function(X, y_score)
shape1 = fn_train(np.random.rand(12, 28, 28, 3)).shape
shape2 = fn_score(np.random.rand(12, 28, 28, 3)).shape
self.assertTrue(shape1 == shape2 and shape1 == (12, 10))
def gender(X, f, **kwargs):
nb_gender = kwargs.get('nb_gender', 4)
if f is None:
f = N.Sequence([
N.Dimshuffle(pattern=(0, 1, 2, 'x')),
N.Conv(num_filters=32,
filter_size=3,
strides=1,
b_init=None,
pad='valid'),
N.BatchNorm(activation=K.relu),
N.Pool(pool_size=2, mode='avg'),
N.Conv(num_filters=64,
filter_size=3,
strides=1,
b_init=None,
pad='valid'),
N.BatchNorm(activation=K.relu),
N.Pool(pool_size=2, mode='avg'),
N.Flatten(outdim=3),
N.Dense(num_units=512, b_init=None),
N.BatchNorm(axes=(0, 1)),
N.AutoRNN(num_units=128,
rnn_mode='gru',
num_layers=2,
input_mode='linear',
direction_mode='unidirectional'),
N.Flatten(outdim=2),
N.Dense(num_units=nb_gender, activation=K.softmax)
],
debug=True)
return f(X), f
def cnn(X, y):
nb_classes = y.shape.as_list()[-1]
with N.args_scope(['Conv', dict(b_init=None, activation=K.linear)],
['BatchNorm', dict(activation=K.relu)]):
f = N.Sequence([
N.Dimshuffle(pattern=(0, 2, 3, 1)),
N.Conv(32, (3, 3), pad='same', stride=(1, 1)),
N.BatchNorm(),
N.Conv(32, (3, 3), pad='same', stride=(1, 1),
b_init=0, activation=K.relu),
N.Pool(pool_size=(2, 2), strides=None, mode='max'),
N.Dropout(level=0.25),
#
N.Conv(64, (3, 3), pad='same', stride=(1, 1)),
N.BatchNorm(),
N.Conv(64, (3, 3), pad='same', stride=(1, 1),
b_init=0., activation=K.relu),
N.Pool(pool_size=(2, 2), strides=None, mode='max'),
N.Dropout(level=0.25),
#
N.Flatten(outdim=2),
N.Dense(512, activation=K.relu),
N.Dropout(level=0.5),
N.Dense(nb_classes, activation=K.linear)
], debug=1)
logit = f(X)
prob = tf.nn.softmax(logit)
return {'logit': logit, 'prob': prob}
def test_batch_norm(self):
K.set_training(True)
x = K.placeholder((None, 8, 12))
y = N.BatchNorm()(x)
f = K.function(x, y)
z = f(np.random.rand(25, 8, 12))
self.assertEquals(z.shape, (25, 8, 12))
# ====== Not training ====== #
K.set_training(False)
x = K.placeholder((None, 8, 12))
y = N.BatchNorm()(x)
f = K.function(x, y)
z = f(np.random.rand(25, 8, 12))
self.assertEquals(z.shape, (25, 8, 12))
def dense_creator():
net = N.Sequence([
N.Dense(int(args.hdim),
b_init=0 if args.no_batchnorm else None,
activation=K.relu if args.no_batchnorm else K.linear),
None if args.no_batchnorm else N.BatchNorm(activation=K.relu)
],
debug=True,
name="DenseBatchNorm%d" % index[0])
index[0] += 1
return net
def test_helper_ops_variables(self):
X = K.placeholder(shape=(10, 20))
f = N.Sequence([
N.Dense(12),
N.Dense(8),
N.BatchNorm(),
N.Dense(25, W_init=tf.zeros(shape=(8, 25)))
])
y = f(X)
self.assertEqual(y.shape.as_list(), [10, 25])
self.assertEqual(len(f.variables), 10)
self.assertEqual(len(f.parameters), 7)
self.assertEqual(len(f.trainable_variables), 9)
def test_slice_ops(self):
X = K.placeholder(shape=(None, 28, 28, 28, 3))
f = N.Sequence([
N.Conv(32, 3, pad='same', activation=K.linear),
N.BatchNorm(activation=tf.nn.relu),
N.Flatten(outdim=4)[:, 8:12, 18:25, 13:],
])
y = f(X)
fn = K.function(X, y)
self.assertTrue(
fn(np.random.rand(12, 28, 28, 28, 3)).shape[1:] == tuple(
y.shape.as_list()[1:]))
self.assertEqual(y.shape.as_list()[1:], [4, 7, 883])
def ladder1(X, y, states, **kwargs):
noise = kwargs.get('noise', 0.3)
# hyperparameters that denote the importance of each layer
denoising_cost = [1000.0, 10.0, 0.10, 0.10, 0.10]
if states is None:
#
f_encoder = N.Sequence([
N.Flatten(outdim=2),
N.Dense(num_units=1024, b_init=None),
N.BatchNorm(
axes=0, noise_level=noise, noise_dims=None, activation=K.relu),
N.Dense(num_units=512, b_init=None),
N.BatchNorm(
axes=0, noise_level=noise, noise_dims=None, activation=K.relu),
N.Dense(num_units=256, b_init=None),
N.BatchNorm(
axes=0, noise_level=noise, noise_dims=None, activation=K.relu),
N.Dense(num_units=128, b_init=None),
N.BatchNorm(
axes=0, noise_level=noise, noise_dims=None, activation=K.relu),
N.Dense(num_units=10, activation=K.softmax),
],
all_layers=True,
debug=True,
name='Encoder')
#
f_decoder = N.Sequence([
N.Dense(num_units=128, b_init=None),
N.BatchNorm(axes=0, activation=K.relu),
N.Dense(num_units=256, b_init=None),
N.BatchNorm(axes=0, activation=K.relu),
N.Dense(num_units=512, b_init=None),
N.BatchNorm(axes=0, activation=K.relu),
N.Dense(num_units=1024, b_init=None),
N.BatchNorm(axes=0, activation=K.relu),
N.Reshape(shape=(-1, 28, 28)),
],
all_layers=True,
debug=True,
name='Decoder')
else:
f_encoder, f_decoder = states
y_encoder_clean = f_encoder(X, noise=-1)[2::2]
y_encoder_corrp = f_encoder(X, noise=1)[2::2]
print(len(y_encoder_clean), len(y_encoder_corrp))
exit()
return (None, None), [f_encoder, f_decoder]
X = inputs[0]
y = inputs[1]
print("Inputs:", ctext(inputs, 'cyan'))
# ====== create the networks ====== #
with N.args_scope(
['TimeDelayedConv',
dict(time_pool='none', activation=K.relu)],
['Dense', dict(activation=K.linear, b_init=None)]):
f = N.Sequence([
N.Dropout(level=0.3),
N.TimeDelayedConv(n_new_features=512, n_time_context=5),
N.TimeDelayedConv(n_new_features=512, n_time_context=5),
N.TimeDelayedConv(
n_new_features=512, n_time_context=7, name="LatentTDNN"),
N.Dense(512),
N.BatchNorm(activation=K.relu),
N.Dense(1500),
N.BatchNorm(activation=K.relu),
N.StatsPool(axes=1, output_mode='concat'),
N.Flatten(outdim=2, name="StatsPooling"),
N.Dense(512, name="LatentDense"),
N.BatchNorm(activation=K.relu),
N.Dense(512),
N.BatchNorm(activation=K.relu),
N.Dense(num_units=n_classes,
activation=K.linear,
b_init=init_ops.constant_initializer(0))
],
debug=1)
# ====== create outputs ====== #
y_logit = f(X)
USE_MNIST_DATA = True if 'mnist' in arg['ds'].lower() else False
if USE_MNIST_DATA:
ds = fuel.load_mnist()
else:
ds = fuel.load_cifar10()
X = K.placeholder(shape=(None, ) + ds['X_train'].shape[1:], name='X')
y = K.placeholder(shape=(None, ), name='y', dtype='int32')
# ===========================================================================
# Build network
# ===========================================================================
ops = N.Sequence([
N.Dimshuffle((0, 1, 2, 'x')) if USE_MNIST_DATA else None,
N.BatchNorm(axes='auto'),
N.Conv(32, (3, 3), strides=(1, 1), pad='same', activation=K.relu),
N.Pool(pool_size=(2, 2), strides=None),
N.Conv(64, (3, 3), strides=(1, 1), pad='same', activation=K.relu),
N.Pool(pool_size=(2, 2), strides=None),
N.Flatten(outdim=2),
N.Dense(256, activation=K.relu),
N.Dense(10, activation=K.softmax)
],
debug=True)
ops = cPickle.loads(cPickle.dumps(ops)) # test if the ops is pickle-able
K.set_training(True)
y_pred_train = ops(X)
K.set_training(False)
y_pred_score = ops(X)
x_vec = N.deserialize(path=all_models[SCORE_SYSTEM_ID],
force_restore_vars=True)
else:
with N.args_scope(
['TimeDelayedConv', dict(time_pool='none', activation=K.relu)],
['Dense', dict(activation=K.linear, b_init=None)],
['BatchNorm', dict(activation=K.relu)]
):
x_vec = N.Sequence([
N.Dropout(level=0.3),
N.TimeDelayedConv(n_new_features=512, n_time_context=5),
N.TimeDelayedConv(n_new_features=512, n_time_context=5),
N.TimeDelayedConv(n_new_features=512, n_time_context=7),
N.Dense(512), N.BatchNorm(),
N.Dense(1500), N.BatchNorm(),
N.StatsPool(axes=1, output_mode='concat'),
N.Flatten(outdim=2),
N.Dense(512, name="LatentOutput"), N.BatchNorm(),
N.Dense(512), N.BatchNorm(),
N.Dense(n_speakers, activation=K.linear,
b_init=init_ops.constant_initializer(value=0))
], debug=1, name='XNetwork')
# ====== create outputs ====== #
y_logit = x_vec(X)
y_proba = tf.nn.softmax(y_logit)
z = K.ComputationGraph(y_proba).get(roles=N.Dense, scope='LatentOutput',
def convolutional_vae(X, saved_states, **kwargs):
""" convolutional_vae
Return
------
[y_encoder, y_decoder]
States
------
[f_inference (encoder), f_generative (decoder)]
"""
n = kwargs.get('n', 10)
batch_size = K.get_shape(X)[0]
if batch_size is None:
raise ValueError(
"You must specify batch_size dimension for the input placeholder.")
# ====== init ====== #
if saved_states is None:
# Encoder
f_inference = N.Sequence([
N.Reshape(shape=(-1, 28, 28, 1)),
N.Conv(num_filters=32,
filter_size=3,
strides=1,
pad='valid',
b_init=init_ops.constant_initializer(0.),
activation=K.elu),
N.Conv(num_filters=64,
filter_size=5,
strides=2,
pad='same',
b_init=init_ops.constant_initializer(0.),
activation=K.elu),
N.Dropout(level=0.1),
N.Flatten(outdim=2),
N.Dense(num_units=n * 2, b_init=None),
N.BatchNorm(axes=0)
],
debug=True,
name='Encoder')
# Decoder
f_generative = N.Sequence([
N.Dimshuffle(pattern=(0, 'x', 'x', 1)),
N.TransposeConv(num_filters=64,
filter_size=3,
strides=1,
pad='valid',
b_init=init_ops.constant_initializer(0.),
activation=K.elu),
N.TransposeConv(num_filters=32,
filter_size=5,
strides=2,
pad='same',
b_init=init_ops.constant_initializer(0.),
activation=K.elu),
N.TransposeConv(num_filters=1,
filter_size=13,
strides=3,
pad='valid',
b_init=None),
N.BatchNorm(activation=K.linear),
N.Flatten(outdim=3)
],
debug=True,
name="Decoder")
else:
f_inference, f_generative = saved_states
# ====== Perfrom ====== #
# Encoder
y_encoder = f_inference(K.cast(X, 'float32'))
mu = y_encoder[:, :n]
sigma = K.softplus(y_encoder[:, n:])
qz = Normal(mu=mu, sigma=sigma, name='Normal_qz')
# Decoder
z = Normal(mu=K.zeros(shape=(batch_size, n)),
sigma=K.ones(shape=(batch_size, n)),
name="Normal_pz")
logits = f_generative(z)
X_reconstruct = Bernoulli(logits=logits)
# inference
params = f_inference.parameters + f_generative.parameters
inference = ed.KLqp(latent_vars={z: qz}, data={X_reconstruct: X})
# ====== get cost for training ====== #
# Bind p(x, z) and q(z | x) to the same placeholder for x.
if K.is_training():
import tensorflow as tf
inference.initialize()
if True:
optimizer = tf.train.AdamOptimizer(0.01, epsilon=1.0)
updates = optimizer.apply_gradients(
optimizer.compute_gradients(inference.loss, var_list=params))
init = tf.global_variables_initializer()
init.run()
f_train = K.function(X, inference.loss, updates)
else:
optimizer = tf.train.AdamOptimizer(0.01, epsilon=1.0)
inference.initialize(optimizer=optimizer, var_list=params)
init = tf.global_variables_initializer()
init.run()
f_train = lambda x: inference.update(feed_dict={X: x})['loss']
samples = K.sigmoid(logits)
return (samples, z, qz), (f_inference, f_generative)
test.set_recipes(recipes)
# ===========================================================================
# Create model
# ===========================================================================
inputs = [
K.placeholder(shape=(None, ) + shape[1:],
dtype='float32',
name='input%d' % i) for i, shape in enumerate(train.shape)
]
print("Inputs:", ctext(inputs, 'cyan'))
# ====== create the network ====== #
f_encoder = N.Sequence([
N.Dimshuffle(pattern=(0, 1, 2, 'x')),
N.Conv(
num_filters=32, filter_size=(7, 7), b_init=None, activation=K.linear),
N.BatchNorm(),
N.Pool(pool_size=(3, 2), strides=2),
],
debug=True,
name='Encoder')
f_latent = N.Sequence([
N.Flatten(outdim=3),
N.CudnnRNN(
num_units=128, num_layers=1, is_bidirectional=False, rnn_mode='lstm'),
],
debug=True,
name='Latent')
f_decoder = N.Sequence([
N.Flatten(outdim=2),
N.Dense(num_units=1024, b_init=None, activation=K.linear),
N.BatchNorm(axes=0, activation=K.relu)
ds = fuel.load_mnist()
else:
ds = fuel.load_cifar10()
print(ds)
X = K.placeholder(shape=(None, ) + ds['X_train'].shape[1:], name='X')
y = K.placeholder(shape=(None, ), name='y', dtype='int32')
# ===========================================================================
# Build network
# ===========================================================================
ops = N.Sequence([
N.Dimshuffle((0, 1, 2, 'x')) if USE_MNIST_DATA else N.Dimshuffle(
(0, 2, 3, 1)),
N.Conv(32, filter_size=3, strides=1, pad='same', activation=K.linear),
N.BatchNorm(axes='auto', activation=K.relu),
N.Pool(pool_size=2, strides=None),
N.Dimshuffle(pattern=(0, 3, 1, 2)),
N.Flatten(outdim=3),
N.CudnnRNN(18,
initial_states=None,
rnn_mode='lstm',
num_layers=2,
input_mode='linear',
direction_mode='unidirectional',
params_split=False),
N.Flatten(outdim=2),
N.Dense(128, activation=K.relu),
N.Dense(10, activation=K.softmax)
],
debug=True)
K.placeholder(shape=(None, ) + shape[1:],
dtype='float32',
name='input%d' % i)
for i, shape in enumerate(as_tuple_of_shape(train.shape))
]
X = inputs[0]
y = inputs[1]
print("Inputs:", ctext(inputs, 'cyan'))
# ====== create the networks ====== #
with N.args_scope([('Conv', 'Dense'),
dict(b_init=None, activation=K.linear, pad='same')],
['BatchNorm', dict(activation=K.relu)]):
f = N.Sequence([
N.Dimshuffle(pattern=(0, 1, 2, 'x')),
N.Conv(num_filters=32, filter_size=(9, 7)),
N.BatchNorm(),
N.Pool(pool_size=(3, 2), strides=2),
N.Conv(num_filters=64, filter_size=(5, 3)),
N.BatchNorm(),
N.Pool(pool_size=(3, 1), strides=(2, 1), name='PoolOutput1'),
N.Conv(num_filters=64, filter_size=(5, 3)),
N.BatchNorm(),
N.Pool(pool_size=(3, 2), strides=(2, 2), name='PoolOutput2'),
N.Flatten(outdim=2),
N.Dense(512, name="LatentDense"),
N.BatchNorm(),
N.Dense(512),
N.BatchNorm(),
N.Dense(n_classes)
],
debug=1)
y = K.placeholder(shape=(None,), name='y_input')
# ===========================================================================
# Create the network
# ===========================================================================
LATENT_DROPOUT = 0.3
if args.cnn:
with N.args_scope(([N.Conv, N.Dense], dict(b_init=None, activation=K.linear)),
(N.BatchNorm, dict(activation=tf.nn.elu)),
(N.Pool, dict(mode='max', pool_size=2))):
f_encoder = N.Sequence([
N.Dropout(level=0.5),
N.Dimshuffle((0, 2, 3, 1)) if is_cifar10 else N.Dimshuffle((0, 1, 2, 'x')),
N.Conv(num_filters=32, filter_size=3, pad='valid'),
N.Pool(),
N.BatchNorm(),
N.Conv(num_filters=64, filter_size=3, pad='same'),
N.BatchNorm(),
N.Conv(num_filters=64, filter_size=3, pad='valid'),
N.BatchNorm(activation=tf.nn.elu),
N.Pool(),
N.Flatten(outdim=2),
N.Dense(num_units=args.dim)
], debug=True, name='EncoderNetwork')
f_decoder = N.Sequence([
N.Dropout(level=LATENT_DROPOUT, noise_type='uniform'),
N.Noise(level=1.0, noise_type='gaussian'),
# ===========================================================================
# Create model
# ===========================================================================
inputs = [K.placeholder(shape=(None,) + shape[1:], dtype='float32', name='input%d' % i)
for i, shape in enumerate(as_tuple_of_shape(train.shape))]
X = inputs[0]
y = inputs[1]
print("Inputs:", ctext(inputs, 'cyan'))
# ====== create the networks ====== #
with N.args_scope(
[('Conv', 'Dense'), dict(b_init=None, activation=K.linear, pad='same')],
['BatchNorm', dict(activation=K.relu)]):
f = N.Sequence([
N.Dimshuffle(pattern=(0, 1, 2, 'x')),
N.Conv(num_filters=32, filter_size=(9, 7)), N.BatchNorm(),
N.Pool(pool_size=(3, 2), strides=2),
N.Conv(num_filters=64, filter_size=(5, 3)), N.BatchNorm(),
N.Pool(pool_size=(3, 1), strides=(2, 1), name='PoolOutput1'),
N.Conv(num_filters=64, filter_size=(5, 3)), N.BatchNorm(),
N.Pool(pool_size=(3, 2), strides=(2, 2), name='PoolOutput2'),
N.Flatten(outdim=2),
N.Dense(512, name="LatentDense"), N.BatchNorm(),
N.Dense(512), N.BatchNorm(),
N.Dense(n_classes)
], debug=1)
# ====== create outputs ====== #
y_logit = f(X)
