def build_pipeline(self, input_shape, params):
from blocks.bricks import Tanh, Sequence
from blocks.bricks.conv import Convolutional, MaxPooling
from blocks.initialization import Uniform
_, num_channels, input_len, num_freqs = input_shape # bc01
# Note: this layer is linear
conv = Convolutional(
name='conv',
filter_size=(params['filter_width_time'],
params['filter_width_freq']),
num_filters=params['num_components'], # out
num_channels=num_channels, # in
image_size=(input_len, num_freqs),
weights_init=Uniform(mean=0, std=0.01),
use_bias=params['use_bias'])
tanh = Tanh()
# optional pooling
if params['pool_width_time'] > 1 or params['pool_width_freq'] > 1:
pool = MaxPooling(
(params['pool_width_time'], params['pool_width_freq']),
step=(params['pool_stride_time'], params['pool_stride_freq']))
pipeline = Sequence([conv.apply, tanh.apply, pool.apply],
name='pipeline')
else:
pipeline = Sequence([conv.apply, tanh.apply], name='pipeline')
pipeline.initialize()
return pipeline
def test_sequence_variable_inputs():
x, y = tensor.matrix(), tensor.matrix()
parallel_1 = Parallel(input_names=['input_1', 'input_2'],
input_dims=dict(input_1=4, input_2=5),
output_dims=dict(input_1=3, input_2=2),
prototype=Linear(), weights_init=Constant(2),
biases_init=Constant(1))
parallel_2 = Parallel(input_names=['input_1', 'input_2'],
input_dims=dict(input_1=3, input_2=2),
output_dims=dict(input_1=5, input_2=4),
prototype=Linear(), weights_init=Constant(2),
biases_init=Constant(1))
sequence = Sequence([parallel_1.apply, parallel_2.apply])
sequence.initialize()
new_x, new_y = sequence.apply(x, y)
x_val = numpy.ones((4, 4), dtype=theano.config.floatX)
y_val = numpy.ones((4, 5), dtype=theano.config.floatX)
assert_allclose(
new_x.eval({x: x_val}),
(x_val.dot(2 * numpy.ones((4, 3))) + numpy.ones((4, 3))).dot(
2 * numpy.ones((3, 5))) + numpy.ones((4, 5)))
assert_allclose(
new_y.eval({y: y_val}),
(y_val.dot(2 * numpy.ones((5, 2))) + numpy.ones((4, 2))).dot(
2 * numpy.ones((2, 4))) + numpy.ones((4, 4)))
def check(bricks):
sequence = Sequence(bricks)
sequence.initialize()
y = sequence.apply(x)
x_val = numpy.ones((4, 16), dtype=theano.config.floatX)
assert_allclose(y.eval({x: x_val}), (x_val.dot(2 * numpy.ones(
(16, 8))) + numpy.ones((4, 8))).dot(3 * numpy.ones(
(8, 4))) + 4 * numpy.ones((4, 4)))
def setUp(self):
self.mlp = MLP([
Sequence([Identity(name='id1').apply,
Tanh(name='tanh1').apply],
name='sequence1'),
Sequence([
Logistic(name='logistic1').apply,
Identity(name='id2').apply,
Tanh(name='tanh2').apply
],
name='sequence2'),
Logistic(name='logistic2'),
Sequence([
Sequence([Logistic(name='logistic3').apply],
name='sequence4').apply
],
name='sequence3')
], [10, 5, 9, 5, 9])
def __init__(self, visual_dim, textual_dim, output_dim, hidden_size,
init_ranges, **kwargs):
(visual_init_range, textual_init_range, gbu_init_range, linear_range_1,
linear_range_2, linear_range_3) = init_ranges
visual_mlp = Sequence([
BatchNormalization(input_dim=visual_dim).apply,
Linear(visual_dim,
hidden_size,
use_bias=False,
weights_init=initialization.Uniform(
width=visual_init_range)).apply,
],
name='visual_mlp')
textual_mlp = Sequence([
BatchNormalization(input_dim=textual_dim).apply,
Linear(textual_dim,
hidden_size,
use_bias=False,
weights_init=initialization.Uniform(
width=textual_init_range)).apply,
],
name='textual_mlp')
gbu = GatedBimodal(
hidden_size,
weights_init=initialization.Uniform(width=gbu_init_range))
logistic_mlp = MLPGenreClassifier(
hidden_size, output_dim, hidden_size,
[linear_range_1, linear_range_2, linear_range_3])
# logistic_mlp = Sequence([
# BatchNormalization(input_dim=hidden_size, name='bn1').apply,
# Linear(hidden_size, output_dim, name='linear_output', use_bias=False,
# weights_init=initialization.Uniform(width=linear_range_1)).apply,
# Logistic().apply
#], name='logistic_mlp')
children = [visual_mlp, textual_mlp, gbu, logistic_mlp]
kwargs.setdefault('use_bias', False)
kwargs.setdefault('children', children)
super(GatedClassifier, self).__init__(**kwargs)
def test_sequence():
x = tensor.matrix()
linear_1 = Linear(input_dim=16, output_dim=8, weights_init=Constant(2),
biases_init=Constant(1))
linear_2 = Linear(input_dim=8, output_dim=4, weights_init=Constant(3),
biases_init=Constant(4))
sequence = Sequence([linear_1.apply, linear_2.apply])
sequence.initialize()
y = sequence.apply(x)
x_val = numpy.ones((4, 16), dtype=theano.config.floatX)
assert_allclose(
y.eval({x: x_val}),
(x_val.dot(2 * numpy.ones((16, 8))) + numpy.ones((4, 8))).dot(
3 * numpy.ones((8, 4))) + 4 * numpy.ones((4, 4)))
def test_batch_normalization_nested():
x = tensor.tensor4()
eps = 1e-4
r_dims = (0, 2, 3)
batch_dims = (5, 4, 3, 2)
bn = BatchNormalization(input_dim=batch_dims[1:],
broadcastable=(False, True, True),
epsilon=eps)
seq = Sequence([bn.apply, Tanh().apply])
seq.initialize()
with batch_normalization(seq):
y = seq.apply(x)
rng = numpy.random.RandomState((2016, 1, 18))
x_ = rng.uniform(size=batch_dims).astype(theano.config.floatX)
y_ = y.eval({x: x_})
y_expected = numpy.tanh(
(x_ - x_.mean(axis=r_dims, keepdims=True)) /
numpy.sqrt(x_.var(axis=r_dims, keepdims=True) + eps))
assert_allclose(y_, y_expected, rtol=1e-4)
def test_sequence_variable_outputs():
x = tensor.matrix()
linear_1 = Linear(input_dim=16, output_dim=8, weights_init=Constant(2),
biases_init=Constant(1))
fork = Fork(input_dim=8, output_names=['linear_2_1', 'linear_2_2'],
output_dims=[4, 5], prototype=Linear(),
weights_init=Constant(3), biases_init=Constant(4))
sequence = Sequence([linear_1.apply, fork.apply])
sequence.initialize()
y_1, y_2 = sequence.apply(x)
x_val = numpy.ones((4, 16), dtype=theano.config.floatX)
assert_allclose(
y_1.eval({x: x_val}),
(x_val.dot(2 * numpy.ones((16, 8))) + numpy.ones((4, 8))).dot(
3 * numpy.ones((8, 4))) + 4 * numpy.ones((4, 4)))
assert_allclose(
y_2.eval({x: x_val}),
(x_val.dot(2 * numpy.ones((16, 8))) + numpy.ones((4, 8))).dot(
3 * numpy.ones((8, 5))) + 4 * numpy.ones((4, 5)))
def build_autoencoder(features, labels_num, labels_cat):
mlp_bottom = MLP(activations=[
Rectifier(),
Rectifier(),
Rectifier(),
Rectifier(),
Rectifier()
],
dims=[24033, 5000, 1000, 100, 1000, 5000],
weights_init=IsotropicGaussian(),
biases_init=Constant(1))
mlp_bottom.initialize()
mlp_top = build_top_mlp()
mlp_top.push_initialization_config()
mlp_top.initialize()
# a = mlp_bottom.apply(features)
# b = mlp_top.apply(a)
# Construct feedforward sequence
ss_seq = Sequence([mlp_bottom.apply, mlp_top.apply])
ss_seq.push_initialization_config()
ss_seq.initialize()
[outputs_numerical, outputs_categorical] = ss_seq.apply(features)
cost = SquaredError().apply(
labels_num, outputs_numerical) + BinaryCrossEntropy().apply(
labels_cat, outputs_categorical)
cg = ComputationGraph(cost)
#cg_dropout0 = apply_dropout(cg, [VariableFilter(roles=[INPUT])(cg.variables)[1]], .2)
#cg_dropout1 = apply_dropout(cg, [VariableFilter(roles=[OUTPUT])(cg.variables)[1], VariableFilter(roles=[OUTPUT])(cg.variables)[3]], .2)
#cost_dropout1 = cg_dropout1.outputs[0]
return cost, cg.parameters
def __init__(self, visual_dim, textual_dim, output_dim, hidden_size,
init_ranges, **kwargs):
(visual_range, textual_range, linear_range_1, linear_range_2,
linear_range_3) = init_ranges
manager_dim = visual_dim + textual_dim
visual_mlp = MLPGenreClassifier(
visual_dim,
output_dim,
hidden_size, [linear_range_1, linear_range_2, linear_range_3],
name='visual_mlp')
textual_mlp = MLPGenreClassifier(
textual_dim,
output_dim,
hidden_size, [linear_range_1, linear_range_2, linear_range_3],
name='textual_mlp')
# manager_mlp = MLPGenreClassifier(manager_dim, 2, hidden_size, [
# linear_range_1, linear_range_2, linear_range_3], output_act=Softmax,
# name='manager_mlp')
bn = BatchNormalization(input_dim=manager_dim, name='bn3')
manager_mlp = Sequence([
Linear(manager_dim,
2,
name='linear_output',
use_bias=False,
weights_init=initialization.Uniform(
width=linear_range_1)).apply,
],
name='manager_mlp')
fork = Fork(
input_dim=manager_dim,
output_dims=[2] * output_dim,
prototype=manager_mlp,
output_names=['linear_' + str(i) for i in range(output_dim)])
children = [visual_mlp, textual_mlp, fork, bn, NDimensionalSoftmax()]
kwargs.setdefault('use_bias', False)
kwargs.setdefault('children', children)
super(MoEClassifier, self).__init__(**kwargs)
def test_apply_batch_normalization_nested():
x = tensor.matrix()
eps = 1e-8
batch_dims = (3, 9)
bn = BatchNormalization(input_dim=5, epsilon=eps)
mlp = MLP([Sequence([bn.apply, Tanh().apply])], [9, 5],
weights_init=Constant(0.4),
biases_init=Constant(1))
mlp.initialize()
y = mlp.apply(x)
cg = apply_batch_normalization(ComputationGraph([y]))
y_bn = cg.outputs[0]
rng = numpy.random.RandomState((2016, 1, 18))
x_ = rng.uniform(size=batch_dims).astype(theano.config.floatX)
y_ = y_bn.eval({x: x_})
W_, b_ = map(
lambda s:
(getattr(mlp.linear_transformations[0], s).get_value(borrow=True)),
['W', 'b'])
z_ = numpy.dot(x_, W_) + b_
y_expected = numpy.tanh(
(z_ - z_.mean(axis=0)) / numpy.sqrt(z_.var(axis=0) + eps))
assert_allclose(y_, y_expected, rtol=1e-3)
def create_model_bricks(z_dim, image_size, depth):
g_image_size = image_size
g_image_size2 = g_image_size / 2
g_image_size3 = g_image_size / 4
g_image_size4 = g_image_size / 8
g_image_size5 = g_image_size / 16
encoder_layers = []
if depth > 0:
encoder_layers = encoder_layers + [
Convolutional(filter_size=(3, 3),
border_mode=(1, 1),
num_filters=32,
name='conv1'),
SpatialBatchNormalization(name='batch_norm1'),
Rectifier(),
Convolutional(filter_size=(3, 3),
border_mode=(1, 1),
num_filters=32,
name='conv2'),
SpatialBatchNormalization(name='batch_norm2'),
Rectifier(),
Convolutional(
filter_size=(2, 2), step=(2, 2), num_filters=32, name='conv3'),
SpatialBatchNormalization(name='batch_norm3'),
Rectifier()
]
if depth > 1:
encoder_layers = encoder_layers + [
Convolutional(filter_size=(3, 3),
border_mode=(1, 1),
num_filters=64,
name='conv4'),
SpatialBatchNormalization(name='batch_norm4'),
Rectifier(),
Convolutional(filter_size=(3, 3),
border_mode=(1, 1),
num_filters=64,
name='conv5'),
SpatialBatchNormalization(name='batch_norm5'),
Rectifier(),
Convolutional(
filter_size=(2, 2), step=(2, 2), num_filters=64, name='conv6'),
SpatialBatchNormalization(name='batch_norm6'),
Rectifier()
]
if depth > 2:
encoder_layers = encoder_layers + [
Convolutional(filter_size=(3, 3),
border_mode=(1, 1),
num_filters=128,
name='conv7'),
SpatialBatchNormalization(name='batch_norm7'),
Rectifier(),
Convolutional(filter_size=(3, 3),
border_mode=(1, 1),
num_filters=128,
name='conv8'),
SpatialBatchNormalization(name='batch_norm8'),
Rectifier(),
Convolutional(
filter_size=(2, 2), step=(2, 2), num_filters=128,
name='conv9'),
SpatialBatchNormalization(name='batch_norm9'),
Rectifier()
]
if depth > 3:
encoder_layers = encoder_layers + [
Convolutional(filter_size=(3, 3),
border_mode=(1, 1),
num_filters=256,
name='conv10'),
SpatialBatchNormalization(name='batch_norm10'),
Rectifier(),
Convolutional(filter_size=(3, 3),
border_mode=(1, 1),
num_filters=256,
name='conv11'),
SpatialBatchNormalization(name='batch_norm11'),
Rectifier(),
Convolutional(filter_size=(2, 2),
step=(2, 2),
num_filters=256,
name='conv12'),
SpatialBatchNormalization(name='batch_norm12'),
Rectifier(),
]
if depth > 4:
encoder_layers = encoder_layers + [
Convolutional(filter_size=(3, 3),
border_mode=(1, 1),
num_filters=512,
name='conv13'),
SpatialBatchNormalization(name='batch_norm13'),
Rectifier(),
Convolutional(filter_size=(3, 3),
border_mode=(1, 1),
num_filters=512,
name='conv14'),
SpatialBatchNormalization(name='batch_norm14'),
Rectifier(),
Convolutional(filter_size=(2, 2),
step=(2, 2),
num_filters=512,
name='conv15'),
SpatialBatchNormalization(name='batch_norm15'),
Rectifier()
]
decoder_layers = []
if depth > 4:
decoder_layers = decoder_layers + [
Convolutional(filter_size=(3, 3),
border_mode=(1, 1),
num_filters=512,
name='conv_n3'),
SpatialBatchNormalization(name='batch_norm_n3'),
Rectifier(),
Convolutional(filter_size=(3, 3),
border_mode=(1, 1),
num_filters=512,
name='conv_n2'),
SpatialBatchNormalization(name='batch_norm_n2'),
Rectifier(),
ConvolutionalTranspose(
filter_size=(2, 2),
step=(2, 2),
original_image_size=(g_image_size5, g_image_size5),
num_filters=512,
name='conv_n1'),
SpatialBatchNormalization(name='batch_norm_n1'),
Rectifier()
]
if depth > 3:
decoder_layers = decoder_layers + [
Convolutional(filter_size=(3, 3),
border_mode=(1, 1),
num_filters=256,
name='conv1'),
SpatialBatchNormalization(name='batch_norm1'),
Rectifier(),
Convolutional(filter_size=(3, 3),
border_mode=(1, 1),
num_filters=256,
name='conv2'),
SpatialBatchNormalization(name='batch_norm2'),
Rectifier(),
ConvolutionalTranspose(
filter_size=(2, 2),
step=(2, 2),
original_image_size=(g_image_size4, g_image_size4),
num_filters=256,
name='conv3'),
SpatialBatchNormalization(name='batch_norm3'),
Rectifier()
]
if depth > 2:
decoder_layers = decoder_layers + [
Convolutional(filter_size=(3, 3),
border_mode=(1, 1),
num_filters=128,
name='conv4'),
SpatialBatchNormalization(name='batch_norm4'),
Rectifier(),
Convolutional(filter_size=(3, 3),
border_mode=(1, 1),
num_filters=128,
name='conv5'),
SpatialBatchNormalization(name='batch_norm5'),
Rectifier(),
ConvolutionalTranspose(
filter_size=(2, 2),
step=(2, 2),
original_image_size=(g_image_size3, g_image_size3),
num_filters=128,
name='conv6'),
SpatialBatchNormalization(name='batch_norm6'),
Rectifier()
]
if depth > 1:
decoder_layers = decoder_layers + [
Convolutional(filter_size=(3, 3),
border_mode=(1, 1),
num_filters=64,
name='conv7'),
SpatialBatchNormalization(name='batch_norm7'),
Rectifier(),
Convolutional(filter_size=(3, 3),
border_mode=(1, 1),
num_filters=64,
name='conv8'),
SpatialBatchNormalization(name='batch_norm8'),
Rectifier(),
ConvolutionalTranspose(
filter_size=(2, 2),
step=(2, 2),
original_image_size=(g_image_size2, g_image_size2),
num_filters=64,
name='conv9'),
SpatialBatchNormalization(name='batch_norm9'),
Rectifier()
]
if depth > 0:
decoder_layers = decoder_layers + [
Convolutional(filter_size=(3, 3),
border_mode=(1, 1),
num_filters=32,
name='conv10'),
SpatialBatchNormalization(name='batch_norm10'),
Rectifier(),
Convolutional(filter_size=(3, 3),
border_mode=(1, 1),
num_filters=32,
name='conv11'),
SpatialBatchNormalization(name='batch_norm11'),
Rectifier(),
ConvolutionalTranspose(
filter_size=(2, 2),
step=(2, 2),
original_image_size=(g_image_size, g_image_size),
num_filters=32,
name='conv12'),
SpatialBatchNormalization(name='batch_norm12'),
Rectifier()
]
decoder_layers = decoder_layers + [
Convolutional(filter_size=(1, 1), num_filters=3, name='conv_out'),
Logistic()
]
print("creating model of depth {} with {} encoder and {} decoder layers".
format(depth, len(encoder_layers), len(decoder_layers)))
encoder_convnet = ConvolutionalSequence(
layers=encoder_layers,
num_channels=3,
image_size=(g_image_size, g_image_size),
use_bias=False,
weights_init=IsotropicGaussian(0.033),
biases_init=Constant(0),
name='encoder_convnet')
encoder_convnet.initialize()
encoder_filters = numpy.prod(encoder_convnet.get_dim('output'))
encoder_mlp = MLP(
dims=[encoder_filters, 1000, z_dim],
activations=[
Sequence([BatchNormalization(1000).apply,
Rectifier().apply],
name='activation1'),
Identity().apply
],
weights_init=IsotropicGaussian(0.033),
biases_init=Constant(0),
name='encoder_mlp')
encoder_mlp.initialize()
decoder_mlp = BatchNormalizedMLP(
activations=[Rectifier(), Rectifier()],
dims=[encoder_mlp.output_dim // 2, 1000, encoder_filters],
weights_init=IsotropicGaussian(0.033),
biases_init=Constant(0),
name='decoder_mlp')
decoder_mlp.initialize()
decoder_convnet = ConvolutionalSequence(
layers=decoder_layers,
num_channels=encoder_convnet.get_dim('output')[0],
image_size=encoder_convnet.get_dim('output')[1:],
use_bias=False,
weights_init=IsotropicGaussian(0.033),
biases_init=Constant(0),
name='decoder_convnet')
decoder_convnet.initialize()
return encoder_convnet, encoder_mlp, decoder_convnet, decoder_mlp
def __init__(self):
srng = MRG_RandomStreams(seed=123)
X = T.matrix('features')
self.X = X
#drop = Dropout(p_drop=0.5)
#o = drop.apply(X)
o = (X - 128) / 128.0
self.scaled = o
#n_hidden = 64
n_hidden = 2048 * 2
n_zs = 1024
self.n_zs = n_zs
self.n_hidden = n_hidden
l = Linear(input_dim=32 * 32 * 3,
output_dim=n_hidden,
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0))
l.initialize()
o = l.apply(o)
o = Rectifier().apply(o)
l = Linear(input_dim=n_hidden,
output_dim=n_hidden,
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0))
l.initialize()
o = l.apply(o)
o = Rectifier().apply(o)
l = Linear(input_dim=n_hidden,
output_dim=n_zs,
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0))
l.initialize()
mu_encoder = l.apply(o)
l = Linear(input_dim=n_hidden,
output_dim=n_zs,
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0))
l.initialize()
log_sigma_encoder = l.apply(o)
eps = srng.normal(log_sigma_encoder.shape)
z = eps * T.exp(log_sigma_encoder) + mu_encoder
z_to_h1_decode = Linear(input_dim=n_zs,
output_dim=n_hidden,
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0))
z_to_h1_decode.initialize()
h1_decode_to_h_decode = Linear(input_dim=n_hidden,
output_dim=n_hidden,
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0))
h1_decode_to_h_decode.initialize()
h_decode_produce = Linear(input_dim=n_hidden,
output_dim=32 * 32 * 3,
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0),
name="linear4")
h_decode_produce.initialize()
#o = h_decode_produce.apply(h_decoder)
h_decode_produce = Linear(input_dim=n_hidden,
output_dim=32 * 32 * 3,
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0),
name="linear4")
h_decode_produce.initialize()
#self.produced = Sigmoid().apply(o)
seq = Sequence([
z_to_h1_decode.apply,
Rectifier().apply, h1_decode_to_h_decode.apply,
Rectifier().apply, h_decode_produce.apply,
Sigmoid().apply
])
seq.initialize()
self.produced = seq.apply(z)
self.cost = T.mean(T.sqr(self.produced - self.scaled))
#self.cost = T.sum(T.nnet.binary_crossentropy(self.produced, self.scaled)) #T.sum(T.sqr(self.produced - self.scaled))
self.cost.name = "cost"
self.variational_cost = - 0.5 * T.mean(1 + 2*log_sigma_encoder - mu_encoder * mu_encoder\
- T.exp(2 * log_sigma_encoder)) + self.cost
self.variational_cost.name = "variational_cost"
self.Z = T.matrix('z')
self.sampled = seq.apply(self.Z)
cg = ComputationGraph([self.variational_cost])
bricks = [
get_brick(var) for var in cg.variables + cg.scan_variables
if get_brick(var)
]
for i, b in enumerate(bricks):
b.name = b.name + "_" + str(i)
def create_model_bricks():
encoder_convnet = ConvolutionalSequence(
layers=[
Convolutional(filter_size=(3, 3),
border_mode=(1, 1),
num_filters=32,
name='conv1'),
SpatialBatchNormalization(name='batch_norm1'),
Rectifier(),
Convolutional(filter_size=(3, 3),
border_mode=(1, 1),
num_filters=32,
name='conv2'),
SpatialBatchNormalization(name='batch_norm2'),
Rectifier(),
Convolutional(filter_size=(2, 2),
step=(2, 2),
num_filters=32,
name='conv3'),
SpatialBatchNormalization(name='batch_norm3'),
Rectifier(),
Convolutional(filter_size=(3, 3),
border_mode=(1, 1),
num_filters=64,
name='conv4'),
SpatialBatchNormalization(name='batch_norm4'),
Rectifier(),
Convolutional(filter_size=(3, 3),
border_mode=(1, 1),
num_filters=64,
name='conv5'),
SpatialBatchNormalization(name='batch_norm5'),
Rectifier(),
Convolutional(filter_size=(2, 2),
step=(2, 2),
num_filters=64,
name='conv6'),
SpatialBatchNormalization(name='batch_norm6'),
Rectifier(),
Convolutional(filter_size=(3, 3),
border_mode=(1, 1),
num_filters=128,
name='conv7'),
SpatialBatchNormalization(name='batch_norm7'),
Rectifier(),
Convolutional(filter_size=(3, 3),
border_mode=(1, 1),
num_filters=128,
name='conv8'),
SpatialBatchNormalization(name='batch_norm8'),
Rectifier(),
Convolutional(filter_size=(2, 2),
step=(2, 2),
num_filters=128,
name='conv9'),
SpatialBatchNormalization(name='batch_norm9'),
Rectifier(),
Convolutional(filter_size=(3, 3),
border_mode=(1, 1),
num_filters=256,
name='conv10'),
SpatialBatchNormalization(name='batch_norm10'),
Rectifier(),
Convolutional(filter_size=(3, 3),
border_mode=(1, 1),
num_filters=256,
name='conv11'),
SpatialBatchNormalization(name='batch_norm11'),
Rectifier(),
Convolutional(filter_size=(2, 2),
step=(2, 2),
num_filters=256,
name='conv12'),
SpatialBatchNormalization(name='batch_norm12'),
Rectifier(),
],
num_channels=3,
image_size=(64, 64),
use_bias=False,
weights_init=IsotropicGaussian(0.033),
biases_init=Constant(0),
name='encoder_convnet')
encoder_convnet.initialize()
encoder_filters = numpy.prod(encoder_convnet.get_dim('output'))
encoder_mlp = MLP(
dims=[encoder_filters, 1000, 1000],
activations=[
Sequence([BatchNormalization(1000).apply,
Rectifier().apply],
name='activation1'),
Identity().apply
],
weights_init=IsotropicGaussian(0.033),
biases_init=Constant(0),
name='encoder_mlp')
encoder_mlp.initialize()
decoder_mlp = BatchNormalizedMLP(
activations=[Rectifier(), Rectifier()],
dims=[encoder_mlp.output_dim // 2, 1000, encoder_filters],
weights_init=IsotropicGaussian(0.033),
biases_init=Constant(0),
name='decoder_mlp')
decoder_mlp.initialize()
decoder_convnet = ConvolutionalSequence(
layers=[
Convolutional(filter_size=(3, 3),
border_mode=(1, 1),
num_filters=256,
name='conv1'),
SpatialBatchNormalization(name='batch_norm1'),
Rectifier(),
Convolutional(filter_size=(3, 3),
border_mode=(1, 1),
num_filters=256,
name='conv2'),
SpatialBatchNormalization(name='batch_norm2'),
Rectifier(),
ConvolutionalTranspose(filter_size=(2, 2),
step=(2, 2),
original_image_size=(8, 8),
num_filters=256,
name='conv3'),
SpatialBatchNormalization(name='batch_norm3'),
Rectifier(),
Convolutional(filter_size=(3, 3),
border_mode=(1, 1),
num_filters=128,
name='conv4'),
SpatialBatchNormalization(name='batch_norm4'),
Rectifier(),
Convolutional(filter_size=(3, 3),
border_mode=(1, 1),
num_filters=128,
name='conv5'),
SpatialBatchNormalization(name='batch_norm5'),
Rectifier(),
ConvolutionalTranspose(filter_size=(2, 2),
step=(2, 2),
original_image_size=(16, 16),
num_filters=128,
name='conv6'),
SpatialBatchNormalization(name='batch_norm6'),
Rectifier(),
Convolutional(filter_size=(3, 3),
border_mode=(1, 1),
num_filters=64,
name='conv7'),
SpatialBatchNormalization(name='batch_norm7'),
Rectifier(),
Convolutional(filter_size=(3, 3),
border_mode=(1, 1),
num_filters=64,
name='conv8'),
SpatialBatchNormalization(name='batch_norm8'),
Rectifier(),
ConvolutionalTranspose(filter_size=(2, 2),
step=(2, 2),
original_image_size=(32, 32),
num_filters=64,
name='conv9'),
SpatialBatchNormalization(name='batch_norm9'),
Rectifier(),
Convolutional(filter_size=(3, 3),
border_mode=(1, 1),
num_filters=32,
name='conv10'),
SpatialBatchNormalization(name='batch_norm10'),
Rectifier(),
Convolutional(filter_size=(3, 3),
border_mode=(1, 1),
num_filters=32,
name='conv11'),
SpatialBatchNormalization(name='batch_norm11'),
Rectifier(),
ConvolutionalTranspose(filter_size=(2, 2),
step=(2, 2),
original_image_size=(64, 64),
num_filters=32,
name='conv12'),
SpatialBatchNormalization(name='batch_norm12'),
Rectifier(),
Convolutional(filter_size=(1, 1), num_filters=3, name='conv_out'),
Logistic(),
],
num_channels=encoder_convnet.get_dim('output')[0],
image_size=encoder_convnet.get_dim('output')[1:],
use_bias=False,
weights_init=IsotropicGaussian(0.033),
biases_init=Constant(0),
name='decoder_convnet')
decoder_convnet.initialize()
return encoder_convnet, encoder_mlp, decoder_convnet, decoder_mlp
def __init__(self):
srng = MRG_RandomStreams(seed=123)
X = T.matrix('features')
self.X = X
#drop = Dropout(p_drop=0.5)
#o = drop.apply(X)
o = X
self.noisy = o
#n_hidden = 64
n_hidden = 128
n_zs = 2
self.n_zs = n_zs
self.n_hidden = n_hidden
l = Linear(input_dim=28 * 28,
output_dim=n_hidden,
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0))
l.initialize()
o = l.apply(o)
o = Tanh().apply(o)
l = Linear(input_dim=n_hidden,
output_dim=n_hidden,
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0))
l.initialize()
o = l.apply(o)
o = Tanh().apply(o)
l = Linear(input_dim=n_hidden,
output_dim=n_zs,
weights_init=IsotropicGaussian(.101),
biases_init=Constant(0))
l.initialize()
mu_encoder = l.apply(o)
l = Linear(input_dim=n_hidden,
output_dim=n_zs,
weights_init=IsotropicGaussian(0.1),
biases_init=Constant(0))
l.initialize()
log_sigma_encoder = l.apply(o)
eps = srng.normal(log_sigma_encoder.shape)
z = eps * T.exp(log_sigma_encoder) + mu_encoder
z_to_h1_decode = Linear(input_dim=n_zs,
output_dim=n_hidden,
weights_init=IsotropicGaussian(0.1),
biases_init=Constant(0))
z_to_h1_decode.initialize()
h1_decode_to_h_decode = Linear(input_dim=n_hidden,
output_dim=n_hidden,
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0))
h1_decode_to_h_decode.initialize()
#o = z_to_h_decode.apply(z)
#h_decoder = Tanh().apply(o)
h_decode_produce = Linear(input_dim=n_hidden,
output_dim=28 * 28,
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0),
name="linear4")
h_decode_produce.initialize()
#o = h_decode_produce.apply(h_decoder)
#self.produced = Sigmoid().apply(o)
seq = Sequence([
z_to_h1_decode.apply,
Tanh().apply, h1_decode_to_h_decode.apply,
Tanh().apply, h_decode_produce.apply,
Sigmoid().apply
])
seq.initialize()
self.produced = seq.apply(z)
self.cost = T.sum(T.sqr(self.produced - X)) #regular old mean squared
#self.cost = T.sum(T.nnet.binary_crossentropy(self.produced, X)) #T.sum(T.sqr(self.produced - X))
self.cost.name = "cost"
# Computed with L = 1, only one sample of produced.
logpxz = T.sum(-1 * log_sigma_encoder * T.log(2 * np.pi) -
T.sqr((self.produced - X) /
(2 * T.exp(log_sigma_encoder))))
self.variational_cost = - 0.5 * T.sum(1 + 2*log_sigma_encoder - mu_encoder * mu_encoder\
- T.exp(2 * log_sigma_encoder)) + logpxz
self.variational_cost.name = "variational_cost"
self.Z = T.matrix('z')
self.sampled = seq.apply(self.Z)
cg = ComputationGraph([self.variational_cost])
bricks = [
get_brick(var) for var in cg.variables + cg.scan_variables
if get_brick(var)
]
for i, b in enumerate(bricks):
b.name = b.name + "_" + str(i)
