def build_pipeline(self, input_shape, params): from blocks.bricks import Tanh, Sequence from blocks.bricks.conv import Convolutional, MaxPooling from blocks.initialization import Uniform _, num_channels, input_len, num_freqs = input_shape # bc01 # Note: this layer is linear conv = Convolutional( name='conv', filter_size=(params['filter_width_time'], params['filter_width_freq']), num_filters=params['num_components'], # out num_channels=num_channels, # in image_size=(input_len, num_freqs), weights_init=Uniform(mean=0, std=0.01), use_bias=params['use_bias']) tanh = Tanh() # optional pooling if params['pool_width_time'] > 1 or params['pool_width_freq'] > 1: pool = MaxPooling( (params['pool_width_time'], params['pool_width_freq']), step=(params['pool_stride_time'], params['pool_stride_freq'])) pipeline = Sequence([conv.apply, tanh.apply, pool.apply], name='pipeline') else: pipeline = Sequence([conv.apply, tanh.apply], name='pipeline') pipeline.initialize() return pipeline
def test_sequence_variable_inputs(): x, y = tensor.matrix(), tensor.matrix() parallel_1 = Parallel(input_names=['input_1', 'input_2'], input_dims=dict(input_1=4, input_2=5), output_dims=dict(input_1=3, input_2=2), prototype=Linear(), weights_init=Constant(2), biases_init=Constant(1)) parallel_2 = Parallel(input_names=['input_1', 'input_2'], input_dims=dict(input_1=3, input_2=2), output_dims=dict(input_1=5, input_2=4), prototype=Linear(), weights_init=Constant(2), biases_init=Constant(1)) sequence = Sequence([parallel_1.apply, parallel_2.apply]) sequence.initialize() new_x, new_y = sequence.apply(x, y) x_val = numpy.ones((4, 4), dtype=theano.config.floatX) y_val = numpy.ones((4, 5), dtype=theano.config.floatX) assert_allclose( new_x.eval({x: x_val}), (x_val.dot(2 * numpy.ones((4, 3))) + numpy.ones((4, 3))).dot( 2 * numpy.ones((3, 5))) + numpy.ones((4, 5))) assert_allclose( new_y.eval({y: y_val}), (y_val.dot(2 * numpy.ones((5, 2))) + numpy.ones((4, 2))).dot( 2 * numpy.ones((2, 4))) + numpy.ones((4, 4)))
def check(bricks): sequence = Sequence(bricks) sequence.initialize() y = sequence.apply(x) x_val = numpy.ones((4, 16), dtype=theano.config.floatX) assert_allclose(y.eval({x: x_val}), (x_val.dot(2 * numpy.ones( (16, 8))) + numpy.ones((4, 8))).dot(3 * numpy.ones( (8, 4))) + 4 * numpy.ones((4, 4)))
def setUp(self): self.mlp = MLP([ Sequence([Identity(name='id1').apply, Tanh(name='tanh1').apply], name='sequence1'), Sequence([ Logistic(name='logistic1').apply, Identity(name='id2').apply, Tanh(name='tanh2').apply ], name='sequence2'), Logistic(name='logistic2'), Sequence([ Sequence([Logistic(name='logistic3').apply], name='sequence4').apply ], name='sequence3') ], [10, 5, 9, 5, 9])
def __init__(self, visual_dim, textual_dim, output_dim, hidden_size, init_ranges, **kwargs): (visual_init_range, textual_init_range, gbu_init_range, linear_range_1, linear_range_2, linear_range_3) = init_ranges visual_mlp = Sequence([ BatchNormalization(input_dim=visual_dim).apply, Linear(visual_dim, hidden_size, use_bias=False, weights_init=initialization.Uniform( width=visual_init_range)).apply, ], name='visual_mlp') textual_mlp = Sequence([ BatchNormalization(input_dim=textual_dim).apply, Linear(textual_dim, hidden_size, use_bias=False, weights_init=initialization.Uniform( width=textual_init_range)).apply, ], name='textual_mlp') gbu = GatedBimodal( hidden_size, weights_init=initialization.Uniform(width=gbu_init_range)) logistic_mlp = MLPGenreClassifier( hidden_size, output_dim, hidden_size, [linear_range_1, linear_range_2, linear_range_3]) # logistic_mlp = Sequence([ # BatchNormalization(input_dim=hidden_size, name='bn1').apply, # Linear(hidden_size, output_dim, name='linear_output', use_bias=False, # weights_init=initialization.Uniform(width=linear_range_1)).apply, # Logistic().apply #], name='logistic_mlp') children = [visual_mlp, textual_mlp, gbu, logistic_mlp] kwargs.setdefault('use_bias', False) kwargs.setdefault('children', children) super(GatedClassifier, self).__init__(**kwargs)
def test_sequence(): x = tensor.matrix() linear_1 = Linear(input_dim=16, output_dim=8, weights_init=Constant(2), biases_init=Constant(1)) linear_2 = Linear(input_dim=8, output_dim=4, weights_init=Constant(3), biases_init=Constant(4)) sequence = Sequence([linear_1.apply, linear_2.apply]) sequence.initialize() y = sequence.apply(x) x_val = numpy.ones((4, 16), dtype=theano.config.floatX) assert_allclose( y.eval({x: x_val}), (x_val.dot(2 * numpy.ones((16, 8))) + numpy.ones((4, 8))).dot( 3 * numpy.ones((8, 4))) + 4 * numpy.ones((4, 4)))
def test_batch_normalization_nested(): x = tensor.tensor4() eps = 1e-4 r_dims = (0, 2, 3) batch_dims = (5, 4, 3, 2) bn = BatchNormalization(input_dim=batch_dims[1:], broadcastable=(False, True, True), epsilon=eps) seq = Sequence([bn.apply, Tanh().apply]) seq.initialize() with batch_normalization(seq): y = seq.apply(x) rng = numpy.random.RandomState((2016, 1, 18)) x_ = rng.uniform(size=batch_dims).astype(theano.config.floatX) y_ = y.eval({x: x_}) y_expected = numpy.tanh( (x_ - x_.mean(axis=r_dims, keepdims=True)) / numpy.sqrt(x_.var(axis=r_dims, keepdims=True) + eps)) assert_allclose(y_, y_expected, rtol=1e-4)
def test_sequence_variable_outputs(): x = tensor.matrix() linear_1 = Linear(input_dim=16, output_dim=8, weights_init=Constant(2), biases_init=Constant(1)) fork = Fork(input_dim=8, output_names=['linear_2_1', 'linear_2_2'], output_dims=[4, 5], prototype=Linear(), weights_init=Constant(3), biases_init=Constant(4)) sequence = Sequence([linear_1.apply, fork.apply]) sequence.initialize() y_1, y_2 = sequence.apply(x) x_val = numpy.ones((4, 16), dtype=theano.config.floatX) assert_allclose( y_1.eval({x: x_val}), (x_val.dot(2 * numpy.ones((16, 8))) + numpy.ones((4, 8))).dot( 3 * numpy.ones((8, 4))) + 4 * numpy.ones((4, 4))) assert_allclose( y_2.eval({x: x_val}), (x_val.dot(2 * numpy.ones((16, 8))) + numpy.ones((4, 8))).dot( 3 * numpy.ones((8, 5))) + 4 * numpy.ones((4, 5)))
def build_autoencoder(features, labels_num, labels_cat): mlp_bottom = MLP(activations=[ Rectifier(), Rectifier(), Rectifier(), Rectifier(), Rectifier() ], dims=[24033, 5000, 1000, 100, 1000, 5000], weights_init=IsotropicGaussian(), biases_init=Constant(1)) mlp_bottom.initialize() mlp_top = build_top_mlp() mlp_top.push_initialization_config() mlp_top.initialize() # a = mlp_bottom.apply(features) # b = mlp_top.apply(a) # Construct feedforward sequence ss_seq = Sequence([mlp_bottom.apply, mlp_top.apply]) ss_seq.push_initialization_config() ss_seq.initialize() [outputs_numerical, outputs_categorical] = ss_seq.apply(features) cost = SquaredError().apply( labels_num, outputs_numerical) + BinaryCrossEntropy().apply( labels_cat, outputs_categorical) cg = ComputationGraph(cost) #cg_dropout0 = apply_dropout(cg, [VariableFilter(roles=[INPUT])(cg.variables)[1]], .2) #cg_dropout1 = apply_dropout(cg, [VariableFilter(roles=[OUTPUT])(cg.variables)[1], VariableFilter(roles=[OUTPUT])(cg.variables)[3]], .2) #cost_dropout1 = cg_dropout1.outputs[0] return cost, cg.parameters
def __init__(self, visual_dim, textual_dim, output_dim, hidden_size, init_ranges, **kwargs): (visual_range, textual_range, linear_range_1, linear_range_2, linear_range_3) = init_ranges manager_dim = visual_dim + textual_dim visual_mlp = MLPGenreClassifier( visual_dim, output_dim, hidden_size, [linear_range_1, linear_range_2, linear_range_3], name='visual_mlp') textual_mlp = MLPGenreClassifier( textual_dim, output_dim, hidden_size, [linear_range_1, linear_range_2, linear_range_3], name='textual_mlp') # manager_mlp = MLPGenreClassifier(manager_dim, 2, hidden_size, [ # linear_range_1, linear_range_2, linear_range_3], output_act=Softmax, # name='manager_mlp') bn = BatchNormalization(input_dim=manager_dim, name='bn3') manager_mlp = Sequence([ Linear(manager_dim, 2, name='linear_output', use_bias=False, weights_init=initialization.Uniform( width=linear_range_1)).apply, ], name='manager_mlp') fork = Fork( input_dim=manager_dim, output_dims=[2] * output_dim, prototype=manager_mlp, output_names=['linear_' + str(i) for i in range(output_dim)]) children = [visual_mlp, textual_mlp, fork, bn, NDimensionalSoftmax()] kwargs.setdefault('use_bias', False) kwargs.setdefault('children', children) super(MoEClassifier, self).__init__(**kwargs)
def test_apply_batch_normalization_nested(): x = tensor.matrix() eps = 1e-8 batch_dims = (3, 9) bn = BatchNormalization(input_dim=5, epsilon=eps) mlp = MLP([Sequence([bn.apply, Tanh().apply])], [9, 5], weights_init=Constant(0.4), biases_init=Constant(1)) mlp.initialize() y = mlp.apply(x) cg = apply_batch_normalization(ComputationGraph([y])) y_bn = cg.outputs[0] rng = numpy.random.RandomState((2016, 1, 18)) x_ = rng.uniform(size=batch_dims).astype(theano.config.floatX) y_ = y_bn.eval({x: x_}) W_, b_ = map( lambda s: (getattr(mlp.linear_transformations[0], s).get_value(borrow=True)), ['W', 'b']) z_ = numpy.dot(x_, W_) + b_ y_expected = numpy.tanh( (z_ - z_.mean(axis=0)) / numpy.sqrt(z_.var(axis=0) + eps)) assert_allclose(y_, y_expected, rtol=1e-3)
def create_model_bricks(z_dim, image_size, depth): g_image_size = image_size g_image_size2 = g_image_size / 2 g_image_size3 = g_image_size / 4 g_image_size4 = g_image_size / 8 g_image_size5 = g_image_size / 16 encoder_layers = [] if depth > 0: encoder_layers = encoder_layers + [ Convolutional(filter_size=(3, 3), border_mode=(1, 1), num_filters=32, name='conv1'), SpatialBatchNormalization(name='batch_norm1'), Rectifier(), Convolutional(filter_size=(3, 3), border_mode=(1, 1), num_filters=32, name='conv2'), SpatialBatchNormalization(name='batch_norm2'), Rectifier(), Convolutional( filter_size=(2, 2), step=(2, 2), num_filters=32, name='conv3'), SpatialBatchNormalization(name='batch_norm3'), Rectifier() ] if depth > 1: encoder_layers = encoder_layers + [ Convolutional(filter_size=(3, 3), border_mode=(1, 1), num_filters=64, name='conv4'), SpatialBatchNormalization(name='batch_norm4'), Rectifier(), Convolutional(filter_size=(3, 3), border_mode=(1, 1), num_filters=64, name='conv5'), SpatialBatchNormalization(name='batch_norm5'), Rectifier(), Convolutional( filter_size=(2, 2), step=(2, 2), num_filters=64, name='conv6'), SpatialBatchNormalization(name='batch_norm6'), Rectifier() ] if depth > 2: encoder_layers = encoder_layers + [ Convolutional(filter_size=(3, 3), border_mode=(1, 1), num_filters=128, name='conv7'), SpatialBatchNormalization(name='batch_norm7'), Rectifier(), Convolutional(filter_size=(3, 3), border_mode=(1, 1), num_filters=128, name='conv8'), SpatialBatchNormalization(name='batch_norm8'), Rectifier(), Convolutional( filter_size=(2, 2), step=(2, 2), num_filters=128, name='conv9'), SpatialBatchNormalization(name='batch_norm9'), Rectifier() ] if depth > 3: encoder_layers = encoder_layers + [ Convolutional(filter_size=(3, 3), border_mode=(1, 1), num_filters=256, name='conv10'), SpatialBatchNormalization(name='batch_norm10'), Rectifier(), Convolutional(filter_size=(3, 3), border_mode=(1, 1), num_filters=256, name='conv11'), SpatialBatchNormalization(name='batch_norm11'), Rectifier(), Convolutional(filter_size=(2, 2), step=(2, 2), num_filters=256, name='conv12'), SpatialBatchNormalization(name='batch_norm12'), Rectifier(), ] if depth > 4: encoder_layers = encoder_layers + [ Convolutional(filter_size=(3, 3), border_mode=(1, 1), num_filters=512, name='conv13'), SpatialBatchNormalization(name='batch_norm13'), Rectifier(), Convolutional(filter_size=(3, 3), border_mode=(1, 1), num_filters=512, name='conv14'), SpatialBatchNormalization(name='batch_norm14'), Rectifier(), Convolutional(filter_size=(2, 2), step=(2, 2), num_filters=512, name='conv15'), SpatialBatchNormalization(name='batch_norm15'), Rectifier() ] decoder_layers = [] if depth > 4: decoder_layers = decoder_layers + [ Convolutional(filter_size=(3, 3), border_mode=(1, 1), num_filters=512, name='conv_n3'), SpatialBatchNormalization(name='batch_norm_n3'), Rectifier(), Convolutional(filter_size=(3, 3), border_mode=(1, 1), num_filters=512, name='conv_n2'), SpatialBatchNormalization(name='batch_norm_n2'), Rectifier(), ConvolutionalTranspose( filter_size=(2, 2), step=(2, 2), original_image_size=(g_image_size5, g_image_size5), num_filters=512, name='conv_n1'), SpatialBatchNormalization(name='batch_norm_n1'), Rectifier() ] if depth > 3: decoder_layers = decoder_layers + [ Convolutional(filter_size=(3, 3), border_mode=(1, 1), num_filters=256, name='conv1'), SpatialBatchNormalization(name='batch_norm1'), Rectifier(), Convolutional(filter_size=(3, 3), border_mode=(1, 1), num_filters=256, name='conv2'), SpatialBatchNormalization(name='batch_norm2'), Rectifier(), ConvolutionalTranspose( filter_size=(2, 2), step=(2, 2), original_image_size=(g_image_size4, g_image_size4), num_filters=256, name='conv3'), SpatialBatchNormalization(name='batch_norm3'), Rectifier() ] if depth > 2: decoder_layers = decoder_layers + [ Convolutional(filter_size=(3, 3), border_mode=(1, 1), num_filters=128, name='conv4'), SpatialBatchNormalization(name='batch_norm4'), Rectifier(), Convolutional(filter_size=(3, 3), border_mode=(1, 1), num_filters=128, name='conv5'), SpatialBatchNormalization(name='batch_norm5'), Rectifier(), ConvolutionalTranspose( filter_size=(2, 2), step=(2, 2), original_image_size=(g_image_size3, g_image_size3), num_filters=128, name='conv6'), SpatialBatchNormalization(name='batch_norm6'), Rectifier() ] if depth > 1: decoder_layers = decoder_layers + [ Convolutional(filter_size=(3, 3), border_mode=(1, 1), num_filters=64, name='conv7'), SpatialBatchNormalization(name='batch_norm7'), Rectifier(), Convolutional(filter_size=(3, 3), border_mode=(1, 1), num_filters=64, name='conv8'), SpatialBatchNormalization(name='batch_norm8'), Rectifier(), ConvolutionalTranspose( filter_size=(2, 2), step=(2, 2), original_image_size=(g_image_size2, g_image_size2), num_filters=64, name='conv9'), SpatialBatchNormalization(name='batch_norm9'), Rectifier() ] if depth > 0: decoder_layers = decoder_layers + [ Convolutional(filter_size=(3, 3), border_mode=(1, 1), num_filters=32, name='conv10'), SpatialBatchNormalization(name='batch_norm10'), Rectifier(), Convolutional(filter_size=(3, 3), border_mode=(1, 1), num_filters=32, name='conv11'), SpatialBatchNormalization(name='batch_norm11'), Rectifier(), ConvolutionalTranspose( filter_size=(2, 2), step=(2, 2), original_image_size=(g_image_size, g_image_size), num_filters=32, name='conv12'), SpatialBatchNormalization(name='batch_norm12'), Rectifier() ] decoder_layers = decoder_layers + [ Convolutional(filter_size=(1, 1), num_filters=3, name='conv_out'), Logistic() ] print("creating model of depth {} with {} encoder and {} decoder layers". format(depth, len(encoder_layers), len(decoder_layers))) encoder_convnet = ConvolutionalSequence( layers=encoder_layers, num_channels=3, image_size=(g_image_size, g_image_size), use_bias=False, weights_init=IsotropicGaussian(0.033), biases_init=Constant(0), name='encoder_convnet') encoder_convnet.initialize() encoder_filters = numpy.prod(encoder_convnet.get_dim('output')) encoder_mlp = MLP( dims=[encoder_filters, 1000, z_dim], activations=[ Sequence([BatchNormalization(1000).apply, Rectifier().apply], name='activation1'), Identity().apply ], weights_init=IsotropicGaussian(0.033), biases_init=Constant(0), name='encoder_mlp') encoder_mlp.initialize() decoder_mlp = BatchNormalizedMLP( activations=[Rectifier(), Rectifier()], dims=[encoder_mlp.output_dim // 2, 1000, encoder_filters], weights_init=IsotropicGaussian(0.033), biases_init=Constant(0), name='decoder_mlp') decoder_mlp.initialize() decoder_convnet = ConvolutionalSequence( layers=decoder_layers, num_channels=encoder_convnet.get_dim('output')[0], image_size=encoder_convnet.get_dim('output')[1:], use_bias=False, weights_init=IsotropicGaussian(0.033), biases_init=Constant(0), name='decoder_convnet') decoder_convnet.initialize() return encoder_convnet, encoder_mlp, decoder_convnet, decoder_mlp
def __init__(self): srng = MRG_RandomStreams(seed=123) X = T.matrix('features') self.X = X #drop = Dropout(p_drop=0.5) #o = drop.apply(X) o = (X - 128) / 128.0 self.scaled = o #n_hidden = 64 n_hidden = 2048 * 2 n_zs = 1024 self.n_zs = n_zs self.n_hidden = n_hidden l = Linear(input_dim=32 * 32 * 3, output_dim=n_hidden, weights_init=IsotropicGaussian(0.01), biases_init=Constant(0)) l.initialize() o = l.apply(o) o = Rectifier().apply(o) l = Linear(input_dim=n_hidden, output_dim=n_hidden, weights_init=IsotropicGaussian(0.01), biases_init=Constant(0)) l.initialize() o = l.apply(o) o = Rectifier().apply(o) l = Linear(input_dim=n_hidden, output_dim=n_zs, weights_init=IsotropicGaussian(0.01), biases_init=Constant(0)) l.initialize() mu_encoder = l.apply(o) l = Linear(input_dim=n_hidden, output_dim=n_zs, weights_init=IsotropicGaussian(0.01), biases_init=Constant(0)) l.initialize() log_sigma_encoder = l.apply(o) eps = srng.normal(log_sigma_encoder.shape) z = eps * T.exp(log_sigma_encoder) + mu_encoder z_to_h1_decode = Linear(input_dim=n_zs, output_dim=n_hidden, weights_init=IsotropicGaussian(0.01), biases_init=Constant(0)) z_to_h1_decode.initialize() h1_decode_to_h_decode = Linear(input_dim=n_hidden, output_dim=n_hidden, weights_init=IsotropicGaussian(0.01), biases_init=Constant(0)) h1_decode_to_h_decode.initialize() h_decode_produce = Linear(input_dim=n_hidden, output_dim=32 * 32 * 3, weights_init=IsotropicGaussian(0.01), biases_init=Constant(0), name="linear4") h_decode_produce.initialize() #o = h_decode_produce.apply(h_decoder) h_decode_produce = Linear(input_dim=n_hidden, output_dim=32 * 32 * 3, weights_init=IsotropicGaussian(0.01), biases_init=Constant(0), name="linear4") h_decode_produce.initialize() #self.produced = Sigmoid().apply(o) seq = Sequence([ z_to_h1_decode.apply, Rectifier().apply, h1_decode_to_h_decode.apply, Rectifier().apply, h_decode_produce.apply, Sigmoid().apply ]) seq.initialize() self.produced = seq.apply(z) self.cost = T.mean(T.sqr(self.produced - self.scaled)) #self.cost = T.sum(T.nnet.binary_crossentropy(self.produced, self.scaled)) #T.sum(T.sqr(self.produced - self.scaled)) self.cost.name = "cost" self.variational_cost = - 0.5 * T.mean(1 + 2*log_sigma_encoder - mu_encoder * mu_encoder\ - T.exp(2 * log_sigma_encoder)) + self.cost self.variational_cost.name = "variational_cost" self.Z = T.matrix('z') self.sampled = seq.apply(self.Z) cg = ComputationGraph([self.variational_cost]) bricks = [ get_brick(var) for var in cg.variables + cg.scan_variables if get_brick(var) ] for i, b in enumerate(bricks): b.name = b.name + "_" + str(i)
def create_model_bricks(): encoder_convnet = ConvolutionalSequence( layers=[ Convolutional(filter_size=(3, 3), border_mode=(1, 1), num_filters=32, name='conv1'), SpatialBatchNormalization(name='batch_norm1'), Rectifier(), Convolutional(filter_size=(3, 3), border_mode=(1, 1), num_filters=32, name='conv2'), SpatialBatchNormalization(name='batch_norm2'), Rectifier(), Convolutional(filter_size=(2, 2), step=(2, 2), num_filters=32, name='conv3'), SpatialBatchNormalization(name='batch_norm3'), Rectifier(), Convolutional(filter_size=(3, 3), border_mode=(1, 1), num_filters=64, name='conv4'), SpatialBatchNormalization(name='batch_norm4'), Rectifier(), Convolutional(filter_size=(3, 3), border_mode=(1, 1), num_filters=64, name='conv5'), SpatialBatchNormalization(name='batch_norm5'), Rectifier(), Convolutional(filter_size=(2, 2), step=(2, 2), num_filters=64, name='conv6'), SpatialBatchNormalization(name='batch_norm6'), Rectifier(), Convolutional(filter_size=(3, 3), border_mode=(1, 1), num_filters=128, name='conv7'), SpatialBatchNormalization(name='batch_norm7'), Rectifier(), Convolutional(filter_size=(3, 3), border_mode=(1, 1), num_filters=128, name='conv8'), SpatialBatchNormalization(name='batch_norm8'), Rectifier(), Convolutional(filter_size=(2, 2), step=(2, 2), num_filters=128, name='conv9'), SpatialBatchNormalization(name='batch_norm9'), Rectifier(), Convolutional(filter_size=(3, 3), border_mode=(1, 1), num_filters=256, name='conv10'), SpatialBatchNormalization(name='batch_norm10'), Rectifier(), Convolutional(filter_size=(3, 3), border_mode=(1, 1), num_filters=256, name='conv11'), SpatialBatchNormalization(name='batch_norm11'), Rectifier(), Convolutional(filter_size=(2, 2), step=(2, 2), num_filters=256, name='conv12'), SpatialBatchNormalization(name='batch_norm12'), Rectifier(), ], num_channels=3, image_size=(64, 64), use_bias=False, weights_init=IsotropicGaussian(0.033), biases_init=Constant(0), name='encoder_convnet') encoder_convnet.initialize() encoder_filters = numpy.prod(encoder_convnet.get_dim('output')) encoder_mlp = MLP( dims=[encoder_filters, 1000, 1000], activations=[ Sequence([BatchNormalization(1000).apply, Rectifier().apply], name='activation1'), Identity().apply ], weights_init=IsotropicGaussian(0.033), biases_init=Constant(0), name='encoder_mlp') encoder_mlp.initialize() decoder_mlp = BatchNormalizedMLP( activations=[Rectifier(), Rectifier()], dims=[encoder_mlp.output_dim // 2, 1000, encoder_filters], weights_init=IsotropicGaussian(0.033), biases_init=Constant(0), name='decoder_mlp') decoder_mlp.initialize() decoder_convnet = ConvolutionalSequence( layers=[ Convolutional(filter_size=(3, 3), border_mode=(1, 1), num_filters=256, name='conv1'), SpatialBatchNormalization(name='batch_norm1'), Rectifier(), Convolutional(filter_size=(3, 3), border_mode=(1, 1), num_filters=256, name='conv2'), SpatialBatchNormalization(name='batch_norm2'), Rectifier(), ConvolutionalTranspose(filter_size=(2, 2), step=(2, 2), original_image_size=(8, 8), num_filters=256, name='conv3'), SpatialBatchNormalization(name='batch_norm3'), Rectifier(), Convolutional(filter_size=(3, 3), border_mode=(1, 1), num_filters=128, name='conv4'), SpatialBatchNormalization(name='batch_norm4'), Rectifier(), Convolutional(filter_size=(3, 3), border_mode=(1, 1), num_filters=128, name='conv5'), SpatialBatchNormalization(name='batch_norm5'), Rectifier(), ConvolutionalTranspose(filter_size=(2, 2), step=(2, 2), original_image_size=(16, 16), num_filters=128, name='conv6'), SpatialBatchNormalization(name='batch_norm6'), Rectifier(), Convolutional(filter_size=(3, 3), border_mode=(1, 1), num_filters=64, name='conv7'), SpatialBatchNormalization(name='batch_norm7'), Rectifier(), Convolutional(filter_size=(3, 3), border_mode=(1, 1), num_filters=64, name='conv8'), SpatialBatchNormalization(name='batch_norm8'), Rectifier(), ConvolutionalTranspose(filter_size=(2, 2), step=(2, 2), original_image_size=(32, 32), num_filters=64, name='conv9'), SpatialBatchNormalization(name='batch_norm9'), Rectifier(), Convolutional(filter_size=(3, 3), border_mode=(1, 1), num_filters=32, name='conv10'), SpatialBatchNormalization(name='batch_norm10'), Rectifier(), Convolutional(filter_size=(3, 3), border_mode=(1, 1), num_filters=32, name='conv11'), SpatialBatchNormalization(name='batch_norm11'), Rectifier(), ConvolutionalTranspose(filter_size=(2, 2), step=(2, 2), original_image_size=(64, 64), num_filters=32, name='conv12'), SpatialBatchNormalization(name='batch_norm12'), Rectifier(), Convolutional(filter_size=(1, 1), num_filters=3, name='conv_out'), Logistic(), ], num_channels=encoder_convnet.get_dim('output')[0], image_size=encoder_convnet.get_dim('output')[1:], use_bias=False, weights_init=IsotropicGaussian(0.033), biases_init=Constant(0), name='decoder_convnet') decoder_convnet.initialize() return encoder_convnet, encoder_mlp, decoder_convnet, decoder_mlp
def __init__(self): srng = MRG_RandomStreams(seed=123) X = T.matrix('features') self.X = X #drop = Dropout(p_drop=0.5) #o = drop.apply(X) o = X self.noisy = o #n_hidden = 64 n_hidden = 128 n_zs = 2 self.n_zs = n_zs self.n_hidden = n_hidden l = Linear(input_dim=28 * 28, output_dim=n_hidden, weights_init=IsotropicGaussian(0.01), biases_init=Constant(0)) l.initialize() o = l.apply(o) o = Tanh().apply(o) l = Linear(input_dim=n_hidden, output_dim=n_hidden, weights_init=IsotropicGaussian(0.01), biases_init=Constant(0)) l.initialize() o = l.apply(o) o = Tanh().apply(o) l = Linear(input_dim=n_hidden, output_dim=n_zs, weights_init=IsotropicGaussian(.101), biases_init=Constant(0)) l.initialize() mu_encoder = l.apply(o) l = Linear(input_dim=n_hidden, output_dim=n_zs, weights_init=IsotropicGaussian(0.1), biases_init=Constant(0)) l.initialize() log_sigma_encoder = l.apply(o) eps = srng.normal(log_sigma_encoder.shape) z = eps * T.exp(log_sigma_encoder) + mu_encoder z_to_h1_decode = Linear(input_dim=n_zs, output_dim=n_hidden, weights_init=IsotropicGaussian(0.1), biases_init=Constant(0)) z_to_h1_decode.initialize() h1_decode_to_h_decode = Linear(input_dim=n_hidden, output_dim=n_hidden, weights_init=IsotropicGaussian(0.01), biases_init=Constant(0)) h1_decode_to_h_decode.initialize() #o = z_to_h_decode.apply(z) #h_decoder = Tanh().apply(o) h_decode_produce = Linear(input_dim=n_hidden, output_dim=28 * 28, weights_init=IsotropicGaussian(0.01), biases_init=Constant(0), name="linear4") h_decode_produce.initialize() #o = h_decode_produce.apply(h_decoder) #self.produced = Sigmoid().apply(o) seq = Sequence([ z_to_h1_decode.apply, Tanh().apply, h1_decode_to_h_decode.apply, Tanh().apply, h_decode_produce.apply, Sigmoid().apply ]) seq.initialize() self.produced = seq.apply(z) self.cost = T.sum(T.sqr(self.produced - X)) #regular old mean squared #self.cost = T.sum(T.nnet.binary_crossentropy(self.produced, X)) #T.sum(T.sqr(self.produced - X)) self.cost.name = "cost" # Computed with L = 1, only one sample of produced. logpxz = T.sum(-1 * log_sigma_encoder * T.log(2 * np.pi) - T.sqr((self.produced - X) / (2 * T.exp(log_sigma_encoder)))) self.variational_cost = - 0.5 * T.sum(1 + 2*log_sigma_encoder - mu_encoder * mu_encoder\ - T.exp(2 * log_sigma_encoder)) + logpxz self.variational_cost.name = "variational_cost" self.Z = T.matrix('z') self.sampled = seq.apply(self.Z) cg = ComputationGraph([self.variational_cost]) bricks = [ get_brick(var) for var in cg.variables + cg.scan_variables if get_brick(var) ] for i, b in enumerate(bricks): b.name = b.name + "_" + str(i)