def test_activations(): x = tensor.vector() x_val = numpy.random.rand(8).astype(theano.config.floatX) exp_x_val = numpy.exp(x_val) assert_allclose(x_val, Identity().apply(x).eval({x: x_val})) assert_allclose(numpy.tanh(x_val), Tanh().apply(x).eval({x: x_val}), rtol=1e-06) assert_allclose(numpy.log(1 + exp_x_val), Softplus(x).apply(x).eval({x: x_val}), rtol=1e-6) assert_allclose(exp_x_val / numpy.sum(exp_x_val), Softmax(x).apply(x).eval({ x: x_val }).flatten(), rtol=1e-6) assert_allclose(1.0 / (1.0 + numpy.exp(-x_val)), Logistic(x).apply(x).eval({x: x_val}), rtol=1e-6) leaky_out_1 = x_val - 0.5 leaky_out_1[leaky_out_1 < 0] *= 0.01 assert_allclose(leaky_out_1, LeakyRectifier().apply(x).eval({x: x_val - 0.5}), rtol=1e-5) leaky_out_2 = x_val - 0.5 leaky_out_2[leaky_out_2 < 0] *= 0.05 assert_allclose(leaky_out_2, LeakyRectifier(leak=0.05).apply(x).eval({x: x_val - 0.5}), rtol=1e-5)
def build_model(images, labels): # Construct a bottom convolutional sequence bottom_conv_sequence = convolutional_sequence((3, 3), 64, (150, 150)) bottom_conv_sequence._push_allocation_config() # Flatten layer flattener = Flattener() # Construct a top MLP conv_out_dim = numpy.prod(bottom_conv_sequence.get_dim('output')) top_mlp = MLP([ LeakyRectifier(name='non_linear_9'), LeakyRectifier(name='non_linear_10'), Softmax(name='non_linear_11') ], [conv_out_dim, 2048, 612, 10], weights_init=IsotropicGaussian(), biases_init=Constant(1)) # Construct feedforward sequence ss_seq = FeedforwardSequence( [bottom_conv_sequence.apply, flattener.apply, top_mlp.apply]) ss_seq.push_initialization_config() ss_seq.initialize() prediction = ss_seq.apply(images) cost = CategoricalCrossEntropy().apply(labels.flatten(), prediction) return cost
def build_model(self, hidden_dim): board_input = T.vector('input') mlp = MLP(activations=[LeakyRectifier(0.1), LeakyRectifier(0.1)], dims=[9, hidden_dim, 9], weights_init=IsotropicGaussian(0.00001), biases_init=Constant(0.01)) output = mlp.apply(board_input) masked_output = Softmax().apply(output * T.eq(board_input, 0) * 1000) mlp.initialize() cost, chosen = self.get_cost(masked_output) return board_input, mlp, cost, chosen, output
def create_model_brick(): layers = [ conv_brick(2, 1, 64), bn_brick(), LeakyRectifier(leak=LEAK), conv_brick(7, 2, 128), bn_brick(), LeakyRectifier(leak=LEAK), conv_brick(5, 2, 256), bn_brick(), LeakyRectifier(leak=LEAK), conv_brick(7, 2, 256), bn_brick(), LeakyRectifier(leak=LEAK), conv_brick(4, 1, 512), bn_brick(), LeakyRectifier(leak=LEAK), conv_brick(1, 1, 2 * NLAT) ] encoder_mapping = ConvolutionalSequence(layers=layers, num_channels=NUM_CHANNELS, image_size=IMAGE_SIZE, use_bias=False, name='encoder_mapping') encoder = GaussianConditional(encoder_mapping, name='encoder') layers = [ conv_transpose_brick(4, 1, 512), bn_brick(), LeakyRectifier(leak=LEAK), conv_transpose_brick(7, 2, 256), bn_brick(), LeakyRectifier(leak=LEAK), conv_transpose_brick(5, 2, 256), bn_brick(), LeakyRectifier(leak=LEAK), conv_transpose_brick(7, 2, 128), bn_brick(), LeakyRectifier(leak=LEAK), conv_transpose_brick(2, 1, 64), bn_brick(), LeakyRectifier(leak=LEAK), conv_brick(1, 1, NUM_CHANNELS), Logistic() ] decoder_mapping = ConvolutionalSequence(layers=layers, num_channels=NLAT, image_size=(1, 1), use_bias=False, name='decoder_mapping') decoder = DeterministicConditional(decoder_mapping, name='decoder') layers = [ conv_brick(2, 1, 64), LeakyRectifier(leak=LEAK), conv_brick(7, 2, 128), bn_brick(), LeakyRectifier(leak=LEAK), conv_brick(5, 2, 256), bn_brick(), LeakyRectifier(leak=LEAK), conv_brick(7, 2, 256), bn_brick(), LeakyRectifier(leak=LEAK), conv_brick(4, 1, 512), bn_brick(), LeakyRectifier(leak=LEAK) ] x_discriminator = ConvolutionalSequence(layers=layers, num_channels=NUM_CHANNELS, image_size=IMAGE_SIZE, use_bias=False, name='x_discriminator') x_discriminator.push_allocation_config() layers = [ conv_brick(1, 1, 1024), LeakyRectifier(leak=LEAK), conv_brick(1, 1, 1024), LeakyRectifier(leak=LEAK) ] z_discriminator = ConvolutionalSequence(layers=layers, num_channels=NLAT, image_size=(1, 1), use_bias=False, name='z_discriminator') z_discriminator.push_allocation_config() layers = [ conv_brick(1, 1, 2048), LeakyRectifier(leak=LEAK), conv_brick(1, 1, 2048), LeakyRectifier(leak=LEAK), conv_brick(1, 1, 1) ] joint_discriminator = ConvolutionalSequence( layers=layers, num_channels=(x_discriminator.get_dim('output')[0] + z_discriminator.get_dim('output')[0]), image_size=(1, 1), name='joint_discriminator') discriminator = XZJointDiscriminator(x_discriminator, z_discriminator, joint_discriminator, name='discriminator') ali = ALI(encoder, decoder, discriminator, weights_init=GAUSSIAN_INIT, biases_init=ZERO_INIT, name='ali') ali.push_allocation_config() encoder_mapping.layers[-1].use_bias = True encoder_mapping.layers[-1].tied_biases = False decoder_mapping.layers[-2].use_bias = True decoder_mapping.layers[-2].tied_biases = False x_discriminator.layers[0].use_bias = True x_discriminator.layers[0].tied_biases = True ali.initialize() raw_marginals, = next( create_celeba_data_streams(500, 500)[0].get_epoch_iterator()) b_value = get_log_odds(raw_marginals) decoder_mapping.layers[-2].b.set_value(b_value) return ali
idx = npr.randint(0, 9, size=5) for n, id in enumerate(idx): test_labels[n, id] = 1 embeddings = embedder_test(test_labels) print(embeddings) print(embeddings.shape) # Generate synthetic 4D tensor features = npr.random(size=(5, 3, 32, 32)) # Testing Encoder layers = [ # 32 X 32 X 3 conv_brick(5, 1, 32), bn_brick(), LeakyRectifier(leak=LEAK), # 28 X 28 X 32 conv_brick(4, 2, 64), bn_brick(), LeakyRectifier(leak=LEAK), # 13 X 13 X 64 conv_brick(4, 1, 128), bn_brick(), LeakyRectifier(leak=LEAK), # 10 X 10 X 128 conv_brick(4, 2, 256), bn_brick(), LeakyRectifier(leak=LEAK), # 4 X 4 X 256 conv_brick(4, 1, 512), bn_brick(),
def convolutional_sequence(filter_size, num_filters, image_size, num_channels=1): layers = [] layers.append( Convolutional(filter_size=filter_size, num_filters=num_filters, num_channels=num_channels, use_bias=True, tied_biases=True, weights_init=IsotropicGaussian(0.01), image_size=image_size, name='conv_1')) layers.append(LeakyRectifier(name='non_linear_1')) layers.append( BatchNormalization(input_dim=layers[0].get_dim('output'), name='batchnorm_1')) layers.append( Convolutional(filter_size=filter_size, num_filters=num_filters, num_channels=num_channels, use_bias=True, tied_biases=True, weights_init=IsotropicGaussian(0.01), image_size=image_size, name='conv_2')) layers.append(LeakyRectifier(name='non_linear_2')) layers.append( BatchNormalization(input_dim=layers[3].get_dim('output'), name='batchnorm_2')) layers.append( MaxPooling(pooling_size=(2, 2), padding=(1, 1), weights_init=IsotropicGaussian(0.01), name='maxpool_2')) layers.append( Convolutional(filter_size=filter_size, num_filters=num_filters * 2, num_channels=num_channels, use_bias=True, tied_biases=True, weights_init=IsotropicGaussian(0.01), image_size=image_size, name='conv_3')) layers.append(LeakyRectifier(name='non_linear_3')) layers.append( BatchNormalization(input_dim=layers[7].get_dim('output'), name='batchnorm_3')) layers.append( Convolutional(filter_size=filter_size, num_filters=num_filters * 2, num_channels=num_channels, use_bias=True, tied_biases=True, weights_init=IsotropicGaussian(0.01), image_size=image_size, name='conv_4')) layers.append(LeakyRectifier(name='non_linear_4')) layers.append( BatchNormalization(input_dim=layers[10].get_dim('output'), name='batchnorm_4')) layers.append( MaxPooling(pooling_size=(2, 2), padding=(1, 1), weights_init=IsotropicGaussian(0.01), name='maxpool_4')) layers.append( Convolutional(filter_size=filter_size, num_filters=num_filters * 4, num_channels=num_channels, use_bias=True, tied_biases=True, weights_init=IsotropicGaussian(0.01), image_size=image_size, name='conv_5')) layers.append(LeakyRectifier(name='non_linear_5')) layers.append( BatchNormalization(input_dim=layers[14].get_dim('output'), name='batchnorm_5')) layers.append( Convolutional(filter_size=filter_size, num_filters=num_filters * 4, num_channels=num_channels, use_bias=True, tied_biases=True, weights_init=IsotropicGaussian(0.01), image_size=image_size, name='conv_6')) layers.append(LeakyRectifier(name='non_linear_6')) layers.append( BatchNormalization(input_dim=layers[17].get_dim('output'), name='batchnorm_6')) layers.append( MaxPooling(pooling_size=(2, 2), padding=(1, 1), weights_init=IsotropicGaussian(0.01), name='maxpool_6')) layers.append( Convolutional(filter_size=filter_size, num_filters=num_filters * 8, num_channels=num_channels, use_bias=True, tied_biases=True, weights_init=IsotropicGaussian(0.01), image_size=image_size, name='conv_7')) layers.append(LeakyRectifier(name='non_linear_7')) layers.append( BatchNormalization(input_dim=layers[21].get_dim('output'), name='batchnorm_7')) layers.append( Convolutional(filter_size=filter_size, num_filters=num_filters * 8, num_channels=num_channels, use_bias=True, tied_biases=True, weights_init=IsotropicGaussian(0.01), image_size=image_size, name='conv_8')) layers.append(LeakyRectifier(name='non_linear_8')) layers.append( BatchNormalization(input_dim=layers[24].get_dim('output'), name='batchnorm_8')) layers.append( MaxPooling(pooling_size=(2, 2), padding=(1, 1), weights_init=IsotropicGaussian(0.01), name='maxpool_8')) return ConvolutionalSequence(layers, num_channels=num_channels, image_size=image_size, biases_init=Uniform(width=.1))