def test_activations():
x = tensor.vector()
x_val = numpy.random.rand(8).astype(theano.config.floatX)
exp_x_val = numpy.exp(x_val)
assert_allclose(x_val, Identity().apply(x).eval({x: x_val}))
assert_allclose(numpy.tanh(x_val),
Tanh().apply(x).eval({x: x_val}),
rtol=1e-06)
assert_allclose(numpy.log(1 + exp_x_val),
Softplus(x).apply(x).eval({x: x_val}),
rtol=1e-6)
assert_allclose(exp_x_val / numpy.sum(exp_x_val),
Softmax(x).apply(x).eval({
x: x_val
}).flatten(),
rtol=1e-6)
assert_allclose(1.0 / (1.0 + numpy.exp(-x_val)),
Logistic(x).apply(x).eval({x: x_val}),
rtol=1e-6)
leaky_out_1 = x_val - 0.5
leaky_out_1[leaky_out_1 < 0] *= 0.01
assert_allclose(leaky_out_1,
LeakyRectifier().apply(x).eval({x: x_val - 0.5}),
rtol=1e-5)
leaky_out_2 = x_val - 0.5
leaky_out_2[leaky_out_2 < 0] *= 0.05
assert_allclose(leaky_out_2,
LeakyRectifier(leak=0.05).apply(x).eval({x: x_val - 0.5}),
rtol=1e-5)
def build_model(images, labels):
# Construct a bottom convolutional sequence
bottom_conv_sequence = convolutional_sequence((3, 3), 64, (150, 150))
bottom_conv_sequence._push_allocation_config()
# Flatten layer
flattener = Flattener()
# Construct a top MLP
conv_out_dim = numpy.prod(bottom_conv_sequence.get_dim('output'))
top_mlp = MLP([
LeakyRectifier(name='non_linear_9'),
LeakyRectifier(name='non_linear_10'),
Softmax(name='non_linear_11')
], [conv_out_dim, 2048, 612, 10],
weights_init=IsotropicGaussian(),
biases_init=Constant(1))
# Construct feedforward sequence
ss_seq = FeedforwardSequence(
[bottom_conv_sequence.apply, flattener.apply, top_mlp.apply])
ss_seq.push_initialization_config()
ss_seq.initialize()
prediction = ss_seq.apply(images)
cost = CategoricalCrossEntropy().apply(labels.flatten(), prediction)
return cost
def build_model(self, hidden_dim):
board_input = T.vector('input')
mlp = MLP(activations=[LeakyRectifier(0.1),
LeakyRectifier(0.1)],
dims=[9, hidden_dim, 9],
weights_init=IsotropicGaussian(0.00001),
biases_init=Constant(0.01))
output = mlp.apply(board_input)
masked_output = Softmax().apply(output * T.eq(board_input, 0) * 1000)
mlp.initialize()
cost, chosen = self.get_cost(masked_output)
return board_input, mlp, cost, chosen, output
def create_model_brick():
layers = [
conv_brick(2, 1, 64),
bn_brick(),
LeakyRectifier(leak=LEAK),
conv_brick(7, 2, 128),
bn_brick(),
LeakyRectifier(leak=LEAK),
conv_brick(5, 2, 256),
bn_brick(),
LeakyRectifier(leak=LEAK),
conv_brick(7, 2, 256),
bn_brick(),
LeakyRectifier(leak=LEAK),
conv_brick(4, 1, 512),
bn_brick(),
LeakyRectifier(leak=LEAK),
conv_brick(1, 1, 2 * NLAT)
]
encoder_mapping = ConvolutionalSequence(layers=layers,
num_channels=NUM_CHANNELS,
image_size=IMAGE_SIZE,
use_bias=False,
name='encoder_mapping')
encoder = GaussianConditional(encoder_mapping, name='encoder')
layers = [
conv_transpose_brick(4, 1, 512),
bn_brick(),
LeakyRectifier(leak=LEAK),
conv_transpose_brick(7, 2, 256),
bn_brick(),
LeakyRectifier(leak=LEAK),
conv_transpose_brick(5, 2, 256),
bn_brick(),
LeakyRectifier(leak=LEAK),
conv_transpose_brick(7, 2, 128),
bn_brick(),
LeakyRectifier(leak=LEAK),
conv_transpose_brick(2, 1, 64),
bn_brick(),
LeakyRectifier(leak=LEAK),
conv_brick(1, 1, NUM_CHANNELS),
Logistic()
]
decoder_mapping = ConvolutionalSequence(layers=layers,
num_channels=NLAT,
image_size=(1, 1),
use_bias=False,
name='decoder_mapping')
decoder = DeterministicConditional(decoder_mapping, name='decoder')
layers = [
conv_brick(2, 1, 64),
LeakyRectifier(leak=LEAK),
conv_brick(7, 2, 128),
bn_brick(),
LeakyRectifier(leak=LEAK),
conv_brick(5, 2, 256),
bn_brick(),
LeakyRectifier(leak=LEAK),
conv_brick(7, 2, 256),
bn_brick(),
LeakyRectifier(leak=LEAK),
conv_brick(4, 1, 512),
bn_brick(),
LeakyRectifier(leak=LEAK)
]
x_discriminator = ConvolutionalSequence(layers=layers,
num_channels=NUM_CHANNELS,
image_size=IMAGE_SIZE,
use_bias=False,
name='x_discriminator')
x_discriminator.push_allocation_config()
layers = [
conv_brick(1, 1, 1024),
LeakyRectifier(leak=LEAK),
conv_brick(1, 1, 1024),
LeakyRectifier(leak=LEAK)
]
z_discriminator = ConvolutionalSequence(layers=layers,
num_channels=NLAT,
image_size=(1, 1),
use_bias=False,
name='z_discriminator')
z_discriminator.push_allocation_config()
layers = [
conv_brick(1, 1, 2048),
LeakyRectifier(leak=LEAK),
conv_brick(1, 1, 2048),
LeakyRectifier(leak=LEAK),
conv_brick(1, 1, 1)
]
joint_discriminator = ConvolutionalSequence(
layers=layers,
num_channels=(x_discriminator.get_dim('output')[0] +
z_discriminator.get_dim('output')[0]),
image_size=(1, 1),
name='joint_discriminator')
discriminator = XZJointDiscriminator(x_discriminator,
z_discriminator,
joint_discriminator,
name='discriminator')
ali = ALI(encoder,
decoder,
discriminator,
weights_init=GAUSSIAN_INIT,
biases_init=ZERO_INIT,
name='ali')
ali.push_allocation_config()
encoder_mapping.layers[-1].use_bias = True
encoder_mapping.layers[-1].tied_biases = False
decoder_mapping.layers[-2].use_bias = True
decoder_mapping.layers[-2].tied_biases = False
x_discriminator.layers[0].use_bias = True
x_discriminator.layers[0].tied_biases = True
ali.initialize()
raw_marginals, = next(
create_celeba_data_streams(500, 500)[0].get_epoch_iterator())
b_value = get_log_odds(raw_marginals)
decoder_mapping.layers[-2].b.set_value(b_value)
return ali
idx = npr.randint(0, 9, size=5)
for n, id in enumerate(idx):
test_labels[n, id] = 1
embeddings = embedder_test(test_labels)
print(embeddings)
print(embeddings.shape)
# Generate synthetic 4D tensor
features = npr.random(size=(5, 3, 32, 32))
# Testing Encoder
layers = [
# 32 X 32 X 3
conv_brick(5, 1, 32),
bn_brick(),
LeakyRectifier(leak=LEAK),
# 28 X 28 X 32
conv_brick(4, 2, 64),
bn_brick(),
LeakyRectifier(leak=LEAK),
# 13 X 13 X 64
conv_brick(4, 1, 128),
bn_brick(),
LeakyRectifier(leak=LEAK),
# 10 X 10 X 128
conv_brick(4, 2, 256),
bn_brick(),
LeakyRectifier(leak=LEAK),
# 4 X 4 X 256
conv_brick(4, 1, 512),
bn_brick(),
def convolutional_sequence(filter_size,
num_filters,
image_size,
num_channels=1):
layers = []
layers.append(
Convolutional(filter_size=filter_size,
num_filters=num_filters,
num_channels=num_channels,
use_bias=True,
tied_biases=True,
weights_init=IsotropicGaussian(0.01),
image_size=image_size,
name='conv_1'))
layers.append(LeakyRectifier(name='non_linear_1'))
layers.append(
BatchNormalization(input_dim=layers[0].get_dim('output'),
name='batchnorm_1'))
layers.append(
Convolutional(filter_size=filter_size,
num_filters=num_filters,
num_channels=num_channels,
use_bias=True,
tied_biases=True,
weights_init=IsotropicGaussian(0.01),
image_size=image_size,
name='conv_2'))
layers.append(LeakyRectifier(name='non_linear_2'))
layers.append(
BatchNormalization(input_dim=layers[3].get_dim('output'),
name='batchnorm_2'))
layers.append(
MaxPooling(pooling_size=(2, 2),
padding=(1, 1),
weights_init=IsotropicGaussian(0.01),
name='maxpool_2'))
layers.append(
Convolutional(filter_size=filter_size,
num_filters=num_filters * 2,
num_channels=num_channels,
use_bias=True,
tied_biases=True,
weights_init=IsotropicGaussian(0.01),
image_size=image_size,
name='conv_3'))
layers.append(LeakyRectifier(name='non_linear_3'))
layers.append(
BatchNormalization(input_dim=layers[7].get_dim('output'),
name='batchnorm_3'))
layers.append(
Convolutional(filter_size=filter_size,
num_filters=num_filters * 2,
num_channels=num_channels,
use_bias=True,
tied_biases=True,
weights_init=IsotropicGaussian(0.01),
image_size=image_size,
name='conv_4'))
layers.append(LeakyRectifier(name='non_linear_4'))
layers.append(
BatchNormalization(input_dim=layers[10].get_dim('output'),
name='batchnorm_4'))
layers.append(
MaxPooling(pooling_size=(2, 2),
padding=(1, 1),
weights_init=IsotropicGaussian(0.01),
name='maxpool_4'))
layers.append(
Convolutional(filter_size=filter_size,
num_filters=num_filters * 4,
num_channels=num_channels,
use_bias=True,
tied_biases=True,
weights_init=IsotropicGaussian(0.01),
image_size=image_size,
name='conv_5'))
layers.append(LeakyRectifier(name='non_linear_5'))
layers.append(
BatchNormalization(input_dim=layers[14].get_dim('output'),
name='batchnorm_5'))
layers.append(
Convolutional(filter_size=filter_size,
num_filters=num_filters * 4,
num_channels=num_channels,
use_bias=True,
tied_biases=True,
weights_init=IsotropicGaussian(0.01),
image_size=image_size,
name='conv_6'))
layers.append(LeakyRectifier(name='non_linear_6'))
layers.append(
BatchNormalization(input_dim=layers[17].get_dim('output'),
name='batchnorm_6'))
layers.append(
MaxPooling(pooling_size=(2, 2),
padding=(1, 1),
weights_init=IsotropicGaussian(0.01),
name='maxpool_6'))
layers.append(
Convolutional(filter_size=filter_size,
num_filters=num_filters * 8,
num_channels=num_channels,
use_bias=True,
tied_biases=True,
weights_init=IsotropicGaussian(0.01),
image_size=image_size,
name='conv_7'))
layers.append(LeakyRectifier(name='non_linear_7'))
layers.append(
BatchNormalization(input_dim=layers[21].get_dim('output'),
name='batchnorm_7'))
layers.append(
Convolutional(filter_size=filter_size,
num_filters=num_filters * 8,
num_channels=num_channels,
use_bias=True,
tied_biases=True,
weights_init=IsotropicGaussian(0.01),
image_size=image_size,
name='conv_8'))
layers.append(LeakyRectifier(name='non_linear_8'))
layers.append(
BatchNormalization(input_dim=layers[24].get_dim('output'),
name='batchnorm_8'))
layers.append(
MaxPooling(pooling_size=(2, 2),
padding=(1, 1),
weights_init=IsotropicGaussian(0.01),
name='maxpool_8'))
return ConvolutionalSequence(layers,
num_channels=num_channels,
image_size=image_size,
biases_init=Uniform(width=.1))
